diff --git a/.DS_Store b/.DS_Store index 254976283..91ea25e8f 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/backend/.env.example b/backend/.env.example index c1eb0b480..438ea21cc 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -57,6 +57,10 @@ BACKEND_CV_STORAGE_BUCKET= BACKEND_CV_MAX_UPLOADS_PER_USER= BACKEND_CV_RATE_LIMIT_PER_MINUTE= +# SSE streaming (optional; defaults: chunk size 10, mode chars, delay 12ms) +BACKEND_STREAM_CHUNK_SIZE= +BACKEND_STREAM_CHUNK_MODE= +BACKEND_STREAM_DELTA_DELAY_MS= # Locales BACKEND_LANGUAGE_CONFIG='{"default_locale":"en-US","available_locales":[{"locale":"en-US","date_format":"MM/DD/YYYY"}]}' diff --git a/backend/app/agent/agent_director/llm_agent_director.py b/backend/app/agent/agent_director/llm_agent_director.py index efb7fcbed..4385ffef2 100644 --- a/backend/app/agent/agent_director/llm_agent_director.py +++ b/backend/app/agent/agent_director/llm_agent_director.py @@ -18,7 +18,7 @@ from app.vector_search.vector_search_dependencies import SearchServices from app.user_recommendations.services.service import IUserRecommendationsService from app.i18n.translation_service import t -from app.context_vars import phase_ctx_var, agent_type_ctx_var # for observability logging +from app.context_vars import phase_ctx_var, agent_type_ctx_var, get_stream_sink class LLMAgentDirector(AbstractAgentDirector): """ @@ -238,6 +238,21 @@ async def execute(self, user_input: AgentInput) -> AgentOutput: phase=self._state.current_phase, context=context) self._logger.debug("Running agent: %s", {suitable_agent_type}) + _AGENT_STATUS_LABELS = { + AgentType.WELCOME_AGENT: "preparing_welcome", + AgentType.EXPLORE_EXPERIENCES_AGENT: "exploring_experiences", + AgentType.EXPLORE_SKILLS_AGENT: "exploring_skills_in_depth", + AgentType.PREFERENCE_ELICITATION_AGENT: "understanding_preferences", + AgentType.RECOMMENDER_ADVISOR_AGENT: "preparing_recommendations", + AgentType.FAREWELL_AGENT: "wrapping_up", + } + stream_sink = get_stream_sink() + if stream_sink is not None: + await stream_sink.emit_status_update( + label=_AGENT_STATUS_LABELS.get(suitable_agent_type, "running_agent"), + status="started", + agent_type=suitable_agent_type.value, + ) # Track routed agent for sticky routing and observability if suitable_agent_type != self._state.last_routed_agent: @@ -253,6 +268,14 @@ async def execute(self, user_input: AgentInput) -> AgentOutput: # Perform the task agent_output = await agent_for_task.execute(clean_input, context) + stream_sink = get_stream_sink() + if stream_sink is not None: + await stream_sink.emit_status_update( + label="running_agent", + status="completed", + agent_type=suitable_agent_type.value, + detail="response_ready", + ) if not agent_for_task.is_responsible_for_conversation_history(): await self._conversation_manager.update_history(clean_input, agent_output) @@ -276,6 +299,14 @@ async def execute(self, user_input: AgentInput) -> AgentOutput: if transitioned_to_new_phase: self._state.current_phase = new_phase phase_ctx_var.set(new_phase.value if new_phase else ":none:") + stream_sink = get_stream_sink() + if stream_sink is not None: + await stream_sink.emit_status_update( + label="phase_transition", + status="completed", + agent_type=suitable_agent_type.value, + detail=new_phase.name, + ) if get_application_config().inline_phase_transition: # Inline transition: skip the (silence) loop re-invocation. diff --git a/backend/app/agent/collect_experiences_agent/_conversation_llm.py b/backend/app/agent/collect_experiences_agent/_conversation_llm.py index 6b520c3a4..bb7e93ec1 100644 --- a/backend/app/agent/collect_experiences_agent/_conversation_llm.py +++ b/backend/app/agent/collect_experiences_agent/_conversation_llm.py @@ -17,6 +17,7 @@ from app.countries import Country from app.i18n.locale_date_format import get_locale_date_format, format_date_value_for_locale from app.i18n.translation_service import t +from app.conversations.streaming import ConversationStreamingSink from common_libs.llm.generative_models import GeminiGenerativeLLM from common_libs.llm.models_utils import LLMConfig, LLMResponse, get_config_variation, LLMInput from common_libs.retry import Retry @@ -24,6 +25,10 @@ _NO_EXPERIENCE_COLLECTED = "No experience data has been collected yet" _FINAL_MESSAGE = "Thank you for sharing your experiences. Let's move on to the next step." +_END_OF_WORKTYPE_TOKEN = "" # nosec B105 - LLM control token, not a password +_END_OF_CONVERSATION_TOKEN = "" # nosec B105 - LLM control token, not a password +_CONTROL_TOKENS = (_END_OF_WORKTYPE_TOKEN, _END_OF_CONVERSATION_TOKEN) +_MAX_CONTROL_TOKEN_LENGTH = max(len(token) for token in _CONTROL_TOKENS) def _find_incomplete_experiences(collected_data: list[CollectedData]) -> list[tuple[int, CollectedData, list[str]]]: @@ -123,6 +128,77 @@ class ConversationLLMAgentOutput(AgentOutput): exploring_type_finished: bool = False +class _SafeStreamingAccumulator: + def __init__(self, *, stream_sink: ConversationStreamingSink | None, message_id: str): + self._stream_sink = stream_sink + self._message_id = message_id + self._buffer = "" + self._emitted_length = 0 + + async def on_chunk(self, chunk: str) -> None: + if self._stream_sink is None or not chunk: + return + self._buffer += chunk + safe_prefix_length = max(0, len(self._buffer) - (_MAX_CONTROL_TOKEN_LENGTH - 1)) + if safe_prefix_length <= 0: + return + safe_text = self._buffer[:safe_prefix_length] + self._buffer = self._buffer[safe_prefix_length:] + if safe_text: + self._emitted_length += len(safe_text) + await self._stream_sink.append_text(message_id=self._message_id, delta=safe_text) + + async def flush_remaining(self, final_text: str) -> None: + if self._stream_sink is None: + return + remaining = final_text[self._emitted_length:] + if remaining: + self._emitted_length += len(remaining) + await self._stream_sink.append_text(message_id=self._message_id, delta=remaining) + + +def _normalize_response_text( + response_text: str, + unexplored_types: list[WorkType], + logger: logging.Logger, +) -> tuple[str, bool, bool, float, BaseException | None]: + exploring_type_finished = False + finished = False + penalty = 0.0 + error: BaseException | None = None + normalized_text = response_text.strip() + conversation_prematurely_ended_penalty_level = 0 + + if _END_OF_WORKTYPE_TOKEN in normalized_text: + if normalized_text != _END_OF_WORKTYPE_TOKEN: + logger.warning("The response contains '%s' and additional text: %s", _END_OF_WORKTYPE_TOKEN, normalized_text) + exploring_type_finished = True + finished = False + normalized_text = t("messages", "collectExperiences.moveToOtherExperiences") + + if _END_OF_CONVERSATION_TOKEN in normalized_text: + if normalized_text != _END_OF_CONVERSATION_TOKEN: + logger.warning("The response contains '%s' and additional text: %s", _END_OF_CONVERSATION_TOKEN, normalized_text) + + pre_token_text, _, _ = normalized_text.partition(_END_OF_CONVERSATION_TOKEN) + pre_token_text = pre_token_text.strip() + if pre_token_text: + normalized_text = pre_token_text + else: + normalized_text = t("messages", "collectExperiences.finalMessage") + + exploring_type_finished = False + if len(unexplored_types) > 0: + penalty = get_penalty(conversation_prematurely_ended_penalty_level) + error = ValueError(f"LLM response contains '{_END_OF_CONVERSATION_TOKEN}' but there are unexplored types: {unexplored_types}") + logger.error(error) + finished = False + else: + finished = True + + return normalized_text, exploring_type_finished, finished, penalty, error + + class _ConversationLLM: @staticmethod @@ -137,17 +213,19 @@ async def execute(*, unexplored_types: list[WorkType], explored_types: list[WorkType], last_referenced_experience_index: int, - logger: logging.Logger) -> ConversationLLMAgentOutput: + logger: logging.Logger, + stream_sink: ConversationStreamingSink | None = None, + message_id: str | None = None) -> ConversationLLMAgentOutput: async def _callback(attempt: int, max_retries: int) -> tuple[ConversationLLMAgentOutput, float, BaseException | None]: - # Call the LLM to get the next message for the user - # Add some temperature and top_p variation to prompt the LLM to return different results on each retry. - # Exponentially increase the temperature and top_p to avoid the LLM returning the same result every time. temperature_config = get_config_variation(start_temperature=0.25, end_temperature=0.5, start_top_p=0.8, end_top_p=1, attempt=attempt, max_retries=max_retries) logger.debug("Calling _ConversationLLM with temperature: %s, top_p: %s", temperature_config["temperature"], temperature_config["top_p"]) + # Only stream on the first attempt. Retries use buffered generation + # to avoid emitting duplicate/garbled deltas to the client. + attempt_sink = stream_sink if attempt == 1 else None return await _ConversationLLM._internal_execute( temperature_config=temperature_config, first_time_visit=first_time_visit, @@ -160,7 +238,9 @@ async def _callback(attempt: int, max_retries: int) -> tuple[ConversationLLMAgen unexplored_types=unexplored_types, explored_types=explored_types, last_referenced_experience_index=last_referenced_experience_index, - logger=logger + logger=logger, + stream_sink=attempt_sink, + message_id=message_id, ) result, _result_penalty, _error = await Retry[ConversationLLMAgentOutput].call_with_penalty(callback=_callback, logger=logger) @@ -179,7 +259,9 @@ async def _internal_execute(*, unexplored_types: list[WorkType], explored_types: list[WorkType], last_referenced_experience_index: int, - logger: logging.Logger) -> tuple[ConversationLLMAgentOutput, float, BaseException | None]: + logger: logging.Logger, + stream_sink: ConversationStreamingSink | None = None, + message_id: str | None = None) -> tuple[ConversationLLMAgentOutput, float, BaseException | None]: """ Converses with the user and asks probing questions to collect experiences. :param first_time_visit: If this is the first time the user is visiting the agent. @@ -204,22 +286,25 @@ async def _internal_execute(*, msg = user_input.message.strip() # Remove leading and trailing whitespaces llm_start_time = time.time() - exploring_type_finished = False - finished = False + if message_id is None: + message_id = user_input.message_id + llm_response: LLMResponse llm_input: LLMInput | str system_instructions: list[str] | str | None = None + streamer = _SafeStreamingAccumulator(stream_sink=stream_sink, message_id=message_id) + if stream_sink is not None: + await stream_sink.start_message(message_id=message_id) if first_time_visit: - # If this is the first time the user has visited the agent, the agent should get to the point - # and not introduce itself or ask how the user is doing. + first_visit_config = {**temperature_config, "max_output_tokens": 512} llm = GeminiGenerativeLLM(config=LLMConfig( - generation_config=temperature_config + generation_config=first_visit_config )) llm_input = _ConversationLLM._get_first_time_generative_prompt( country_of_user=country_of_user, persona_type=persona_type, exploring_type=exploring_type) - llm_response = await llm.generate_content(llm_input=llm_input) + llm_response = await llm.stream_content(llm_input=llm_input, on_chunk=streamer.on_chunk) else: system_instructions = _ConversationLLM._get_system_instructions(country_of_user=country_of_user, persona_type=persona_type, @@ -228,11 +313,12 @@ async def _internal_execute(*, unexplored_types=unexplored_types, explored_types=explored_types, last_referenced_experience_index=last_referenced_experience_index) + conversation_config = {**temperature_config, "max_output_tokens": 512} llm = GeminiGenerativeLLM( system_instructions=system_instructions, config=LLMConfig( - language_model_name=AgentsConfig.deep_reasoning_model, - generation_config=temperature_config + language_model_name=AgentsConfig.default_model, + generation_config=conversation_config )) # Drop the first message from the conversation history, which is the welcome message from the welcome agent. # This message is treated as an instruction and causes the conversation to go off track. @@ -245,10 +331,9 @@ async def _internal_execute(*, model_response_instructions=None, context=filtered_context, user_input=msg) - llm_response = await llm.generate_content(llm_input=llm_input) + llm_response = await llm.stream_content(llm_input=llm_input, on_chunk=streamer.on_chunk) llm_output_empty_penalty_level = 1 - conversation_prematurely_ended_penalty_level = 0 llm_end_time = time.time() llm_stats = LLMStats(prompt_token_count=llm_response.prompt_token_count, @@ -266,6 +351,7 @@ async def _internal_execute(*, ) _didnt_understand = t("messages", "collectExperiences.didNotUnderstand") return ConversationLLMAgentOutput( + message_id=message_id, message_for_user=_didnt_understand, exploring_type_finished=False, finished=False, @@ -273,39 +359,15 @@ async def _internal_execute(*, agent_response_time_in_sec=round(llm_end_time - llm_start_time, 2), llm_stats=[llm_stats]), get_penalty(llm_output_empty_penalty_level), ValueError("Conversation LLM response is empty") - penalty: float = 0 - error: BaseException | None = None - - # Test if the response is the same as the previous two - if llm_response.text.find("") != -1: - # We finished a work type (and it is not the last one) we need to move to the next one - if llm_response.text != "": - logger.warning("The response contains '' and additional text: %s", llm_response.text) - exploring_type_finished = True - finished = False - llm_response.text = t("messages", "collectExperiences.moveToOtherExperiences") - - if llm_response.text.find("") != -1: - if llm_response.text != "": - logger.warning("The response contains '' and additional text: %s", llm_response.text) - - pre_token_text, _, _ = llm_response.text.partition("") - pre_token_text = pre_token_text.strip() - if pre_token_text: - llm_response.text = pre_token_text - else: - llm_response.text = t("messages", "collectExperiences.finalMessage") - - exploring_type_finished = False - if len(unexplored_types) > 0: - penalty = get_penalty(conversation_prematurely_ended_penalty_level) - error = ValueError(f"LLM response contains '' but there are unexplored types: {unexplored_types}") - logger.error(error) - finished = False - else: - finished = True + llm_response.text, exploring_type_finished, finished, penalty, error = _normalize_response_text( + llm_response.text, + unexplored_types, + logger, + ) + await streamer.flush_remaining(llm_response.text) return ConversationLLMAgentOutput( + message_id=message_id, message_for_user=llm_response.text, exploring_type_finished=exploring_type_finished, finished=finished, diff --git a/backend/app/agent/collect_experiences_agent/_dataextraction_llm.py b/backend/app/agent/collect_experiences_agent/_dataextraction_llm.py index a6ab64ab2..fb16ef084 100644 --- a/backend/app/agent/collect_experiences_agent/_dataextraction_llm.py +++ b/backend/app/agent/collect_experiences_agent/_dataextraction_llm.py @@ -1,5 +1,6 @@ import asyncio import logging +from collections.abc import Awaitable, Callable from typing import Optional from pydantic import BaseModel @@ -93,7 +94,8 @@ async def execute(self, *, user_input: AgentInput, context: ConversationContext, - collected_experience_data_so_far: list[CollectedData] + collected_experience_data_so_far: list[CollectedData], + on_status: Callable[[str], Awaitable[None]] | None = None, ) -> tuple[int, list[LLMStats]]: """ Given the last user input, a conversation history and the experience data collected so far. @@ -107,6 +109,8 @@ async def execute(self, """ # 1. Get the list of operations from the user's input + if on_status is not None: + await on_status("analyzing_your_input") commands, llm_stats = await self._intent_analyzer_tool.execute( collected_experience_data_so_far=collected_experience_data_so_far, conversation_context=context, @@ -151,6 +155,8 @@ async def execute(self, user_statement=command.users_statement )) + if update_tasks and on_status is not None: + await on_status("extracting_experience_details") update_results = await asyncio.gather(*update_tasks) for (result, _llm_stats) in update_results: llm_stats.extend(_llm_stats) diff --git a/backend/app/agent/collect_experiences_agent/_transition_decision_tool.py b/backend/app/agent/collect_experiences_agent/_transition_decision_tool.py index a850bddec..49ec13f70 100644 --- a/backend/app/agent/collect_experiences_agent/_transition_decision_tool.py +++ b/backend/app/agent/collect_experiences_agent/_transition_decision_tool.py @@ -84,9 +84,10 @@ def _get_llm(collected_data_json: str, temperature_config: Optional[dict] = None language_style=get_language_style() ), config=LLMConfig( - language_model_name=AgentsConfig.deep_reasoning_model, + language_model_name=AgentsConfig.default_model, generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG + | {"max_output_tokens": 512} | temperature_config | with_response_schema(_TransitionDecisionOutput) )) diff --git a/backend/app/agent/collect_experiences_agent/collect_experiences_agent.py b/backend/app/agent/collect_experiences_agent/collect_experiences_agent.py index c59bc44a1..42adc9c84 100644 --- a/backend/app/agent/collect_experiences_agent/collect_experiences_agent.py +++ b/backend/app/agent/collect_experiences_agent/collect_experiences_agent.py @@ -1,6 +1,7 @@ import asyncio from typing import Optional, Mapping, Any +from bson import ObjectId from pydantic import BaseModel, Field, field_serializer, field_validator from app.agent.agent import Agent @@ -20,6 +21,7 @@ from app.conversation_memory.conversation_memory_types import ConversationContext from app.countries import Country from app.i18n.translation_service import t +from app.context_vars import get_stream_sink from app.vector_search.esco_entities import OccupationSkillEntity from app.vector_search.vector_search_dependencies import SearchServices @@ -179,6 +181,20 @@ def __init__(self, self._search_services = search_services self._experience_pipeline_config = experience_pipeline_config + async def _emit_stream_status(self, label: str) -> None: + stream_sink = get_stream_sink() + if stream_sink is None: + return + await stream_sink.emit_status_update( + label=label, + status="running", + agent_type=self.agent_type.value, + ) + + @staticmethod + def _has_pending_title_normalization(collected_data: list[CollectedData]) -> bool: + return any(elem.experience_title and not elem.normalized_experience_title for elem in collected_data) + async def _normalize_experience_titles(self, *, collected_data: list[CollectedData]): if not self._search_services or self._state is None: return @@ -249,7 +265,10 @@ async def execute(self, user_input: AgentInput, # TODO: the LLM can and will fail with an exception or even return None, we need to handle this last_referenced_experience_index, data_extraction_llm_stats = await data_extraction_llm.execute(user_input=user_input, context=context, - collected_experience_data_so_far=collected_data) + collected_experience_data_so_far=collected_data, + on_status=self._emit_stream_status) + if self._has_pending_title_normalization(collected_data): + await self._emit_stream_status(label="matching_job_titles") await self._normalize_experience_titles(collected_data=collected_data) # TODO: Keep track of the last_referenced_experience_index and if it has changed it means that the user has # provided a new experience, we need to handle this as @@ -257,8 +276,10 @@ async def execute(self, user_input: AgentInput, # b) the model may have made a mistake interpreting the user input as we need to clarify conversation_llm = _ConversationLLM() exploring_type = self._state.unexplored_types[0] if len(self._state.unexplored_types) > 0 else None + conversation_message_id = str(ObjectId()) transition_decision_tool = TransitionDecisionTool(self.logger) + await self._emit_stream_status(label="composing_response") # Both are pure readers of collected_data/context/user_input -- safe to parallelize conversation_llm_output, (transition_decision, transition_reasoning, transition_llm_stats) = await asyncio.gather( @@ -273,7 +294,9 @@ async def execute(self, user_input: AgentInput, exploring_type=exploring_type, unexplored_types=self._state.unexplored_types, explored_types=self._state.explored_types, - logger=self.logger), + logger=self.logger, + stream_sink=get_stream_sink(), + message_id=conversation_message_id), transition_decision_tool.execute( collected_data=collected_data, exploring_type=exploring_type, @@ -325,9 +348,16 @@ async def execute(self, user_input: AgentInput, else: transition_text = t('messages', 'collectExperiences.recapPrompt') + transition_delta = f"\n\n{transition_text}" conversation_llm_output.message_for_user = ( - f"{conversation_llm_output.message_for_user}\n\n{transition_text}" + f"{conversation_llm_output.message_for_user}{transition_delta}" ) + stream_sink = get_stream_sink() + if stream_sink is not None: + await stream_sink.append_text( + message_id=conversation_llm_output.message_id, + delta=transition_delta, + ) elif transition_decision == TransitionDecision.END_CONVERSATION: conversation_llm_output.finished = True diff --git a/backend/app/agent/collect_experiences_agent/data_extraction_llm/_intent_analyzer_tool.py b/backend/app/agent/collect_experiences_agent/data_extraction_llm/_intent_analyzer_tool.py index db9030d60..e2e304bd6 100644 --- a/backend/app/agent/collect_experiences_agent/data_extraction_llm/_intent_analyzer_tool.py +++ b/backend/app/agent/collect_experiences_agent/data_extraction_llm/_intent_analyzer_tool.py @@ -76,8 +76,8 @@ def _get_llm(previously_extracted_data: str, temperature_config: Optional[dict] language_style=get_language_style()), config=LLMConfig( generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG | { - "max_output_tokens": 3000 - # Limit the output to 3000 tokens to avoid the "reasoning recursion issues" + "max_output_tokens": 1024 + # Intent analysis emits a small JSON payload, so a lower cap improves latency. } | temperature_config | with_response_schema(_LLMOutput) )) diff --git a/backend/app/agent/collect_experiences_agent/data_extraction_llm/_temporal_classifier_tool.py b/backend/app/agent/collect_experiences_agent/data_extraction_llm/_temporal_classifier_tool.py index 1065ff3b6..609628cb9 100644 --- a/backend/app/agent/collect_experiences_agent/data_extraction_llm/_temporal_classifier_tool.py +++ b/backend/app/agent/collect_experiences_agent/data_extraction_llm/_temporal_classifier_tool.py @@ -98,7 +98,7 @@ def _get_llm(self, temperature_config: Optional[dict] = None) -> GeminiGenerativ return GeminiGenerativeLLM( system_instructions=self._get_system_instructions(), config=LLMConfig( - language_model_name=AgentsConfig.deep_reasoning_model, + language_model_name=AgentsConfig.default_model, generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG | { "max_output_tokens": 3000 # Limit the output to 3000 tokens to avoid the "reasoning recursion issues" diff --git a/backend/app/agent/explore_experiences_agent_director.py b/backend/app/agent/explore_experiences_agent_director.py index fe62f9f06..2f55752aa 100644 --- a/backend/app/agent/explore_experiences_agent_director.py +++ b/backend/app/agent/explore_experiences_agent_director.py @@ -21,6 +21,8 @@ from app.vector_search.esco_entities import SkillEntity from app.i18n.translation_service import t from app.vector_search.vector_search_dependencies import SearchServices +from app.conversations.constants import DIVE_IN_EXPERIENCES_PERCENTAGE, PREFERENCE_ELICITATION_PERCENTAGE +from app.context_vars import get_stream_sink class ConversationPhase(Enum): @@ -176,6 +178,63 @@ class ExploreExperiencesAgentDirector(Agent): This is a stateful agent. """ + def _build_dive_in_stream_phase(self, state: ExploreExperiencesAgentDirectorState) -> dict[str, int | str | None]: + total_experiences = len(state.experiences_state) + total_explored_experiences = sum( + 1 for exp in state.experiences_state.values() if exp.dive_in_phase == DiveInPhase.PROCESSED + ) + dive_in_gap = PREFERENCE_ELICITATION_PERCENTAGE - DIVE_IN_EXPERIENCES_PERCENTAGE + if total_experiences == 0: + percentage = DIVE_IN_EXPERIENCES_PERCENTAGE + current = None + total = None + else: + percentage = round( + DIVE_IN_EXPERIENCES_PERCENTAGE + ((total_explored_experiences / total_experiences) * dive_in_gap) + ) + current = min(total_explored_experiences + 1, total_experiences) + total = total_experiences + return { + "phase": "DIVE_IN", + "percentage": percentage, + "current": current, + "total": total, + } + + async def _emit_stream_status( + self, + *, + label: str, + status: str = "running", + detail: str | None = None, + current_phase: dict[str, int | str | None] | None = None, + ) -> None: + stream_sink = get_stream_sink() + if stream_sink is None: + return + await stream_sink.emit_status_update( + label=label, + status=status, + agent_type=self.agent_type.value, + detail=detail, + current_phase=current_phase, + ) + + async def _emit_dive_in_phase_update( + self, + state: ExploreExperiencesAgentDirectorState, + *, + detail: str | None = None, + ) -> None: + stream_sink = get_stream_sink() + if stream_sink is None: + return + await stream_sink.emit_phase_update( + current_phase=self._build_dive_in_stream_phase(state), + agent_type=self.agent_type.value, + detail=detail, + ) + async def _dive_into_experiences(self, *, user_input: AgentInput, context: ConversationContext, @@ -190,6 +249,11 @@ async def _dive_into_experiences(self, *, current_experience = state.experiences_state.get(state.current_experience_uuid, None) if not current_experience: + await self._emit_stream_status( + label="transitioning_to_preferences", + status="running", + detail="all_experiences_processed", + ) message = AgentOutput( message_for_user=t("messages", "exploreExperiences.transitionToPreferences"), finished=True, @@ -208,9 +272,25 @@ async def _dive_into_experiences(self, *, # Start the first sub-phase current_experience.dive_in_phase = DiveInPhase.EXPLORING_SKILLS picked_new_experience = True + await self._emit_stream_status( + label="diving_into_experiences", + status="running", + detail=current_experience.experience.experience_title, + current_phase=self._build_dive_in_stream_phase(state), + ) + await self._emit_dive_in_phase_update( + state, + detail=current_experience.experience.experience_title, + ) # Sub-phase 2 if current_experience.dive_in_phase == DiveInPhase.EXPLORING_SKILLS: + await self._emit_stream_status( + label="exploring_skills", + status="running", + detail=current_experience.experience.experience_title, + current_phase=self._build_dive_in_stream_phase(state), + ) if picked_new_experience: # When transitioning between states set this message to "" and handle it in the execute method of the agent @@ -230,6 +310,12 @@ async def _dive_into_experiences(self, *, if current_experience.dive_in_phase == DiveInPhase.LINKING_RANKING: if current_experience.experience.responsibilities.responsibilities: + await self._emit_stream_status( + label="linking_and_ranking", + status="running", + detail=current_experience.experience.experience_title, + current_phase=self._build_dive_in_stream_phase(state), + ) # Infer the occupations for the experience and update the experience entity # , then link the skills and rank them agent_output = await self._link_and_rank( @@ -244,11 +330,21 @@ async def _dive_into_experiences(self, *, # completed processing this experience current_experience.dive_in_phase = DiveInPhase.PROCESSED state.current_experience_uuid = None + await self._emit_dive_in_phase_update( + state, + detail=current_experience.experience.experience_title, + ) else: # if the current experience does not have any responsibilities, then we should skip this experience # as there is no information to link and ran, and we should move to the next experience current_experience.dive_in_phase = DiveInPhase.PROCESSED state.current_experience_uuid = None + await self._emit_stream_status( + label="skipping_experience", + status="completed", + detail=current_experience.experience.experience_title, + current_phase=self._build_dive_in_stream_phase(state), + ) agent_output = AgentOutput( message_for_user=t( "messages", @@ -266,6 +362,10 @@ async def _dive_into_experiences(self, *, ), agent_output) # get the context again after updating the history context = await self._conversation_manager.get_conversation_context() + await self._emit_dive_in_phase_update( + state, + detail=current_experience.experience.experience_title, + ) if current_experience.dive_in_phase == DiveInPhase.PROCESSED: # Add the experience to the list of explored experiences @@ -276,6 +376,11 @@ async def _dive_into_experiences(self, *, # then we are done _next_experience = _pick_next_experience_to_process(state.experiences_state) if not _next_experience: + await self._emit_stream_status( + label="transitioning_to_preferences", + status="completed", + detail="all_experiences_processed", + ) return AgentOutput( message_for_user=t("messages", "exploreExperiences.transitionToPreferences"), finished=True, @@ -321,6 +426,12 @@ async def execute(self, user_input: AgentInput, context: ConversationContext) -> # and transition to the next phase state.conversation_phase = ConversationPhase.DIVE_IN transitioned_between_states = True + await self._emit_stream_status( + label="diving_into_experiences", + status="started", + current_phase=self._build_dive_in_stream_phase(state), + ) + await self._emit_dive_in_phase_update(state, detail="phase_entered") # Then dive into each of the experiences collected if state.conversation_phase == ConversationPhase.DIVE_IN: diff --git a/backend/app/agent/preference_elicitation_agent/agent.py b/backend/app/agent/preference_elicitation_agent/agent.py index aada515eb..e21d0c912 100644 --- a/backend/app/agent/preference_elicitation_agent/agent.py +++ b/backend/app/agent/preference_elicitation_agent/agent.py @@ -47,6 +47,7 @@ STD_AGENT_CHARACTER, STD_LANGUAGE_STYLE ) +from app.context_vars import get_stream_sink # Adaptive D-efficiency imports try: @@ -65,6 +66,7 @@ StoppingCriterion = None AdaptiveConfig = None from app.agent.simple_llm_agent.prompt_response_template import get_json_response_instructions +from app.conversations.constants import FINISHED_CONVERSATION_PERCENTAGE, PREFERENCE_ELICITATION_PERCENTAGE from app.conversation_memory.conversation_formatter import ConversationHistoryFormatter from app.conversation_memory.conversation_memory_manager import ConversationContext from app.agent.experience.experience_entity import ExperienceEntity @@ -457,6 +459,108 @@ async def _extract_user_context(self) -> None: f"industry={self._user_context.industry}, level={self._user_context.experience_level}" ) + def _build_stream_current_phase(self) -> dict[str, int | str | None]: + total_categories = 6 + min_vignettes = self._state.minimum_vignettes_completed + phase_name = self._state.conversation_phase + + if phase_name == "COMPLETE": + return { + "phase": "PREFERENCE_ELICITATION", + "percentage": FINISHED_CONVERSATION_PERCENTAGE, + "current": None, + "total": None, + } + + total_indicators = 0 + completed_indicators = 0 + + if phase_name == "BWS": + total_indicators += 12 + completed_indicators += self._state.bws_tasks_completed + elif self._state.bws_phase_complete: + total_indicators += 12 + completed_indicators += 12 + + total_indicators += min_vignettes + completed_indicators += min(len(self._state.completed_vignettes), min_vignettes) + + total_indicators += total_categories + completed_indicators += len(self._state.categories_covered) + + if total_indicators == 0: + percentage = PREFERENCE_ELICITATION_PERCENTAGE + else: + pref_gap = FINISHED_CONVERSATION_PERCENTAGE - PREFERENCE_ELICITATION_PERCENTAGE + percentage = round( + PREFERENCE_ELICITATION_PERCENTAGE + ((completed_indicators / total_indicators) * pref_gap) + ) + + current: int | None = None + total: int | None = None + if phase_name == "EXPERIENCE_QUESTIONS": + total = total_categories + current = min(len(self._state.categories_covered) + 1, total) + elif phase_name == "BWS": + total = 12 + current = min(max(self._state.bws_tasks_completed, 1), total) + elif phase_name in ("VIGNETTES", "FOLLOW_UP"): + total = min_vignettes + current = min(len(self._state.completed_vignettes) + 1, total) if total > 0 else None + elif phase_name == "GATE": + total = 3 + current = min(self._state.gate_interventions_completed + 1, total) + elif phase_name == "WRAPUP": + total = 1 + current = 1 + + return { + "phase": "PREFERENCE_ELICITATION", + "percentage": percentage, + "current": current, + "total": total, + } + + def _get_stream_status_label(self, phase_name: str) -> str: + return { + "INTRO": "introducing_preferences", + "EXPERIENCE_QUESTIONS": "asking_preference_questions", + "VIGNETTES": "presenting_vignette", + "FOLLOW_UP": "asking_follow_up", + "GATE": "asking_clarifying_question", + "BWS": "ranking_work_activities", + "WRAPUP": "summarizing_preferences", + "COMPLETE": "preferences_complete", + }.get(phase_name, "preference_elicitation") + + async def _emit_status_update( + self, + *, + label: str, + status: str = "running", + detail: str | None = None, + ) -> None: + stream_sink = get_stream_sink() + if stream_sink is None: + return + await stream_sink.emit_status_update( + label=label, + status=status, + agent_type=self.agent_type.value, + detail=detail, + current_phase=self._build_stream_current_phase(), + ) + + async def _emit_phase_update(self, *, detail: str | None = None) -> None: + stream_sink = get_stream_sink() + if stream_sink is None: + return + await stream_sink.emit_phase_update( + current_phase=self._build_stream_current_phase(), + agent_type=self.agent_type.value, + detail=detail, + ) + async def execute( self, user_input: AgentInput, @@ -494,6 +598,13 @@ async def execute( # Execute based on current phase try: + phase_before_execution = self._state.conversation_phase + await self._emit_status_update( + label=self._get_stream_status_label(phase_before_execution), + status="running", + detail="phase_started", + ) + await self._emit_phase_update(detail="phase_started") if self._state.conversation_phase == "INTRO": response, llm_stats = await self._handle_intro_phase(msg, context) elif self._state.conversation_phase == "EXPERIENCE_QUESTIONS": @@ -513,6 +624,13 @@ async def execute( else: self.logger.error(f"Unknown phase: {self._state.conversation_phase}") return self._create_error_response(agent_start_time) + await self._emit_phase_update(detail="phase_progressed") + if phase_before_execution != self._state.conversation_phase: + await self._emit_status_update( + label=self._get_stream_status_label(self._state.conversation_phase), + status="running", + detail="phase_entered", + ) except Exception as e: self.logger.exception("Error in preference elicitation agent: %s", str(e)) @@ -1502,7 +1620,17 @@ async def _handle_wrapup_phase( ) # Save preference vector to youth database (DB6) + await self._emit_status_update( + label="saving_preferences", + status="running", + detail="db6_save_started", + ) await self._save_preference_vector_to_db6() + await self._emit_status_update( + label="saving_preferences", + status="completed", + detail="db6_save_finished", + ) self._state.conversation_phase = "COMPLETE" diff --git a/backend/app/agent/skill_explorer_agent/_conversation_llm.py b/backend/app/agent/skill_explorer_agent/_conversation_llm.py index ad140fa06..25681fd99 100644 --- a/backend/app/agent/skill_explorer_agent/_conversation_llm.py +++ b/backend/app/agent/skill_explorer_agent/_conversation_llm.py @@ -9,6 +9,7 @@ from app.agent.prompt_template.format_prompt import replace_placeholders_with_indent from app.conversation_memory.conversation_formatter import ConversationHistoryFormatter from app.conversation_memory.conversation_memory_types import ConversationContext +from app.conversations.streaming import ConversationStreamingSink from app.countries import Country from app.i18n.translation_service import t from app.agent.persona_detector import PersonaType, get_persona_prompt_section @@ -19,6 +20,38 @@ # centralize use for skill_explorer_agent and conversation_llm_test _FINAL_MESSAGE_KEY = "exploreSkills.finalMessage" +_END_OF_CONVERSATION_TOKEN = "" # nosec B105 - LLM control token, not a password +_MAX_CONTROL_TOKEN_LENGTH = len(_END_OF_CONVERSATION_TOKEN) + + +class _SafeStreamingAccumulator: + def __init__(self, *, stream_sink: ConversationStreamingSink | None, message_id: str): + self._stream_sink = stream_sink + self._message_id = message_id + self._buffer = "" + self._emitted_length = 0 + + async def on_chunk(self, chunk: str) -> None: + if self._stream_sink is None or not chunk: + return + self._buffer += chunk + safe_prefix_length = max(0, len(self._buffer) - (_MAX_CONTROL_TOKEN_LENGTH - 1)) + if safe_prefix_length <= 0: + return + safe_text = self._buffer[:safe_prefix_length] + self._buffer = self._buffer[safe_prefix_length:] + if safe_text: + self._emitted_length += len(safe_text) + await self._stream_sink.append_text(message_id=self._message_id, delta=safe_text) + + async def flush_remaining(self, final_text: str) -> None: + if self._stream_sink is None: + return + remaining = final_text[self._emitted_length:] + if remaining: + self._emitted_length += len(remaining) + await self._stream_sink.append_text(message_id=self._message_id, delta=remaining) + class _ConversationLLM: @@ -35,18 +68,18 @@ async def execute(*, rich_response: bool, experience_title, work_type: WorkType, - logger: logging.Logger) -> AgentOutput: + logger: logging.Logger, + stream_sink: ConversationStreamingSink | None = None, + message_id: str | None = None) -> AgentOutput: async def _callback(attempt: int, max_retries: int) -> tuple[AgentOutput, float, BaseException | None]: - # Call the LLM to get the next message for the user. - # Add some temperature and top_p variation to prompt the LLM to return different results on each retry. - # Exponentially increase the temperature and top_p to avoid the LLM returning the same result every time. temperature_config = get_config_variation(start_temperature=0.25, end_temperature=0.5, start_top_p=0.8, end_top_p=1, attempt=attempt, max_retries=max_retries) logger.debug("Calling _ConversationLLM with temperature: %s, top_p: %s", temperature_config["temperature"], temperature_config["top_p"]) + attempt_sink = stream_sink if attempt == 1 else None return await _ConversationLLM._internal_execute( temperature_config=temperature_config, experiences_explored=experiences_explored, @@ -60,7 +93,9 @@ async def _callback(attempt: int, max_retries: int) -> tuple[AgentOutput, float, rich_response=rich_response, experience_title=experience_title, work_type=work_type, - logger=logger + logger=logger, + stream_sink=attempt_sink, + message_id=message_id, ) result, _result_penalty, _error = await Retry[AgentOutput].call_with_penalty(callback=_callback, logger=logger) @@ -80,32 +115,29 @@ async def _internal_execute(*, rich_response: bool, experience_title, work_type: WorkType, - logger: logging.Logger) -> tuple[AgentOutput, float, BaseException | None]: + logger: logging.Logger, + stream_sink: ConversationStreamingSink | None = None, + message_id: str | None = None) -> tuple[AgentOutput, float, BaseException | None]: """ The main conversation logic for the skill explorer agent. """ if user_input.message == "": - # If the user input is empty, set it to "(silence)" - # This is to avoid the agent failing to respond to an empty input user_input.message = "(silence)" user_input.is_artificial = True - msg = user_input.message.strip() # Remove leading and trailing whitespaces + msg = user_input.message.strip() llm_start_time = time.time() + if message_id is None: + message_id = user_input.message_id + llm_response: LLMResponse llm_input: LLMInput | str system_instructions: list[str] | str | None = None + streamer = _SafeStreamingAccumulator(stream_sink=stream_sink, message_id=message_id) + if stream_sink is not None: + await stream_sink.start_message(message_id=message_id) if first_time_for_experience: - # If this is the first experience, generate only a response without passing the conversation history - # or user message. Including these can confuse the model, potentially leading to responses about previous experiences. - # - # Various approaches have been tested, including using artificial prompts like "I am ready to share my experience as a ...", - # but this added unnecessary complexity. - # - # While earlier mitigations helped reduce this issue, they are no longer required, though we are keeping them for now - # in case they prove useful in the future. - llm = GeminiGenerativeLLM( config=LLMConfig( generation_config=temperature_config @@ -119,7 +151,7 @@ async def _internal_execute(*, rich_response=rich_response, work_type=work_type ) - llm_response = await llm.generate_content(llm_input=llm_input) + llm_response = await llm.stream_content(llm_input=llm_input, on_chunk=streamer.on_chunk) else: system_instructions = _ConversationLLM._create_conversation_system_instructions( question_asked_until_now=question_asked_until_now, @@ -137,7 +169,7 @@ async def _internal_execute(*, llm_input = ConversationHistoryFormatter.format_for_agent_generative_prompt( model_response_instructions=None, context=context, user_input=msg) - llm_response = await llm.generate_content(llm_input=llm_input) + llm_response = await llm.stream_content(llm_input=llm_input, on_chunk=streamer.on_chunk) llm_end_time = time.time() llm_stats = LLMStats(prompt_token_count=llm_response.prompt_token_count, @@ -159,15 +191,18 @@ async def _internal_execute(*, agent_response_time_in_sec=round(llm_end_time - llm_start_time, 2), llm_stats=[llm_stats]), 100, ValueError("LLM response is empty") - if llm_response.text == "": + if llm_response.text == _END_OF_CONVERSATION_TOKEN: llm_response.text = t("messages", _FINAL_MESSAGE_KEY) finished = True - if llm_response.text.find("") != -1: + if _END_OF_CONVERSATION_TOKEN in llm_response.text: llm_response.text = t("messages", _FINAL_MESSAGE_KEY) finished = True - logger.warning("The response contains '' and additional text: %s", llm_response.text) + logger.warning("The response contains '%s' and additional text: %s", _END_OF_CONVERSATION_TOKEN, llm_response.text) + + await streamer.flush_remaining(llm_response.text) return AgentOutput( + message_id=message_id, message_for_user=llm_response.text, finished=finished, agent_type=AgentType.EXPLORE_SKILLS_AGENT, diff --git a/backend/app/agent/skill_explorer_agent/skill_explorer_agent.py b/backend/app/agent/skill_explorer_agent/skill_explorer_agent.py index 4e7ccdbb0..7db8ac413 100644 --- a/backend/app/agent/skill_explorer_agent/skill_explorer_agent.py +++ b/backend/app/agent/skill_explorer_agent/skill_explorer_agent.py @@ -10,6 +10,7 @@ from app.countries import Country from app.i18n.translation_service import t from app.agent.persona_detector import PersonaType +from app.context_vars import get_stream_sink from ._conversation_llm import _ConversationLLM, _FINAL_MESSAGE_KEY from ._responsibilities_extraction_tool import _ResponsibilitiesExtractionTool @@ -203,7 +204,9 @@ async def execute(self, rich_response=rich_response, experience_title=self.experience_entity.experience_title, work_type=self.experience_entity.work_type, - logger=self.logger) + logger=self.logger, + stream_sink=get_stream_sink(), + message_id=user_input.message_id) if conversation_llm_output.message_for_user != t("messages", _FINAL_MESSAGE_KEY): # don't add the final message to the list of questions asked, since it is not a question diff --git a/backend/app/app_config.py b/backend/app/app_config.py index f9a2e369b..0eb029d29 100644 --- a/backend/app/app_config.py +++ b/backend/app/app_config.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Literal, Optional from pydantic import BaseModel, Field, model_validator @@ -109,6 +109,26 @@ class ApplicationConfig(BaseModel): Corresponds to the COMPASS_INLINE_PHASE_TRANSITION environment variable. """ + stream_chunk_size: int = Field(default=10, gt=0) + """ + Size of each text chunk when streaming SSE messages. + In chars mode: approximate character count per chunk. + In words mode: number of words per chunk. + Corresponds to BACKEND_STREAM_CHUNK_SIZE. + """ + + stream_chunk_mode: Literal["chars", "words"] = "chars" + """ + How to chunk text for simulated streaming: by characters or by words. + Corresponds to BACKEND_STREAM_CHUNK_MODE. + """ + + stream_delta_delay_ms: int = Field(default=12, ge=0) + """ + Delay between stream deltas in milliseconds (typing effect). + Corresponds to BACKEND_STREAM_DELTA_DELAY_MS. + """ + @model_validator(mode='after') def check_cv_upload_configurations(self) -> "ApplicationConfig": # Check that the CV upload configurations are valid. diff --git a/backend/app/context_vars.py b/backend/app/context_vars.py index 8e116a548..0f26d5294 100644 --- a/backend/app/context_vars.py +++ b/backend/app/context_vars.py @@ -32,3 +32,11 @@ # Detected language for the current turn (ENGLISH, SWAHILI, or MIXED) detected_language_ctx_var = contextvars.ContextVar("detected_language", default="english") + +# Streaming sink for the current turn (set at start of stream_send, cleared in finally) +# Components read via get_stream_sink() instead of setter injection +stream_sink_ctx_var = contextvars.ContextVar("stream_sink", default=None) + + +def get_stream_sink(): + return stream_sink_ctx_var.get() diff --git a/backend/app/conversation_memory/conversation_memory_manager.py b/backend/app/conversation_memory/conversation_memory_manager.py index 9bf96365e..6a5f83113 100644 --- a/backend/app/conversation_memory/conversation_memory_manager.py +++ b/backend/app/conversation_memory/conversation_memory_manager.py @@ -1,11 +1,11 @@ import logging from abc import ABC, abstractmethod - from app.agent.agent_types import AgentInput, AgentOutput from app.conversation_memory.conversation_memory_types import ConversationHistory, \ ConversationContext, ConversationTurn, ConversationMemoryManagerState from app.conversation_memory.summarizer import Summarizer +from app.context_vars import get_stream_sink class IConversationMemoryManager(ABC): @@ -54,7 +54,7 @@ async def is_user_message(self, message_id: str) -> bool: class ConversationMemoryManager(IConversationMemoryManager): """ - Manages the conversation history + Manages the conversation history. """ def __init__(self, unsummarized_window_size, to_be_summarized_window_size): @@ -103,6 +103,9 @@ async def update_history(self, user_input: AgentInput, agent_output: AgentOutput # If the to_be_summarized_history window is full, we perform summarization if len(self._state.to_be_summarized_history.turns) == self._to_be_summarized_window_size: await self._summarize() + stream_sink = get_stream_sink() + if stream_sink is not None: + await stream_sink.emit_agent_output(agent_output) async def is_user_message(self, message_id: str) -> bool: # find out if the message_id of the message to react to is found an input message diff --git a/backend/app/conversations/routes.py b/backend/app/conversations/routes.py index ab9018fb9..9bc019b5b 100644 --- a/backend/app/conversations/routes.py +++ b/backend/app/conversations/routes.py @@ -6,6 +6,7 @@ from typing import Annotated from fastapi import FastAPI, APIRouter, Request, Depends, HTTPException, Path +from fastapi.responses import StreamingResponse from motor.motor_asyncio import AsyncIOMotorDatabase from app.agent.agent_director.llm_agent_director import LLMAgentDirector @@ -70,7 +71,7 @@ def add_conversation_routes(app: FastAPI, authentication: Authentication): conversation_router = APIRouter(prefix="/conversations/{session_id}", tags=["conversations"]) - @conversation_router.post(path="/messages", status_code=HTTPStatus.CREATED, response_model=ConversationResponse, + @conversation_router.post(path="/messages", status_code=HTTPStatus.CREATED, responses={HTTPStatus.BAD_REQUEST: {"model": HTTPErrorResponse}, # user is not allowed to send a message on a session where the conversation is already concluded HTTPStatus.FORBIDDEN: {"model": HTTPErrorResponse}, @@ -112,9 +113,26 @@ async def _send_message(request: Request, body: ConversationInput, session_id: A # set the client_id in the context variable. client_id_ctx_var.set(current_user_preferences.client_id) - return await service.send(user_id, session_id, user_input, clear_memory, filter_pii, + await service.ensure_conversation_not_concluded(session_id, clear_memory) + + async def _event_stream(): + async for chunk in service.stream_send(user_id, session_id, user_input, clear_memory, filter_pii, city=current_user_preferences.city, - province=current_user_preferences.province) + province=current_user_preferences.province): + if await request.is_disconnected(): + break + yield chunk + + return StreamingResponse( + _event_stream(), + status_code=HTTPStatus.OK, + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) except ConversationAlreadyConcludedError as e: warning_msg = str(e) logger.warning(warning_msg) diff --git a/backend/app/conversations/service.py b/backend/app/conversations/service.py index e09fb7bc6..16a871c8b 100644 --- a/backend/app/conversations/service.py +++ b/backend/app/conversations/service.py @@ -1,8 +1,11 @@ """ This module contains the service layer for handling conversations. """ +import asyncio import logging from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from contextlib import suppress from datetime import datetime, timezone from app.agent.agent_director.abstract_agent_director import ConversationPhase @@ -16,6 +19,7 @@ from app.conversations.types import ConversationResponse from app.conversations.utils import get_messages_from_conversation_manager, filter_conversation_history, \ get_total_explored_experiences, get_current_conversation_phase_response +from app.conversations.streaming import ConversationStreamingSink from app.sensitive_filter import sensitive_filter from app.metrics.application_state_metrics_recorder.recorder import IApplicationStateMetricsRecorder from app.job_preferences.service import IJobPreferencesService @@ -68,6 +72,30 @@ async def send(self, user_id: str, session_id: int, user_input: str, clear_memor """ raise NotImplementedError() + @abstractmethod + async def stream_send( + self, + user_id: str, + session_id: int, + user_input: str, + clear_memory: bool, + filter_pii: bool, + city: str | None = None, + province: str | None = None, + ) -> AsyncIterator[str]: + """ + Stream a conversation turn over SSE while preserving the same final conversation state. + """ + raise NotImplementedError() + + @abstractmethod + async def ensure_conversation_not_concluded(self, session_id: int, clear_memory: bool) -> None: + """ + Raise ConversationAlreadyConcludedError if the conversation is already concluded. + Call before starting a stream to return HTTP 400 instead of 201. + """ + raise NotImplementedError() + @abstractmethod async def get_history_by_session_id(self, user_id: str, session_id: int) -> ConversationResponse: """ @@ -96,10 +124,91 @@ def __init__(self, *, self._job_preferences_service = job_preferences_service self._user_recommendations_service = user_recommendations_service + async def ensure_conversation_not_concluded(self, session_id: int, clear_memory: bool) -> None: + if clear_memory: + return + state = await self._application_state_metrics_recorder.get_state(session_id) + if state.agent_director_state.current_phase == ConversationPhase.ENDED: + raise ConversationAlreadyConcludedError(session_id) + async def send(self, user_id: str, session_id: int, user_input: str, clear_memory: bool, filter_pii: bool, city: str | None = None, province: str | None = None) -> ConversationResponse: + return await self._execute_turn( + user_id=user_id, + session_id=session_id, + user_input=user_input, + clear_memory=clear_memory, + filter_pii=filter_pii, + stream_sink=None, + city=city, + province=province, + ) + + async def stream_send( + self, + user_id: str, + session_id: int, + user_input: str, + clear_memory: bool, + filter_pii: bool, + city: str | None = None, + province: str | None = None, + ) -> AsyncIterator[str]: + stream_sink = ConversationStreamingSink() + + async def _run_streamed_turn() -> None: + try: + await self._execute_turn( + user_id=user_id, + session_id=session_id, + user_input=user_input, + clear_memory=clear_memory, + filter_pii=filter_pii, + stream_sink=stream_sink, + city=city, + province=province, + ) + except ConversationAlreadyConcludedError as e: + await stream_sink.emit_error( + code="conversation_already_concluded", + message=str(e), + recoverable=False, + ) + except Exception: + self._logger.exception("Unexpected failure during SSE turn execution") + await stream_sink.emit_error( + code="unexpected_failure", + message="Oops! something went wrong", + recoverable=False, + ) + finally: + await stream_sink.close() + + task = asyncio.create_task(_run_streamed_turn()) + try: + async for chunk in stream_sink.iter_sse(): + yield chunk + await task + finally: + if not task.done(): + task.cancel() + with suppress(asyncio.CancelledError): + await task + + async def _execute_turn( + self, + *, + user_id: str, + session_id: int, + user_input: str, + clear_memory: bool, + filter_pii: bool, + stream_sink: ConversationStreamingSink | None, + city: str | None = None, + province: str | None = None, + ) -> ConversationResponse: if clear_memory: await self._application_state_metrics_recorder.delete_state(session_id) if filter_pii: @@ -186,28 +295,65 @@ async def send(self, user_id: str, session_id: int, user_input: str, clear_memor ) user_language_ctx_var.set(default_locale) - await self._agent_director.execute(user_input=user_input) - # get the context again after updating the history - context = await self._conversation_memory_manager.get_conversation_context() - response = await get_messages_from_conversation_manager(context, from_index=current_index) - - # Save preference vector to JobPreferences if preference elicitation just completed - if self._should_save_preference_vector(state): - await self._save_preference_vector_to_job_preferences(state) - - # get the date when the conversation was conducted - state.agent_director_state.conversation_conducted_at = datetime.now(timezone.utc) + from app.context_vars import stream_sink_ctx_var - # save the state, before responding to the user - await self._application_state_metrics_recorder.save_state(state, user_id) + stream_sink_token = None + if stream_sink is not None: + stream_sink_token = stream_sink_ctx_var.set(stream_sink) + try: + if stream_sink is not None: + initial_phase = get_current_conversation_phase_response(state, self._logger).model_dump(mode="json") + await stream_sink.emit_turn_started( + session_id=session_id, + user_message_id=user_input.message_id, + current_phase=initial_phase, + ) + await stream_sink.emit_status_update( + label="preparing_state", + status="started", + current_phase=initial_phase, + ) + await stream_sink.emit_status_update( + label="routing", + status="running", + current_phase=initial_phase, + ) - return ConversationResponse( - messages=response, - conversation_completed=state.agent_director_state.current_phase == ConversationPhase.ENDED, - conversation_conducted_at=state.agent_director_state.conversation_conducted_at, - experiences_explored=get_total_explored_experiences(state), - current_phase=get_current_conversation_phase_response(state, self._logger) - ) + await self._agent_director.execute(user_input=user_input) + # get the context again after updating the history + context = await self._conversation_memory_manager.get_conversation_context() + response = await get_messages_from_conversation_manager(context, from_index=current_index) + + # Save preference vector to JobPreferences if preference elicitation just completed + if self._should_save_preference_vector(state): + await self._save_preference_vector_to_job_preferences(state) + + # get the date when the conversation was conducted + state.agent_director_state.conversation_conducted_at = datetime.now(timezone.utc) + + # save the state, before responding to the user + current_phase = get_current_conversation_phase_response(state, self._logger) + if stream_sink is not None: + await stream_sink.emit_status_update( + label="saving_state", + status="running", + current_phase=current_phase.model_dump(mode="json"), + ) + await self._application_state_metrics_recorder.save_state(state, user_id) + + conversation_response = ConversationResponse( + messages=response, + conversation_completed=state.agent_director_state.current_phase == ConversationPhase.ENDED, + conversation_conducted_at=state.agent_director_state.conversation_conducted_at, + experiences_explored=get_total_explored_experiences(state), + current_phase=current_phase + ) + if stream_sink is not None: + await stream_sink.emit_turn_completed(conversation_response) + return conversation_response + finally: + if stream_sink_token is not None: + stream_sink_ctx_var.reset(stream_sink_token) async def get_history_by_session_id(self, user_id: str, session_id: int) -> ConversationResponse: state = await self._application_state_metrics_recorder.get_state(session_id) diff --git a/backend/app/conversations/streaming.py b/backend/app/conversations/streaming.py new file mode 100644 index 000000000..dfb8660ad --- /dev/null +++ b/backend/app/conversations/streaming.py @@ -0,0 +1,270 @@ +import asyncio +import json +from enum import Enum +from typing import Any + +from pydantic import BaseModel + +from app.agent.agent_types import AgentOutput +from app.app_config import get_application_config +from app.conversations.types import ConversationMessage, ConversationMessageSender, ConversationResponse + + +class ConversationStreamEventType(str, Enum): + TURN_STARTED = "turn_started" + STATUS_UPDATED = "status_updated" + PHASE_UPDATED = "phase_updated" + MESSAGE_STARTED = "message_started" + MESSAGE_DELTA = "message_delta" + MESSAGE_COMPLETED = "message_completed" + TURN_COMPLETED = "turn_completed" + ERROR = "error" + + +class TurnStartedEvent(BaseModel): + session_id: int + user_message_id: str + current_phase: dict[str, Any] | None = None + + +class MessageStartedEvent(BaseModel): + message_id: str + sender: str + message_type: str = "TEXT" + metadata: dict[str, Any] | None = None + + +class MessageDeltaEvent(BaseModel): + message_id: str + delta: str + + +class StatusUpdatedEvent(BaseModel): + label: str + status: str = "running" + agent_type: str | None = None + detail: str | None = None + current_phase: dict[str, Any] | None = None + + +class PhaseUpdatedEvent(BaseModel): + current_phase: dict[str, Any] + agent_type: str | None = None + detail: str | None = None + + +class TurnCompletedEvent(BaseModel): + conversation_completed: bool + conversation_conducted_at: str | None = None + experiences_explored: int + current_phase: dict[str, Any] + + +class ErrorEvent(BaseModel): + code: str + message: str + recoverable: bool = False + + +def format_sse_event(event: str, data: dict[str, Any]) -> str: + payload = json.dumps(data, ensure_ascii=True) + return f"event: {event}\ndata: {payload}\n\n" + + +def _normalize_optional_text(value: Any) -> str | None: + if value is None: + return None + return str(value) + + +def _get_message_type_and_metadata(agent_output: AgentOutput) -> tuple[str, dict[str, Any] | None]: + output_metadata = agent_output.metadata if hasattr(agent_output, "metadata") else None + message_type = "TEXT" + if output_metadata and output_metadata.get("task_id") is not None and "alternatives" in output_metadata: + message_type = "BWS_TASK" + return message_type, output_metadata + + +def conversation_message_from_agent_output(agent_output: AgentOutput) -> ConversationMessage: + message_type, metadata = _get_message_type_and_metadata(agent_output) + return ConversationMessage( + message_id=agent_output.message_id, + message=agent_output.message_for_user, + sent_at=agent_output.sent_at, + sender=ConversationMessageSender.COMPASS, + reaction=None, + message_type=message_type, + metadata=metadata, + ) + + +class ConversationStreamingSink: + def __init__(self): + self._queue: asyncio.Queue[str | None] = asyncio.Queue() + self._started_message_ids: set[str] = set() + + async def _emit_model(self, event_type: ConversationStreamEventType, model: BaseModel | ConversationMessage) -> None: + await self._queue.put(format_sse_event(event_type.value, model.model_dump(mode="json"))) + + async def emit_turn_started(self, *, session_id: int, user_message_id: str, current_phase: dict[str, Any] | None) -> None: + await self._emit_model( + ConversationStreamEventType.TURN_STARTED, + TurnStartedEvent( + session_id=session_id, + user_message_id=user_message_id, + current_phase=current_phase, + ), + ) + + async def emit_status_update( + self, + *, + label: str, + status: str = "running", + agent_type: str | None = None, + detail: Any | None = None, + current_phase: dict[str, Any] | None = None, + ) -> None: + await self._emit_model( + ConversationStreamEventType.STATUS_UPDATED, + StatusUpdatedEvent( + label=label, + status=status, + agent_type=agent_type, + detail=_normalize_optional_text(detail), + current_phase=current_phase, + ), + ) + + async def emit_phase_update( + self, + *, + current_phase: dict[str, Any], + agent_type: str | None = None, + detail: Any | None = None, + ) -> None: + await self._emit_model( + ConversationStreamEventType.PHASE_UPDATED, + PhaseUpdatedEvent( + current_phase=current_phase, + agent_type=agent_type, + detail=_normalize_optional_text(detail), + ), + ) + + async def start_message( + self, + *, + message_id: str, + sender: str = "COMPASS", + message_type: str = "TEXT", + metadata: dict[str, Any] | None = None, + ) -> None: + if message_id in self._started_message_ids: + return + self._started_message_ids.add(message_id) + await self._emit_model( + ConversationStreamEventType.MESSAGE_STARTED, + MessageStartedEvent( + message_id=message_id, + sender=sender, + message_type=message_type, + metadata=metadata, + ), + ) + + async def append_text(self, *, message_id: str, delta: str) -> None: + if not delta: + return + await self.start_message(message_id=message_id) + await self._emit_model( + ConversationStreamEventType.MESSAGE_DELTA, + MessageDeltaEvent(message_id=message_id, delta=delta), + ) + + async def complete_message(self, message: ConversationMessage) -> None: + await self.start_message( + message_id=message.message_id, + sender=message.sender.name, + message_type=message.message_type, + metadata=message.metadata, + ) + await self._emit_model(ConversationStreamEventType.MESSAGE_COMPLETED, message) + + async def emit_agent_output(self, agent_output: AgentOutput) -> None: + message = conversation_message_from_agent_output(agent_output) + already_streamed = message.message_id in self._started_message_ids + if already_streamed: + return + if message.message_type != "TEXT" or not message.message: + await self.complete_message(message) + return + await self._stream_text_then_complete(message) + + async def _stream_text_then_complete(self, message: ConversationMessage) -> None: + app_config = get_application_config() + chunk_size = app_config.stream_chunk_size + chunk_mode = app_config.stream_chunk_mode + delay_sec = app_config.stream_delta_delay_ms / 1000.0 + text = message.message + await self.start_message( + message_id=message.message_id, + sender=message.sender.name, + message_type=message.message_type, + metadata=message.metadata, + ) + if chunk_mode == "words": + words = text.split() + for i in range(0, len(words), chunk_size): + chunk = " ".join(words[i : i + chunk_size]) + if i + chunk_size < len(words): + chunk += " " + await self._emit_model( + ConversationStreamEventType.MESSAGE_DELTA, + MessageDeltaEvent(message_id=message.message_id, delta=chunk), + ) + await asyncio.sleep(delay_sec) + else: + offset = 0 + while offset < len(text): + end = min(offset + chunk_size, len(text)) + if end < len(text): + space = text.rfind(" ", offset, end + 1) + if space > offset: + end = space + 1 + chunk = text[offset:end] + await self._emit_model( + ConversationStreamEventType.MESSAGE_DELTA, + MessageDeltaEvent(message_id=message.message_id, delta=chunk), + ) + offset = end + await asyncio.sleep(delay_sec) + await self._emit_model(ConversationStreamEventType.MESSAGE_COMPLETED, message) + + async def emit_turn_completed(self, response: ConversationResponse) -> None: + serialized_response = response.model_dump(mode="json") + await self._emit_model( + ConversationStreamEventType.TURN_COMPLETED, + TurnCompletedEvent( + conversation_completed=serialized_response["conversation_completed"], + conversation_conducted_at=serialized_response["conversation_conducted_at"], + experiences_explored=serialized_response["experiences_explored"], + current_phase=serialized_response["current_phase"], + ), + ) + + async def emit_error(self, *, code: str, message: str, recoverable: bool = False) -> None: + await self._emit_model( + ConversationStreamEventType.ERROR, + ErrorEvent(code=code, message=message, recoverable=recoverable), + ) + + async def close(self) -> None: + await self._queue.put(None) + + async def iter_sse(self): + while True: + item = await self._queue.get() + if item is None: + break + yield item diff --git a/backend/app/conversations/test_routes.py b/backend/app/conversations/test_routes.py index 25885cea1..143c52187 100644 --- a/backend/app/conversations/test_routes.py +++ b/backend/app/conversations/test_routes.py @@ -1,7 +1,7 @@ from datetime import datetime from http import HTTPStatus from typing import Generator -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock import pytest import pytest_mock @@ -10,6 +10,7 @@ from app.conversations.types import ConversationResponse, ConversationMessage, ConversationInput, \ ConversationMessageSender, ConversationPhaseResponse, CurrentConversationPhaseResponse from app.conversations.constants import MAX_MESSAGE_LENGTH +from app.conversations.streaming import format_sse_event from app.conversations.routes import get_conversation_service, add_conversation_routes from app.conversations.service import IConversationService, ConversationAlreadyConcludedError from app.users.auth import UserInfo @@ -25,6 +26,11 @@ TestClientWithMocks = tuple[TestClient, IConversationService, IUserPreferenceRepository, UserInfo | None] +async def _stream_events(*events: str): + for event in events: + yield event + + def get_mock_user_preferences(session_id: int): return UserPreferences( language="en", @@ -46,6 +52,13 @@ async def send(self, user_id: str, session_id: int, user_input: str, clear_memor city: str | None = None, province: str | None = None): raise NotImplementedError + async def stream_send(self, user_id: str, session_id: int, user_input: str, clear_memory: bool, filter_pii: bool, + city: str | None = None, province: str | None = None): + raise NotImplementedError + + async def ensure_conversation_not_concluded(self, session_id: int, clear_memory: bool): + raise NotImplementedError + async def get_history_by_session_id(self, user_id: str, session_id: int): raise NotImplementedError @@ -133,38 +146,53 @@ async def test_send_successful(self, authenticated_client_with_mocks: TestClient # AND the user has a valid session given_session_id = 123 - # AND a ConversationService that will return a valid conversation response - expected_response = ConversationResponse( - messages=[ - ConversationMessage( - message_id="foo_id", - message=given_user_message.user_input, - sender=ConversationMessageSender.USER, - sent_at=datetime.now().isoformat() - ), - ConversationMessage( - message_id="bar_id", - message="Hello, I'm compass", - sender=ConversationMessageSender.COMPASS, - sent_at=datetime.now().isoformat() - ), - ], - conversation_completed=False, - conversation_conducted_at=datetime.now().isoformat(), - experiences_explored=0, - current_phase=ConversationPhaseResponse( + # AND a ConversationService that will return a valid SSE stream + expected_turn_completed = { + "conversation_completed": False, + "conversation_conducted_at": datetime.now().isoformat(), + "experiences_explored": 0, + "current_phase": ConversationPhaseResponse( percentage=0, phase=CurrentConversationPhaseResponse.INTRO - ) + ).model_dump(mode="json"), + } + expected_message = ConversationMessage( + message_id="bar_id", + message="Hello, I'm compass", + sender=ConversationMessageSender.COMPASS, + sent_at=datetime.now().isoformat() ) + expected_stream = "".join([ + format_sse_event("status_updated", { + "label": "routing", + "status": "running", + "agent_type": "welcome_agent", + "detail": "INTRO", + "current_phase": expected_turn_completed["current_phase"], + }), + format_sse_event("phase_updated", { + "current_phase": expected_turn_completed["current_phase"], + "agent_type": "preference_elicitation_agent", + "detail": "phase_progressed", + }), + format_sse_event("message_started", { + "message_id": expected_message.message_id, + "sender": "COMPASS", + "message_type": "TEXT", + "metadata": None, + }), + format_sse_event("message_completed", expected_message.model_dump(mode="json")), + format_sse_event("turn_completed", expected_turn_completed), + ]) # AND mock the repository and service responses mocked_preferences_repository.get_user_preference_by_user_id = AsyncMock( return_value=get_mock_user_preferences(given_session_id)) preferences_spy = mocker.spy(mocked_preferences_repository, "get_user_preference_by_user_id") - mocked_service.send = AsyncMock(return_value=expected_response) - service_spy = mocker.spy(mocked_service, "send") + mocked_service.ensure_conversation_not_concluded = AsyncMock() + mocked_service.stream_send = Mock(return_value=_stream_events(expected_stream)) + service_spy = mocker.spy(mocked_service, "stream_send") # WHEN a POST request where the session_id is in the Path response = client.post( @@ -172,11 +200,12 @@ async def test_send_successful(self, authenticated_client_with_mocks: TestClient json=given_user_message.model_dump(), ) - # THEN the response is CREATED - assert response.status_code == HTTPStatus.CREATED + # THEN the response is OK + assert response.status_code == HTTPStatus.OK - # AND the response is the expected response - assert response.json() == expected_response.model_dump() + # AND the response is an SSE stream with the expected payload + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == expected_stream # AND the user preferences repository was called with the correct user_id preferences_spy.assert_called_once_with(mocked_user.user_id) @@ -280,11 +309,15 @@ async def test_send_conversation_already_concluded(self, authenticated_client_wi # AND the user has a valid session given_session_id = 123 - # AND a ConversationService that will raise a ConversationAlreadyConcludedError + # AND ensure_conversation_not_concluded raises (conversation already concluded) mocked_preferences_repository.get_user_preference_by_user_id = AsyncMock( return_value=get_mock_user_preferences(given_session_id)) - send_spy = mocker.spy(mocked_service, "send") - send_spy.side_effect = ConversationAlreadyConcludedError(given_session_id) + + async def _raise_concluded(session_id: int, clear_memory: bool): + raise ConversationAlreadyConcludedError(session_id) + + mocked_service.ensure_conversation_not_concluded = AsyncMock(side_effect=_raise_concluded) + stream_spy = mocker.spy(mocked_service, "stream_send") # WHEN a POST request where the session_id is in the Path response = client.post( @@ -292,19 +325,12 @@ async def test_send_conversation_already_concluded(self, authenticated_client_wi json=given_user_message.model_dump(), ) - # THEN the response is BAD_REQUEST + # THEN the response is BAD_REQUEST (400) assert response.status_code == HTTPStatus.BAD_REQUEST + assert str(given_session_id) in response.json()["detail"] - # AND the conversation service was called with the correct arguments - send_spy.assert_called_once_with( - mocked_user.user_id, - given_session_id, - given_user_message.user_input, - False, # clear_memory - False, # filter_pii - city=None, - province=None - ) + # AND stream_send was never called + stream_spy.assert_not_called() @pytest.mark.asyncio async def test_send_service_internal_server_error(self, authenticated_client_with_mocks: TestClientWithMocks, @@ -319,11 +345,17 @@ async def test_send_service_internal_server_error(self, authenticated_client_wit # AND the user has a valid session given_session_id = 123 - # AND a ConversationService that will raise an unexpected error + # AND a ConversationService that will emit an unexpected SSE error event mocked_preferences_repository.get_user_preference_by_user_id = AsyncMock( return_value=get_mock_user_preferences(given_session_id)) - send_spy = mocker.spy(mocked_service, "send") - send_spy.side_effect = Exception("Unexpected error") + mocked_service.ensure_conversation_not_concluded = AsyncMock() + expected_stream = format_sse_event("error", { + "code": "unexpected_failure", + "message": "Oops! something went wrong", + "recoverable": False, + }) + mocked_service.stream_send = Mock(return_value=_stream_events(expected_stream)) + stream_spy = mocker.spy(mocked_service, "stream_send") # WHEN a POST request where the session_id is in the Path response = client.post( @@ -331,11 +363,12 @@ async def test_send_service_internal_server_error(self, authenticated_client_wit json=given_user_message.model_dump(), ) - # THEN the response is INTERNAL_SERVER_ERROR - assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + # THEN the response is OK and contains an SSE error payload + assert response.status_code == HTTPStatus.OK + assert response.text == expected_stream # AND the conversation service was called with the correct arguments - send_spy.assert_called_once_with( + stream_spy.assert_called_once_with( mocked_user.user_id, given_session_id, given_user_message.user_input, diff --git a/backend/app/conversations/test_streaming.py b/backend/app/conversations/test_streaming.py new file mode 100644 index 000000000..61dd5807d --- /dev/null +++ b/backend/app/conversations/test_streaming.py @@ -0,0 +1,155 @@ +import json +from datetime import datetime, timezone + +import pytest + +from app.agent.agent_types import AgentOutput +from app.conversations.streaming import ConversationStreamingSink, format_sse_event +from app.conversations.types import ConversationPhaseResponse, ConversationResponse, CurrentConversationPhaseResponse + + +def test_format_sse_event(): + payload = {"message": "hello"} + assert format_sse_event("message_completed", payload) == ( + 'event: message_completed\n' + 'data: {"message": "hello"}\n\n' + ) + + +@pytest.mark.asyncio +async def test_streaming_sink_emits_message_and_turn_events(setup_application_config): + sink = ConversationStreamingSink() + agent_output = AgentOutput( + message_id="msg-1", + message_for_user="Hello from Compass", + finished=False, + agent_response_time_in_sec=0.42, + llm_stats=[], + ) + response = ConversationResponse( + messages=[], + conversation_completed=False, + conversation_conducted_at=datetime.now(timezone.utc), + experiences_explored=1, + current_phase=ConversationPhaseResponse( + percentage=25, + phase=CurrentConversationPhaseResponse.COLLECT_EXPERIENCES, + ), + ) + + await sink.emit_turn_started( + session_id=123, + user_message_id="user-1", + current_phase=response.current_phase.model_dump(mode="json"), + ) + await sink.emit_agent_output(agent_output) + await sink.emit_turn_completed(response) + await sink.close() + + events = [] + async for chunk in sink.iter_sse(): + lines = [line for line in chunk.split("\n") if line] + event_name_line, data_line = lines[:2] + events.append((event_name_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: ")))) + + event_names = [event_name for event_name, _ in events] + assert event_names[0] == "turn_started" + assert event_names[-1] == "turn_completed" + assert "message_started" in event_names + assert "message_completed" in event_names + # message_delta events appear when streaming text chunks + assert event_names.count("message_delta") >= 1 + + message_completed = next(data for name, data in events if name == "message_completed") + assert message_completed["message"] == "Hello from Compass" + + turn_completed = next(data for name, data in events if name == "turn_completed") + assert turn_completed["experiences_explored"] == 1 + + +@pytest.mark.asyncio +async def test_streaming_sink_emits_status_and_phase_updates(): + sink = ConversationStreamingSink() + + await sink.emit_status_update( + label="routing", + status="running", + agent_type="welcome_agent", + detail="INTRO", + current_phase={"phase": "INTRO", "percentage": 0, "current": None, "total": None}, + ) + await sink.emit_phase_update( + current_phase={"phase": "PREFERENCE_ELICITATION", "percentage": 72, "current": 1, "total": 6}, + agent_type="preference_elicitation_agent", + detail="phase_progressed", + ) + await sink.close() + + events = [] + async for chunk in sink.iter_sse(): + lines = [line for line in chunk.split("\n") if line] + event_name_line, data_line = lines[:2] + events.append((event_name_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: ")))) + + assert [event_name for event_name, _ in events] == [ + "status_updated", + "phase_updated", + ] + assert events[0][1]["label"] == "routing" + assert events[1][1]["current_phase"]["phase"] == "PREFERENCE_ELICITATION" + + +@pytest.mark.asyncio +async def test_streaming_sink_coerces_non_string_detail_values(): + sink = ConversationStreamingSink() + + await sink.emit_status_update( + label="running_agent", + status="started", + agent_type="welcome_agent", + detail=0, + ) + await sink.close() + + events = [] + async for chunk in sink.iter_sse(): + lines = [line for line in chunk.split("\n") if line] + event_name_line, data_line = lines[:2] + events.append((event_name_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: ")))) + + assert events == [ + ( + "status_updated", + { + "label": "running_agent", + "status": "started", + "agent_type": "welcome_agent", + "detail": "0", + "current_phase": None, + }, + ) + ] + + +@pytest.mark.asyncio +async def test_streaming_sink_appends_multiple_deltas_once_per_message_start(): + sink = ConversationStreamingSink() + + await sink.append_text(message_id="msg-1", delta="Hello") + await sink.append_text(message_id="msg-1", delta=", world") + await sink.append_text(message_id="msg-1", delta="") + await sink.close() + + events = [] + async for chunk in sink.iter_sse(): + lines = [line for line in chunk.split("\n") if line] + event_name_line, data_line = lines[:2] + events.append((event_name_line.removeprefix("event: "), json.loads(data_line.removeprefix("data: ")))) + + assert [event_name for event_name, _ in events] == [ + "message_started", + "message_delta", + "message_delta", + ] + assert events[1][1]["delta"] == "Hello" + assert events[2][1]["delta"] == ", world" diff --git a/backend/app/metrics/routes/test_routes.py b/backend/app/metrics/routes/test_routes.py index f5905b924..34b1cd7e0 100644 --- a/backend/app/metrics/routes/test_routes.py +++ b/backend/app/metrics/routes/test_routes.py @@ -21,6 +21,7 @@ CVDownloadedEvent, DemographicsEvent, DeviceSpecificationEvent, + NetworkInformationEvent, UserLocationEvent, UIInteractionEvent ) @@ -72,8 +73,8 @@ def get_network_information_request() -> dict: return { "event_type": EventType.NETWORK_INFORMATION.value, "user_id": get_random_user_id(), - "session_id": get_random_session_id(), "effective_connection_type": get_random_printable_string(10), + "connection_type": 4, } @@ -181,6 +182,7 @@ class TestMetricsRoutes: (get_cv_downloaded_request, CVDownloadedEvent, EventType.CV_DOWNLOADED), (get_demographics_request, DemographicsEvent, EventType.DEMOGRAPHICS), (get_device_specification_request, DeviceSpecificationEvent, EventType.DEVICE_SPECIFICATION), + (get_network_information_request, NetworkInformationEvent, EventType.NETWORK_INFORMATION), (get_user_location_request, UserLocationEvent, EventType.USER_LOCATION), (get_ui_interaction_request, UIInteractionEvent, EventType.UI_INTERACTION), ], @@ -188,6 +190,7 @@ class TestMetricsRoutes: "CV Downloaded", "Demographics", "Device Specification", + "Network Information", "User Location", "UI Interaction" ] @@ -238,6 +241,7 @@ async def test_record_multiple_metric_events_successful( get_cv_downloaded_request(), get_demographics_request(), get_device_specification_request(), + get_network_information_request(), get_user_location_request(), get_ui_interaction_request() ] @@ -265,6 +269,7 @@ async def test_record_multiple_metric_events_successful( (get_cv_downloaded_request, CVDownloadedEvent, EventType.CV_DOWNLOADED), (get_demographics_request, DemographicsEvent, EventType.DEMOGRAPHICS), (get_device_specification_request, DeviceSpecificationEvent, EventType.DEVICE_SPECIFICATION), + (get_network_information_request, NetworkInformationEvent, EventType.NETWORK_INFORMATION), (get_user_location_request, UserLocationEvent, EventType.USER_LOCATION), (get_ui_interaction_request, UIInteractionEvent, EventType.UI_INTERACTION), ], @@ -272,6 +277,7 @@ async def test_record_multiple_metric_events_successful( "CV Downloaded", "Demographics", "Device Specification", + "Network Information", "User Location", "UI Interaction" ] diff --git a/backend/app/metrics/types.py b/backend/app/metrics/types.py index 741a9a631..a597afed0 100644 --- a/backend/app/metrics/types.py +++ b/backend/app/metrics/types.py @@ -684,8 +684,20 @@ class NetworkInformationEvent(AbstractUserAccountEvent): """ effective_connection_type - the network classification of the user's connection: 2g, 3g, 4g, 5g... """ + connection_type: int | None = None + """ + connection_type - numeric connection type from Network Information API: 0=unknown, 1=bluetooth, 2=cellular, 3=ethernet, 4=wifi, 5=wimax, 6=none + """ - def __init__(self, *, user_id: str, effective_connection_type: str, client_id: str | None = None, relevant_experiments: dict[str, str] = None): + def __init__( + self, + *, + user_id: str, + effective_connection_type: str, + client_id: str | None = None, + relevant_experiments: dict[str, str] | None = None, + connection_type: int | None = None, + ): super().__init__( user_id=user_id, client_id=client_id, @@ -693,6 +705,7 @@ def __init__(self, *, user_id: str, effective_connection_type: str, client_id: s effective_connection_type=effective_connection_type, relevant_experiments=relevant_experiments or {} ) + self.connection_type = connection_type class Config: extra = "forbid" diff --git a/backend/app/server.py b/backend/app/server.py index 1f7d0d7b3..fd5aabd1d 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -234,6 +234,9 @@ def setup_sentry(): matching_service_url=os.getenv("MATCHING_SERVICE_URL"), matching_service_api_key=os.getenv("MATCHING_SERVICE_API_KEY"), inline_phase_transition=os.getenv("COMPASS_INLINE_PHASE_TRANSITION", "").lower() in ("1", "true"), + stream_chunk_size=int(os.getenv("BACKEND_STREAM_CHUNK_SIZE") or 10), + stream_chunk_mode="words" if (os.getenv("BACKEND_STREAM_CHUNK_MODE") or "").lower() == "words" else "chars", + stream_delta_delay_ms=int(os.getenv("BACKEND_STREAM_DELTA_DELAY_MS") or 12), ) set_application_config(application_config) diff --git a/backend/app/users/auth.py b/backend/app/users/auth.py index 09fe423ea..cc2d0a648 100644 --- a/backend/app/users/auth.py +++ b/backend/app/users/auth.py @@ -64,35 +64,24 @@ def _get_user_info(decoded_token: Any, token: str) -> UserInfo: ) -def _decode_user_info_api_gateway(auth_info_b64): +def _decode_user_info_espv2(auth_info_b64): """ - Decodes a base64-encoded string containing user information and returns it as a dictionary. - The api-gateway does not allways send the correct padding, so this function - handles partial padding (missing one or two `=` characters). Rejects invalid input. + Decodes a base64url-encoded string containing user information and returns it as a dictionary. + ESPv2 sends the JWT payload as base64url without padding, so we add == unconditionally + before decoding (urlsafe_b64decode ignores extra padding). Args: - auth_info_b64 (str): The base64-encoded string containing JSON user data. + auth_info_b64 (str): The base64url-encoded string containing JSON user data. Returns: dict: Decoded user information as a Python dictionary. Raises: - ValueError: If the input is not valid base64 or if the JSON is invalid. + ValueError: If the input is not valid base64url or if the JSON is invalid. """ try: - # Check if padding is needed and apply it - padding_needed = len(auth_info_b64) % 4 - if padding_needed == 1: - raise ValueError("Invalid base64 input: length is not compatible with base64 encoding") - elif padding_needed == 2: - padded_base64 = auth_info_b64 + '==' # Add two `=` characters - elif padding_needed == 3: - padded_base64 = auth_info_b64 + '=' # Add one `=` character - else: - padded_base64 = auth_info_b64 # No padding needed - - # Decode the base64 string - decoded_bytes = base64.b64decode(padded_base64.encode('utf-8')) + # ESPv2 sends base64url with no padding — add == unconditionally (ignored if not needed). + decoded_bytes = base64.urlsafe_b64decode(auth_info_b64 + '==') # Convert bytes to string decoded_string = decoded_bytes.decode('utf-8') @@ -103,7 +92,7 @@ def _decode_user_info_api_gateway(auth_info_b64): return user_info except Exception as e: - raise ValueError("Invalid base64 or JSON input") from e + raise ValueError("Invalid base64url or JSON input") from e class Authentication: @@ -146,20 +135,20 @@ def construct_user_info(request: Request, provider: HTTPAuthorizationCredentials credentials: str = provider.credentials token_info: Any if target_env != "local": - # When deployed, the credentials are verified by the API Gateway, which sends - # the decoded user information in a Base64-encoded format through the `x-apigateway-api-userinfo` header. - # If the header is missing, the user should be treated as unauthenticated, and a 403 error must be raised. - # While this scenario should not occur under normal circumstances, since the - # API Gateway is expected to always include the header, we implement this check - # as a precautionary measure. For example, such an issue could arise if the API - # Gateway is improperly configured or if the incoming request does not originate - # from the API Gateway. - auth_info_b64 = request.headers.get('x-apigateway-api-userinfo') + # When deployed, the credentials are verified by ESPv2, which sends + # the decoded user information in a base64url-encoded format through the + # `X-Endpoint-API-UserInfo` header. + # If the header is missing, the user should be treated as unauthenticated, and a 401 error must be raised. + # While this scenario should not occur under normal circumstances, since ESPv2 + # is expected to always include the header, we implement this check + # as a precautionary measure. For example, such an issue could arise if ESPv2 + # is improperly configured or if the incoming request does not originate from ESPv2. + auth_info_b64 = request.headers.get('X-Endpoint-API-UserInfo') if not auth_info_b64: raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="forbidden") - # The user info is encoded as base 64 string by the api-gateway - token_info = _decode_user_info_api_gateway(auth_info_b64) + # The user info is encoded as base64url string by ESPv2 + token_info = _decode_user_info_espv2(auth_info_b64) else: # When running locally, use the jwt token from Authorization header. if not credentials: @@ -182,11 +171,17 @@ def construct_user_info(request: Request, provider: HTTPAuthorizationCredentials class ApiKeyAuth: """ - This class is used to api calls using API keys in the header. - see https://cloud.google.com/endpoints/docs/openapi/authentication-method for more details. + Enforces API key authentication via the ``x-api-key`` request header. + + ESPv2 validates the GCP API key against Service Control before the request reaches the backend. + This dependency registers the ``gcp_api_key`` security scheme in FastAPI's OpenAPI spec so that + ``_construct_espv2_cfg.py`` can detect API-key-protected routes and emit the correct ESPv2 + security definition for them. It also rejects requests that somehow bypass ESPv2 (e.g. direct + Cloud Run calls during development) by raising a 403 if the header is absent. + + See https://cloud.google.com/endpoints/docs/openapi/authentication-method for more details. """ async def __call__(self, api_key: str = Depends(APIKeyHeader(scheme_name="gcp_api_key", name="x-api-key", auto_error=True))): - # Currently, there's not much to do here as the API Gateway validates the key. - # Additionally, since APIKeyHeader is used with auto_error=True, it will raise an exception if the header key is missing. - # In the future, the key can be checked against roles and permissions. + # ESPv2 has already validated the key value before this point. + # APIKeyHeader with auto_error=True raises HTTP 403 if the header is missing entirely. return api_key diff --git a/backend/app/vector_search/occupation_search_routes.py b/backend/app/vector_search/occupation_search_routes.py index 67ad95682..bb39e8ea3 100644 --- a/backend/app/vector_search/occupation_search_routes.py +++ b/backend/app/vector_search/occupation_search_routes.py @@ -23,9 +23,10 @@ class OccupationsResponse(BaseModel): @app.get("/search/occupations", response_model=OccupationsResponse, description=""" - Semantically search for occupations based on a query. + Semantically search for occupations based on a query. The search is based on the embeddings of the occupations, and uses the cosine similarity to find the most similar occupations.""", + openapi_extra={"security": [{"gcp_api_key": []}]}, ) async def _search_occupations( query: Annotated[str, Query(max_length=3000, description="The text to search for matching occupations")], @@ -47,9 +48,10 @@ class OccupationsSkillsResponse(BaseModel): @app.get("/search/occupations-skills", response_model=OccupationsSkillsResponse, description=""" - Semantically search for occupations based on a query. + Semantically search for occupations based on a query. The search is based on the embeddings of the occupations, and uses the cosine similarity to find the most similar occupations.""", + openapi_extra={"security": [{"gcp_api_key": []}]}, ) async def _search_occupations( query: Annotated[str, Query(max_length=3000, description="The text to search for matching occupations")], diff --git a/backend/app/vector_search/skill_search_routes.py b/backend/app/vector_search/skill_search_routes.py index 0dccc0fe7..55b6b3303 100644 --- a/backend/app/vector_search/skill_search_routes.py +++ b/backend/app/vector_search/skill_search_routes.py @@ -26,8 +26,9 @@ class SkillsResponse(BaseModel): @app.get("/search/skills", response_model=SkillsResponse, description=""" - Semantically search for skills based on a query. The search is based on the embeddings of the skills, + Semantically search for skills based on a query. The search is based on the embeddings of the skills, and uses the cosine similarity to find the most similar skills.""", + openapi_extra={"security": [{"gcp_api_key": []}]}, ) async def _search_skills( query: Annotated[str, Query(max_length=3000, description="The text to search for matching skills")], diff --git a/backend/common_libs/llm/generative_models.py b/backend/common_libs/llm/generative_models.py index 95e423839..7180f9907 100644 --- a/backend/common_libs/llm/generative_models.py +++ b/backend/common_libs/llm/generative_models.py @@ -1,14 +1,20 @@ import logging import traceback +import inspect from vertexai.generative_models import GenerativeModel, Content, Part, GenerationConfig from vertexai.language_models import TextGenerationModel -from common_libs.llm.models_utils import LLMConfig, LLMInput, LLMResponse, BasicLLM +from common_libs.llm.models_utils import LLMConfig, LLMInput, LLMResponse, BasicLLM, LLMStreamingCallback logger = logging.getLogger(__name__) +async def _maybe_await(callback_result): + if inspect.isawaitable(callback_result): + await callback_result + + class GeminiGenerativeLLM(BasicLLM): """ A wrapper for the Gemini LLM that provides retry logic with exponential backoff and jitter for generating content. @@ -27,14 +33,47 @@ def __init__(self, *, # noinspection PyProtectedMember self._resource_name = self._model._prediction_resource_name # pylint: disable=protected-access - async def internal_generate_content(self, llm_input: LLMInput | str) -> LLMResponse: - contents = llm_input if isinstance(llm_input, str) else [ + @staticmethod + def _build_contents(llm_input: LLMInput | str): + return llm_input if isinstance(llm_input, str) else [ Content(role=turn.role, parts=[Part.from_text(turn.content)]) for turn in llm_input.turns] + + async def internal_generate_content(self, llm_input: LLMInput | str) -> LLMResponse: + contents = self._build_contents(llm_input) response = await self._model.generate_content_async(contents=contents) return LLMResponse(text=response.text, prompt_token_count=response.usage_metadata.prompt_token_count, response_token_count=response.usage_metadata.candidates_token_count) + async def internal_stream_content( + self, + llm_input: LLMInput | str, + on_chunk: LLMStreamingCallback | None = None, + ) -> LLMResponse: + contents = self._build_contents(llm_input) + stream = await self._model.generate_content_async(contents=contents, stream=True) + + chunks: list[str] = [] + final_response = None + async for response in stream: + final_response = response + text = response.text or "" + if not text: + continue + chunks.append(text) + if on_chunk is not None: + await _maybe_await(on_chunk(text)) + + if final_response is None: + return LLMResponse(text="", prompt_token_count=0, response_token_count=0) + + usage_metadata = final_response.usage_metadata + return LLMResponse( + text="".join(chunks), + prompt_token_count=usage_metadata.prompt_token_count if usage_metadata else 0, + response_token_count=usage_metadata.candidates_token_count if usage_metadata else 0, + ) + class PalmTextGenerativeLLM(BasicLLM): """ @@ -61,3 +100,13 @@ async def internal_generate_content(self, llm_input: LLMInput | str) -> LLMRespo prompt_token_count=token_meta_data['inputTokenCount']['totalTokens'], response_token_count=token_meta_data['outputTokenCount']['totalTokens'] ) + + async def internal_stream_content( + self, + llm_input: LLMInput | str, + on_chunk: LLMStreamingCallback | None = None, + ) -> LLMResponse: + response = await self.internal_generate_content(llm_input) + if on_chunk is not None and response.text: + await _maybe_await(on_chunk(response.text)) + return response diff --git a/backend/common_libs/llm/models_utils.py b/backend/common_libs/llm/models_utils.py index 1b747b9a6..3f049f821 100644 --- a/backend/common_libs/llm/models_utils.py +++ b/backend/common_libs/llm/models_utils.py @@ -1,6 +1,7 @@ import logging import os from abc import ABC, abstractmethod +from collections.abc import Awaitable, Callable import vertexai from dotenv import load_dotenv @@ -189,6 +190,9 @@ class LLMResponse(BaseModel): """The number of tokens in the response.""" +LLMStreamingCallback = Callable[[str], Awaitable[None] | None] + + class LLM(ABC): """ An abstract class for a LLM. @@ -205,6 +209,14 @@ async def generate_content(self, llm_input: LLMInput | str) -> LLMResponse: """ raise NotImplementedError() + @abstractmethod + async def stream_content(self, llm_input: LLMInput | str, on_chunk: LLMStreamingCallback | None = None) -> LLMResponse: + """ + Stream generated content and optionally invoke a callback for each text chunk. + Returns the final aggregated response once streaming completes. + """ + raise NotImplementedError() + class BasicLLM(LLM): def __init__(self, *, config: LLMConfig = LLMConfig()): @@ -231,6 +243,26 @@ async def _generate_content() -> LLMResponse: return await Retry[str].call_with_exponential_backoff(callback=_generate_content, logger=logger) + async def stream_content(self, llm_input: LLMInput | str, on_chunk: LLMStreamingCallback | None = None) -> LLMResponse: + async def _stream_content() -> LLMResponse: + try: + logger.debug("Streaming content with resource:%s", self._resource_name) + return await self.internal_stream_content(llm_input, on_chunk=on_chunk) + except Exception as e: + logger.error("An error occurred while streaming content with resource:%s", + self._resource_name, exc_info=True) + raise e + + return await Retry[str].call_with_exponential_backoff(callback=_stream_content, logger=logger) + @abstractmethod async def internal_generate_content(self, llm_input: LLMInput | str) -> LLMResponse: raise NotImplementedError() + + @abstractmethod + async def internal_stream_content( + self, + llm_input: LLMInput | str, + on_chunk: LLMStreamingCallback | None = None, + ) -> LLMResponse: + raise NotImplementedError() diff --git a/backend/poetry.lock b/backend/poetry.lock index f92071da3..1b6f53d97 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1030,7 +1030,7 @@ files = [ [package.dependencies] cachecontrol = ">=0.12.6" -google-api-core = {version = ">=1.22.1,<3.0.0.dev0", extras = ["grpc"], markers = "platform_python_implementation != \"PyPy\""} +google-api-core = {version = ">=1.22.1,<3.0.0dev", extras = ["grpc"], markers = "platform_python_implementation != \"PyPy\""} google-api-python-client = ">=1.7.8" google-cloud-firestore = {version = ">=2.9.1", markers = "platform_python_implementation != \"PyPy\""} google-cloud-storage = ">=1.37.1" @@ -1267,14 +1267,14 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" -grpcio = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +grpcio = {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} grpcio-status = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} -proto-plus = ">=1.22.3,<2.0.0.dev0" +proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] @@ -1363,26 +1363,26 @@ files = [ [package.dependencies] docstring-parser = "<1" -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.8.dev0,<3.0.0.dev0", extras = ["grpc"]} -google-auth = ">=2.14.1,<3.0.0.dev0" -google-cloud-bigquery = ">=1.15.0,<3.20.0 || >3.20.0,<4.0.0.dev0" -google-cloud-resource-manager = ">=1.3.3,<3.0.0.dev0" -google-cloud-storage = ">=1.32.0,<3.0.0.dev0" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.8.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" +google-cloud-bigquery = ">=1.15.0,<3.20.0 || >3.20.0,<4.0.0dev" +google-cloud-resource-manager = ">=1.3.3,<3.0.0dev" +google-cloud-storage = ">=1.32.0,<3.0.0dev" packaging = ">=14.3" -proto-plus = ">=1.22.0,<2.0.0.dev0" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +proto-plus = ">=1.22.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" pydantic = "<3" -shapely = "<3.0.0.dev0" +shapely = "<3.0.0dev" [package.extras] autologging = ["mlflow (>=1.27.0,<=2.1.1)"] -cloud-profiler = ["tensorboard-plugin-profile (>=2.4.0,<3.0.0.dev0)", "tensorflow (>=2.4.0,<3.0.0.dev0)", "werkzeug (>=2.0.0,<2.1.0.dev0)"] -datasets = ["pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.dev0) ; python_version < \"3.11\""] +cloud-profiler = ["tensorboard-plugin-profile (>=2.4.0,<3.0.0dev)", "tensorflow (>=2.4.0,<3.0.0dev)", "werkzeug (>=2.0.0,<2.1.0dev)"] +datasets = ["pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0dev) ; python_version < \"3.11\""] endpoint = ["requests (>=2.28.1)"] -full = ["cloudpickle (<3.0)", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0)", "fastapi (>=0.71.0,<=0.109.1)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-cloud-logging (<4.0)", "google-vizier (>=0.1.6)", "httpx (>=0.23.0,<0.25.0)", "immutabledict", "lit-nlp (==0.4.0)", "mlflow (>=1.27.0,<=2.1.1)", "nest-asyncio (>=1.0.0,<1.6.0)", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.dev0) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pydantic (<2)", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "requests (>=2.28.1)", "setuptools (<70.0.0)", "starlette (>=0.17.1)", "tensorflow (>=2.3.0,<3.0.0.dev0)", "tensorflow (>=2.3.0,<3.0.0.dev0) ; python_version <= \"3.11\"", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)"] +full = ["cloudpickle (<3.0)", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0)", "fastapi (>=0.71.0,<=0.109.1)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-cloud-logging (<4.0)", "google-vizier (>=0.1.6)", "httpx (>=0.23.0,<0.25.0)", "immutabledict", "lit-nlp (==0.4.0)", "mlflow (>=1.27.0,<=2.1.1)", "nest-asyncio (>=1.0.0,<1.6.0)", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0dev) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pydantic (<2)", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "requests (>=2.28.1)", "setuptools (<70.0.0)", "starlette (>=0.17.1)", "tensorflow (>=2.3.0,<3.0.0dev)", "tensorflow (>=2.3.0,<3.0.0dev) ; python_version <= \"3.11\"", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)"] langchain = ["langchain (>=0.1.16,<0.2)", "langchain-core (<0.2)", "langchain-google-vertexai (<2)"] langchain-testing = ["absl-py", "cloudpickle (>=2.2.1,<4.0)", "langchain (>=0.1.16,<0.2)", "langchain-core (<0.2)", "langchain-google-vertexai (<2)", "pydantic (>=2.6.3,<3)", "pytest-xdist"] -lit = ["explainable-ai-sdk (>=1.0.0)", "lit-nlp (==0.4.0)", "pandas (>=1.0.0)", "tensorflow (>=2.3.0,<3.0.0.dev0)"] +lit = ["explainable-ai-sdk (>=1.0.0)", "lit-nlp (==0.4.0)", "pandas (>=1.0.0)", "tensorflow (>=2.3.0,<3.0.0dev)"] metadata = ["numpy (>=1.15.0)", "pandas (>=1.0.0)"] pipelines = ["pyyaml (>=5.3.1,<7)"] prediction = ["docker (>=5.0.3)", "fastapi (>=0.71.0,<=0.109.1)", "httpx (>=0.23.0,<0.25.0)", "starlette (>=0.17.1)", "uvicorn[standard] (>=0.16.0)"] @@ -1392,10 +1392,10 @@ rapid-evaluation = ["nest-asyncio (>=1.0.0,<1.6.0)", "pandas (>=1.0.0,<2.2.0)"] ray = ["google-cloud-bigquery", "google-cloud-bigquery-storage", "immutabledict", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=6.0.1)", "pydantic (<2)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "setuptools (<70.0.0)"] ray-testing = ["google-cloud-bigquery", "google-cloud-bigquery-storage", "immutabledict", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=6.0.1)", "pydantic (<2)", "pytest-xdist", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "ray[train] (==2.9.3)", "scikit-learn", "setuptools (<70.0.0)", "tensorflow", "torch (>=2.0.0,<2.1.0)", "xgboost", "xgboost-ray"] reasoningengine = ["cloudpickle (>=2.2.1,<4.0)", "pydantic (>=2.6.3,<3)"] -tensorboard = ["tensorflow (>=2.3.0,<3.0.0.dev0) ; python_version <= \"3.11\""] -testing = ["bigframes ; python_version >= \"3.10\"", "cloudpickle (<3.0)", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0)", "fastapi (>=0.71.0,<=0.109.1)", "google-api-core (>=2.11,<3.0.0)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-cloud-logging (<4.0)", "google-vizier (>=0.1.6)", "grpcio-testing", "httpx (>=0.23.0,<0.25.0)", "immutabledict", "ipython", "kfp (>=2.6.0,<3.0.0)", "lit-nlp (==0.4.0)", "mlflow (>=1.27.0,<=2.1.1)", "nest-asyncio (>=1.0.0,<1.6.0)", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.dev0) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pydantic (<2)", "pyfakefs", "pytest-asyncio", "pytest-xdist", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<1.0.0)", "scikit-learn", "setuptools (<70.0.0)", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<3.0.0.dev0)", "tensorflow (==2.13.0) ; python_version <= \"3.11\"", "tensorflow (==2.16.1) ; python_version > \"3.11\"", "tensorflow (>=2.3.0,<3.0.0.dev0)", "tensorflow (>=2.3.0,<3.0.0.dev0) ; python_version <= \"3.11\"", "tensorflow (>=2.4.0,<3.0.0.dev0)", "torch (>=2.0.0,<2.1.0) ; python_version <= \"3.11\"", "torch (>=2.2.0) ; python_version > \"3.11\"", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<2.1.0.dev0)", "xgboost"] +tensorboard = ["tensorflow (>=2.3.0,<3.0.0dev) ; python_version <= \"3.11\""] +testing = ["bigframes ; python_version >= \"3.10\"", "cloudpickle (<3.0)", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0)", "fastapi (>=0.71.0,<=0.109.1)", "google-api-core (>=2.11,<3.0.0)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-cloud-logging (<4.0)", "google-vizier (>=0.1.6)", "grpcio-testing", "httpx (>=0.23.0,<0.25.0)", "immutabledict", "ipython", "kfp (>=2.6.0,<3.0.0)", "lit-nlp (==0.4.0)", "mlflow (>=1.27.0,<=2.1.1)", "nest-asyncio (>=1.0.0,<1.6.0)", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pandas (>=1.0.0,<2.2.0)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0dev) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pydantic (<2)", "pyfakefs", "pytest-asyncio", "pytest-xdist", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<=2.9.3) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.9.3) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<1.0.0)", "scikit-learn", "setuptools (<70.0.0)", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<3.0.0dev)", "tensorflow (==2.13.0) ; python_version <= \"3.11\"", "tensorflow (==2.16.1) ; python_version > \"3.11\"", "tensorflow (>=2.3.0,<3.0.0dev)", "tensorflow (>=2.3.0,<3.0.0dev) ; python_version <= \"3.11\"", "tensorflow (>=2.4.0,<3.0.0dev)", "torch (>=2.0.0,<2.1.0) ; python_version <= \"3.11\"", "torch (>=2.2.0) ; python_version > \"3.11\"", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<2.1.0dev)", "xgboost"] vizier = ["google-vizier (>=0.1.6)"] -xai = ["tensorflow (>=2.3.0,<3.0.0.dev0)"] +xai = ["tensorflow (>=2.3.0,<3.0.0dev)"] [[package]] name = "google-cloud-bigquery" @@ -1410,24 +1410,24 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} -google-auth = ">=2.14.1,<3.0.0.dev0" -google-cloud-core = ">=1.6.0,<3.0.0.dev0" -google-resumable-media = ">=0.6.0,<3.0.dev0" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<3.0.0dev" +google-cloud-core = ">=1.6.0,<3.0.0dev" +google-resumable-media = ">=0.6.0,<3.0dev" packaging = ">=20.0.0" -python-dateutil = ">=2.7.2,<3.0.dev0" -requests = ">=2.21.0,<3.0.0.dev0" +python-dateutil = ">=2.7.2,<3.0dev" +requests = ">=2.21.0,<3.0.0dev" [package.extras] -all = ["Shapely (>=1.8.4,<3.0.0.dev0)", "db-dtypes (>=0.3.0,<2.0.0.dev0)", "geopandas (>=0.9.0,<1.0.dev0)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0.dev0)", "grpcio (>=1.47.0,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "importlib-metadata (>=1.0.0) ; python_version < \"3.8\"", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0.dev0)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0.dev0)"] -bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0.dev0)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0)"] -bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0.dev0)", "grpcio (>=1.47.0,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "pyarrow (>=3.0.0)"] -geopandas = ["Shapely (>=1.8.4,<3.0.0.dev0)", "geopandas (>=0.9.0,<1.0.dev0)"] +all = ["Shapely (>=1.8.4,<3.0.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "importlib-metadata (>=1.0.0) ; python_version < \"3.8\"", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)"] +bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "pyarrow (>=3.0.0)"] +geopandas = ["Shapely (>=1.8.4,<3.0.0dev)", "geopandas (>=0.9.0,<1.0dev)"] ipython = ["ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)"] ipywidgets = ["ipykernel (>=6.0.0)", "ipywidgets (>=7.7.0)"] opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)"] -pandas = ["db-dtypes (>=0.3.0,<2.0.0.dev0)", "importlib-metadata (>=1.0.0) ; python_version < \"3.8\"", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] -tqdm = ["tqdm (>=4.7.4,<5.0.0.dev0)"] +pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0) ; python_version < \"3.8\"", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] +tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] [[package]] name = "google-cloud-core" @@ -1442,11 +1442,11 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0.dev0" -google-auth = ">=1.25.0,<3.0.dev0" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0.dev0)", "grpcio-status (>=1.38.0,<2.0.dev0)"] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-dlp" @@ -1461,10 +1461,10 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0" -proto-plus = ">=1.22.3,<2.0.0.dev0" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" +proto-plus = ">=1.22.3,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] name = "google-cloud-firestore" @@ -1480,11 +1480,11 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0" -google-cloud-core = ">=1.4.1,<3.0.0.dev0" -proto-plus = {version = ">=1.22.2,<2.0.0.dev0", markers = "python_version >= \"3.11\""} -protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" +google-cloud-core = ">=1.4.1,<3.0.0dev" +proto-plus = {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""} +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" [[package]] name = "google-cloud-resource-manager" @@ -1499,11 +1499,11 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0" -grpc-google-iam-v1 = ">=0.12.4,<1.0.0.dev0" -proto-plus = ">=1.22.3,<2.0.0.dev0" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" +proto-plus = ">=1.22.3,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] name = "google-cloud-storage" @@ -1518,15 +1518,15 @@ files = [ ] [package.dependencies] -google-api-core = ">=2.15.0,<3.0.0.dev0" -google-auth = ">=2.26.1,<3.0.dev0" -google-cloud-core = ">=2.3.0,<3.0.dev0" -google-crc32c = ">=1.0,<2.0.dev0" +google-api-core = ">=2.15.0,<3.0.0dev" +google-auth = ">=2.26.1,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-crc32c = ">=1.0,<2.0dev" google-resumable-media = ">=2.7.2" -requests = ">=2.18.0,<3.0.0.dev0" +requests = ">=2.18.0,<3.0.0dev" [package.extras] -protobuf = ["protobuf (<6.0.0.dev0)"] +protobuf = ["protobuf (<6.0.0dev)"] tracing = ["opentelemetry-api (>=1.1.0)"] [[package]] @@ -1648,11 +1648,11 @@ files = [ ] [package.dependencies] -google-crc32c = ">=1.0,<2.0.dev0" +google-crc32c = ">=1.0,<2.0dev" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "google-auth (>=1.22.0,<2.0.dev0)"] -requests = ["requests (>=2.18.0,<3.0.0.dev0)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" @@ -1686,9 +1686,9 @@ files = [ ] [package.dependencies] -googleapis-common-protos = {version = ">=1.56.0,<2.0.0.dev0", extras = ["grpc"]} -grpcio = ">=1.44.0,<2.0.0.dev0" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} +grpcio = ">=1.44.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] name = "grpcio" @@ -3193,7 +3193,7 @@ files = [ ] [package.dependencies] -protobuf = ">=3.19.0,<5.0.0.dev0" +protobuf = ">=3.19.0,<5.0.0dev" [package.extras] testing = ["google-api-core[grpc] (>=1.31.5)"] @@ -3501,7 +3501,7 @@ files = [ ] [package.dependencies] -astroid = ">=3.2.2,<=3.3.0.dev0" +astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, diff --git a/frontend-new/src/chat/Chat.test.tsx b/frontend-new/src/chat/Chat.test.tsx index 34f260755..c01e314e5 100644 --- a/frontend-new/src/chat/Chat.test.tsx +++ b/frontend-new/src/chat/Chat.test.tsx @@ -605,7 +605,11 @@ describe("Chat", () => { // AND expect a typing indicator to have been shown assertTypingMessageWasShown(); // AND expect an empty message to be sent to the chat service - expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenActiveSessionId, ""); + expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith( + givenActiveSessionId, + "", + expect.any(Object) + ); await expect((ChatService.getInstance().sendMessage as jest.Mock).mock.results[0].value).resolves.toBe( givenSendMessageResponse ); @@ -651,7 +655,11 @@ describe("Chat", () => { // AND expect a typing indicator to be shown assertTypingMessageWasShown(); // AND expect an empty message to be sent to the chat service - expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenActiveSessionId, ""); + expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith( + givenActiveSessionId, + "", + expect.any(Object) + ); // AND an error message to be shown in the chat list expect(ChatList as jest.Mock).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -859,7 +867,7 @@ describe("Chat", () => { // AND expect the chat history to be fetched for the new session expect(ChatService.getInstance().getChatHistory).toHaveBeenCalledWith(givenNewSessionId); // AND expect an empty message to be sent to the chat service for the new session - expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenNewSessionId, ""); + expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenNewSessionId, "", expect.any(Object)); // AND expect the chat list to be updated with the new (empty) messages list expect(ChatList as jest.Mock).toHaveBeenNthCalledWith(1, expect.objectContaining({ messages: [] }), {}); // AND expect no errors or warning to have occurred @@ -1216,7 +1224,11 @@ describe("Chat", () => { }); // THEN expect the send message method to be called with the user's message - expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenActiveSessionId, givenMessage); + expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith( + givenActiveSessionId, + givenMessage, + expect.any(Object) + ); // AND expect the user's message and a typing indicator to be shown in the chat await waitFor(() => { assertMessagesAreShown( @@ -1443,7 +1455,11 @@ describe("Chat", () => { }); // THEN expect the send message method to be called with the user's message - expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith(givenActiveSessionId, givenMessage); + expect(ChatService.getInstance().sendMessage).toHaveBeenCalledWith( + givenActiveSessionId, + givenMessage, + expect.any(Object) + ); // AND expect the user's message and a typing indicator to be shown in the chat await waitFor(() => { assertMessagesAreShown( @@ -1579,6 +1595,183 @@ describe("Chat", () => { expect(useSnackbar().enqueueSnackbar).not.toHaveBeenCalled(); }); + test("should keep typing visible until the first streamed delta and update progress live", async () => { + const givenUser: TabiyaUser = getMockUser(); + AuthenticationStateService.getInstance().setUser(givenUser); + const givenActiveSessionId = 123; + UserPreferencesStateService.getInstance().setUserPreferences( + getMockUserPreferences(givenUser, givenActiveSessionId) + ); + + const givenPreviousConversation: ConversationResponse = getMockConversationResponse( + [ + { + message_id: nanoid(), + message: "Welcome", + sent_at: new Date().toISOString(), + sender: ConversationMessageSender.COMPASS, + reaction: null, + }, + ], + ConversationPhase.INTRO, + 0 + ); + jest.spyOn(ChatService.getInstance(), "getChatHistory").mockResolvedValueOnce(givenPreviousConversation); + + const streamedPhase = { + phase: ConversationPhase.PREFERENCE_ELICITATION, + percentage: 72, + current: 1, + total: 6, + }; + const finalResponse: ConversationResponse = { + messages: [ + { + message_id: "stream-msg", + message: "Hello there", + sent_at: new Date().toISOString(), + sender: ConversationMessageSender.COMPASS, + reaction: null, + }, + ], + conversation_completed: false, + conversation_conducted_at: null, + experiences_explored: 0, + current_phase: streamedPhase, + }; + + let resolveSendMessage!: (value: ConversationResponse) => void; + let streamHandlers: any; + jest.spyOn(ChatService.getInstance(), "sendMessage").mockImplementationOnce( + (_sessionId, _message, handlers) => + new Promise((resolve) => { + streamHandlers = handlers; + resolveSendMessage = resolve; + }) + ); + + render(); + await assertChatInitialized(); + + const givenMessage = "Tell me what happens next"; + act(() => { + (ChatMessageField as jest.Mock).mock.calls.at(-1)[0].handleSend(givenMessage); + }); + + await waitFor(() => { + expect((ChatMessageField as jest.Mock).mock.calls.at(-1)[0].aiIsTyping).toBe(true); + }); + + act(() => { + streamHandlers?.onStatusUpdated?.({ + label: "routing", + status: "running", + agent_type: "welcome_agent", + detail: "INTRO", + current_phase: givenPreviousConversation.current_phase, + }); + }); + + await waitFor(() => { + expect(ChatList as jest.Mock).toHaveBeenLastCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + type: TYPING_CHAT_MESSAGE_TYPE, + payload: expect.objectContaining({ + status: "Choosing the best next step", + }), + }), + ]), + }), + {} + ); + }); + + act(() => { + streamHandlers?.onPhaseUpdated?.({ + current_phase: streamedPhase, + agent_type: "preference_elicitation_agent", + detail: "phase_progressed", + }); + streamHandlers?.onMessageStarted?.({ + message_id: "stream-msg", + sender: ConversationMessageSender.COMPASS, + message_type: "TEXT", + }); + }); + + await waitFor(() => { + expect((ChatMessageField as jest.Mock).mock.calls.at(-1)[0].aiIsTyping).toBe(true); + }); + expect(ChatProgressBar as jest.Mock).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: streamedPhase.phase, + percentage: streamedPhase.percentage, + current: streamedPhase.current, + total: streamedPhase.total, + }), + {} + ); + + act(() => { + streamHandlers?.onMessageDelta?.({ + message_id: "stream-msg", + delta: "Hello", + }); + }); + + await waitFor(() => { + expect((ChatMessageField as jest.Mock).mock.calls.at(-1)[0].aiIsTyping).toBe(false); + }); + await waitFor(() => { + expect(ChatList as jest.Mock).toHaveBeenLastCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + type: COMPASS_CHAT_MESSAGE_TYPE, + payload: expect.objectContaining({ + message_id: "stream-msg", + message: "Hello", + }), + }), + ]), + }), + {} + ); + }); + + act(() => { + streamHandlers?.onMessageCompleted?.(finalResponse.messages[0]); + streamHandlers?.onTurnCompleted?.({ + conversation_completed: false, + conversation_conducted_at: null, + experiences_explored: 0, + current_phase: streamedPhase, + }); + resolveSendMessage(finalResponse); + }); + + await waitFor(() => { + expect(ChatList as jest.Mock).toHaveBeenLastCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + type: COMPASS_CHAT_MESSAGE_TYPE, + payload: expect.objectContaining({ + message_id: "stream-msg", + message: "Hello there", + }), + }), + ]), + }), + {} + ); + }); + expect(console.error).not.toHaveBeenCalled(); + expect(console.warn).not.toHaveBeenCalled(); + }); + test("should handle send message error", async () => { // GIVEN a user is logged in const givenUser: TabiyaUser = getMockUser(); @@ -1644,7 +1837,11 @@ describe("Chat", () => { // THEN expect the send message method to be called expect(ChatService.getInstance().sendMessage).toHaveBeenCalledTimes(1); - expect(ChatService.getInstance().sendMessage).toHaveBeenLastCalledWith(givenActiveSessionId, givenMessage); + expect(ChatService.getInstance().sendMessage).toHaveBeenLastCalledWith( + givenActiveSessionId, + givenMessage, + expect.any(Object) + ); // AND expect an error to have been logged await waitFor(() => { expect(console.error).toHaveBeenCalledWith(new ChatError("Failed to send message:", givenError)); diff --git a/frontend-new/src/chat/Chat.tsx b/frontend-new/src/chat/Chat.tsx index 1c5d6f64f..095e42e1d 100644 --- a/frontend-new/src/chat/Chat.tsx +++ b/frontend-new/src/chat/Chat.tsx @@ -1,5 +1,6 @@ import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; +import { TranslationKey } from "src/react-i18next"; import ChatService from "src/chat/ChatService/ChatService"; import ChatList from "src/chat/chatList/ChatList"; import { IChatMessage } from "src/chat/Chat.types"; @@ -135,6 +136,7 @@ export const Chat: React.FC> = ({ ); const [currentUserId] = useState(authenticationStateService.getInstance().getUser()?.id ?? null); const [currentPhase, setCurrentPhase] = useState(defaultCurrentPhase); + const [streamStatusMessage, setStreamStatusMessage] = useState(null); // CV upload states const [isUploadingCv, setIsUploadingCv] = useState(false); const [cvUploadError, setCvUploadError] = useState(null); @@ -165,12 +167,142 @@ export const Chat: React.FC> = ({ setMessages((prevMessages) => [...prevMessages, message]); }, []); + const upsertMessageInChat = useCallback((message: IChatMessage) => { + setMessages((prevMessages) => { + const existingIndex = prevMessages.findIndex((msg) => msg.message_id === message.message_id); + if (existingIndex === -1) { + return [...prevMessages, message]; + } + const nextMessages = [...prevMessages]; + nextMessages[existingIndex] = message; + return nextMessages; + }); + }, []); + + const pendingDeltasRef = useRef>(new Map()); + const rafIdRef = useRef(null); + + const flushPendingDeltas = useCallback(() => { + rafIdRef.current = null; + const pending = pendingDeltasRef.current; + if (pending.size === 0) { + return; + } + const snapshot = new Map(pending); + pending.clear(); + + setMessages((prevMessages) => { + const appendedMessageIds = new Set(); + const nextMessages = prevMessages.map((message) => { + const delta = snapshot.get(message.message_id); + if (!delta || !message.type.startsWith("compass-message-")) { + return message; + } + appendedMessageIds.add(message.message_id); + return { + ...message, + payload: { + ...(message.payload as CompassChatMessageProps), + message: ((message.payload as CompassChatMessageProps).message ?? "") + delta, + animateChunks: true, + streamVersion: ((message.payload as CompassChatMessageProps).streamVersion ?? 0) + 1, + }, + }; + }); + + snapshot.forEach((delta, messageId) => { + if (!appendedMessageIds.has(messageId)) { + nextMessages.push( + generateCompassMessage(messageId, delta, new Date().toISOString(), null, { + animateChunks: true, + streamVersion: 1, + }) + ); + } + }); + + return nextMessages; + }); + }, []); + + const appendTextToCompassMessage = useCallback( + (messageId: string, delta: string) => { + if (!delta) { + return; + } + const pending = pendingDeltasRef.current; + pending.set(messageId, (pending.get(messageId) ?? "") + delta); + + if (rafIdRef.current === null) { + rafIdRef.current = requestAnimationFrame(flushPendingDeltas); + } + }, + [flushPendingDeltas] + ); + + const clearPendingDeltasForMessage = useCallback((messageId: string) => { + pendingDeltasRef.current.delete(messageId); + }, []); + const removeMessageFromChat = useCallback((messageId: string) => { setMessages((prevMessages) => prevMessages.filter((msg) => msg.message_id !== messageId)); }, []); + const createChatMessageFromConversationMessage = useCallback( + (messageItem: ConversationMessage): IChatMessage => { + if (messageItem.message_type === "BWS_TASK" && messageItem.metadata) { + return generateBWSTaskMessage( + messageItem.message_id, + messageItem.metadata, + (t, b, w) => handleBWSSubmitRef.current?.(t, b, w) ?? Promise.resolve() + ); + } + + return generateCompassMessage( + messageItem.message_id, + messageItem.message, + messageItem.sent_at, + messageItem.reaction + ); + }, + [] + ); + const { showSkillsRanking } = useSkillsRanking(addMessageToChat, removeMessageFromChat); + const getActiveTypingMessage = useCallback(() => { + if (currentPhase.phase === ConversationPhase.PREFERENCE_ELICITATION) { + return t("chat.chatMessage.typingChatMessage.thinkingPreferenceElicitation"); + } + return undefined; + }, [currentPhase.phase, t]); + + const formatStreamStatusMessage = useCallback( + (label: string, detail?: string | null) => { + const i18nKey = `chat.chatMessage.streamingStatus.${label}` as TranslationKey; + const translated = t(i18nKey); + const baseMessage = + translated !== i18nKey + ? translated + : label + .split("_") + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); + + if (!detail || detail === "phase_started" || detail === "phase_progressed" || detail === "phase_entered") { + return baseMessage; + } + if (detail === "response_ready" || detail === "db6_save_started" || detail === "db6_save_finished") { + return baseMessage; + } + if (/^\d+$/.test(detail) || /^[A-Z_]+$/.test(detail)) { + return baseMessage; + } + return `${baseMessage}: ${detail}`; + }, + [t] + ); + useEffect(() => { if (!currentUserId || networkInfoSentRef.current) { return; @@ -189,30 +321,35 @@ export const Chat: React.FC> = ({ } }, [currentUserId]); - // Depending on the typing state, add or remove the typing message from the messages list const addOrRemoveTypingMessage = useCallback( - (userIsTyping: boolean) => { + (userIsTyping: boolean, typingMessage?: string, statusMessage?: string) => { if (userIsTyping) { - // Only add typing message if it doesn't already exist - const thinkingMessage = - currentPhase.phase === ConversationPhase.PREFERENCE_ELICITATION - ? t("chat.chatMessage.typingChatMessage.thinkingPreferenceElicitation") - : undefined; setMessages((prevMessages) => { const lastMessage = prevMessages[prevMessages.length - 1]; const hasTypingMessage = lastMessage?.type?.startsWith("typing-message-") ?? false; if (!hasTypingMessage) { - return [...prevMessages, generateTypingMessage(undefined, thinkingMessage)]; + return [...prevMessages, generateTypingMessage(undefined, undefined, typingMessage, statusMessage)]; } - return prevMessages; + return prevMessages.map((message) => { + if (!message.type.startsWith("typing-message-")) { + return message; + } + return { + ...message, + payload: { + ...message.payload, + message: typingMessage, + status: statusMessage, + }, + }; + }); }); } else { - // filter out the typing message setMessages((prevMessages) => prevMessages.filter((message) => !message.type.startsWith("typing-message-"))); } }, - [currentPhase.phase, t] + [] ); const recordChatResponseMetrics = useCallback( @@ -648,6 +785,7 @@ export const Chat: React.FC> = ({ const sendMessage = useCallback( async (userMessage: string, sessionId: number, displayMessage?: string) => { setAiIsTyping(true); + setStreamStatusMessage(null); // displayMessage="" suppresses the bubble; undefined = use userMessage as display const chatText = displayMessage !== undefined ? displayMessage : userMessage; if (chatText) { @@ -658,10 +796,64 @@ export const Chat: React.FC> = ({ const startTimeMs = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); const previousExploredExperiences = exploredExperiences; + let sawVisibleAssistantContent = false; + + const markAssistantVisible = () => { + if (sawVisibleAssistantContent) { + return; + } + sawVisibleAssistantContent = true; + setStreamStatusMessage(null); + setAiIsTyping(false); + }; try { // Send the user's message - const response = await ChatService.getInstance().sendMessage(sessionId, userMessage); + const response = await ChatService.getInstance().sendMessage(sessionId, userMessage, { + onTurnStarted: (event) => { + if (event.current_phase) { + setCurrentPhase((_previousCurrentPhase) => + parseConversationPhase(event.current_phase!, _previousCurrentPhase) + ); + } + }, + onStatusUpdated: (event) => { + if (event.current_phase) { + setCurrentPhase((_previousCurrentPhase) => + parseConversationPhase(event.current_phase!, _previousCurrentPhase) + ); + } + setStreamStatusMessage(formatStreamStatusMessage(event.label, event.detail)); + }, + onPhaseUpdated: (event) => { + setCurrentPhase((_previousCurrentPhase) => + parseConversationPhase(event.current_phase, _previousCurrentPhase) + ); + }, + onMessageStarted: (event) => { + if (event.message_type === "BWS_TASK") { + setStreamStatusMessage(formatStreamStatusMessage("ranking_work_activities")); + } + }, + onMessageDelta: (event) => { + markAssistantVisible(); + appendTextToCompassMessage(event.message_id, event.delta); + }, + onMessageCompleted: (messageItem) => { + markAssistantVisible(); + clearPendingDeltasForMessage(messageItem.message_id); + upsertMessageInChat(createChatMessageFromConversationMessage(messageItem)); + }, + onTurnCompleted: (event) => { + setCurrentPhase((_previousCurrentPhase) => + parseConversationPhase(event.current_phase, _previousCurrentPhase) + ); + setStreamStatusMessage(null); + }, + onError: () => { + setStreamStatusMessage(null); + }, + }); const endTimeMs = typeof performance !== "undefined" && performance.now ? performance.now() : Date.now(); const durationMs = Math.round(endTimeMs - startTimeMs); recordChatResponseMetrics({ @@ -679,42 +871,39 @@ export const Chat: React.FC> = ({ await fetchExperiences(); } - response.messages.forEach((messageItem, idx) => { - const isConclusionMessage = response.conversation_completed && idx === response.messages.length - 1; - if (!isConclusionMessage) { - if (messageItem.message_type === "BWS_TASK" && messageItem.metadata) { - if (messageItem.metadata.task_number === 1) { + if (!sawVisibleAssistantContent) { + response.messages.forEach((messageItem, idx) => { + const isConclusionMessage = response.conversation_completed && idx === response.messages.length - 1; + if (!isConclusionMessage) { + if (messageItem.message_type === "BWS_TASK" && messageItem.metadata) { + if (messageItem.metadata.task_number === 1) { + addMessageToChat( + generateCompassMessage( + `bws-transition-${messageItem.message_id}`, + "Now I'd like to understand what you value most in a job. I'll show you a few sets of work activities — for each, choose the one you'd **most** prefer and the one you'd **least** prefer.", + messageItem.sent_at, + null + ) + ); + } addMessageToChat( - generateCompassMessage( - `bws-transition-${messageItem.message_id}`, - "Now I'd like to understand what you value most in a job. I'll show you a few sets of work activities — for each, choose the one you'd **most** prefer and the one you'd **least** prefer.", - messageItem.sent_at, - null + generateBWSTaskMessage( + messageItem.message_id, + messageItem.metadata, + (t, b, w) => handleBWSSubmitRef.current?.(t, b, w) ?? Promise.resolve() ) ); + } else { + upsertMessageInChat(createChatMessageFromConversationMessage(messageItem)); } - addMessageToChat( - generateBWSTaskMessage( - messageItem.message_id, - messageItem.metadata, - (t, b, w) => handleBWSSubmitRef.current?.(t, b, w) ?? Promise.resolve() - ) - ); - } else { - addMessageToChat( - generateCompassMessage( - messageItem.message_id, - messageItem.message, - messageItem.sent_at, - messageItem.reaction - ) - ); } - } - }); + }); + } + // Handle the conclusion message and skills ranking flow for new messages if (response.conversation_completed && response.messages.length) { const lastMessage = response.messages[response.messages.length - 1]; + removeMessageFromChat(lastMessage.message_id); if (SkillsRankingService.getInstance().isSkillsRankingFeatureEnabled()) { // Check if skill ranking is already completed @@ -749,15 +938,22 @@ export const Chat: React.FC> = ({ console.error(new ChatError("Failed to send message:", error)); addMessageToChat(generatePleaseRepeatMessage()); } finally { + setStreamStatusMessage(null); setAiIsTyping(false); } }, [ addMessageToChat, + appendTextToCompassMessage, + clearPendingDeltasForMessage, + createChatMessageFromConversationMessage, exploredExperiences, fetchExperiences, + formatStreamStatusMessage, activeSessionId, + removeMessageFromChat, showSkillsRanking, + upsertMessageInChat, recordChatResponseMetrics, ] ); @@ -1013,10 +1209,9 @@ export const Chat: React.FC> = ({ } }, [exploredExperiencesNotification]); - // add a message when the compass is typing useEffect(() => { - addOrRemoveTypingMessage(aiIsTyping); - }, [aiIsTyping, addOrRemoveTypingMessage]); + addOrRemoveTypingMessage(aiIsTyping, getActiveTypingMessage(), streamStatusMessage ?? undefined); + }, [aiIsTyping, addOrRemoveTypingMessage, getActiveTypingMessage, streamStatusMessage]); // Fetch experiences when the active session id changes // And when not currentPhase is not `INITIALIZING`. diff --git a/frontend-new/src/chat/ChatService/ChatService.test.ts b/frontend-new/src/chat/ChatService/ChatService.test.ts index 45ec071f7..fb890d594 100644 --- a/frontend-new/src/chat/ChatService/ChatService.test.ts +++ b/frontend-new/src/chat/ChatService/ChatService.test.ts @@ -4,11 +4,17 @@ import { StatusCodes } from "http-status-codes"; import { RestAPIError } from "src/error/restAPIError/RestAPIError"; import { setupAPIServiceSpy } from "src/_test_utilities/fetchSpy"; import ErrorConstants from "src/error/restAPIError/RestAPIError.constants"; +import { ConversationResponse } from "./ChatService.types"; +import { ConversationPhase } from "src/chat/chatProgressbar/types"; import { generateTestChatResponses, generateTestHistory, } from "src/chat/ChatService/_test_utilities/generateTestChatResponses"; +const serializeSSEEvent = (event: string, data: unknown) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; +const serializeCRLFSSEEvent = (event: string, data: unknown) => + `event: ${event}\r\ndata: ${JSON.stringify(data)}\r\n\r\n`; + describe("ChatService", () => { let givenApiServerUrl: string = "/path/to/api"; beforeEach(() => { @@ -43,12 +49,29 @@ describe("ChatService", () => { // GIVEN some message specification to send const givenMessage = "Hello"; // AND the send message REST API will respond with OK and some message response - const expectedRootMessageResponse = generateTestChatResponses(); - const fetchSpy = setupAPIServiceSpy( - StatusCodes.CREATED, - expectedRootMessageResponse, - "application/json;charset=UTF-8" - ); + const expectedMessages = generateTestChatResponses(); + const expectedRootMessageResponse: ConversationResponse = { + messages: expectedMessages, + conversation_completed: false, + conversation_conducted_at: null, + experiences_explored: 0, + current_phase: { + phase: ConversationPhase.INTRO, + percentage: 0, + current: null, + total: null, + }, + }; + const sseResponseBody = [ + ...expectedMessages.map((message) => serializeSSEEvent("message_completed", message)), + serializeSSEEvent("turn_completed", { + conversation_completed: expectedRootMessageResponse.conversation_completed, + conversation_conducted_at: expectedRootMessageResponse.conversation_conducted_at, + experiences_explored: expectedRootMessageResponse.experiences_explored, + current_phase: expectedRootMessageResponse.current_phase, + }), + ].join(""); + const fetchSpy = setupAPIServiceSpy(StatusCodes.OK, sseResponseBody, "text/event-stream;charset=UTF-8"); // WHEN the sendMessage function is called with the given arguments const givenSessionId = 1234; @@ -62,11 +85,11 @@ describe("ChatService", () => { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ user_input: givenMessage }), - expectedStatusCode: StatusCodes.CREATED, + expectedStatusCode: StatusCodes.OK, serviceName: "ChatService", serviceFunction: "sendMessage", failureMessage: `Failed to send message with session id ${givenSessionId}`, - expectedContentType: "application/json", + expectedContentType: "text/event-stream", }); // AND returns the message response @@ -103,12 +126,12 @@ describe("ChatService", () => { ["is a malformed json", "{"], ["is a string", "foo"], ])( - "on 201, should reject with an error ERROR_CODE.INVALID_RESPONSE_BODY if response %s", + "on 200, should reject with an error ERROR_CODE.INVALID_RESPONSE_BODY if response %s", async (_description, givenResponse) => { // GIVEN some message specification to send const givenMessage = "Hello"; - // AND the send message REST API will respond with OK and some response that does conform to the messageResponseSchema even if it states that it is application/json - setupAPIServiceSpy(StatusCodes.CREATED, givenResponse, "application/json;charset=UTF-8"); + // AND the send message REST API will respond with an invalid SSE payload + setupAPIServiceSpy(StatusCodes.OK, givenResponse, "text/event-stream;charset=UTF-8"); // WHEN the sendMessage function is called with the given arguments const givenSessionId = 1234; @@ -122,7 +145,7 @@ describe("ChatService", () => { "sendMessage", "POST", `${givenApiServerUrl}/conversations`, - StatusCodes.CREATED, + StatusCodes.OK, ErrorConstants.ErrorCodes.INVALID_RESPONSE_BODY, "", "" @@ -136,6 +159,109 @@ describe("ChatService", () => { expect(console.warn).not.toHaveBeenCalled(); } ); + + test("should dispatch live stream handlers for status, phase, and delta events", async () => { + const givenMessage = "Hello"; + const expectedMessages = generateTestChatResponses(); + const expectedRootMessageResponse: ConversationResponse = { + messages: [expectedMessages[0]], + conversation_completed: false, + conversation_conducted_at: null, + experiences_explored: 1, + current_phase: { + phase: ConversationPhase.PREFERENCE_ELICITATION, + percentage: 72, + current: 1, + total: 6, + }, + }; + const sseResponseBody = [ + serializeSSEEvent("turn_started", { + session_id: 1234, + user_message_id: "user-1", + current_phase: { + phase: ConversationPhase.INTRO, + percentage: 0, + current: null, + total: null, + }, + }), + serializeCRLFSSEEvent("status_updated", { + label: "routing", + status: "running", + agent_type: "welcome_agent", + detail: "INTRO", + current_phase: { + phase: ConversationPhase.INTRO, + percentage: 0, + current: null, + total: null, + }, + }), + serializeSSEEvent("phase_updated", { + current_phase: expectedRootMessageResponse.current_phase, + agent_type: "preference_elicitation_agent", + detail: "phase_progressed", + }), + serializeSSEEvent("message_started", { + message_id: expectedMessages[0].message_id, + sender: expectedMessages[0].sender, + message_type: expectedMessages[0].message_type ?? "TEXT", + metadata: expectedMessages[0].metadata ?? null, + }), + serializeSSEEvent("message_delta", { + message_id: expectedMessages[0].message_id, + delta: expectedMessages[0].message.slice(0, 5), + }), + serializeSSEEvent("message_completed", expectedMessages[0]), + serializeSSEEvent("turn_completed", { + conversation_completed: expectedRootMessageResponse.conversation_completed, + conversation_conducted_at: expectedRootMessageResponse.conversation_conducted_at, + experiences_explored: expectedRootMessageResponse.experiences_explored, + current_phase: expectedRootMessageResponse.current_phase, + }), + ].join(""); + setupAPIServiceSpy(StatusCodes.OK, sseResponseBody, "text/event-stream;charset=UTF-8"); + + const handlers = { + onTurnStarted: jest.fn(), + onStatusUpdated: jest.fn(), + onPhaseUpdated: jest.fn(), + onMessageStarted: jest.fn(), + onMessageDelta: jest.fn(), + onMessageCompleted: jest.fn(), + onTurnCompleted: jest.fn(), + }; + + const response = await ChatService.getInstance().sendMessage(1234, givenMessage, handlers); + + expect(response).toEqual(expectedRootMessageResponse); + expect(handlers.onTurnStarted).toHaveBeenCalledTimes(1); + expect(handlers.onStatusUpdated).toHaveBeenCalledWith( + expect.objectContaining({ + label: "routing", + status: "running", + }) + ); + expect(handlers.onPhaseUpdated).toHaveBeenCalledWith( + expect.objectContaining({ + current_phase: expectedRootMessageResponse.current_phase, + }) + ); + expect(handlers.onMessageStarted).toHaveBeenCalledTimes(1); + expect(handlers.onMessageDelta).toHaveBeenCalledWith( + expect.objectContaining({ + message_id: expectedMessages[0].message_id, + delta: expectedMessages[0].message.slice(0, 5), + }) + ); + expect(handlers.onMessageCompleted).toHaveBeenCalledWith(expectedMessages[0]); + expect(handlers.onTurnCompleted).toHaveBeenCalledWith( + expect.objectContaining({ + current_phase: expectedRootMessageResponse.current_phase, + }) + ); + }); }); describe("getChatHistory", () => { diff --git a/frontend-new/src/chat/ChatService/ChatService.ts b/frontend-new/src/chat/ChatService/ChatService.ts index cf2854965..a3412303c 100644 --- a/frontend-new/src/chat/ChatService/ChatService.ts +++ b/frontend-new/src/chat/ChatService/ChatService.ts @@ -3,7 +3,66 @@ import { StatusCodes } from "http-status-codes"; import { customFetch } from "src/utils/customFetch/customFetch"; import ErrorConstants from "src/error/restAPIError/RestAPIError.constants"; import { getBackendUrl } from "src/envService"; -import { ConversationResponse } from "./ChatService.types"; +import { + ConversationMessage, + ConversationResponse, + ErrorEventData, + MessageDeltaEventData, + MessageStartedEventData, + PhaseUpdatedEventData, + SendMessageStreamHandlers, + StatusUpdatedEventData, + TurnCompletedEventData, + TurnStartedEventData, +} from "./ChatService.types"; + +type ParsedSSEEvent = { + event: string; + data: unknown; +}; + +const getSSEBoundary = (buffer: string): { index: number; length: number } | null => { + const unixBoundary = buffer.indexOf("\n\n"); + const windowsBoundary = buffer.indexOf("\r\n\r\n"); + if (unixBoundary === -1 && windowsBoundary === -1) { + return null; + } + if (unixBoundary === -1) { + return { index: windowsBoundary, length: 4 }; + } + if (windowsBoundary === -1) { + return { index: unixBoundary, length: 2 }; + } + return unixBoundary < windowsBoundary ? { index: unixBoundary, length: 2 } : { index: windowsBoundary, length: 4 }; +}; + +const parseSSEEvent = (rawEvent: string): ParsedSSEEvent | null => { + const lines = rawEvent.split(/\r?\n/); + let event = "message"; + const dataLines: string[] = []; + + lines.forEach((line) => { + if (!line || line.startsWith(":")) { + return; + } + if (line.startsWith("event:")) { + event = line.slice("event:".length).trim(); + return; + } + if (line.startsWith("data:")) { + dataLines.push(line.slice("data:".length).trim()); + } + }); + + if (dataLines.length === 0) { + return null; + } + + return { + event, + data: JSON.parse(dataLines.join("\n")), + }; +}; export default class ChatService { private static instance: ChatService; @@ -25,7 +84,11 @@ export default class ChatService { return ChatService.instance; } - public async sendMessage(sessionId: number, message: string): Promise { + public async sendMessage( + sessionId: number, + message: string, + handlers?: SendMessageStreamHandlers + ): Promise { const serviceName = "ChatService"; const serviceFunction = "sendMessage"; const method = "POST"; @@ -40,31 +103,133 @@ export default class ChatService { body: JSON.stringify({ user_input: message, }), - expectedStatusCode: StatusCodes.CREATED, + expectedStatusCode: StatusCodes.OK, serviceName, serviceFunction, failureMessage: `Failed to send message with session id ${sessionId}`, - expectedContentType: "application/json", + expectedContentType: "text/event-stream", }); - const responseBody = await response.text(); + const completedMessages: ConversationMessage[] = []; + let completedTurn: TurnCompletedEventData | null = null; + let streamError: ErrorEventData | null = null; - let messageResponse: ConversationResponse; - try { - messageResponse = JSON.parse(responseBody); - } catch (e: any) { + const processRawEvent = (rawEvent: string) => { + if (!rawEvent.trim()) { + return; + } + let parsedEvent: ParsedSSEEvent | null = null; + try { + parsedEvent = parseSSEEvent(rawEvent); + } catch (e: any) { + throw errorFactory( + response.status, + ErrorConstants.ErrorCodes.INVALID_RESPONSE_BODY, + "Response did not contain valid SSE JSON payloads", + { + rawEvent, + error: e, + } + ); + } + if (!parsedEvent) { + return; + } + + switch (parsedEvent.event) { + case "turn_started": + handlers?.onTurnStarted?.(parsedEvent.data as TurnStartedEventData); + break; + case "status_updated": + handlers?.onStatusUpdated?.(parsedEvent.data as StatusUpdatedEventData); + break; + case "phase_updated": + handlers?.onPhaseUpdated?.(parsedEvent.data as PhaseUpdatedEventData); + break; + case "message_started": + handlers?.onMessageStarted?.(parsedEvent.data as MessageStartedEventData); + break; + case "message_delta": + handlers?.onMessageDelta?.(parsedEvent.data as MessageDeltaEventData); + break; + case "message_completed": { + const completedMessage = parsedEvent.data as ConversationMessage; + completedMessages.push(completedMessage); + handlers?.onMessageCompleted?.(completedMessage); + break; + } + case "turn_completed": + completedTurn = parsedEvent.data as TurnCompletedEventData; + handlers?.onTurnCompleted?.(completedTurn); + break; + case "error": + streamError = parsedEvent.data as ErrorEventData; + handlers?.onError?.(streamError); + break; + default: + break; + } + }; + + const processBuffer = (buffer: string) => { + let boundary = getSSEBoundary(buffer); + while (boundary) { + processRawEvent(buffer.slice(0, boundary.index)); + buffer = buffer.slice(boundary.index + boundary.length); + boundary = getSSEBoundary(buffer); + } + if (buffer.trim()) { + processRawEvent(buffer); + } + }; + + if (response.body) { + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + buffer += decoder.decode(value || new Uint8Array(), { stream: !done }); + const boundary = getSSEBoundary(buffer); + if (boundary) { + processRawEvent(buffer.slice(0, boundary.index)); + buffer = buffer.slice(boundary.index + boundary.length); + } + if (done) { + processBuffer(buffer); + break; + } + } + } else { + const text = await response.text(); + processBuffer(text); + } + + if (streamError) { + const errorEvent = streamError as ErrorEventData; + throw errorFactory(response.status, ErrorConstants.ErrorCodes.API_ERROR, errorEvent.message, errorEvent); + } + + if (!completedTurn) { throw errorFactory( response.status, ErrorConstants.ErrorCodes.INVALID_RESPONSE_BODY, - "Response did not contain valid JSON", + "SSE stream ended without turn completion", { - responseBody, - error: e, + messagesReceived: completedMessages.length, } ); } - return messageResponse; + const completedTurnData = completedTurn as TurnCompletedEventData; + return { + messages: completedMessages, + conversation_completed: completedTurnData.conversation_completed, + conversation_conducted_at: completedTurnData.conversation_conducted_at, + experiences_explored: completedTurnData.experiences_explored, + current_phase: completedTurnData.current_phase, + }; } public async getChatHistory(sessionId: number): Promise { diff --git a/frontend-new/src/chat/ChatService/ChatService.types.ts b/frontend-new/src/chat/ChatService/ChatService.types.ts index 9917816d4..82eee13d4 100644 --- a/frontend-new/src/chat/ChatService/ChatService.types.ts +++ b/frontend-new/src/chat/ChatService/ChatService.types.ts @@ -32,3 +32,69 @@ export interface ConversationResponse { experiences_explored: number; // a count for all the experiences explored (processed) current_phase: CurrentPhase; // The current conversation phase } + +export type ConversationStreamEventType = + | "turn_started" + | "status_updated" + | "phase_updated" + | "message_started" + | "message_delta" + | "message_completed" + | "turn_completed" + | "error"; + +export interface TurnStartedEventData { + session_id: number; + user_message_id: string; + current_phase: CurrentPhase | null; +} + +export interface MessageStartedEventData { + message_id: string; + sender: ConversationMessageSender; + message_type?: ConversationMessageType; + metadata?: BWSTaskMetadata; +} + +export interface MessageDeltaEventData { + message_id: string; + delta: string; +} + +export interface StatusUpdatedEventData { + label: string; + status: string; + agent_type?: string | null; + detail?: string | null; + current_phase?: CurrentPhase | null; +} + +export interface PhaseUpdatedEventData { + current_phase: CurrentPhase; + agent_type?: string | null; + detail?: string | null; +} + +export interface TurnCompletedEventData { + conversation_completed: boolean; + conversation_conducted_at: string | null; + experiences_explored: number; + current_phase: CurrentPhase; +} + +export interface ErrorEventData { + code: string; + message: string; + recoverable: boolean; +} + +export interface SendMessageStreamHandlers { + onTurnStarted?: (event: TurnStartedEventData) => void; + onStatusUpdated?: (event: StatusUpdatedEventData) => void; + onPhaseUpdated?: (event: PhaseUpdatedEventData) => void; + onMessageStarted?: (event: MessageStartedEventData) => void; + onMessageDelta?: (event: MessageDeltaEventData) => void; + onMessageCompleted?: (event: ConversationMessage) => void; + onTurnCompleted?: (event: TurnCompletedEventData) => void; + onError?: (event: ErrorEventData) => void; +} diff --git a/frontend-new/src/chat/chatMessage/compassChatMessage/CompassChatMessage.tsx b/frontend-new/src/chat/chatMessage/compassChatMessage/CompassChatMessage.tsx index e99905dd8..ffc76ef80 100644 --- a/frontend-new/src/chat/chatMessage/compassChatMessage/CompassChatMessage.tsx +++ b/frontend-new/src/chat/chatMessage/compassChatMessage/CompassChatMessage.tsx @@ -1,4 +1,5 @@ -import React from "react"; +import React, { useEffect, useMemo, useRef } from "react"; +import { keyframes } from "@emotion/react"; import { Box, styled } from "@mui/material"; import { ConversationMessageSender, MessageReaction } from "src/chat/ChatService/ChatService.types"; import ChatBubble from "src/chat/chatMessage/components/chatBubble/ChatBubble"; @@ -27,9 +28,57 @@ export interface CompassChatMessageProps { message: string; sent_at: string; // ISO formatted datetime string reaction: MessageReaction | null; + animateChunks?: boolean; + streamVersion?: number; } -const CompassChatMessage: React.FC = ({ message_id, message, sent_at, reaction }) => { +const chunkReveal = keyframes` + from { + opacity: 0; + transform: translateY(2px); + } + to { + opacity: 1; + transform: translateY(0); + } +`; + +const AnimatedSuffix = styled("span")({ + display: "inline", + animation: `${chunkReveal} 180ms ease-out`, + whiteSpace: "pre-wrap", +}); + +const CompassChatMessage: React.FC = ({ + message_id, + message, + sent_at, + reaction, + animateChunks = false, + streamVersion, +}) => { + const previousMessageRef = useRef(""); + + const renderedMessage = useMemo(() => { + const previousMessage = previousMessageRef.current; + if (!animateChunks || !message.startsWith(previousMessage) || message === previousMessage) { + return message; + } + + return ( + <> + {previousMessage} + + {message.slice(previousMessage.length)} + + + ); + }, [animateChunks, message, message_id, streamVersion]); + + useEffect(() => { + previousMessageRef.current = message; + }, [message]); + return ( = ({ message_id, mes }} > - + diff --git a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.stories.tsx b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.stories.tsx index 50ea08ec0..83d17464a 100644 --- a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.stories.tsx +++ b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.stories.tsx @@ -32,3 +32,31 @@ export const ShownWhenThinking: Story = { waitBeforeThinking: 0, }, }; + +export const WithStreamingStatusPreparing: Story = { + args: { + waitBeforeThinking: 0, + status: "Preparing your response", + }, +}; + +export const WithStreamingStatusExploring: Story = { + args: { + waitBeforeThinking: 0, + status: "Exploring your experiences", + }, +}; + +export const WithStreamingStatusUnderstandingPreferences: Story = { + args: { + waitBeforeThinking: 0, + status: "Understanding what matters most to you", + }, +}; + +export const WithStreamingStatusPreparingRecommendations: Story = { + args: { + waitBeforeThinking: 0, + status: "Preparing your recommendations", + }, +}; diff --git a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.test.tsx b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.test.tsx index f10d89028..f6d975524 100644 --- a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.test.tsx +++ b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.test.tsx @@ -120,4 +120,13 @@ describe("TypingChatMessage", () => { expect(console.error).not.toHaveBeenCalled(); expect(console.warn).not.toHaveBeenCalled(); }); + + test("should render status above typing bubble when status prop is set", () => { + render(); + + expect(screen.getByText("Preparing your response")).toBeInTheDocument(); + expect(screen.getByText(i18n.t(UI_TEXT_KEYS.TYPING))).toBeInTheDocument(); + expect(console.error).not.toHaveBeenCalled(); + expect(console.warn).not.toHaveBeenCalled(); + }); }); diff --git a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.tsx b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.tsx index cfa6329e4..ca930fe8f 100644 --- a/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.tsx +++ b/frontend-new/src/chat/chatMessage/typingChatMessage/TypingChatMessage.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; -import { Box, keyframes, Typography } from "@mui/material"; +import { Box, keyframes, Typography, useTheme } from "@mui/material"; import ChatBubble from "src/chat/chatMessage/components/chatBubble/ChatBubble"; import { MessageContainer } from "src/chat/chatMessage/userChatMessage/UserChatMessage"; import { ConversationMessageSender } from "src/chat/ChatService/ChatService.types"; @@ -25,6 +25,8 @@ export const TYPING_CHAT_MESSAGE_TYPE = `typing-message-${uniqueId}`; export interface TypingChatMessageProps { waitBeforeThinking?: number; thinkingMessage?: string; + message?: string; + status?: string; } const dotAnimation = keyframes` @@ -46,12 +48,20 @@ const textVariants = { const TypingChatMessage: React.FC = ({ waitBeforeThinking = WAIT_BEFORE_THINKING, thinkingMessage, + message, + status, }) => { const { t } = useTranslation(); - const [displayText, setDisplayText] = useState(t(UI_TEXT_KEYS.TYPING)); + const theme = useTheme(); + const [displayText, setDisplayText] = useState(message ?? t(UI_TEXT_KEYS.TYPING)); useEffect(() => { - // Change text after waitBeforeThinking duration + if (message) { + setDisplayText(message); + return; + } + + setDisplayText(t(UI_TEXT_KEYS.TYPING)); const textChangeTimer = setTimeout( () => { setDisplayText(thinkingMessage ?? t(UI_TEXT_KEYS.THINKING)); @@ -63,21 +73,33 @@ const TypingChatMessage: React.FC = ({ return () => { clearTimeout(textChangeTimer); }; - }, [waitBeforeThinking, t, thinkingMessage]); + }, [waitBeforeThinking, t, thinkingMessage, message]); return ( - - - + {status && ( + + {status} + + )} + + @@ -101,9 +123,9 @@ const TypingChatMessage: React.FC = ({ - - - + + + ); }; diff --git a/frontend-new/src/chat/chatMessage/typingChatMessage/__snapshots__/TypingChatMessage.test.tsx.snap b/frontend-new/src/chat/chatMessage/typingChatMessage/__snapshots__/TypingChatMessage.test.tsx.snap index bf06e4c4f..dc4a83d22 100644 --- a/frontend-new/src/chat/chatMessage/typingChatMessage/__snapshots__/TypingChatMessage.test.tsx.snap +++ b/frontend-new/src/chat/chatMessage/typingChatMessage/__snapshots__/TypingChatMessage.test.tsx.snap @@ -7,35 +7,43 @@ exports[`TypingChatMessage should render the Typing Chat message correctly when origin="COMPASS" >
-

- Typing -

- - - . - - - . - + Please wait, I'm thinking +

- . + + . + + + . + + + . + - +
@@ -48,35 +56,43 @@ exports[`TypingChatMessage should render the Typing Chat message correctly when origin="COMPASS" >
-

- Typing -

- - - . - - - . - + Typing +

- . + + . + + + . + + + . + - +
diff --git a/frontend-new/src/chat/util.tsx b/frontend-new/src/chat/util.tsx index 28f2f6d06..d30bc7f37 100644 --- a/frontend-new/src/chat/util.tsx +++ b/frontend-new/src/chat/util.tsx @@ -78,13 +78,16 @@ export const generateCompassMessage = ( message_id: string, message: string, sent_at: string, - reaction: MessageReaction | null + reaction: MessageReaction | null, + options?: { animateChunks?: boolean; streamVersion?: number } ): IChatMessage => { const payload: CompassChatMessageProps = { message_id: message_id, message: message, sent_at: sent_at, reaction: reaction, + animateChunks: options?.animateChunks, + streamVersion: options?.streamVersion, }; return { type: COMPASS_CHAT_MESSAGE_TYPE, @@ -110,11 +113,15 @@ export const generateErrorMessage = (message: string): IChatMessage => { const payload: TypingChatMessageProps = { waitBeforeThinking: waitBeforeThinking, thinkingMessage: thinkingMessage, + message: message, + status: status, }; return { type: TYPING_CHAT_MESSAGE_TYPE, diff --git a/frontend-new/src/chat/utils.test.tsx b/frontend-new/src/chat/utils.test.tsx index 2eb95dbd7..aa6fe3b0a 100644 --- a/frontend-new/src/chat/utils.test.tsx +++ b/frontend-new/src/chat/utils.test.tsx @@ -123,6 +123,8 @@ describe("Chat Utils", () => { message: givenMessage, sent_at: givenSentAt, reaction: givenReaction, + animateChunks: undefined, + streamVersion: undefined, }, component: expect.any(Function), }); @@ -153,6 +155,8 @@ describe("Chat Utils", () => { message: givenMessage, sent_at: givenSentAt, reaction: null, + animateChunks: undefined, + streamVersion: undefined, }, component: expect.any(Function), }); @@ -179,6 +183,8 @@ describe("Chat Utils", () => { type: TYPING_CHAT_MESSAGE_TYPE, payload: { waitBeforeThinking: undefined, + thinkingMessage: undefined, + message: undefined, }, component: expect.any(Function), }); diff --git a/frontend-new/src/features/skillsRanking/components/skillsRankingPrompt/__snapshots__/SkillsRankingPrompt.test.tsx.snap b/frontend-new/src/features/skillsRanking/components/skillsRankingPrompt/__snapshots__/SkillsRankingPrompt.test.tsx.snap index d2ec53804..c4b280d0d 100644 --- a/frontend-new/src/features/skillsRanking/components/skillsRankingPrompt/__snapshots__/SkillsRankingPrompt.test.tsx.snap +++ b/frontend-new/src/features/skillsRanking/components/skillsRankingPrompt/__snapshots__/SkillsRankingPrompt.test.tsx.snap @@ -14,16 +14,16 @@ exports[`SkillsRankingPrompt snapshot: initial render in INITIAL phase (non-repl transition="[object Object]" >
str: - """ - Construct the API Gateway configuration from the Cloud Run service's (Backend App) OpenAPI 3 specification. - """ - pulumi.info("Constructing API Gateway configuration...") - pulumi.info(f"cloud_run_url: {cloud_run_url}") - - # Get the current credentials; pulumi is using to create resources, - # specifically with the scope of Cloud Platform. - # For more information about scopes see: https://developers.google.com/identity/protocols/googlescopes - _credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) - - # Initialize the ID token Feather - request = Request() - _credentials.refresh(Request()) - - # Fetch the ID token for the Cloud Run URL - _id_token = id_token.fetch_id_token(request, cloud_run_url) - - # get the open api JSON content from the cloud run url - open_api_3_json = _get_open_api_config(cloud_run_url, _id_token, expected_version) - - # convert the openapi JSON to YAML bytes - yaml_config = yaml.dump(_convert_open_api_3_to_2(open_api_3_json), - None, - encoding='utf-8', - allow_unicode=True, - indent=2) - - # return the YAML config as a string - return yaml_config.decode('utf-8') diff --git a/iac/backend/_construct_espv2_cfg.py b/iac/backend/_construct_espv2_cfg.py new file mode 100644 index 000000000..c2a24ff4a --- /dev/null +++ b/iac/backend/_construct_espv2_cfg.py @@ -0,0 +1,202 @@ +import os +import time + +import pulumi +import yaml +import requests + +import google +from google.oauth2 import id_token +from google.auth.transport.requests import Request + +from lib import Version + + +def _get_open_api_config(cloud_run_url: str, _id_token: str, expected_version: Version) -> dict: + """ + Fetch the OpenAPI 3 spec from the Cloud Run backend, retrying until the expected version is live. + + :param cloud_run_url: The URL of the Cloud Run service (internal, requires Bearer token). + :param _id_token: ID token for authenticating to the private Cloud Run URL. + :param expected_version: The deployed version; used to confirm the right image is serving. + :return: Parsed OpenAPI 3 JSON as a dict. + """ + attempts = 5 + + open_api_3_json = None + for attempt in range(attempts): + response = requests.get( + f"{cloud_run_url}/openapi.json", + headers={"Authorization": f"Bearer {_id_token}"}, + ) + + if response.status_code == 200: + open_api_3_json = response.json() + else: + raise ValueError(f"Failed to fetch OpenAPI JSON from {cloud_run_url}: {response.status_code}") + + versions_match = ( + open_api_3_json.get("info", {}).get("version") + == f"{expected_version.git_branch_name}-{expected_version.git_sha}" + ) + + # In preview/dry-run mode skip the version check to allow pulumi preview without a live service. + if pulumi.runtime.is_dry_run() or versions_match: + return open_api_3_json + + if attempt < attempts - 1: + # Exponential backoff: 3, 6, 12, 24 seconds + wait_time = 3 * (2 ** attempt) + pulumi.info(f"Waiting {wait_time}s before retrying (attempt {attempt + 1}/{attempts})...") + pulumi.info( + f"Expected version: {expected_version.git_branch_name}-{expected_version.git_sha}, " + f"got: {open_api_3_json.get('info', {}).get('version')}" + ) + time.sleep(wait_time) + + return open_api_3_json + + +def _convert_params(parameters: list) -> list: + """ + Convert OpenAPI 3 parameters to Swagger 2.0 format. + Flattens schema.type up to the parameter level (required by Swagger 2.0). + """ + result = [] + for param in parameters: + p = {k: v for k, v in param.items() if k != "schema"} + schema = param.get("schema", {}) + if "type" in schema: + p["type"] = schema["type"] + else: + p["type"] = "string" # default; ESPv2 needs a type + result.append(p) + return result + + +def _build_espv2_spec( + openapi3: dict, + template_str: str, + espv2_hostname: str, + backend_uri: str, + firebase_project_id: str, +) -> str: + """ + Build the ESPv2 Swagger 2.0 OpenAPI spec. + + Substitutes hostname/backend/firebase values into the template, then walks + the FastAPI OpenAPI 3 spec to classify each path and inject explicit entries + before the wildcard ``/{path=**}`` catch-all: + + - Public paths (no ``security`` on any operation) → ``security: []`` + - API-key paths (``gcp_api_key`` security on every operation) → ``security: [{api_key: []}]`` + ESPv2 validates the GCP API key from the ``x-api-key`` request header. + - Firebase-protected paths → no injection; handled by the wildcard catch-all. + + :param openapi3: Parsed OpenAPI 3 spec from the FastAPI backend. + :param template_str: Raw YAML text of espv2_openapi_template.yaml. + :param espv2_hostname: Cloud Endpoints hostname (e.g. espv2-gateway.endpoints..cloud.goog). + :param backend_uri: Cloud Run backend URI. + :param firebase_project_id: Firebase / GCP project ID for JWT validation. + :return: YAML string of the completed ESPv2 Swagger 2.0 spec. + """ + spec = yaml.load(template_str, Loader=yaml.SafeLoader) + + # Fill in the placeholder values. + spec["host"] = espv2_hostname + spec["x-google-backend"]["address"] = backend_uri + spec["x-google-backend"]["jwt_audience"] = backend_uri + spec["securityDefinitions"]["firebase"]["x-google-issuer"] = ( + f"https://securetoken.google.com/{firebase_project_id}" + ) + spec["securityDefinitions"]["firebase"]["x-google-audiences"] = firebase_project_id + + # The FastAPI scheme name used on API-key-protected routes (must match ApiKeyAuth's scheme_name). + _FASTAPI_API_KEY_SCHEME = "gcp_api_key" + # The ESPv2 security definition name for the x-api-key header (must match espv2_openapi_template.yaml). + _ESPV2_API_KEY_SCHEME = "api_key" + + # Walk FastAPI's OpenAPI 3 paths and inject explicit entries into the ESPv2 Swagger 2.0 spec for: + # - Public paths: every operation has no ``security`` field → inject security: [] + # - API-key paths: every operation declares gcp_api_key security → inject security: [{api_key: []}] + # Firebase-protected paths are handled by the wildcard ``/{path=**}`` catch-all and need no injection. + for path, path_item in openapi3.get("paths", {}).items(): + # path_item values can be non-dict (e.g. summary strings) — skip those. + operations = { + method: op + for method, op in path_item.items() + if isinstance(op, dict) + } + if not operations: + continue + + is_public = all("security" not in op for op in operations.values()) + is_api_key_protected = all( + op.get("security") == [{_FASTAPI_API_KEY_SCHEME: []}] + for op in operations.values() + ) + + if not is_public and not is_api_key_protected: + continue + + espv2_security: list = [] if is_public else [{_ESPV2_API_KEY_SCHEME: []}] + + for method, op_obj in operations.items(): + entry: dict = { + "operationId": op_obj.get("operationId") or f"{method}_{path.replace('/', '_')}", + "security": espv2_security, + "responses": {"200": {"description": "OK"}}, + } + if "parameters" in op_obj: + entry["parameters"] = _convert_params(op_obj["parameters"]) + spec["paths"].setdefault(path, {})[method] = entry + + label = "public (no-auth)" if is_public else "api-key-protected (x-api-key header)" + pulumi.info(f"ESPv2: injecting {label} path: {path}") + + yaml_bytes = yaml.dump( + spec, + None, + encoding="utf-8", + allow_unicode=True, + indent=2, + ) + return yaml_bytes.decode("utf-8") + + +def construct_espv2_cfg( + *, + cloud_run_url: str, + espv2_hostname: str, + backend_uri: str, + firebase_project_id: str, + expected_version: Version, +) -> str: + """ + Entry point called from deploy_backend.py via pulumi.Output.apply(). + + Fetches the FastAPI OpenAPI spec, extracts public paths, and returns the + fully-rendered ESPv2 Swagger 2.0 YAML string. + """ + pulumi.info("Constructing ESPv2 OpenAPI configuration...") + pulumi.info(f"cloud_run_url: {cloud_run_url}") + + # Authenticate to the private Cloud Run URL using Application Default Credentials. + _credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) + request = Request() + _credentials.refresh(request) + _id_token = id_token.fetch_id_token(request, cloud_run_url) + + open_api_3_json = _get_open_api_config(cloud_run_url, _id_token, expected_version) + + template_path = os.path.join(os.path.dirname(__file__), "espv2_openapi_template.yaml") + with open(template_path) as f: + template_str = f.read() + + return _build_espv2_spec( + openapi3=open_api_3_json, + template_str=template_str, + espv2_hostname=espv2_hostname, + backend_uri=backend_uri, + firebase_project_id=firebase_project_id, + ) diff --git a/iac/backend/deploy_backend.py b/iac/backend/deploy_backend.py index 75d6cb80b..c30f4a849 100644 --- a/iac/backend/deploy_backend.py +++ b/iac/backend/deploy_backend.py @@ -1,4 +1,4 @@ -import base64 +import os from dataclasses import dataclass from typing import Optional @@ -7,12 +7,11 @@ from pulumi import Output -from backend._construct_api_gateway_cfg import construct_api_gateway_cfg from lib import ProjectBaseConfig, get_resource_name, get_project_base_config, Version from scripts.formatters import construct_docker_tag from backend.cv_bucket import _create_cv_upload_bucket, _grant_cloud_run_sa_access_to_cv_bucket - -api_gateway_config_file_name = "api_gateway_config.yaml" +from espv2_image_builder import EspV2ImageBuilder +from _construct_espv2_cfg import construct_espv2_cfg @dataclass(frozen=True) @@ -49,11 +48,13 @@ class BackendServiceConfig: cloudrun_request_timeout: str cloudrun_memory_limit: str cloudrun_cpu_limit: str - api_gateway_timeout: str features: Optional[str] experience_pipeline_config: Optional[str] cv_max_uploads_per_user: Optional[str] cv_rate_limit_per_minute: Optional[str] + stream_chunk_size: Optional[str] + stream_chunk_mode: Optional[str] + stream_delta_delay_ms: Optional[str] enable_cv_upload: Optional[str] language_config: str global_product_name: Optional[str] @@ -63,119 +64,221 @@ class BackendServiceConfig: """ -# Set up GCP API Gateway. -# The API Gateway will route the requests to the Compass Cloudrun instance. Additionally, it will verify the incoming -# JWT tokens and add a new header x-apigateway-api-userinfo that will contain the JWT claims. -# The Compass Cloudrun instance is publicly accessible over internet, otherwise it cannot be behind API Gateway. -# To prevent direct calls to the Compass Cloudrun instance, the Cloudrun instance will require GCP IAM based -# authentication. A service account with 'roles/run.invoker' permission is created for the API Gateway -# which will allow it to call the Cloudrun instance. +# Deploy an ESPv2 proxy on Cloud Run for the Compass backend. +# +# ESPv2 (Envoy-based) is a self-managed alternative to Apigee/API Gateway. +# Unlike API Gateway (which buffers full responses), ESPv2 passes bytes through +# without buffering — required for SSE/streaming endpoints. +# +# The proxy verifies Firebase JWTs and injects the X-Endpoint-API-UserInfo header +# (base64url-encoded JWT payload) so the backend can identify the authenticated user. +# +# Bootstrap challenge: ESPv2's image must have the Endpoints CONFIG_ID baked in. +# We solve this with a pulumi.dynamic.Resource (EspV2ImageBuilder in espv2_image_builder.py) that calls +# gcloud_build_image during create(). +# +# Two-phase approach to break the circular dependency: +# 1. Deploy ESPv2 Cloud Run with a placeholder image → get stable URI/hostname. +# 2. Deploy gcp.endpoints.Service with the OpenAPI spec (using that hostname). +# 3. EspV2ImageBuilder runs gcloud_build_image → produces the real ESPv2 image URI. +# 4. The ESPv2 Cloud Run service is updated to use the real image. """ -def _setup_api_gateway(*, - basic_config: ProjectBaseConfig, - cloudrun: gcp.cloudrunv2.Service, - backend_service_cfg: BackendServiceConfig, - dependencies: list[pulumi.Resource], - artifacts_version: Version): - apigw_service_account = gcp.serviceaccount.Account( - resource_name=get_resource_name(resource="api-gateway", resource_type="sa"), - account_id="api-gateway-sa", +def _deploy_espv2_proxy(*, + basic_config: ProjectBaseConfig, + cloudrun: gcp.cloudrunv2.Service, + firebase_project_id: pulumi.Output[str], + deployable_version: Version, + min_instance_count: int, + max_instance_count: int, + dependencies: list[pulumi.Resource]) -> gcp.cloudrunv2.Service: + """ + Deploy ESPv2 as a Cloud Run service proxying the backend Cloud Run service. + + Steps: + 1. Create espv2-proxy-sa in the env project. + 2. Grant espv2-proxy-sa roles/run.invoker on the backend Cloud Run service. + 3. Grant espv2-proxy-sa roles/servicemanagement.serviceController on the project. + 4. Deploy the Endpoints OpenAPI spec (gcp.endpoints.Service). + 5. Build the ESPv2 image with the config ID baked in (EspV2ImageBuilder). + 6. Deploy ESPv2 as a Cloud Run service using the built image. + 7. Allow unauthenticated invocations on the ESPv2 Cloud Run service (ESPv2 handles auth). + 8. Lock down the backend Cloud Run service to internal-only + espv2-proxy-sa invoker. + """ + # 1. Dedicated service account for ESPv2. + proxy_sa = gcp.serviceaccount.Account( + resource_name=get_resource_name(resource="espv2-proxy", resource_type="sa"), + account_id="espv2-proxy-sa", project=basic_config.project, - display_name="API Gateway Service Account", + display_name="ESPv2 Proxy Service Account — used by ESPv2 to call Cloud Run backend", create_ignore_already_exists=True, opts=pulumi.ResourceOptions(depends_on=dependencies, provider=basic_config.provider), ) - apigw_api = gcp.apigateway.Api( - resource_name=get_resource_name(resource="api-gateway", resource_type="api"), - api_id="backend-api-gateway-api", + # 2. Allow ESPv2 SA to invoke the backend Cloud Run service. + gcp.cloudrunv2.ServiceIamMember( + resource_name=get_resource_name(resource="espv2-proxy-sa-run-invoker", resource_type="iam-member"), + project=basic_config.project, + location=basic_config.location, + name=cloudrun.name, + role="roles/run.invoker", + member=proxy_sa.email.apply(lambda email: f"serviceAccount:{email}"), + opts=pulumi.ResourceOptions(depends_on=[proxy_sa, cloudrun], provider=basic_config.provider), + ) + + # 3. Allow ESPv2 SA to report metrics/validate auth via Service Control API. + gcp.projects.IAMMember( + get_resource_name(resource="espv2-proxy-sa-service-controller", resource_type="iam-member"), + project=basic_config.project, + role="roles/servicemanagement.serviceController", + member=proxy_sa.email.apply(lambda email: f"serviceAccount:{email}"), + opts=pulumi.ResourceOptions(depends_on=[proxy_sa], provider=basic_config.provider), + ) + + # Enable the Cloud API Keys API — required to create and validate GCP API keys. + apikeys_service = gcp.projects.Service( + get_resource_name(resource="apikeys-api", resource_type="project-service"), + service="apikeys.googleapis.com", project=basic_config.project, opts=pulumi.ResourceOptions(depends_on=dependencies, provider=basic_config.provider), ) - # The GCP API Gateway uses OpenAPI 2.0 yaml files for the configurations. - # The yaml must be base64 encoded. - apigw_config_yml_string = cloudrun.uri.apply( - lambda cloudrun_url: construct_api_gateway_cfg(cloud_run_url=cloudrun_url, - expected_version=artifacts_version)) - - # update the yaml with the correct values - # we are not using pulumi.Output.format because with path variables they are encapsulated in {} - # which causes issues with pulumi.Output.format to throw because we want to keep them as {path variable}. - apigw_config_yaml = pulumi.Output.all(basic_config.project, cloudrun.uri, apigw_config_yml_string).apply( - lambda args: - args[2] - # project ID - .replace('__PROJECT_ID__', args[0]) - - # cloud run uri - .replace('__BACKEND_URI__', args[1]) - - # replace the backend api gateway timeout. - .replace("__API_GATEWAY_TIMEOUT__", backend_service_cfg.api_gateway_timeout) - - # replace the environment name in the api gateway config - .replace("__ENVIRONMENT_NAME__", backend_service_cfg.target_environment_name) + # ESPv2 Cloud Run service name — set explicitly and stable across deploys. + espv2_service_name = "espv2-gateway" + + # The Endpoints service name uses the canonical Cloud Endpoints DNS pattern: + # {service}.endpoints.{project}.cloud.goog + # This is deterministic (no GCP-generated hash), which avoids the circular dependency + # where the Cloud Run URL is unknown until after first deploy. + endpoints_service_name = pulumi.Output.concat( + espv2_service_name, ".endpoints.", basic_config.project, ".cloud.goog" ) - apigw_config_yaml_b64encoded = apigw_config_yaml.apply(lambda yaml: base64.b64encode(yaml.encode()).decode()) + # 4. Deploy the Endpoints OpenAPI spec. + # The spec is built dynamically from the FastAPI /openapi.json, so public paths + # (those without security: [...] in any operation) are injected with security: [] + # before the wildcard catch-all. This allows unauthenticated access to routes + # like /user-invitations/check-status without hardcoding them in the template. + openapi_spec = pulumi.Output.all( + backend_uri=cloudrun.uri, + firebase_project_id=firebase_project_id, + endpoints_service_name=endpoints_service_name, + ).apply(lambda args: construct_espv2_cfg( + cloud_run_url=args['backend_uri'], + espv2_hostname=args['endpoints_service_name'], + backend_uri=args['backend_uri'], + firebase_project_id=args['firebase_project_id'], + expected_version=deployable_version, + )) + + endpoints_service = gcp.endpoints.Service( + get_resource_name(resource="espv2-endpoints", resource_type="endpoints-service"), + service_name=endpoints_service_name, + openapi_config=openapi_spec, + opts=pulumi.ResourceOptions(depends_on=[cloudrun] + dependencies, provider=basic_config.provider), + ) + + # 5. Build the ESPv2 image with the config ID baked in. + image_builder = EspV2ImageBuilder( + get_resource_name(resource="espv2-image-builder", resource_type="dynamic"), + service_name=endpoints_service_name, + config_id=endpoints_service.config_id, + project_id=basic_config.project, + opts=pulumi.ResourceOptions(depends_on=[endpoints_service]), + ) - apigw_config = gcp.apigateway.ApiConfig( - resource_name=get_resource_name(resource="api-gateway", resource_type="api-config"), - api=apigw_api.api_id, + # 6. Deploy ESPv2 as a Cloud Run service. + espv2_cloudrun = gcp.cloudrunv2.Service( + get_resource_name(resource="espv2-gateway", resource_type="service"), + name=espv2_service_name, project=basic_config.project, - openapi_documents=[ - gcp.apigateway.ApiConfigOpenapiDocumentArgs( - document=gcp.apigateway.ApiConfigOpenapiDocumentDocumentArgs( - # this is the file name used in the API Gateway - # This is typically the path of the file when it is uploaded. - path=api_gateway_config_file_name, - contents=apigw_config_yaml_b64encoded, - ), - ) - ], - gateway_config=gcp.apigateway.ApiConfigGatewayConfigArgs( - backend_config=gcp.apigateway.ApiConfigGatewayConfigBackendConfigArgs( - google_service_account=apigw_service_account.email - ) + location=basic_config.location, + ingress="INGRESS_TRAFFIC_ALL", + template=gcp.cloudrunv2.ServiceTemplateArgs( + scaling=gcp.cloudrunv2.ServiceTemplateScalingArgs( + min_instance_count=min_instance_count, + max_instance_count=max_instance_count, + ), + # Match the backend timeout so Cloud Run doesn't cut SSE streams short. + timeout="3600s", + service_account=proxy_sa.email, + containers=[ + gcp.cloudrunv2.ServiceTemplateContainerArgs( + image=image_builder.image_uri, + args=[ + "--listener_port=8080", + "--backend=grpc://localhost:9090", + pulumi.Output.concat("--service=", endpoints_service_name), + "--rollout_strategy=managed", + "--cors_preset=basic", + ], + envs=[ + gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( + name="ESPv2_ARGS", + value="--http_request_timeout_s=3600", + ), + ], + ports=[gcp.cloudrunv2.ServiceTemplateContainerPortArgs(container_port=8080)], + ) + ], + ), + opts=pulumi.ResourceOptions( + depends_on=[image_builder, proxy_sa] + dependencies, + provider=basic_config.provider, ), - opts=pulumi.ResourceOptions(provider=basic_config.provider) ) - api_gateway = gcp.apigateway.Gateway( - resource_name=get_resource_name(resource="api-gateway"), - api_config=apigw_config.id, - display_name="Backend API Gateway", - gateway_id="backend-api-gateway", + # 7. Allow unauthenticated invocations on the ESPv2 Cloud Run service. + # ESPv2 itself handles Firebase JWT validation. + gcp.cloudrunv2.ServiceIamMember( + resource_name=get_resource_name(resource="espv2-allusers-invoker", resource_type="iam-member"), project=basic_config.project, - region=basic_config.location, - opts=pulumi.ResourceOptions(depends_on=dependencies, provider=basic_config.provider), + location=basic_config.location, + name=espv2_cloudrun.name, + role="roles/run.invoker", + member="allUsers", + opts=pulumi.ResourceOptions(depends_on=[espv2_cloudrun], provider=basic_config.provider), ) - # Only allow access (roles/run.invoker permission) to apigw_service_account - # This prevents the service from being accessed directly from the internet - gcp.cloudrun.IamMember( - resource_name=get_resource_name(resource="api-gateway-sa", resource_type="iam-member"), + # 8. Lock down the backend Cloud Run service: only allow ESPv2 SA to invoke it. + # Change ingress to internal + load balancer only (blocks direct internet access). + # NOTE: Pulumi will update the existing backend Cloud Run service in-place. + gcp.cloudrunv2.ServiceIamMember( + resource_name=get_resource_name(resource="backend-espv2-invoker", resource_type="iam-member"), project=basic_config.project, location=basic_config.location, - service=cloudrun.name, + name=cloudrun.name, role="roles/run.invoker", - member=apigw_service_account.email.apply(lambda email: f"serviceAccount:{email}"), - opts=pulumi.ResourceOptions(depends_on=dependencies, provider=basic_config.provider), + member=proxy_sa.email.apply(lambda email: f"serviceAccount:{email}"), + opts=pulumi.ResourceOptions(depends_on=[proxy_sa, cloudrun, espv2_cloudrun], provider=basic_config.provider), ) - # Enable the private service access for the API Gateway - gcp.projects.Service( - get_resource_name(resource="compass-backend-api", resource_type="service"), + + pulumi.export("espv2_cloud_run_name", espv2_cloudrun.name) + + # Create a GCP API key for callers of the search endpoints. + # Restricted to the Cloud Endpoints service so it cannot be used against other GCP APIs. + search_api_key = gcp.apikeys.Key( + get_resource_name(resource="search-api-key", resource_type="api-key"), project=basic_config.project, - service=apigw_api.managed_service, - opts=pulumi.ResourceOptions(depends_on=[api_gateway], provider=basic_config.provider) + display_name="Search API Key — grants access to /search/* endpoints via x-api-key header", + restrictions=gcp.apikeys.KeyRestrictionsArgs( + api_targets=[ + gcp.apikeys.KeyRestrictionsApiTargetArgs( + service=endpoints_service_name, + ) + ] + ), + opts=pulumi.ResourceOptions( + depends_on=[apikeys_service, endpoints_service], + provider=basic_config.provider, + ), ) + # Export the key string so it can be distributed to authorised callers. + # Treat this output as a secret — do not log or commit it. + pulumi.export("search_api_key_string", pulumi.Output.secret(search_api_key.key_string)) - pulumi.export("apigateway_url", api_gateway.default_hostname.apply(lambda hostname: f"https://{hostname}")) - pulumi.export("apigateway_id", api_gateway.gateway_id) - return api_gateway + return espv2_cloudrun def _grant_docker_repository_access_to_project_service_account( @@ -414,6 +517,15 @@ def _deploy_cloud_run_service( gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( name="BACKEND_CV_RATE_LIMIT_PER_MINUTE", value=backend_service_cfg.cv_rate_limit_per_minute), + gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( + name="BACKEND_STREAM_CHUNK_SIZE", + value=backend_service_cfg.stream_chunk_size or "10"), + gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( + name="BACKEND_STREAM_CHUNK_MODE", + value=backend_service_cfg.stream_chunk_mode or "chars"), + gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( + name="BACKEND_STREAM_DELTA_DELAY_MS", + value=backend_service_cfg.stream_delta_delay_ms or "12"), gcp.cloudrunv2.ServiceTemplateContainerEnvArgs( name="BACKEND_LANGUAGE_CONFIG", value=backend_service_cfg.language_config), @@ -463,6 +575,7 @@ def deploy_backend( backend_service_cfg: BackendServiceConfig, docker_repository: pulumi.Output[gcp.artifactregistry.Repository], deployable_version: Version, + firebase_project_id: pulumi.Output[str], ): """ Deploy the backend infrastructure @@ -506,10 +619,12 @@ def deploy_backend( service_account=cloud_run_sa, ) - _api_gateway = _setup_api_gateway( + _deploy_espv2_proxy( basic_config=basic_config, cloudrun=cloud_run, - artifacts_version=deployable_version, - backend_service_cfg=backend_service_cfg, - dependencies=[cloud_run] + firebase_project_id=firebase_project_id, + deployable_version=deployable_version, + min_instance_count=backend_service_cfg.cloudrun_min_instance_count, + max_instance_count=backend_service_cfg.cloudrun_max_instance_count, + dependencies=[cloud_run], ) diff --git a/iac/backend/espv2_image_builder.py b/iac/backend/espv2_image_builder.py new file mode 100644 index 000000000..355ddf0fe --- /dev/null +++ b/iac/backend/espv2_image_builder.py @@ -0,0 +1,175 @@ +""" +ESPv2 image build dynamic resource. + +Kept free of `lib` and other iac-local imports: Pulumi runs dynamic providers in a +subprocess that does not execute __main__.py, so sys.path may not include the iac root. +""" + +from typing import Any, Optional + +import pulumi +import pulumi.dynamic + +_ESPV2_DIAG_MAX_CHARS = 100_000 + + +def _espv2_build_diagnostic_text(combined_stdout_stderr: str) -> str: + s = combined_stdout_stderr.strip() + if len(s) <= _ESPV2_DIAG_MAX_CHARS: + return s + omitted = len(s) - _ESPV2_DIAG_MAX_CHARS + return f"[... {omitted} characters omitted ...]\n" + s[-_ESPV2_DIAG_MAX_CHARS:] + + +_GCLOUD_BUILD_IMAGE_NEW_IMAGE_LINE = ( + 'NEW_IMAGE="${IMAGE_REPOSITORY}/endpoints-runtime-serverless:${ESP_FULL_VERSION}-${SERVICE}-${CONFIG_ID}"' +) +# Upstream tag is ${ESP_FULL_VERSION}-${SERVICE}-${CONFIG_ID}; SERVICE is the full *.endpoints.*.cloud.goog name +# and can exceed GCR/Cloud Build tag limits. Use a short deterministic tag (still unique per config). +_GCLOUD_BUILD_IMAGE_NEW_IMAGE_PATCHED = ( + 'NEW_IMAGE="${IMAGE_REPOSITORY}/endpoints-runtime-serverless:espv2-' + '$(printf \'%s\' "${ESP_FULL_VERSION}-${SERVICE}-${CONFIG_ID}" | sha256sum | cut -c1-32)"' +) + + +def _patch_gcloud_build_image_script(script_bytes: bytes) -> bytes: + text = script_bytes.decode("utf-8") + if _GCLOUD_BUILD_IMAGE_NEW_IMAGE_LINE not in text: + raise RuntimeError( + "ESPv2: upstream gcloud_build_image script changed; expected NEW_IMAGE line not found. " + "Update _GCLOUD_BUILD_IMAGE_NEW_IMAGE_LINE in espv2_image_builder.py." + ) + patched = text.replace(_GCLOUD_BUILD_IMAGE_NEW_IMAGE_LINE, _GCLOUD_BUILD_IMAGE_NEW_IMAGE_PATCHED, 1) + return patched.encode("utf-8") + + +class _EspV2ImageBuilderProvider(pulumi.dynamic.ResourceProvider): + """ + Runs `gcloud_build_image` to build an ESPv2 container image with the + given Endpoints service name and config ID baked in. + Uses the same GCP project as the rest of the Pulumi stack. + """ + + _GCLOUD_BUILD_IMAGE_URL = ( + "https://raw.githubusercontent.com/GoogleCloudPlatform/esp-v2/master" + "/docker/serverless/gcloud_build_image" + ) + + def _run_build_image(self, service_name: str, config_id: str, project_id: str) -> str: + import concurrent.futures + import os + import stat + import subprocess + import tempfile + import time + import urllib.request + + # CI often uses a service account whose *home* project differs from the env project. gcloud then + # attributes API quota to that home project; if Cloud Build is only enabled on the env project, + # you see: "cloudbuild.googleapis.com not enabled on project []". Point quota at + # the env project where this stack enables Cloud Build (see environment create_new_environment). + subprocess.run( + ["gcloud", "config", "set", "billing/quota_project", project_id], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + subprocess_env = os.environ.copy() + subprocess_env["GOOGLE_CLOUD_QUOTA_PROJECT"] = project_id + + with tempfile.NamedTemporaryFile(suffix=".sh", delete=False, mode="wb") as f: + script_path = f.name + with urllib.request.urlopen(self._GCLOUD_BUILD_IMAGE_URL) as resp: + f.write(_patch_gcloud_build_image_script(resp.read())) + os.chmod(script_path, os.stat(script_path).st_mode | stat.S_IEXEC) + + cmd = ["/bin/bash", script_path, "-s", service_name, "-c", config_id, "-p", project_id] + pulumi.log.info("ESPv2 image build: started") + start = time.monotonic() + + def _run_gcloud_build_image() -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + capture_output=True, + text=True, + env=subprocess_env, + ) + + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(_run_gcloud_build_image) + while True: + try: + result = future.result(timeout=120) + break + except concurrent.futures.TimeoutError: + pulumi.log.info( + f"ESPv2 image build: still running ({int(time.monotonic() - start)}s elapsed)" + ) + finally: + os.unlink(script_path) + + combined = (result.stdout or "") + "\n" + (result.stderr or "") + if result.returncode != 0: + detail = _espv2_build_diagnostic_text(combined) + raise RuntimeError( + f"gcloud_build_image failed with exit code {result.returncode}:\n{detail}" + ) + + pulumi.log.info("ESPv2 image build: gcloud_build_image subprocess exited") + + for line in reversed(combined.splitlines()): + for token in line.split(): + token = token.strip("'\"") + if "endpoints-runtime-serverless" in token and ("gcr.io" in token or ".pkg.dev" in token): + return token + detail = _espv2_build_diagnostic_text(combined) + raise RuntimeError( + "Could not extract ESPv2 image URI from gcloud_build_image output:\n" + detail + ) + + def create(self, props: dict[str, Any]) -> pulumi.dynamic.CreateResult: + service_name: str = props["service_name"] + config_id: str = props["config_id"] + project_id: str = props["project_id"] + image_uri = self._run_build_image(service_name, config_id, project_id) + resource_id = f"{project_id}/{service_name}/{config_id}" + return pulumi.dynamic.CreateResult(id_=resource_id, outs={**props, "image_uri": image_uri}) + + def diff(self, id: str, olds: dict[str, Any], news: dict[str, Any]) -> pulumi.dynamic.DiffResult: + changes = olds.get("config_id") != news.get("config_id") + return pulumi.dynamic.DiffResult(changes=changes, replaces=[] if not changes else ["config_id"]) + + def update(self, id: str, olds: dict[str, Any], news: dict[str, Any]) -> pulumi.dynamic.UpdateResult: + result = self.create(news) + return pulumi.dynamic.UpdateResult(outs=result.outs) + + def delete(self, id: str, props: dict[str, Any]) -> None: + pass + + +class EspV2ImageBuilder(pulumi.dynamic.Resource): + """ + Builds an ESPv2 container image with the Endpoints config_id baked in. + Exposes `image_uri` as an Output. + """ + image_uri: pulumi.Output[str] + + def __init__(self, + name: str, + service_name: pulumi.Input[str], + config_id: pulumi.Input[str], + project_id: pulumi.Input[str], + opts: Optional[pulumi.ResourceOptions] = None): + super().__init__( + _EspV2ImageBuilderProvider(), + name, + { + "service_name": service_name, + "config_id": config_id, + "project_id": project_id, + "image_uri": None, + }, + opts, + ) diff --git a/iac/backend/espv2_openapi_template.yaml b/iac/backend/espv2_openapi_template.yaml new file mode 100644 index 000000000..ce739a51d --- /dev/null +++ b/iac/backend/espv2_openapi_template.yaml @@ -0,0 +1,110 @@ +# Multi-segment catch-all: `/{path=**}` (Swagger `/{any}` only matches one segment). +# Wildcard methods (except OPTIONS) require Firebase JWT at ESPv2; OPTIONS stays open for CORS preflight. +swagger: "2.0" +info: + title: compass-backend + version: "1.0" +host: __ESPV2_HOSTNAME__ +x-google-backend: + address: __BACKEND_URI__ + protocol: http/1.1 + deadline: 3600.0 + jwt_audience: __BACKEND_URI__ +x-google-management: + metrics: + - name: "request-metric" + displayName: "Request metric" + valueType: INT64 + metricKind: DELTA + quota: + limits: + - name: "request-limit" + metric: "request-metric" + unit: "1/min/{project}" + values: + STANDARD: 1200 +paths: + "/{path=**}": + get: + operationId: getWildcard + parameters: + - name: path + in: path + required: true + type: string + security: + - firebase: [] + responses: + "200": + description: OK + post: + operationId: postWildcard + parameters: + - name: path + in: path + required: true + type: string + security: + - firebase: [] + responses: + "200": + description: OK + put: + operationId: putWildcard + parameters: + - name: path + in: path + required: true + type: string + security: + - firebase: [] + responses: + "200": + description: OK + patch: + operationId: patchWildcard + parameters: + - name: path + in: path + required: true + type: string + security: + - firebase: [] + responses: + "200": + description: OK + delete: + operationId: deleteWildcard + parameters: + - name: path + in: path + required: true + type: string + security: + - firebase: [] + responses: + "200": + description: OK + options: + operationId: optionsWildcard + parameters: + - name: path + in: path + required: true + type: string + security: [] + responses: + "200": + description: OK +securityDefinitions: + firebase: + authorizationUrl: "" + flow: implicit + type: oauth2 + x-google-issuer: "https://securetoken.google.com/__FIREBASE_PROJECT_ID__" + x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/jwk/securetoken@system.gserviceaccount.com" + x-google-audiences: "__FIREBASE_PROJECT_ID__" + api_key: + type: "apiKey" + name: "x-api-key" + in: "header" diff --git a/iac/backend/openapi2_template.yaml b/iac/backend/openapi2_template.yaml deleted file mode 100644 index 958031894..000000000 --- a/iac/backend/openapi2_template.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# openapi2-functions.yaml -swagger: '2.0' -info: - title: Compass Backend API __ENVIRONMENT_NAME__ - description: The backend api for the Compass application - version: 1.0.0 -schemes: - - https -produces: - - application/json -x-google-management: - # Read https://cloud.google.com/endpoints/docs/openapi/quotas-configure for more information - # on how to configure quotas - metrics: - - name: "request-metric" - displayName: "Request metric" - valueType: INT64 - metricKind: DELTA - quota: - limits: - - name: "request-limit" - metric: "request-metric" - unit: "1/min/{project}" - values: - STANDARD: 120 # requests per minute -x-google-backend: - address: "__BACKEND_URI__" - deadline: "__API_GATEWAY_TIMEOUT__" # timeout as the cloud run container may take a while to start, and some requests may take a while to process. Reference: https://cloud.google.com/endpoints/docs/openapi/openapi-extensions#deadline -securityDefinitions: - gcp_api_key: - # Read https://cloud.google.com/endpoints/docs/openapi/restricting-api-access-with-api-keys for more information - "type": apiKey - "name": "x-api-key" - "in": "header" - firebase: - # Read https://cloud.google.com/endpoints/docs/openapi/authenticating-users-firebase - authorizationUrl: "" - flow: "implicit" - type: "oauth2" - x-google-issuer: "https://securetoken.google.com/__PROJECT_ID__" - x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/metadata/x509/securetoken@system.gserviceaccount.com" - x-google-audiences: "__PROJECT_ID__" -paths: - /poc/conversation: - get: - summary: Chat Conversation - operationId: conversation - responses: - '200': - description: A successful response - schema: - type: string - /docs: - get: - summary: Docs endpoint - operationId: api_docs - responses: - '200': - description: Docs page - schema: - type: string - /openapi.json: - get: - summary: OAS file - operationId: oas_file - responses: - '200': - description: OAS file - schema: - type: string diff --git a/iac/common/__main__.py b/iac/common/__main__.py index 3a579e20e..a90d085c8 100644 --- a/iac/common/__main__.py +++ b/iac/common/__main__.py @@ -14,7 +14,7 @@ def main(): - _, _, stack_name = parse_realm_env_name_from_stack() + realm_name, environment_name, stack_name = parse_realm_env_name_from_stack() # Load environment variables. load_dot_realm_env(stack_name) @@ -39,10 +39,19 @@ def main(): frontend_bucket_name = getstackref(frontend_stack_ref, "bucket_name") frontend_bucket_name.apply(lambda name: print(f"Using frontend bucket name: {name}")) - # Get stack reference for backend + # Get the ESPv2 Cloud Run service from the backend stack. backend_stack_ref = pulumi.StackReference(f"tabiya-tech/compass-backend/{stack_name}") - api_gateway_id = getstackref(backend_stack_ref, "apigateway_id") - api_gateway_id.apply(lambda _id: print(f"Using API gateway id: {_id}")) + espv2_cloud_run_name = getstackref(backend_stack_ref, "espv2_cloud_run_name") + espv2_cloud_run_name.apply(lambda name: print(f"Using ESPv2 Cloud Run service: {name}")) + + # Look up the ESPv2 Cloud Run service resource by name so we can pass it to deploy_common. + import pulumi_gcp as gcp + espv2_cloudrun_service = gcp.cloudrunv2.Service.get( + "espv2-gateway-lookup", + id=pulumi.Output.all(project=project, location=location, name=espv2_cloud_run_name).apply( + lambda args: f"projects/{args['project']}/locations/{args['location']}/services/{args['name']}" + ), + ) # Deploy common deploy_common( @@ -53,7 +62,7 @@ def main(): frontend_bucket_name=frontend_bucket_name, frontend_url=frontend_url, backend_url=backend_url, - api_gateway_id=api_gateway_id) + espv2_cloudrun_service=espv2_cloudrun_service) if __name__ == "__main__": diff --git a/iac/common/deploy_common.py b/iac/common/deploy_common.py index 53d8a2d86..c6f507d65 100644 --- a/iac/common/deploy_common.py +++ b/iac/common/deploy_common.py @@ -14,7 +14,7 @@ def _setup_loadbalancer(*, frontend_url: pulumi.Output[str], frontend_bucket_name: pulumi.Output[str], backend_url: pulumi.Output[str], - api_gateway_id: pulumi.Output[str]) -> pulumi.Resource: + espv2_cloudrun_service: gcp.cloudrunv2.Service) -> pulumi.Resource: # Create a global IP address for the load balancer ipaddress = gcp.compute.GlobalAddress( get_resource_name(resource="lb", resource_type="global-ip-address"), @@ -46,26 +46,29 @@ def _setup_loadbalancer(*, opts=pulumi.ResourceOptions(provider=basic_config.provider) ) - # create a region network endpoint group for connecting to the api gateway. + # Serverless NEG pointing at the ESPv2 Cloud Run service. + # SERVERLESS NEGs allow the load balancer to route to Cloud Run services directly. endpoint_group = gcp.compute.RegionNetworkEndpointGroup( get_resource_name(resource="lb", resource_type="endpoint-group"), network_endpoint_type="SERVERLESS", project=basic_config.project, region=basic_config.location, - serverless_deployment=gcp.compute.RegionNetworkEndpointGroupServerlessDeploymentArgs( - platform="apigateway.googleapis.com", - resource=api_gateway_id + cloud_run=gcp.compute.RegionNetworkEndpointGroupCloudRunArgs( + service=espv2_cloudrun_service.name, ), - opts=pulumi.ResourceOptions(provider=basic_config.provider) + opts=pulumi.ResourceOptions( + depends_on=[espv2_cloudrun_service], + provider=basic_config.provider, + ) ) - # Create a backend service for the api gateway - api_gateway_backend_service = gcp.compute.BackendService( + # Create a backend service for ESPv2 Cloud Run. + espv2_backend_service = gcp.compute.BackendService( get_resource_name(resource="lb", resource_type="backend-service"), project=basic_config.project, connection_draining_timeout_sec=10, - protocol="HTTP", - enable_cdn=False, # the responses will not be cached. + protocol="HTTPS", + enable_cdn=False, # SSE responses must not be cached. load_balancing_scheme="EXTERNAL_MANAGED", backends=[gcp.compute.BackendServiceBackendArgs(group=endpoint_group.id)], log_config=gcp.compute.BackendServiceLogConfigArgs(enable=True), @@ -95,12 +98,12 @@ def _get_path_rule(_service_url: str): # Map /* -> /* of the backend service. backend_path_rule = backend_url.apply(_get_path_rule) backend_rule = gcp.compute.URLMapPathMatcherPathRuleArgs(paths=[backend_path_rule], - service=api_gateway_backend_service.id, + service=espv2_backend_service.id, route_action=route_action) # Swagger UI will request /openapi.json, so we need to rewrite the path to the correct one. backend_openapi_rule = gcp.compute.URLMapPathMatcherPathRuleArgs(paths=["/openapi.json"], - service=api_gateway_backend_service.id) + service=espv2_backend_service.id) https_url_map = gcp.compute.URLMap( get_resource_name(resource="lb", resource_type="https-urlmap"), @@ -194,7 +197,7 @@ def deploy_common(*, frontend_url: pulumi.Output[str], frontend_bucket_name: pulumi.Output[str], backend_url: pulumi.Output[str], - api_gateway_id: pulumi.Output[str]): + espv2_cloudrun_service: gcp.cloudrunv2.Service): basic_config = get_project_base_config(project=project, location=location) # Create the Global Load Balancer @@ -205,4 +208,4 @@ def deploy_common(*, frontend_url=frontend_url, frontend_bucket_name=frontend_bucket_name, backend_url=backend_url, - api_gateway_id=api_gateway_id) + espv2_cloudrun_service=espv2_cloudrun_service) diff --git a/iac/realm/__main__.py b/iac/realm/__main__.py index 03b75a647..cb86e4876 100644 --- a/iac/realm/__main__.py +++ b/iac/realm/__main__.py @@ -55,7 +55,7 @@ def main(): root_project_id=root_project_id, upper_env_google_oauth_projects_folder_id=upper_env_google_oauth_projects_folder_id, lower_env_google_oauth_projects_folder_id=lower_env_google_oauth_projects_folder_id, - roots_path=libs_dir + roots_path=libs_dir, ) diff --git a/iac/realm/create_realm.py b/iac/realm/create_realm.py index ba72c7bb2..b691a31b8 100644 --- a/iac/realm/create_realm.py +++ b/iac/realm/create_realm.py @@ -454,7 +454,10 @@ def _enable_required_services(*, provider: gcp.Provider) -> time.Sleep: # Required for enabling the identity toolkit in the environment projects "identitytoolkit.googleapis.com", # GCP Secret Manager — Required for creating config secrets. - "secretmanager.googleapis.com" + "secretmanager.googleapis.com", + # Required by Pulumi's gcp.endpoints.Service provider (Service Management API calls + # are made from the realm root project's credentials). + "servicemanagement.googleapis.com", ] services = enable_services(provider=provider, service_names=_REQUIRED_SERVICES, dependencies=[]) diff --git a/iac/scripts/up.py b/iac/scripts/up.py index fe4a8d73d..94012d60d 100755 --- a/iac/scripts/up.py +++ b/iac/scripts/up.py @@ -32,22 +32,40 @@ def _run_smoke_tests(version_json_url: str, max_retries: int = 10): print(f"info: running the smoke tests on {version_json_url} and expected version is {artifacts_version}") - version_json_response: Optional[requests.Response] = None - - for _i in range(max_retries): + last_response: Optional[requests.Response] = None + for attempt in range(max_retries): try: - version_json_response = requests.get(version_json_url) - break + last_response = requests.get(version_json_url, timeout=60) + if last_response.status_code == 200: + break + print( + f"info: smoke GET status {last_response.status_code}, " + f"attempt {attempt + 1}/{max_retries}; retry in 10s" + ) + time.sleep(10) except requests.exceptions.SSLError: print(f"info: retrying after 10 seconds the request to {version_json_url} due to SSL error.") time.sleep(10) - assert version_json_response is not None - assert version_json_response.status_code == 200 - - version_json = version_json_response.json() - assert version_json["sha"] == artifacts_version.git_sha - assert version_json["branch"] == artifacts_version.git_branch_name + assert last_response is not None, "no HTTP response from smoke URL" + if last_response.status_code != 200: + body_preview = (last_response.text or "")[:800] + raise AssertionError( + f"smoke GET expected 200, got {last_response.status_code} from {version_json_url!r}. " + f"Body (truncated): {body_preview!r}" + ) + + version_json = last_response.json() + if version_json.get("sha") != artifacts_version.git_sha: + raise AssertionError( + f"smoke version sha mismatch: got {version_json.get('sha')!r}, " + f"expected {artifacts_version.git_sha!r}" + ) + if version_json.get("branch") != artifacts_version.git_branch_name: + raise AssertionError( + f"smoke version branch mismatch: got {version_json.get('branch')!r}, " + f"expected {artifacts_version.git_branch_name!r}" + ) print("info: smoke tests passed successfully.") @@ -65,12 +83,10 @@ def _deploy_frontend(stack_name: str): def _deploy_backend(stack_name: str): - # run pulumi up on the backend - up_results = run_pulumi_up(stack_name, IaCModules.BACKEND) - - # run the smoke tests for the backend. - apigateway_url = up_results.outputs["apigateway_url"].value - _run_smoke_tests(f"{apigateway_url}/version") + run_pulumi_up(stack_name, IaCModules.BACKEND) + # Do not smoke-test the public backend_url here: the URL map / ESPv2 wiring lives in the + # common stack, which runs later in this pipeline. _deploy_common smoke-tests the same URL + # after the load balancer is applied. def _deploy_common(stack_name: str): diff --git a/iac/templates/env.template b/iac/templates/env.template index 776f375b5..bc1693c8b 100644 --- a/iac/templates/env.template +++ b/iac/templates/env.template @@ -48,6 +48,11 @@ BACKEND_EXPERIENCE_PIPELINE_CONFIG=/.*/ BACKEND_CV_MAX_UPLOADS_PER_USER=/^[0-9]*$/ BACKEND_CV_RATE_LIMIT_PER_MINUTE=/^[0-9]*$/ +# SSE streaming (optional; chunk size in chars/words, chunk mode chars|words, delay between deltas in ms) +BACKEND_STREAM_CHUNK_SIZE=/^[0-9]*$/ +BACKEND_STREAM_CHUNK_MODE=/^(chars|words)?$/ +BACKEND_STREAM_DELTA_DELAY_MS=/^[0-9]*$/ + ################### # Frontend ################### diff --git a/iac/templates/stack_config.template.yaml b/iac/templates/stack_config.template.yaml index 277c6bef8..864d52b41 100644 --- a/iac/templates/stack_config.template.yaml +++ b/iac/templates/stack_config.template.yaml @@ -23,7 +23,6 @@ backend: cloudrun:max_instance_count: # See https://cloud.google.com/run/docs/configuring/request-timeout # In seconds e.g. 60s or 3.05s - # This should be equal or less than the timeout of the API Gateway cloudrun:request_timeout: # See https://cloud.google.com/run/docs/configuring/services/memory-limits @@ -35,13 +34,6 @@ backend: cloudrun:cpu_limit: ############################ - ############################ - # Api Gateway settings - # See https://cloud.google.com/endpoints/docs/openapi/openapi-extensions#deadline - # In seconds as a double e.g. 15.0 - api_gateway:timeout: - ############################ - frontend: config: gcp:region: