From 2e3c50bc8213637f84f52659e29c58f4192c5882 Mon Sep 17 00:00:00 2001 From: tobiasfremming Date: Tue, 2 Jun 2026 02:26:07 +0200 Subject: [PATCH] feat: enhance chat service with user context and progress tracking features --- Dockerfile | 2 + .../user_guide_agents_external_access.md | 243 +++++++++++++++++- src/models/chat/command.py | 12 + src/models/training/progress.py | 5 + src/pipeline.py | 55 +++- src/routes/chat.py | 82 +++--- src/routes/progress.py | 74 +++++- src/transcribe.py | 77 +++--- 8 files changed, 468 insertions(+), 82 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff75f36..949d627 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,11 @@ ENV PYTHONUNBUFFERED=1 \ # - libgl1, libglib2.0-0: required by OpenCV (cv2) used by unstructured library # - poppler-utils: required for PDF processing (pdftotext, pdfinfo, etc.) # - tesseract-ocr: optional OCR support for scanned PDFs +# - ffmpeg: required by Whisper for browser-recorded formats such as WebM/Opus RUN apt-get update && \ apt-get install -y \ curl \ + ffmpeg \ git \ libgl1 \ libglib2.0-0 \ diff --git a/docs/manuals/user_guide_agents_external_access.md b/docs/manuals/user_guide_agents_external_access.md index a0d66eb..672d064 100644 --- a/docs/manuals/user_guide_agents_external_access.md +++ b/docs/manuals/user_guide_agents_external_access.md @@ -158,6 +158,9 @@ Body fields: | `active_role_id` | string | Yes | The role name to speak as, for example `student`. | | `access_key` | string | Yes | The raw agent access key. | | `chat_log` | array | Yes | The conversation history. Include the latest user message. | +| `user_information` | array of strings | No | Extra user or game-state facts to include in the prompt. | +| `user_actions` | array of strings | No | Recent actions from the external application or game. | +| `progress` | array | No | Structured task progress entries. | Example: @@ -227,6 +230,237 @@ Example multi-turn request: The final item should normally be the newest user question. +### Optional Live Context + +External clients can send extra context with each chat request. This is useful for games, simulators, or training applications where the LLM should know what the user has done. + +Example: + +```json +{ + "agent_id": "6a1d614955f55909e1272f02", + "active_role_id": "instructor", + "access_key": "YOUR_AGENT_ACCESS_KEY", + "chat_log": [ + { + "role": "user", + "content": "What should I do next?" + } + ], + "user_information": [ + "The player is in the engine room.", + "The player has low health." + ], + "user_actions": [ + "Opened the toolbox", + "Inspected the broken cable" + ], + "progress": [ + { + "task_name": "Repair engine", + "description": "Repair the ship engine", + "status": "started", + "subtask_progress": [ + { + "subtask_name": "Find tool", + "description": "Find the wrench", + "completed": true, + "step_progress": [ + { + "step_name": "Open toolbox", + "repetition_number": 0, + "completed": true + } + ] + } + ] + } + ] +} +``` + +These fields are optional. Existing plain chat integrations can keep sending only `agent_id`, `active_role_id`, `access_key`, and `chat_log`. + +## Voice Endpoints + +RAGdoll includes Whisper-based transcription endpoints for external applications that send recorded audio. + +### Transcribe Audio Only + +```http +POST /api/chat/transcribe +``` + +Form fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `audio` | file | Yes | Audio file to transcribe. WAV is recommended. | +| `language` | string | No | Optional language code, for example `en` or `no`. | + +Example: + +```bash +curl https://iplvr.it.ntnu.no/backend/api/chat/transcribe \ + -F "audio=@question.wav" \ + -F "language=en" +``` + +Example response: + +```json +{ + "success": true, + "transcription": "What should I do next?", + "server_processed": true, + "processing_time_seconds": 1.2, + "processor": "Server-based Whisper" +} +``` + +This endpoint does not talk to an agent. It only returns text. + +### Transcribe Audio and Ask Agent + +```http +POST /api/chat/askTranscribe +``` + +Form fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `audio` | file | Yes | Audio file containing the user's question. | +| `data` | JSON string | Yes | A serialized chat command with `agent_id`, `active_role_id`, `access_key`, and optional context fields. | + +Example: + +```bash +curl https://iplvr.it.ntnu.no/backend/api/chat/askTranscribe \ + -F "audio=@question.wav" \ + -F 'data={ + "agent_id": "6a1d614955f55909e1272f02", + "active_role_id": "student", + "access_key": "YOUR_AGENT_ACCESS_KEY", + "chat_log": [] + }' +``` + +The backend transcribes the audio, appends the transcription as the latest user message, validates the access key, and sends the request through the same RAG pipeline as `/api/chat/ask`. + +Example response shape: + +```json +{ + "transcription": "What should I do next?", + "response": { + "response": "The agent response text is here.", + "context_used": [] + } +} +``` + +## Progress Endpoints + +Progress endpoints are intended for external training applications, games, or simulators that want to store task state and reuse it in later chat requests. + +Progress is stored in memory in the backend process. For durable game state, the external application should still keep its own source of truth and send relevant progress in the chat command. + +### Initialize Tasks + +```http +POST /api/progress/initializeTasks +``` + +Body fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `agent_id` | string | Yes | Agent id connected to the access key. | +| `access_key` | string | Yes | Raw agent access key. | +| `items` | array | Yes | List of progress task objects. | + +Example: + +```json +{ + "agent_id": "6a1d614955f55909e1272f02", + "access_key": "YOUR_AGENT_ACCESS_KEY", + "items": [ + { + "task_name": "Repair engine", + "description": "Repair the ship engine", + "status": "pending", + "subtask_progress": [] + } + ] +} +``` + +### Update One Task + +```http +POST /api/progress/updateTask +``` + +Body fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `agent_id` | string | Yes | Agent id connected to the access key. | +| `access_key` | string | Yes | Raw agent access key. | +| `task_name` | string | Yes | Task identifier. | +| `description` | string | Yes | Human-readable task description. | +| `status` | string | Yes | `pending`, `started`, or `complete`. | +| `subtask_progress` | array | No | Subtasks and step progress. | + +Example: + +```json +{ + "agent_id": "6a1d614955f55909e1272f02", + "access_key": "YOUR_AGENT_ACCESS_KEY", + "task_name": "Repair engine", + "description": "Repair the ship engine", + "status": "started", + "subtask_progress": [ + { + "subtask_name": "Find tool", + "description": "Find the wrench", + "completed": true, + "step_progress": [ + { + "step_name": "Open toolbox", + "repetition_number": 0, + "completed": true + } + ] + } + ] +} +``` + +### Fetch Progress + +```http +GET /api/progress?agent_id=AGENT_ID +``` + +Header: + +```text +access-key: YOUR_AGENT_ACCESS_KEY +``` + +Example: + +```bash +curl "https://iplvr.it.ntnu.no/backend/api/progress?agent_id=6a1d614955f55909e1272f02" \ + -H "access-key: YOUR_AGENT_ACCESS_KEY" +``` + +Fetched progress can be passed back into `/api/chat/ask` or `/api/chat/askTranscribe` in the optional `progress` field. + ## External Chat UI for Testing RAGdollChat includes a test page for external access-key use: @@ -257,7 +491,14 @@ Then enter: - Agent access key - Role name -Use this page to verify that a key and role work before integrating an external application. +Use this page to verify that a key and role work before integrating an external application. After connecting, the page also includes endpoint test tools for: + +- `POST /api/chat/ask` +- `POST /api/chat/transcribe` +- `POST /api/chat/askTranscribe` +- `POST /api/progress/initializeTasks` +- `POST /api/progress/updateTask` +- `GET /api/progress` ## Minimal JavaScript Example diff --git a/src/models/chat/command.py b/src/models/chat/command.py index 19016b1..b15c868 100644 --- a/src/models/chat/command.py +++ b/src/models/chat/command.py @@ -37,6 +37,18 @@ class Command(BaseModel): access_key: str | None = Field( default=None, description="API key for agent access authorization" ) + user_information: list[str] = Field( + default_factory=list, + description="Optional external user/game-state facts to include in the prompt", + ) + progress: list[ProgressData] = Field( + default_factory=list, + description="Optional training/task progress data to include in the prompt", + ) + user_actions: list[str] = Field( + default_factory=list, + description="Optional recent user/game actions to include in the prompt", + ) class Prompt(BaseModel): diff --git a/src/models/training/progress.py b/src/models/training/progress.py index 1927e56..e0dbe31 100644 --- a/src/models/training/progress.py +++ b/src/models/training/progress.py @@ -55,9 +55,12 @@ class ProgressData(BaseModel): task_name: str description: str status: str = Field(default="started") + agent_id: str | None = None + access_key: str | None = None user_id: str | None = None subtask_progress: list[SubtaskProgressDTO] = Field(default_factory=list) started_at: datetime | None = None + completed_at: datetime | None = None completet_at: datetime | None = ( None # Note: typo in original, keeping for compatibility ) @@ -70,4 +73,6 @@ class ListProgressData(BaseModel): items: List of progress data entries """ + agent_id: str + access_key: str items: list[ProgressData] = Field(default_factory=list) diff --git a/src/pipeline.py b/src/pipeline.py index 58c9d2c..b5bf495 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -178,6 +178,10 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict: + agent.prompt + "\n" ) + prompt += ( + "Return only the character's spoken response. Do not prefix the answer " + "with AGENT:, ASSISTANT:, the character name, or any role label.\n" + ) prompt += ("Your role: " + role_prompt + "\n") if role_prompt else "" # Add retrieved context to prompt if retrieved_contexts: @@ -187,6 +191,10 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict: for ctx in retrieved_contexts: prompt += f"\n-{ctx.text}\n" + game_context = game_context_prompt_section(command) + if game_context: + prompt += "\n\n" + game_context + prompt = full_chat_history + "\n" + prompt print(f"Prompt sent to LLM:\n{prompt}") @@ -253,5 +261,50 @@ def chat_history_prompt_section( for msg in command.chat_log[ limit_start : (-1 if not include_latest else None) ]: # Exclude latest user message - chat_history += f"{msg.role.upper()}: {msg.content}\n" + role_label = "ASSISTANT" if msg.role.lower() == "agent" else msg.role.upper() + chat_history += f"{role_label}: {msg.content}\n" return chat_history + + +def game_context_prompt_section(command: Command) -> str: + sections: list[str] = [] + + if command.user_information: + sections.append( + "User/game information:\n" + + "\n".join(f"- {item}" for item in command.user_information if item) + ) + + if command.user_actions: + sections.append( + "Recent user/game actions:\n" + + "\n".join(f"- {action}" for action in command.user_actions if action) + ) + + if command.progress: + progress_lines: list[str] = [] + for task in command.progress: + progress_lines.append( + f"- {task.task_name}: {task.status}. {task.description}" + ) + for subtask in task.subtask_progress: + state = "complete" if subtask.completed else "incomplete" + progress_lines.append( + f" - {subtask.subtask_name}: {state}. {subtask.description}" + ) + for step in subtask.step_progress: + step_state = "complete" if step.completed else "incomplete" + progress_lines.append( + f" - {step.step_name} repetition {step.repetition_number}: {step_state}" + ) + if progress_lines: + sections.append("Training/task progress:\n" + "\n".join(progress_lines)) + + if not sections: + return "" + + return ( + "Additional live context from the external application. Use it only when it is relevant " + "to the user's question and the character's role:\n" + + "\n\n".join(sections) + ) diff --git a/src/routes/chat.py b/src/routes/chat.py index fa06638..a4b7a35 100644 --- a/src/routes/chat.py +++ b/src/routes/chat.py @@ -24,6 +24,28 @@ config = Config() +def _get_authorized_agent(command: Command): + agent = agent_dao.get_agent_by_id(command.agent_id) + if agent is None: + return None, JSONResponse( + content={"message": f"Agent with id '{command.agent_id}' not found."}, + status_code=400, + ) + + if not access_service.authenticate(agent.id, command.access_key): + raise HTTPException(status_code=401, detail="Unauthorized, check logs") + + if command.active_role_id and agent.get_role_by_name(command.active_role_id) is None: + return None, JSONResponse( + content={ + "message": f"Role '{command.active_role_id}' not found in agent '{agent.name}'." + }, + status_code=400, + ) + + return agent, None + + @router.post("/ask", response_model=Command) async def ask(command: Command): """Process a user question using the specified agent and roles. @@ -55,29 +77,9 @@ async def ask(command: Command): 500: Other processing errors """ try: - # Retrieve the agent configuration - agent = agent_dao.get_agent_by_id(command.agent_id) - if agent is None: - return JSONResponse( - content={"message": f"Agent with id '{command.agent_id}' not found."}, - status_code=400, - ) - - # Auth - if not access_service.authenticate(agent.id, command.access_key): - raise HTTPException(status_code=401, detail="Unauthorized, check logs") - - # Validate that requested role exists in the agent - if ( - command.active_role_id - and agent.get_role_by_name(command.active_role_id) is None - ): - return JSONResponse( - content={ - "message": f"Role '{command.active_role_id}' not found in agent '{agent.name}'." - }, - status_code=400, - ) + agent, error_response = _get_authorized_agent(command) + if error_response is not None: + return error_response # Generate response using agent configuration and role-based RAG response = assemble_prompt_with_agent(command, agent) @@ -149,8 +151,15 @@ async def ask_transcribe( Returns: - A JSON response with the agent's answer """ - # Transcribe the audio - transcribed = transcribe_from_upload(audio) + try: + transcribed = transcribe_from_upload(audio) + except ValueError as e: + return JSONResponse(content={"message": str(e)}, status_code=400) + except Exception as e: + return JSONResponse( + content={"message": f"Failed to transcribe audio: {e!s}"}, + status_code=500, + ) # Parse command and add transcribed text as user message command = command_from_json_transcribe_version(data, question=transcribed) @@ -159,16 +168,17 @@ async def ask_transcribe( content={"message": "Invalid command format."}, status_code=400 ) - # Retrieve and validate agent (same logic as /ask) - agent = agent_dao.get_agent_by_id(command.agent_id) - if agent is None: + agent, error_response = _get_authorized_agent(command) + if error_response is not None: + return error_response + + try: + response = assemble_prompt_with_agent(command, agent) return JSONResponse( - content={"message": f"Agent with id '{command.agent_id}' not found."}, - status_code=400, + content={"transcription": transcribed, "response": response}, + status_code=200, ) - - # Generate response - response = assemble_prompt_with_agent(command, agent) - return JSONResponse( - content={"transcription": transcribed, "response": response}, status_code=200 - ) + except LLMAPIError as e: + return JSONResponse(content={"message": str(e)}, status_code=e.status_code) + except LLMGenerationError as e: + return JSONResponse(content={"message": str(e)}, status_code=e.status_code) diff --git a/src/routes/progress.py b/src/routes/progress.py index c5aec1e..6190725 100644 --- a/src/routes/progress.py +++ b/src/routes/progress.py @@ -1,7 +1,9 @@ from datetime import UTC, datetime +from typing import Annotated -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Header, HTTPException +from src.globals import access_service, agent_dao from src.models.training import ListProgressData, ProgressData # In-memory log to store progress data @@ -12,38 +14,75 @@ router = APIRouter() +def _authorize_agent_access(agent_id: str | None, access_key: str | None) -> str: + if not agent_id: + raise HTTPException(status_code=400, detail="agent_id is required") + if not access_key: + raise HTTPException(status_code=401, detail="access_key is required") + + agent = agent_dao.get_agent_by_id(agent_id) + if agent is None: + raise HTTPException( + status_code=404, detail=f"Agent with id '{agent_id}' not found" + ) + if not access_service.authenticate(agent_id, access_key): + raise HTTPException(status_code=401, detail="Unauthorized access to agent") + return agent_id + + +def _progress_dump(progress: ProgressData, agent_id: str) -> dict: + entry = progress.model_dump(exclude={"access_key"}) + entry["agent_id"] = agent_id + return entry + + @router.post("/api/progress/initializeTasks") def receive_hierarchy(task_hierarchy: ListProgressData): """Initializes a list of tasks with their subtasks and steps.""" try: + agent_id = _authorize_agent_access( + task_hierarchy.agent_id, task_hierarchy.access_key + ) for task in task_hierarchy.items: - progress_log.append(task.model_dump()) - return {"message": "Tasks initialized", "data": task_hierarchy} + progress_log.append(_progress_dump(task, agent_id)) + return { + "message": "Tasks initialized", + "data": task_hierarchy.model_dump(exclude={"access_key"}), + } except Exception as e: print(f"Error processing task hierarchy: {e}") + if isinstance(e, HTTPException): + raise raise HTTPException(status_code=500, detail=str(e)) from e @router.post("/api/progress/updateTask") def receive_progress(progress: ProgressData): """Handles task progress updates (started/complete) and stores them.""" + agent_id = _authorize_agent_access(progress.agent_id, progress.access_key) + if progress.status == "started" or progress.status == "pending": # Check for existing incomplete task to update for entry in progress_log: if ( - entry["task_name"] == progress.task_name + entry.get("agent_id") == agent_id + and entry["task_name"] == progress.task_name and entry["completed_at"] is None ): entry.update( { - "subtask_progress": progress.subtask_progress, + "subtask_progress": [ + subtask.model_dump() + for subtask in progress.subtask_progress + ], "started_at": datetime.now(UTC), + "status": progress.status, } ) return {"message": "Progress updated", "data": entry} # New task entry - new_entry = progress.model_dump() + new_entry = _progress_dump(progress, agent_id) new_entry["started_at"] = datetime.now(UTC) progress_log.append(new_entry) return {"message": "Progress received", "data": new_entry} @@ -52,13 +91,18 @@ def receive_progress(progress: ProgressData): # Complete existing task for entry in progress_log: if ( - entry["task_name"] == progress.task_name + entry.get("agent_id") == agent_id + and entry["task_name"] == progress.task_name and entry["completed_at"] is None ): entry.update( { - "subtask_progress": progress.subtask_progress, + "subtask_progress": [ + subtask.model_dump() + for subtask in progress.subtask_progress + ], "completed_at": datetime.now(UTC), + "completet_at": datetime.now(UTC), "status": "complete", } ) @@ -66,9 +110,17 @@ def receive_progress(progress: ProgressData): return {"message": f"No active task {progress.task_name} found."} else: - raise HTTPException(400, "Status must be 'started' or 'complete'.") + raise HTTPException(400, "Status must be 'started', 'pending', or 'complete'.") @router.get("/api/progress") -def get_progress_log(): - return progress_log # Returns the entire in-memory list +def get_progress_log( + agent_id: str, + access_key: Annotated[str | None, Header()] = None, +): + authorized_agent_id = _authorize_agent_access(agent_id, access_key) + return [ + entry + for entry in progress_log + if entry.get("agent_id") == authorized_agent_id + ] diff --git a/src/transcribe.py b/src/transcribe.py index 1e33932..032a629 100644 --- a/src/transcribe.py +++ b/src/transcribe.py @@ -2,13 +2,14 @@ import logging import math import os +import tempfile import numpy as np import soundfile as sf import whisper from fastapi import UploadFile from flask import Flask -from scipy.signal import resample_poly # pip install scipy soundfile +from scipy.signal import resample_poly from src.whisper_model import get_whisper_model @@ -21,28 +22,47 @@ # TODO: switch to FastAPI app = Flask(__name__) -TARGET_SR = 16_000 # 16 kHz mono float32 +TARGET_SR = 16_000 + + +def _load_audio_with_whisper(raw: bytes, filename: str | None = None) -> np.ndarray: + suffix = os.path.splitext(filename or "")[1] or ".webm" + temp_path = None + try: + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as temp_file: + temp_file.write(raw) + temp_path = temp_file.name + return whisper.load_audio(temp_path, sr=TARGET_SR).astype("float32") + except Exception as e: + logger.error(f"Whisper/ffmpeg fallback failed when processing audio: {e!s}") + raise ValueError( + "Invalid audio file format. Browser recordings require ffmpeg support in the backend container." + ) from e + finally: + if temp_path: + try: + os.unlink(temp_path) + except OSError: + logger.warning(f"Failed to remove temporary audio file: {temp_path}") def load_audio_from_upload(file) -> np.ndarray: + raw = file.file.read() try: - raw = file.file.read() # UploadFile → bytes - # --- decode ---------------------------------------------------------------------------------- with io.BytesIO(raw) as bio: - audio, sr = sf.read( - bio, dtype="float32" - ) # libsndfile does the heavy lifting - # --- mono ------------------------------------------------------------------------------------ + audio, sr = sf.read(bio, dtype="float32") if audio.ndim > 1: - audio = audio.mean(axis=1) # down-mix - # --- resample ------------------------------------------------------------------------------- + audio = audio.mean(axis=1) if sr != TARGET_SR: - g = math.gcd(sr, TARGET_SR) # polyphase → good quality & fast + g = math.gcd(sr, TARGET_SR) audio = resample_poly(audio, TARGET_SR // g, sr // g).astype("float32") return audio except sf.SoundFileError as e: - logger.error(f"SoundFile error when processing audio: {e!s}") - raise ValueError("Invalid audio file format.") from e + logger.info( + "SoundFile could not decode uploaded audio, trying Whisper/ffmpeg fallback: %s", + e, + ) + return _load_audio_with_whisper(raw, getattr(file, "filename", None)) except Exception as e: logger.error(f"Error loading audio: {e!s}") raise ValueError("Failed to process audio file.") from e @@ -62,44 +82,37 @@ def transcribe_audio(file: UploadFile, language: str | None = None) -> dict: """Transcribe an audio file with specified language. Args: - file (UploadFile): The audio file to transcribe - language (str, optional): Language code (e.g., 'en', 'es', 'fr') + file: The audio file to transcribe. + language: Optional language code, for example "en", "es", or "fr". Returns: - dict: Response containing transcription or error message + Response containing transcription or an error message. """ try: import time start_time = time.time() - # Check file size (limit to 25MB for example) file.file.seek(0, os.SEEK_END) file_size = file.file.tell() file.file.seek(0) - if file_size > 25 * 1024 * 1024: # 25MB + if file_size > 25 * 1024 * 1024: return {"success": False, "error": "File too large. Maximum size is 25MB."} - # Load audio audio = load_audio_from_upload(file) audio = whisper.pad_or_trim(audio) - # Process with whisper mel = whisper.log_mel_spectrogram(audio).to(model.device) - - # Set language in options if provided - if language: - options = whisper.DecodingOptions(language=language) - else: - options = whisper.DecodingOptions() - + options = ( + whisper.DecodingOptions(language=language) + if language + else whisper.DecodingOptions() + ) result = whisper.decode(model, mel, options) - # Calculate processing time processing_time = time.time() - start_time - # Add server identifier to the response return { "success": True, "transcription": result.text, @@ -109,10 +122,8 @@ def transcribe_audio(file: UploadFile, language: str | None = None) -> dict: } except ValueError as e: - # Handle format errors return {"success": False, "error": str(e)} except Exception as e: - # Handle other errors logger.error(f"Transcription error: {e!s}") return {"success": False, "error": f"Failed to transcribe audio: {e!s}"} @@ -121,10 +132,10 @@ def transcribe(audio): """Transcribe an audio file using Whisper model. Args: - audio (str): Path to the audio file. + audio: Path to the audio file. Returns: - str: Transcribed text. + Transcribed text. """ result = model.transcribe(audio) return result["text"]