diff --git a/docs/manuals/user_guide_agents_external_access.md b/docs/manuals/user_guide_agents_external_access.md index 672d064..bb82f44 100644 --- a/docs/manuals/user_guide_agents_external_access.md +++ b/docs/manuals/user_guide_agents_external_access.md @@ -83,6 +83,7 @@ The external application needs: - An agent access key. - A role name. - A chat history. +- A session id if the application wants progress/task state to be remembered during a run. For the production server, the backend base URL is: @@ -161,6 +162,9 @@ Body fields: | `user_information` | array of strings | No | Extra user or game-state facts to include in the prompt. | | `user_actions` | array of strings | No | Recent actions from the external application or game. | | `progress` | array | No | Structured task progress entries. | +| `session_id` | string | No | External session identifier. If supplied, the backend automatically adds recent stored progress for this session. | +| `include_progress` | boolean | No | Defaults to `true`. Set to `false` to prevent automatic session progress from being added. | +| `progress_limit` | number | No | Defaults to `5`. Maximum number of recent stored progress tasks to add. Maximum accepted value is `20`. | Example: @@ -241,6 +245,7 @@ Example: "agent_id": "6a1d614955f55909e1272f02", "active_role_id": "instructor", "access_key": "YOUR_AGENT_ACCESS_KEY", + "session_id": "unity-session-001", "chat_log": [ { "role": "user", @@ -281,6 +286,8 @@ Example: These fields are optional. Existing plain chat integrations can keep sending only `agent_id`, `active_role_id`, `access_key`, and `chat_log`. +If `session_id` is included, the backend loads the most recent stored progress tasks for that agent/session and adds them to the prompt automatically. The prompt tells the LLM to use this progress only when it is relevant to the user's question, current task, or next action. + ## Voice Endpoints RAGdoll includes Whisper-based transcription endpoints for external applications that send recorded audio. @@ -364,7 +371,28 @@ Example response shape: Progress endpoints are intended for external training applications, games, or simulators that want to store task state and reuse it in later chat requests. -Progress is stored in memory in the backend process. For durable game state, the external application should still keep its own source of truth and send relevant progress in the chat command. +Progress is stored in memory in the backend process and is scoped by: + +- `agent_id` +- `session_id` + +Progress entries are pruned after roughly 24 hours and also disappear if the backend restarts. For durable game state, the external application should still keep its own source of truth. + +Recommended external workflow: + +1. Generate a `session_id` when the external run starts. +2. Store that `session_id` in the game/client memory. +3. Send the same `session_id` with progress updates. +4. Send the same `session_id` with `/api/chat/ask` or `/api/chat/askTranscribe`. +5. The backend automatically includes the most recent progress tasks in the LLM prompt. + +Example session id: + +```text +unity-session-2026-06-02-player-123 +``` + +The session id does not need to be secret. The access key is the authorization credential. ### Initialize Tasks @@ -378,6 +406,7 @@ Body fields: | --- | --- | --- | --- | | `agent_id` | string | Yes | Agent id connected to the access key. | | `access_key` | string | Yes | Raw agent access key. | +| `session_id` | string | Recommended | External session identifier. Defaults to `default` if omitted. | | `items` | array | Yes | List of progress task objects. | Example: @@ -386,6 +415,7 @@ Example: { "agent_id": "6a1d614955f55909e1272f02", "access_key": "YOUR_AGENT_ACCESS_KEY", + "session_id": "unity-session-001", "items": [ { "task_name": "Repair engine", @@ -409,6 +439,7 @@ Body fields: | --- | --- | --- | --- | | `agent_id` | string | Yes | Agent id connected to the access key. | | `access_key` | string | Yes | Raw agent access key. | +| `session_id` | string | Recommended | External session identifier. Defaults to `default` if omitted. | | `task_name` | string | Yes | Task identifier. | | `description` | string | Yes | Human-readable task description. | | `status` | string | Yes | `pending`, `started`, or `complete`. | @@ -420,6 +451,7 @@ Example: { "agent_id": "6a1d614955f55909e1272f02", "access_key": "YOUR_AGENT_ACCESS_KEY", + "session_id": "unity-session-001", "task_name": "Repair engine", "description": "Repair the ship engine", "status": "started", @@ -443,7 +475,7 @@ Example: ### Fetch Progress ```http -GET /api/progress?agent_id=AGENT_ID +GET /api/progress?agent_id=AGENT_ID&session_id=SESSION_ID ``` Header: @@ -455,11 +487,18 @@ access-key: YOUR_AGENT_ACCESS_KEY Example: ```bash -curl "https://iplvr.it.ntnu.no/backend/api/progress?agent_id=6a1d614955f55909e1272f02" \ +curl "https://iplvr.it.ntnu.no/backend/api/progress?agent_id=6a1d614955f55909e1272f02&session_id=unity-session-001" \ -H "access-key: YOUR_AGENT_ACCESS_KEY" ``` -Fetched progress can be passed back into `/api/chat/ask` or `/api/chat/askTranscribe` in the optional `progress` field. +Optional query fields: + +| Field | Type | Description | +| --- | --- | --- | +| `session_id` | string | Return progress for this session. Defaults to `default` if omitted. | +| `limit` | number | Return only the newest N progress tasks. | + +Fetched progress can be passed back into `/api/chat/ask` or `/api/chat/askTranscribe` in the optional `progress` field, but this is no longer required when the chat request includes the same `session_id`. The backend automatically loads recent progress for that session. ## External Chat UI for Testing diff --git a/src/models/chat/command.py b/src/models/chat/command.py index b15c868..5160a22 100644 --- a/src/models/chat/command.py +++ b/src/models/chat/command.py @@ -37,6 +37,20 @@ class Command(BaseModel): access_key: str | None = Field( default=None, description="API key for agent access authorization" ) + session_id: str | None = Field( + default=None, + description="External session identifier used to load recent progress", + ) + include_progress: bool = Field( + default=True, + description="If true, include recent stored session progress in the prompt", + ) + progress_limit: int = Field( + default=5, + ge=0, + le=20, + description="Maximum number of recent stored progress tasks to include", + ) user_information: list[str] = Field( default_factory=list, description="Optional external user/game-state facts to include in the prompt", diff --git a/src/models/training/progress.py b/src/models/training/progress.py index e0dbe31..44628c2 100644 --- a/src/models/training/progress.py +++ b/src/models/training/progress.py @@ -57,10 +57,12 @@ class ProgressData(BaseModel): status: str = Field(default="started") agent_id: str | None = None access_key: str | None = None + session_id: str | None = None user_id: str | None = None subtask_progress: list[SubtaskProgressDTO] = Field(default_factory=list) started_at: datetime | None = None completed_at: datetime | None = None + updated_at: datetime | None = None completet_at: datetime | None = ( None # Note: typo in original, keeping for compatibility ) @@ -75,4 +77,5 @@ class ListProgressData(BaseModel): agent_id: str access_key: str + session_id: str | None = None items: list[ProgressData] = Field(default_factory=list) diff --git a/src/pipeline.py b/src/pipeline.py index b5bf495..170eb28 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -244,6 +244,7 @@ def assemble_prompt_with_agent(command: Command, agent: Agent) -> dict: "response_length": len(parsed_response), "agent_name": agent.name, "num_context_retrieved": len(retrieved_contexts), + "num_progress_items": len(command.progress), }, "function_call": function_call, "response": parsed_response, @@ -305,6 +306,8 @@ def game_context_prompt_section(command: Command) -> str: return ( "Additional live context from the external application. Use it only when it is relevant " - "to the user's question and the character's role:\n" + "to the user's question, the user's current task, or the character's role. " + "Do not mention task progress unless it helps answer the user or guide their next action. Tasks may be incomplete, do not assume that it is done entirely unless status is finished, as the user may need guidance in finishing the task: :\n" + "\n\n".join(sections) + ) diff --git a/src/routes/chat.py b/src/routes/chat.py index a4b7a35..d005568 100644 --- a/src/routes/chat.py +++ b/src/routes/chat.py @@ -17,6 +17,7 @@ ) from src.models.errors import LLMAPIError, LLMGenerationError from src.pipeline import assemble_prompt_with_agent +from src.routes.progress import get_recent_progress_for_session from src.transcribe import transcribe_audio, transcribe_from_upload @@ -46,6 +47,24 @@ def _get_authorized_agent(command: Command): return agent, None +def _attach_recent_progress(command: Command) -> None: + if not command.include_progress or not command.session_id or command.progress_limit <= 0: + return + + recent_progress = get_recent_progress_for_session( + command.agent_id, command.session_id, command.progress_limit + ) + if not recent_progress: + return + + explicit_task_names = {task.task_name for task in command.progress} + merged_progress = [ + task for task in recent_progress if task.task_name not in explicit_task_names + ] + merged_progress.extend(command.progress) + command.progress = merged_progress[: command.progress_limit] + + @router.post("/ask", response_model=Command) async def ask(command: Command): """Process a user question using the specified agent and roles. @@ -80,6 +99,7 @@ async def ask(command: Command): agent, error_response = _get_authorized_agent(command) if error_response is not None: return error_response + _attach_recent_progress(command) # Generate response using agent configuration and role-based RAG response = assemble_prompt_with_agent(command, agent) @@ -171,6 +191,7 @@ async def ask_transcribe( agent, error_response = _get_authorized_agent(command) if error_response is not None: return error_response + _attach_recent_progress(command) try: response = assemble_prompt_with_agent(command, agent) diff --git a/src/routes/progress.py b/src/routes/progress.py index 6190725..515c8f3 100644 --- a/src/routes/progress.py +++ b/src/routes/progress.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Annotated from fastapi import APIRouter, Header, HTTPException @@ -12,6 +12,32 @@ # Define a router for progress-related endpoints router = APIRouter() +DEFAULT_SESSION_ID = "default" +PROGRESS_TTL = timedelta(hours=24) + + +def _normalize_session_id(session_id: str | None) -> str: + normalized = session_id.strip() if session_id else "" + return normalized or DEFAULT_SESSION_ID + + +def _entry_timestamp(entry: dict) -> datetime: + timestamp = entry.get("updated_at") or entry.get("started_at") or datetime.min + if isinstance(timestamp, str): + try: + timestamp = datetime.fromisoformat(timestamp) + except ValueError: + return datetime.min.replace(tzinfo=UTC) + if timestamp.tzinfo is None: + timestamp = timestamp.replace(tzinfo=UTC) + return timestamp + + +def _prune_expired_progress() -> None: + cutoff = datetime.now(UTC) - PROGRESS_TTL + progress_log[:] = [ + entry for entry in progress_log if _entry_timestamp(entry) >= cutoff + ] def _authorize_agent_access(agent_id: str | None, access_key: str | None) -> str: @@ -30,24 +56,57 @@ def _authorize_agent_access(agent_id: str | None, access_key: str | None) -> str return agent_id -def _progress_dump(progress: ProgressData, agent_id: str) -> dict: +def _progress_dump(progress: ProgressData, agent_id: str, session_id: str) -> dict: entry = progress.model_dump(exclude={"access_key"}) entry["agent_id"] = agent_id + entry["session_id"] = session_id + entry["updated_at"] = datetime.now(UTC) return entry +def get_recent_progress_for_session( + agent_id: str, session_id: str | None, limit: int = 5 +) -> list[ProgressData]: + if limit <= 0: + return [] + + _prune_expired_progress() + normalized_session_id = _normalize_session_id(session_id) + entries = [ + entry + for entry in progress_log + if entry.get("agent_id") == agent_id + and entry.get("session_id", DEFAULT_SESSION_ID) == normalized_session_id + ] + entries.sort(key=_entry_timestamp, reverse=True) + return [ + ProgressData.model_validate(entry) + for entry in entries[:limit] + ] + + @router.post("/api/progress/initializeTasks") def receive_hierarchy(task_hierarchy: ListProgressData): """Initializes a list of tasks with their subtasks and steps.""" try: + _prune_expired_progress() agent_id = _authorize_agent_access( task_hierarchy.agent_id, task_hierarchy.access_key ) + session_id = _normalize_session_id(task_hierarchy.session_id) for task in task_hierarchy.items: - progress_log.append(_progress_dump(task, agent_id)) + progress_log.append( + _progress_dump( + task, + agent_id, + _normalize_session_id(task.session_id or session_id), + ) + ) return { "message": "Tasks initialized", - "data": task_hierarchy.model_dump(exclude={"access_key"}), + "data": task_hierarchy.model_dump( + exclude={"access_key": True, "items": {"__all__": {"access_key"}}} + ), } except Exception as e: print(f"Error processing task hierarchy: {e}") @@ -59,30 +118,36 @@ def receive_hierarchy(task_hierarchy: ListProgressData): @router.post("/api/progress/updateTask") def receive_progress(progress: ProgressData): """Handles task progress updates (started/complete) and stores them.""" + _prune_expired_progress() agent_id = _authorize_agent_access(progress.agent_id, progress.access_key) + session_id = _normalize_session_id(progress.session_id) if progress.status == "started" or progress.status == "pending": # Check for existing incomplete task to update for entry in progress_log: if ( entry.get("agent_id") == agent_id + and entry.get("session_id", DEFAULT_SESSION_ID) == session_id and entry["task_name"] == progress.task_name and entry["completed_at"] is None ): + now = datetime.now(UTC) entry.update( { "subtask_progress": [ subtask.model_dump() for subtask in progress.subtask_progress ], - "started_at": datetime.now(UTC), + "started_at": entry.get("started_at") or now, + "updated_at": now, "status": progress.status, + "description": progress.description, } ) return {"message": "Progress updated", "data": entry} # New task entry - new_entry = _progress_dump(progress, agent_id) + new_entry = _progress_dump(progress, agent_id, session_id) new_entry["started_at"] = datetime.now(UTC) progress_log.append(new_entry) return {"message": "Progress received", "data": new_entry} @@ -92,18 +157,22 @@ def receive_progress(progress: ProgressData): for entry in progress_log: if ( entry.get("agent_id") == agent_id + and entry.get("session_id", DEFAULT_SESSION_ID) == session_id and entry["task_name"] == progress.task_name and entry["completed_at"] is None ): + now = datetime.now(UTC) entry.update( { "subtask_progress": [ subtask.model_dump() for subtask in progress.subtask_progress ], - "completed_at": datetime.now(UTC), - "completet_at": datetime.now(UTC), + "completed_at": now, + "completet_at": now, + "updated_at": now, "status": "complete", + "description": progress.description, } ) return {"message": "Task completed", "data": entry} @@ -116,11 +185,18 @@ def receive_progress(progress: ProgressData): @router.get("/api/progress") def get_progress_log( agent_id: str, + session_id: str | None = None, + limit: int | None = None, access_key: Annotated[str | None, Header()] = None, ): + _prune_expired_progress() authorized_agent_id = _authorize_agent_access(agent_id, access_key) - return [ + normalized_session_id = _normalize_session_id(session_id) + entries = [ entry for entry in progress_log if entry.get("agent_id") == authorized_agent_id + and entry.get("session_id", DEFAULT_SESSION_ID) == normalized_session_id ] + entries.sort(key=_entry_timestamp, reverse=True) + return entries[:limit] if limit is not None and limit >= 0 else entries