diff --git a/backend/app/agents/orchestrator.py b/backend/app/agents/orchestrator.py index 2e3f6df..4c173fb 100644 --- a/backend/app/agents/orchestrator.py +++ b/backend/app/agents/orchestrator.py @@ -35,6 +35,7 @@ youtube_agent_node, ) from app.agents.supervisor import HANDOFF_TOOL_NAME, supervisor_node +from app.features.audio.tool import GENERATE_TTS_TOOL_NAME, generate_tts_node from app.features.youtube.tool import SAVE_TRANSCRIPT_TOOL_NAME, save_transcript_node logger = logging.getLogger(__name__) @@ -72,6 +73,8 @@ def _should_continue(state: AxonState) -> str: return "youtube_agent" if tc["name"] == SAVE_TRANSCRIPT_TOOL_NAME: return "save_transcript" + if tc["name"] == GENERATE_TTS_TOOL_NAME: + return "generate_tts" return END @@ -82,6 +85,7 @@ def _build_graph() -> CompiledStateGraph: builder.add_node("memory_agent", memory_agent_node) builder.add_node("youtube_agent", youtube_agent_node) builder.add_node("save_transcript", save_transcript_node) + builder.add_node("generate_tts", generate_tts_node) builder.add_edge(START, "supervisor") builder.add_conditional_edges( "supervisor", @@ -90,6 +94,7 @@ def _build_graph() -> CompiledStateGraph: "memory_agent": "memory_agent", "youtube_agent": "youtube_agent", "save_transcript": "save_transcript", + "generate_tts": "generate_tts", END: END, }, ) @@ -97,6 +102,7 @@ def _build_graph() -> CompiledStateGraph: builder.add_edge("memory_agent", "supervisor") builder.add_edge("youtube_agent", "supervisor") builder.add_edge("save_transcript", "supervisor") + builder.add_edge("generate_tts", "supervisor") return builder.compile() diff --git a/backend/app/agents/state.py b/backend/app/agents/state.py index 4586892..168d5a1 100644 --- a/backend/app/agents/state.py +++ b/backend/app/agents/state.py @@ -23,6 +23,10 @@ class AxonState(TypedDict): Holds {video_id, youtube_url, title, channel, duration_s, summary, key_points} for the current turn. Empty string when no video was processed. Persists in conversation state for cross-turn save flow. + pending_audio: JSON payload string set by generate_tts_node. + Holds {filename, signed_url, title, text_preview} after TTS is + generated. Empty string when no audio is pending. Persists for + cross-turn save confirmation (Phase 10D). """ messages: Annotated[list[BaseMessage], add_messages] @@ -32,3 +36,4 @@ class AxonState(TypedDict): memory_threshold: float memory_limit: int youtube_context: str + pending_audio: str diff --git a/backend/app/agents/supervisor.py b/backend/app/agents/supervisor.py index 8ffbb8c..df8dd8d 100644 --- a/backend/app/agents/supervisor.py +++ b/backend/app/agents/supervisor.py @@ -20,6 +20,7 @@ """ import logging +import re import uuid from fastapi import HTTPException, status @@ -37,6 +38,7 @@ from app.agents.state import AxonState from app.agents.subagents.youtube_fetcher import extract_video_id from app.core.config import settings +from app.features.audio.tool import GENERATE_TTS_TOOL_NAME from app.features.youtube.tool import SAVE_TRANSCRIPT_TOOL_NAME logger = logging.getLogger(__name__) @@ -135,6 +137,28 @@ - The conversation history contains no prior assistant YouTube summary. - The user has not explicitly confirmed they want to save. - The user says "no", "skip", "не", or similar. + +## TTS / Audio generation tool + +Call `generate_tts(text=)` when the user asks you to: +- Generate audio, read aloud, or create an audio version of any text. +- Convert the video summary (or any other text) to speech. +- Any phrasing such as "generate audio", "read this", "make an mp3", + "прочети", "генерирай аудио". + +Pass the text to synthesize as the `text` argument. If the user says +"generate audio of the summary", use the summary text from the most recent +YouTube summary in the conversation. If no specific text is mentioned, ask +the user what text they want converted. + +CRITICAL output rule: When calling `generate_tts`, your response MUST consist +ONLY of the tool call — zero text before or after it. + +After `generate_tts` returns, present the markdown link from the ToolMessage +content verbatim and ask: "Would you like me to save this to your audio library?" + +Do NOT call `generate_tts` for general questions about audio or TTS — only +when the user explicitly requests audio generation. """ @@ -174,6 +198,19 @@ def save_video_transcript() -> str: # noqa: D401 return "Save transcript activated." # pragma: no cover +@tool(GENERATE_TTS_TOOL_NAME) +def generate_tts(text: str) -> str: # noqa: D401 + """Convert the given text to speech and upload the result as an mp3. + + Args: + text: The text to synthesize. Will be truncated to 4096 chars if longer. + + Call this when the user asks to generate audio / read aloud any text. + """ + # The tool body is never executed — orchestrator routes to generate_tts_node. + return "TTS generation activated." # pragma: no cover + + # --------------------------------------------------------------------------- # Model cache (keyed by model_id x has_tools to avoid rebuilding per request) # --------------------------------------------------------------------------- @@ -197,6 +234,7 @@ def _get_model(model_id: str, with_tools: bool) -> Runnable: transfer_to_memory_agent, transfer_to_youtube_agent, save_video_transcript, + generate_tts, ] ) if with_tools @@ -323,4 +361,37 @@ async def supervisor_node(state: AxonState, config: RunnableConfig) -> dict: len(str(response.content)), bool(getattr(response, "tool_calls", None)), ) + + # Post-process: some models (e.g. DeepSeek) emit tool calls as text content + # instead of structured tool_calls. Detect these and convert to proper + # tool_call AIMessages so LangGraph routes correctly and no text leaks to + # the frontend. + content_str = response.content if isinstance(response.content, str) else "" + if not getattr(response, "tool_calls", None) and content_str: + match = re.search( + r'generate_tts\(text=["\'](.+?)["\'](\s*,.*?)?\)', + content_str, + re.DOTALL, + ) + if match: + extracted_text = match.group(1) + logger.info( + "[supervisor] text-format generate_tts detected — converting to structured tool_call" + ) + return { + "messages": [ + AIMessage( + content="", + tool_calls=[ + { + "name": GENERATE_TTS_TOOL_NAME, + "args": {"text": extracted_text}, + "id": str(uuid.uuid4()), + "type": "tool_call", + } + ], + ) + ] + } + return {"messages": [response]} diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 1e4f80a..ec1a5c0 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -37,6 +37,12 @@ class Settings(BaseSettings): # Uses a fast, large-context model via OpenRouter; not the user's selected chat model. youtube_summary_model: str = "google/gemini-2.5-flash" + # TTS (Phase 10C) — text-to-speech via OpenRouter's /audio/speech endpoint. + # Uses openrouter_api_key — no separate OpenAI key needed. + tts_model: str = "openai/gpt-4o-mini-tts-2025-12-15" + tts_voice: str = "alloy" # alloy / echo / fable / onyx / nova / shimmer + audio_bucket: str = "audio" # Supabase Storage bucket name + # Rate limiting — optional Redis backend for multi-worker deployments. # Leave unset (default None) to use in-memory storage (single process). # Example: redis://localhost:6379/0 diff --git a/backend/app/features/audio/__init__.py b/backend/app/features/audio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/features/audio/schemas.py b/backend/app/features/audio/schemas.py new file mode 100644 index 0000000..7fa88c6 --- /dev/null +++ b/backend/app/features/audio/schemas.py @@ -0,0 +1,11 @@ +"""Pydantic schemas for the audio feature (Phase 10C).""" + +from pydantic import BaseModel, Field + + +class AudioGenerateResult(BaseModel): + """Result returned after generating and uploading a TTS audio file.""" + + signed_url: str + filename: str + text_preview: str = Field(..., min_length=1, max_length=1024) diff --git a/backend/app/features/audio/service.py b/backend/app/features/audio/service.py new file mode 100644 index 0000000..f59bf2a --- /dev/null +++ b/backend/app/features/audio/service.py @@ -0,0 +1,154 @@ +"""TTS generation + Supabase Storage upload service (Phase 10C). + +Uses OpenRouter's /audio/speech endpoint (openai/gpt-4o-mini-tts-2025-12-15). +The existing openrouter_api_key is reused — no separate OpenAI key needed. + +Public API +---------- +generate_tts(text, user_id) -> tuple[bytes, str] + Call OpenRouter TTS endpoint, return (mp3_bytes, storage_filename). + +upload_audio(audio_bytes, filename) -> str + Upload mp3 bytes to Supabase Storage and return a signed URL (1 week). +""" + +import logging +from uuid import uuid4 + +import httpx + +from app.core.config import settings +from app.core.privacy import mask_pii +from app.db.supabase import get_supabase_client + +logger = logging.getLogger(__name__) + +# OpenAI TTS hard limit — truncate with a warning rather than failing. +_MAX_TTS_CHARS = 4096 + +# Signed URL expiry: 7 days in seconds. +_SIGNED_URL_TTL = 7 * 24 * 60 * 60 # 604 800 s + + +async def generate_tts(text: str, user_id: str) -> tuple[bytes, str]: + """Call OpenRouter TTS endpoint and return (mp3_bytes, storage_filename). + + Args: + text: The text to synthesize. Truncated to 4 096 chars if longer. + user_id: Authenticated user UUID — used to scope the storage path. + + Returns: + A tuple of (raw mp3 bytes, storage filename scoped to user_id). + + Raises: + RuntimeError: If the OpenRouter API call fails or the response is malformed. + """ + if len(text) > _MAX_TTS_CHARS: + logger.warning( + "[generate_tts] text length %d exceeds limit %d — truncating", + len(text), + _MAX_TTS_CHARS, + ) + text = text[:_MAX_TTS_CHARS] + + if not text.strip(): + raise RuntimeError("Cannot generate TTS for empty text.") + + # Mask PII before forwarding text to the external TTS provider. + masked_text = await mask_pii(text) + + # OpenRouter /audio/speech endpoint — returns raw MP3 bytes (not JSON). + payload = { + "model": settings.tts_model, + "input": masked_text, + "voice": settings.tts_voice, + "response_format": "mp3", + } + headers = { + "Authorization": f"Bearer {settings.openrouter_api_key}", + "Content-Type": "application/json", + } + + tts_url = f"{settings.openrouter_base_url}/audio/speech" + try: + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post(tts_url, json=payload, headers=headers) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + logger.error( + "[generate_tts] OpenRouter returned %d url=%s", + exc.response.status_code, + tts_url, + ) + raise RuntimeError( + f"TTS API request failed with status {exc.response.status_code}." + ) from exc + except httpx.RequestError as exc: + logger.error("[generate_tts] request error url=%s: %s", tts_url, exc) + raise RuntimeError("TTS API request failed due to a network error.") from exc + + audio_bytes = response.content + + # Validate the response is non-empty and looks like an MP3 before uploading. + content_type = response.headers.get("content-type", "") + mp3_magic = audio_bytes[:3] if len(audio_bytes) >= 3 else b"" + is_mp3_magic = mp3_magic == b"ID3" or (len(audio_bytes) >= 2 and audio_bytes[0] == 0xFF and (audio_bytes[1] & 0xE0) == 0xE0) + if not audio_bytes or (not content_type.startswith("audio/") and not is_mp3_magic): + logger.error( + "[generate_tts] unexpected TTS response status=%d content-type=%s bytes=%d", + response.status_code, + content_type, + len(audio_bytes), + ) + raise RuntimeError("TTS API returned an invalid or empty audio response.") + + file_uuid = uuid4() + filename = f"{user_id}/{file_uuid}.mp3" + logger.info( + "[generate_tts] synthesized %d bytes uuid=%s", len(audio_bytes), file_uuid + ) + return audio_bytes, filename + + +async def upload_audio(audio_bytes: bytes, filename: str) -> str: + """Upload mp3 bytes to Supabase Storage and return a signed URL (1 week). + + Args: + audio_bytes: Raw MP3 bytes to upload. + filename: Storage path in the form ``{user_id}/{uuid}.mp3``. + + Returns: + A signed URL valid for 7 days. + + Raises: + RuntimeError: If the Supabase Storage upload or URL generation fails. + """ + client = await get_supabase_client() + bucket = settings.audio_bucket + + # Use only the basename (UUID part) in logs — never the full path (contains user_id). + safe_log_name = filename.split("/")[-1] if "/" in filename else filename + + try: + await client.storage.from_(bucket).upload( + path=filename, + file=audio_bytes, + file_options={"content-type": "audio/mpeg"}, + ) + except Exception as exc: + logger.exception("[upload_audio] Storage upload failed uuid=%s", safe_log_name) + raise RuntimeError("Failed to upload audio to storage.") from exc + + try: + result = await client.storage.from_(bucket).create_signed_url( + filename, _SIGNED_URL_TTL + ) + signed_url: str = result["signedURL"] + except Exception as exc: + logger.exception( + "[upload_audio] Failed to create signed URL uuid=%s", safe_log_name + ) + raise RuntimeError("Audio uploaded but could not generate a signed URL.") from exc + + logger.info("[upload_audio] signed URL created uuid=%s", safe_log_name) + return signed_url diff --git a/backend/app/features/audio/tool.py b/backend/app/features/audio/tool.py new file mode 100644 index 0000000..e037894 --- /dev/null +++ b/backend/app/features/audio/tool.py @@ -0,0 +1,97 @@ +"""Generate-TTS LangGraph node (Phase 10C). + +Called by the orchestrator when the supervisor emits a ``generate_tts`` +tool_call. Synthesizes the requested text via OpenRouter TTS, uploads the +resulting mp3 to Supabase Storage, stores the result in ``state["pending_audio"]`` +for the cross-turn save confirmation (Phase 10D), and returns a ToolMessage with +a markdown link the supervisor can present to the user. + +The node body is never called via LangChain tool execution — it is invoked +directly by the orchestrator graph (same pattern as save_transcript_node). +""" + +import json +import logging + +from langchain_core.messages import AIMessage, ToolMessage + +from app.agents.state import AxonState +from app.features.audio import service as audio_service + +logger = logging.getLogger(__name__) + +# Imported by supervisor.py (tool name) and orchestrator.py (routing key). +GENERATE_TTS_TOOL_NAME = "generate_tts" + + +async def generate_tts_node(state: AxonState) -> dict: + """LangGraph node: synthesize text to speech and upload to Storage. + + Reads the ``text`` argument from the supervisor's ``generate_tts`` tool_call, + calls the audio service, and returns: + - A ToolMessage closing the open tool_call (required for valid message history). + - ``pending_audio`` state update with JSON ``{filename, signed_url, text_preview}`` + for the Phase 10D save confirmation flow. + """ + # -- Resolve tool_call_id and text arg ----------------------------------- + tool_call_id: str | None = None + text: str = "" + + last_msg = state["messages"][-1] if state["messages"] else None + if isinstance(last_msg, AIMessage) and last_msg.tool_calls: + for tc in last_msg.tool_calls: + if tc["name"] == GENERATE_TTS_TOOL_NAME: + tool_call_id = tc["id"] + text = tc.get("args", {}).get("text", "") + break + + def _error(msg: str) -> dict: + messages = [] + if tool_call_id: + messages.append(ToolMessage(content=msg, tool_call_id=tool_call_id)) + return {"messages": messages} + + # -- Validate input ------------------------------------------------------- + if not text or not text.strip(): + logger.warning("[generate_tts_node] called with empty text") + return _error( + "No text provided for audio generation. " + "Please specify what text you would like me to convert to speech." + ) + + user_id: str = state["user_id"] + + # -- Generate TTS + upload ------------------------------------------------ + try: + audio_bytes, filename = await audio_service.generate_tts(text, user_id) + signed_url = await audio_service.upload_audio(audio_bytes, filename) + except RuntimeError as exc: + logger.error("[generate_tts_node] service error: %s", exc) + return _error(f"Audio generation failed: {exc}") + except Exception: + logger.exception("[generate_tts_node] unexpected error") + return _error( + "Audio generation failed due to an unexpected error. Please try again." + ) + + # -- Build pending_audio payload for Phase 10D save ---------------------- + pending = { + "filename": filename, + "signed_url": signed_url, + "text_preview": text[:100], + } + pending_audio_json = json.dumps(pending) + + result_msg = ( + f"Audio generated successfully.\n\n" + f"[Play audio]({signed_url})\n\n" + f"Would you like me to save this to your audio library?" + ) + + logger.info("[generate_tts_node] audio ready filename=%s", filename) + + messages = [] + if tool_call_id: + messages.append(ToolMessage(content=result_msg, tool_call_id=tool_call_id)) + + return {"messages": messages, "pending_audio": pending_audio_json} diff --git a/backend/app/features/chat/service.py b/backend/app/features/chat/service.py index d0d9058..2fbc666 100644 --- a/backend/app/features/chat/service.py +++ b/backend/app/features/chat/service.py @@ -149,6 +149,13 @@ async def stream_chat( yield _format(SSEEvent(type="tool_use", content="Saving to library…")) continue + # TTS node starting — emit tool_use status. + if kind == "on_chain_start" and node == "generate_tts": + memory_agent_invoked = True # blocks pass-1 flush + pass1_buffer.clear() + yield _format(SSEEvent(type="tool_use", content="Generating audio…")) + continue + if kind != "on_chat_model_stream": continue if node != "supervisor": diff --git a/frontend/features/chat/components/ChatWindow.tsx b/frontend/features/chat/components/ChatWindow.tsx index 3c08a31..070bb5c 100644 --- a/frontend/features/chat/components/ChatWindow.tsx +++ b/frontend/features/chat/components/ChatWindow.tsx @@ -10,6 +10,7 @@ import { Database, Play, Bookmark, + Volume2, Menu, X, } from 'lucide-react'; @@ -235,6 +236,8 @@ const ChatWindow = () => { ) : toolStatus.startsWith('Saving') ? ( + ) : toolStatus.startsWith('Generating audio') ? ( + ) : ( )} diff --git a/frontend/features/chat/components/MessageBubble.tsx b/frontend/features/chat/components/MessageBubble.tsx index b2163c9..79da945 100644 --- a/frontend/features/chat/components/MessageBubble.tsx +++ b/frontend/features/chat/components/MessageBubble.tsx @@ -60,16 +60,28 @@ const MessageBubble = ({ message }: MessageBubbleProps) => { {children} ), - a: ({ href, children }) => ( - - {children} - - ), + a: ({ href, children }) => + href?.includes('.mp3') ? ( + + + ) : ( + + {children} + + ), p: ({ children }) => (

{children}

), diff --git a/supabase/migrations/20260512100000_create_audio_bucket_and_policies.sql b/supabase/migrations/20260512100000_create_audio_bucket_and_policies.sql new file mode 100644 index 0000000..ea9de17 --- /dev/null +++ b/supabase/migrations/20260512100000_create_audio_bucket_and_policies.sql @@ -0,0 +1,39 @@ +-- Phase 10C: Create private 'audio' bucket for TTS mp3 files +-- and RLS policies scoped to {user_id}/ path prefix. + +INSERT INTO storage.buckets (id, name, public, file_size_limit, allowed_mime_types) +VALUES ( + 'audio', + 'audio', + false, -- private bucket + 10485760, -- 10 MB per file limit + ARRAY['audio/mpeg', 'audio/mp3'] +) +ON CONFLICT (id) DO NOTHING; + +-- Allow authenticated users to upload their own files ({user_id}/*) +CREATE POLICY "audio_insert_own" + ON storage.objects FOR INSERT + TO authenticated + WITH CHECK ( + bucket_id = 'audio' + AND (storage.foldername(name))[1] = (select auth.uid())::text + ); + +-- Allow authenticated users to read their own files +CREATE POLICY "audio_select_own" + ON storage.objects FOR SELECT + TO authenticated + USING ( + bucket_id = 'audio' + AND (storage.foldername(name))[1] = (select auth.uid())::text + ); + +-- Allow authenticated users to delete their own files (Phase 10D) +CREATE POLICY "audio_delete_own" + ON storage.objects FOR DELETE + TO authenticated + USING ( + bucket_id = 'audio' + AND (storage.foldername(name))[1] = (select auth.uid())::text + );