Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions backend/app/agents/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
youtube_agent_node,
)
from app.agents.supervisor import HANDOFF_TOOL_NAME, supervisor_node
from app.features.audio.tool import GENERATE_TTS_TOOL_NAME, generate_tts_node
from app.features.youtube.tool import SAVE_TRANSCRIPT_TOOL_NAME, save_transcript_node

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -72,6 +73,8 @@ def _should_continue(state: AxonState) -> str:
return "youtube_agent"
if tc["name"] == SAVE_TRANSCRIPT_TOOL_NAME:
return "save_transcript"
if tc["name"] == GENERATE_TTS_TOOL_NAME:
return "generate_tts"
return END


Expand All @@ -82,6 +85,7 @@ def _build_graph() -> CompiledStateGraph:
builder.add_node("memory_agent", memory_agent_node)
builder.add_node("youtube_agent", youtube_agent_node)
builder.add_node("save_transcript", save_transcript_node)
builder.add_node("generate_tts", generate_tts_node)
builder.add_edge(START, "supervisor")
builder.add_conditional_edges(
"supervisor",
Expand All @@ -90,13 +94,15 @@ def _build_graph() -> CompiledStateGraph:
"memory_agent": "memory_agent",
"youtube_agent": "youtube_agent",
"save_transcript": "save_transcript",
"generate_tts": "generate_tts",
END: END,
},
)
# All sub-agents loop back to supervisor for the final response.
builder.add_edge("memory_agent", "supervisor")
builder.add_edge("youtube_agent", "supervisor")
builder.add_edge("save_transcript", "supervisor")
builder.add_edge("generate_tts", "supervisor")
return builder.compile()


Expand Down
5 changes: 5 additions & 0 deletions backend/app/agents/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ class AxonState(TypedDict):
Holds {video_id, youtube_url, title, channel, duration_s, summary,
key_points} for the current turn. Empty string when no video was
processed. Persists in conversation state for cross-turn save flow.
pending_audio: JSON payload string set by generate_tts_node.
Holds {filename, signed_url, title, text_preview} after TTS is
generated. Empty string when no audio is pending. Persists for
cross-turn save confirmation (Phase 10D).
"""

messages: Annotated[list[BaseMessage], add_messages]
Expand All @@ -32,3 +36,4 @@ class AxonState(TypedDict):
memory_threshold: float
memory_limit: int
youtube_context: str
pending_audio: str
71 changes: 71 additions & 0 deletions backend/app/agents/supervisor.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"""

import logging
import re
import uuid

from fastapi import HTTPException, status
Expand All @@ -37,6 +38,7 @@
from app.agents.state import AxonState
from app.agents.subagents.youtube_fetcher import extract_video_id
from app.core.config import settings
from app.features.audio.tool import GENERATE_TTS_TOOL_NAME
from app.features.youtube.tool import SAVE_TRANSCRIPT_TOOL_NAME

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -135,6 +137,28 @@
- The conversation history contains no prior assistant YouTube summary.
- The user has not explicitly confirmed they want to save.
- The user says "no", "skip", "не", or similar.

## TTS / Audio generation tool

Call `generate_tts(text=<text_to_synthesize>)` when the user asks you to:
- Generate audio, read aloud, or create an audio version of any text.
- Convert the video summary (or any other text) to speech.
- Any phrasing such as "generate audio", "read this", "make an mp3",
"прочети", "генерирай аудио".

Pass the text to synthesize as the `text` argument. If the user says
"generate audio of the summary", use the summary text from the most recent
YouTube summary in the conversation. If no specific text is mentioned, ask
the user what text they want converted.

CRITICAL output rule: When calling `generate_tts`, your response MUST consist
ONLY of the tool call — zero text before or after it.

After `generate_tts` returns, present the markdown link from the ToolMessage
content verbatim and ask: "Would you like me to save this to your audio library?"

Do NOT call `generate_tts` for general questions about audio or TTS — only
when the user explicitly requests audio generation.
"""


Expand Down Expand Up @@ -174,6 +198,19 @@ def save_video_transcript() -> str: # noqa: D401
return "Save transcript activated." # pragma: no cover


@tool(GENERATE_TTS_TOOL_NAME)
def generate_tts(text: str) -> str: # noqa: D401
"""Convert the given text to speech and upload the result as an mp3.

Args:
text: The text to synthesize. Will be truncated to 4096 chars if longer.

Call this when the user asks to generate audio / read aloud any text.
"""
# The tool body is never executed — orchestrator routes to generate_tts_node.
return "TTS generation activated." # pragma: no cover


# ---------------------------------------------------------------------------
# Model cache (keyed by model_id x has_tools to avoid rebuilding per request)
# ---------------------------------------------------------------------------
Expand All @@ -197,6 +234,7 @@ def _get_model(model_id: str, with_tools: bool) -> Runnable:
transfer_to_memory_agent,
transfer_to_youtube_agent,
save_video_transcript,
generate_tts,
]
)
if with_tools
Expand Down Expand Up @@ -323,4 +361,37 @@ async def supervisor_node(state: AxonState, config: RunnableConfig) -> dict:
len(str(response.content)),
bool(getattr(response, "tool_calls", None)),
)

# Post-process: some models (e.g. DeepSeek) emit tool calls as text content
# instead of structured tool_calls. Detect these and convert to proper
# tool_call AIMessages so LangGraph routes correctly and no text leaks to
# the frontend.
content_str = response.content if isinstance(response.content, str) else ""
if not getattr(response, "tool_calls", None) and content_str:
match = re.search(
r'generate_tts\(text=["\'](.+?)["\'](\s*,.*?)?\)',
content_str,
re.DOTALL,
)
if match:
extracted_text = match.group(1)
logger.info(
"[supervisor] text-format generate_tts detected — converting to structured tool_call"
)
return {
"messages": [
AIMessage(
content="",
tool_calls=[
{
"name": GENERATE_TTS_TOOL_NAME,
"args": {"text": extracted_text},
"id": str(uuid.uuid4()),
"type": "tool_call",
}
],
)
]
}

return {"messages": [response]}
6 changes: 6 additions & 0 deletions backend/app/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ class Settings(BaseSettings):
# Uses a fast, large-context model via OpenRouter; not the user's selected chat model.
youtube_summary_model: str = "google/gemini-2.5-flash"

# TTS (Phase 10C) — text-to-speech via OpenRouter's /audio/speech endpoint.
# Uses openrouter_api_key — no separate OpenAI key needed.
tts_model: str = "openai/gpt-4o-mini-tts-2025-12-15"
tts_voice: str = "alloy" # alloy / echo / fable / onyx / nova / shimmer
audio_bucket: str = "audio" # Supabase Storage bucket name

# Rate limiting — optional Redis backend for multi-worker deployments.
# Leave unset (default None) to use in-memory storage (single process).
# Example: redis://localhost:6379/0
Expand Down
Empty file.
11 changes: 11 additions & 0 deletions backend/app/features/audio/schemas.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
"""Pydantic schemas for the audio feature (Phase 10C)."""

from pydantic import BaseModel, Field


class AudioGenerateResult(BaseModel):
"""Result returned after generating and uploading a TTS audio file."""

signed_url: str
filename: str
text_preview: str = Field(..., min_length=1, max_length=1024)
154 changes: 154 additions & 0 deletions backend/app/features/audio/service.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
"""TTS generation + Supabase Storage upload service (Phase 10C).

Uses OpenRouter's /audio/speech endpoint (openai/gpt-4o-mini-tts-2025-12-15).
The existing openrouter_api_key is reused — no separate OpenAI key needed.

Public API
----------
generate_tts(text, user_id) -> tuple[bytes, str]
Call OpenRouter TTS endpoint, return (mp3_bytes, storage_filename).

upload_audio(audio_bytes, filename) -> str
Upload mp3 bytes to Supabase Storage and return a signed URL (1 week).
"""

import logging
from uuid import uuid4

import httpx

from app.core.config import settings
from app.core.privacy import mask_pii
from app.db.supabase import get_supabase_client

logger = logging.getLogger(__name__)

# OpenAI TTS hard limit — truncate with a warning rather than failing.
_MAX_TTS_CHARS = 4096

# Signed URL expiry: 7 days in seconds.
_SIGNED_URL_TTL = 7 * 24 * 60 * 60 # 604 800 s


async def generate_tts(text: str, user_id: str) -> tuple[bytes, str]:
"""Call OpenRouter TTS endpoint and return (mp3_bytes, storage_filename).

Args:
text: The text to synthesize. Truncated to 4 096 chars if longer.
user_id: Authenticated user UUID — used to scope the storage path.

Returns:
A tuple of (raw mp3 bytes, storage filename scoped to user_id).

Raises:
RuntimeError: If the OpenRouter API call fails or the response is malformed.
"""
if len(text) > _MAX_TTS_CHARS:
logger.warning(
"[generate_tts] text length %d exceeds limit %d — truncating",
len(text),
_MAX_TTS_CHARS,
)
text = text[:_MAX_TTS_CHARS]

if not text.strip():
raise RuntimeError("Cannot generate TTS for empty text.")

# Mask PII before forwarding text to the external TTS provider.
masked_text = await mask_pii(text)

# OpenRouter /audio/speech endpoint — returns raw MP3 bytes (not JSON).
payload = {
"model": settings.tts_model,
"input": masked_text,
"voice": settings.tts_voice,
"response_format": "mp3",
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
headers = {
"Authorization": f"Bearer {settings.openrouter_api_key}",
"Content-Type": "application/json",
}

tts_url = f"{settings.openrouter_base_url}/audio/speech"
try:
async with httpx.AsyncClient(timeout=120.0) as client:
response = await client.post(tts_url, json=payload, headers=headers)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
logger.error(
"[generate_tts] OpenRouter returned %d url=%s",
exc.response.status_code,
tts_url,
)
raise RuntimeError(
f"TTS API request failed with status {exc.response.status_code}."
) from exc
except httpx.RequestError as exc:
logger.error("[generate_tts] request error url=%s: %s", tts_url, exc)
raise RuntimeError("TTS API request failed due to a network error.") from exc

audio_bytes = response.content

# Validate the response is non-empty and looks like an MP3 before uploading.
content_type = response.headers.get("content-type", "")
mp3_magic = audio_bytes[:3] if len(audio_bytes) >= 3 else b""
is_mp3_magic = mp3_magic == b"ID3" or (len(audio_bytes) >= 2 and audio_bytes[0] == 0xFF and (audio_bytes[1] & 0xE0) == 0xE0)
if not audio_bytes or (not content_type.startswith("audio/") and not is_mp3_magic):
logger.error(
"[generate_tts] unexpected TTS response status=%d content-type=%s bytes=%d",
response.status_code,
content_type,
len(audio_bytes),
)
raise RuntimeError("TTS API returned an invalid or empty audio response.")

file_uuid = uuid4()
filename = f"{user_id}/{file_uuid}.mp3"
logger.info(
"[generate_tts] synthesized %d bytes uuid=%s", len(audio_bytes), file_uuid
)
return audio_bytes, filename
Comment thread
coderabbitai[bot] marked this conversation as resolved.


async def upload_audio(audio_bytes: bytes, filename: str) -> str:
"""Upload mp3 bytes to Supabase Storage and return a signed URL (1 week).

Args:
audio_bytes: Raw MP3 bytes to upload.
filename: Storage path in the form ``{user_id}/{uuid}.mp3``.

Returns:
A signed URL valid for 7 days.

Raises:
RuntimeError: If the Supabase Storage upload or URL generation fails.
"""
client = await get_supabase_client()
bucket = settings.audio_bucket

# Use only the basename (UUID part) in logs — never the full path (contains user_id).
safe_log_name = filename.split("/")[-1] if "/" in filename else filename

try:
await client.storage.from_(bucket).upload(
path=filename,
file=audio_bytes,
file_options={"content-type": "audio/mpeg"},
)
except Exception as exc:
logger.exception("[upload_audio] Storage upload failed uuid=%s", safe_log_name)
raise RuntimeError("Failed to upload audio to storage.") from exc

try:
result = await client.storage.from_(bucket).create_signed_url(
filename, _SIGNED_URL_TTL
)
signed_url: str = result["signedURL"]
except Exception as exc:
logger.exception(
"[upload_audio] Failed to create signed URL uuid=%s", safe_log_name
)
raise RuntimeError("Audio uploaded but could not generate a signed URL.") from exc

logger.info("[upload_audio] signed URL created uuid=%s", safe_log_name)
return signed_url
Loading