diff --git a/backend/.env.example b/backend/.env.example index 31dfe8f..8f3be9e 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -6,12 +6,23 @@ OLLAMA_SERVER_PORT=YOUR_PORT LM_STUDIO_SERVER_URL=YOUR_LM_STUDIO_SERVER_URL_HERE LM_STUDIO_SERVER_PORT=YOUR_PORT -# Change to either 'ollama' or 'lm_studio' depending on which LLM server you want to use +# Change to either 'ollama', 'lm_studio', or 'cloud' depending on which LLM server you want to use LLM_PROVIDER=YOUR_LLM_PROVIDER_HERE # Change to the model name you want to use from your LLM server (ensure the model is already installed on the server) LLM_MODEL_NAME=YOUR_LLM_MODEL_NAME_HERE +# Cloud API settings (only needed if using 'cloud' provider for LLM or transcription) +# Use OpenAI API or any OpenAI-compatible API (e.g., Together AI, OpenRouter, Azure OpenAI, etc.) +CLOUD_API_BASE_URL=https://api.openai.com/v1 +CLOUD_API_KEY=YOUR_API_KEY_HERE + +# Transcription settings +# Change to either 'local' or 'cloud' - local uses Whisper models, cloud uses OpenAI-compatible API +TRANSCRIPTION_PROVIDER=local +# Model name to use for cloud transcription (whisper-large-v3 is default, OpenAI uses whisper-1) +TRANSCRIPTION_MODEL_NAME=whisper-large-v3 + # Authorization header for API access # To generate a new secure token, run: openssl rand -hex 32 API_AUTH_TOKEN=YOUR_AUTH_TOKEN_HERE diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2858ba4..0abbc48 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "faster-whisper>=1.2.1", "mlx-whisper>=0.4.3", "python-dotenv>=1.2.1", + "requests>=2.32.0", "torch>=2.8.0", "uvicorn>=0.39.0", ] diff --git a/backend/services/text_cleaner.py b/backend/services/text_cleaner.py index ae7ed85..3161080 100644 --- a/backend/services/text_cleaner.py +++ b/backend/services/text_cleaner.py @@ -7,6 +7,8 @@ llm_provider = os.getenv("LLM_PROVIDER") llm_model_name = os.getenv("LLM_MODEL_NAME") +cloud_api_base_url = os.getenv("CLOUD_API_BASE_URL") +cloud_api_key = os.getenv("CLOUD_API_KEY") # Define the system prompt for disfluency removal, not recommended to modify system_prompt = ( @@ -32,6 +34,13 @@ def remove_disfluencies(text): api_url = os.getenv("OLLAMA_SERVER_URL") api_port = os.getenv("OLLAMA_SERVER_PORT") endpoint = f"{api_url}:{api_port}/api/chat" + elif llm_provider == "cloud": + # Ensure base URL ends with /chat/completions for OpenAI-compatible APIs + base_url = cloud_api_base_url.rstrip("/") + if not base_url.endswith("/chat/completions"): + endpoint = f"{base_url}/chat/completions" + else: + endpoint = base_url else: raise ValueError(f"Unsupported LLM provider: {llm_provider}") @@ -46,12 +55,19 @@ def remove_disfluencies(text): "max_tokens": 2000, } - response = requests.post(endpoint, json=payload, timeout=180) + headers = {} + if llm_provider == "cloud": + if not cloud_api_key: + raise ValueError("CLOUD_API_KEY must be set when using cloud provider") + headers["Authorization"] = f"Bearer {cloud_api_key}" + headers["Content-Type"] = "application/json" + + response = requests.post(endpoint, json=payload, headers=headers, timeout=180) response.raise_for_status() result = response.json() - if llm_provider == "lm_studio": + if llm_provider == "lm_studio" or llm_provider == "cloud": cleaned_text = result["choices"][0]["message"]["content"] elif llm_provider == "ollama": cleaned_text = result["message"]["content"] @@ -68,3 +84,5 @@ def remove_disfluencies(text): raise Exception(f"Error calling LLM API: {str(e)}") except (KeyError, IndexError) as e: raise Exception(f"Error parsing LLM response: {str(e)}") + + diff --git a/backend/services/transcription.py b/backend/services/transcription.py index 8f634db..27d8754 100644 --- a/backend/services/transcription.py +++ b/backend/services/transcription.py @@ -3,9 +3,22 @@ import platform import tempfile +from dotenv import load_dotenv + +load_dotenv() + +transcription_provider = os.getenv("TRANSCRIPTION_PROVIDER", "local") +cloud_api_base_url = os.getenv("CLOUD_API_BASE_URL") +cloud_api_key = os.getenv("CLOUD_API_KEY") +transcription_model_name = os.getenv("TRANSCRIPTION_MODEL_NAME", "whisper-large-v3") + def _init_whisper(): """Initialize the best Whisper engine based on the available hardware.""" + # Skip initialization if using cloud provider + if transcription_provider == "cloud": + return "cloud", None + system = platform.system() machine = platform.machine() @@ -37,9 +50,45 @@ def _init_whisper(): ENGINE, MODEL = _init_whisper() +def _transcribe_cloud(audio_path: str) -> str: + """Transcribe audio using OpenAI-compatible API""" + import requests + + if not cloud_api_key: + raise ValueError("CLOUD_API_KEY must be set when using cloud transcription provider") + + # Prepare the endpoint URL + base_url = cloud_api_base_url.rstrip("/") + if not base_url.endswith("/audio/transcriptions"): + endpoint = f"{base_url}/audio/transcriptions" + else: + endpoint = base_url + + # Prepare the request + headers = { + "Authorization": f"Bearer {cloud_api_key}" + } + + with open(audio_path, "rb") as audio_file: + files = { + "file": audio_file + } + data = { + "model": transcription_model_name + } + + response = requests.post(endpoint, headers=headers, files=files, data=data, timeout=180) + response.raise_for_status() + + result = response.json() + return result["text"] + + def transcribe_audio(audio_path: str) -> str: """Transcribe audio file to text""" - if ENGINE == "mlx": + if ENGINE == "cloud": + return _transcribe_cloud(audio_path) + elif ENGINE == "mlx": import mlx_whisper result = mlx_whisper.transcribe( @@ -65,3 +114,5 @@ def transcribe_base64(audio_base64: str) -> str: return transcribe_audio(temp_path) finally: os.unlink(temp_path) + + diff --git a/docs/API.md b/docs/API.md index bc31aa4..92959fc 100644 --- a/docs/API.md +++ b/docs/API.md @@ -813,16 +813,50 @@ OLLAMA_SERVER_PORT=11434 LM_STUDIO_SERVER_URL= LM_STUDIO_SERVER_PORT= -# LLM Provider selection +# LLM Provider selection (ollama, lm_studio, or cloud) LLM_PROVIDER=ollama # Model name LLM_MODEL_NAME=llama2 +# Cloud API Configuration (for OpenAI-compatible APIs) +# Only needed if using 'cloud' provider for LLM or transcription +CLOUD_API_BASE_URL=https://api.openai.com/v1 +CLOUD_API_KEY=your_api_key_here + +# Transcription Provider (local or cloud) +TRANSCRIPTION_PROVIDER=local +# Model name for cloud transcription (whisper-large-v3 is default, OpenAI uses whisper-1) +TRANSCRIPTION_MODEL_NAME=whisper-large-v3 + # Authorization token (generate with: openssl rand -hex 32) API_AUTH_TOKEN=your_secure_token_here ``` +### Configuration Options + +#### LLM Provider Options +- **`ollama`**: Use local Ollama server for text cleaning +- **`lm_studio`**: Use LM Studio server for text cleaning +- **`cloud`**: Use any OpenAI-compatible API for text cleaning (OpenAI, Together AI, OpenRouter, Azure OpenAI, etc.) + +#### Transcription Provider Options +- **`local`**: Use local Whisper models (mlx_whisper for Apple Silicon, faster-whisper for other systems) +- **`cloud`**: Use OpenAI-compatible transcription API (OpenAI Whisper API, Groq, etc.) + +#### Cloud API Configuration +When using `cloud` for either LLM_PROVIDER or TRANSCRIPTION_PROVIDER, you need to configure: +- **`CLOUD_API_BASE_URL`**: The base URL of your OpenAI-compatible API (e.g., `https://api.openai.com/v1`) +- **`CLOUD_API_KEY`**: Your API key for authentication + +**Supported Cloud Providers:** +- OpenAI (https://api.openai.com/v1) +- Together AI (https://api.together.xyz/v1) +- OpenRouter (https://openrouter.ai/api/v1) +- Azure OpenAI (https://YOUR_RESOURCE.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT) +- Groq (https://api.groq.com/openai/v1) +- Any other OpenAI-compatible API + ## Using Different LLM Providers ### Using External Ollama Service @@ -915,6 +949,133 @@ If you have Ollama or LM Studio running on a remote machine: LLM_PROVIDER=lm_studio ``` +### Using Cloud APIs (OpenAI-Compatible) + +OpenWhisper now supports using any OpenAI-compatible API for both transcription and text cleaning. This allows you to use cloud providers like OpenAI, Together AI, OpenRouter, Groq, and more. + +#### Using Cloud API for Text Cleaning (LLM) + +To use a cloud API for text cleaning/disfluency removal: + +1. Update your `.env` file: + ```dotenv + # Set LLM provider to cloud + LLM_PROVIDER=cloud + + # Specify the model name (e.g., gpt-4, gpt-3.5-turbo, claude-3-opus, etc.) + LLM_MODEL_NAME=gpt-3.5-turbo + + # Configure cloud API settings + CLOUD_API_BASE_URL=https://api.openai.com/v1 + CLOUD_API_KEY=sk-your-api-key-here + + # Keep transcription local if desired + TRANSCRIPTION_PROVIDER=local + ``` + +2. Restart the backend service + +**Example configurations for different providers:** + +**OpenAI:** +```dotenv +CLOUD_API_BASE_URL=https://api.openai.com/v1 +CLOUD_API_KEY=sk-your-openai-key +LLM_MODEL_NAME=gpt-3.5-turbo +``` + +**Together AI:** +```dotenv +CLOUD_API_BASE_URL=https://api.together.xyz/v1 +CLOUD_API_KEY=your-together-key +LLM_MODEL_NAME=meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo +``` + +**OpenRouter:** +```dotenv +CLOUD_API_BASE_URL=https://openrouter.ai/api/v1 +CLOUD_API_KEY=sk-or-your-openrouter-key +LLM_MODEL_NAME=anthropic/claude-3-opus +``` + +**Groq:** +```dotenv +CLOUD_API_BASE_URL=https://api.groq.com/openai/v1 +CLOUD_API_KEY=gsk_your-groq-key +LLM_MODEL_NAME=llama-3.1-70b-versatile +``` + +#### Using Cloud API for Transcription + +To use a cloud API for transcription (instead of local Whisper): + +1. Update your `.env` file: + ```dotenv + # Set transcription provider to cloud + TRANSCRIPTION_PROVIDER=cloud + + # Specify the transcription model (whisper-large-v3 is default, OpenAI uses whisper-1) + TRANSCRIPTION_MODEL_NAME=whisper-large-v3 + + # Configure cloud API settings + CLOUD_API_BASE_URL=https://api.openai.com/v1 + CLOUD_API_KEY=sk-your-api-key-here + + # Keep text cleaning local or cloud as desired + LLM_PROVIDER=ollama + LLM_MODEL_NAME=llama2 + ``` + +2. Restart the backend service + +**Example configurations for different transcription providers:** + +**OpenAI Whisper:** +```dotenv +CLOUD_API_BASE_URL=https://api.openai.com/v1 +CLOUD_API_KEY=sk-your-openai-key +TRANSCRIPTION_MODEL_NAME=whisper-1 +``` + +**Groq Whisper:** +```dotenv +CLOUD_API_BASE_URL=https://api.groq.com/openai/v1 +CLOUD_API_KEY=gsk_your-groq-key +TRANSCRIPTION_MODEL_NAME=whisper-large-v3 +``` + +#### Using Cloud for Both Services + +You can use cloud APIs for both transcription and text cleaning: + +```dotenv +# Transcription settings +TRANSCRIPTION_PROVIDER=cloud +TRANSCRIPTION_MODEL_NAME=whisper-large-v3 + +# Text cleaning settings +LLM_PROVIDER=cloud +LLM_MODEL_NAME=gpt-3.5-turbo + +# Shared cloud API settings +CLOUD_API_BASE_URL=https://api.openai.com/v1 +CLOUD_API_KEY=sk-your-api-key-here +``` + +#### Benefits of Cloud APIs + +- **No local GPU required**: Run on any machine without powerful hardware +- **Faster processing**: Cloud APIs often provide faster inference +- **No model downloads**: No need to download large model files +- **Scalability**: Easy to scale without managing infrastructure +- **Cost-effective**: Pay only for what you use + +#### Considerations + +- **Privacy**: Audio and text data is sent to third-party services +- **Costs**: Cloud APIs charge per request (check pricing for your provider) +- **Internet dependency**: Requires stable internet connection +- **Latency**: Network round-trip may add latency compared to local processing 3. Ensure the remote server is accessible from your Docker network (firewall rules, network connectivity, etc.) ## Building and Running