Add support for Cloud Based LLMs and Cloud based inference

backend/.env.example

@@ -6,12 +6,23 @@ OLLAMA_SERVER_PORT=YOUR_PORT
 LM_STUDIO_SERVER_URL=YOUR_LM_STUDIO_SERVER_URL_HERE
 LM_STUDIO_SERVER_PORT=YOUR_PORT
 
-# Change to either 'ollama' or 'lm_studio' depending on which LLM server you want to use
+# Change to either 'ollama', 'lm_studio', or 'cloud' depending on which LLM server you want to use
 LLM_PROVIDER=YOUR_LLM_PROVIDER_HERE
 
 # Change to the model name you want to use from your LLM server (ensure the model is already installed on the server)
 LLM_MODEL_NAME=YOUR_LLM_MODEL_NAME_HERE
 
+# Cloud API settings (only needed if using 'cloud' provider for LLM or transcription)
+# Use OpenAI API or any OpenAI-compatible API (e.g., Together AI, OpenRouter, Azure OpenAI, etc.)
+CLOUD_API_BASE_URL=https://api.openai.com/v1
+CLOUD_API_KEY=YOUR_API_KEY_HERE
+
+# Transcription settings
+# Change to either 'local' or 'cloud' - local uses Whisper models, cloud uses OpenAI-compatible API
+TRANSCRIPTION_PROVIDER=local
+# Model name to use for cloud transcription (whisper-large-v3 is default, OpenAI uses whisper-1)
+TRANSCRIPTION_MODEL_NAME=whisper-large-v3
+
 # Authorization header for API access
 # To generate a new secure token, run: openssl rand -hex 32
 API_AUTH_TOKEN=YOUR_AUTH_TOKEN_HERE

backend/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "faster-whisper>=1.2.1",
     "mlx-whisper>=0.4.3",
     "python-dotenv>=1.2.1",
+    "requests>=2.32.0",
     "torch>=2.8.0",
     "uvicorn>=0.39.0",
 ]

backend/services/text_cleaner.py

@@ -7,6 +7,8 @@
 
 llm_provider = os.getenv("LLM_PROVIDER")
 llm_model_name = os.getenv("LLM_MODEL_NAME")
+cloud_api_base_url = os.getenv("CLOUD_API_BASE_URL")
+cloud_api_key = os.getenv("CLOUD_API_KEY")
 
 # Define the system prompt for disfluency removal, not recommended to modify
 system_prompt = (
@@ -32,6 +34,13 @@ def remove_disfluencies(text):
         api_url = os.getenv("OLLAMA_SERVER_URL")
         api_port = os.getenv("OLLAMA_SERVER_PORT")
         endpoint = f"{api_url}:{api_port}/api/chat"
+    elif llm_provider == "cloud":
+        # Ensure base URL ends with /chat/completions for OpenAI-compatible APIs
+        base_url = cloud_api_base_url.rstrip("/")
+        if not base_url.endswith("/chat/completions"):
+            endpoint = f"{base_url}/chat/completions"
+        else:
+            endpoint = base_url
     else:
         raise ValueError(f"Unsupported LLM provider: {llm_provider}")
 
@@ -46,12 +55,19 @@ def remove_disfluencies(text):
             "max_tokens": 2000,
         }
 
-        response = requests.post(endpoint, json=payload, timeout=180)
+        headers = {}
+        if llm_provider == "cloud":
+            if not cloud_api_key:
+                raise ValueError("CLOUD_API_KEY must be set when using cloud provider")
+            headers["Authorization"] = f"Bearer {cloud_api_key}"
+            headers["Content-Type"] = "application/json"
+
+        response = requests.post(endpoint, json=payload, headers=headers, timeout=180)
         response.raise_for_status()
 
         result = response.json()
 
-        if llm_provider == "lm_studio":
+        if llm_provider == "lm_studio" or llm_provider == "cloud":
             cleaned_text = result["choices"][0]["message"]["content"]
         elif llm_provider == "ollama":
             cleaned_text = result["message"]["content"]
@@ -68,3 +84,5 @@ def remove_disfluencies(text):
         raise Exception(f"Error calling LLM API: {str(e)}")
     except (KeyError, IndexError) as e:
         raise Exception(f"Error parsing LLM response: {str(e)}")
+
+

backend/services/transcription.py

@@ -3,9 +3,22 @@
 import platform
 import tempfile
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
+transcription_provider = os.getenv("TRANSCRIPTION_PROVIDER", "local")
+cloud_api_base_url = os.getenv("CLOUD_API_BASE_URL")
+cloud_api_key = os.getenv("CLOUD_API_KEY")
+transcription_model_name = os.getenv("TRANSCRIPTION_MODEL_NAME", "whisper-large-v3")
+
 
 def _init_whisper():
     """Initialize the best Whisper engine based on the available hardware."""
+    # Skip initialization if using cloud provider
+    if transcription_provider == "cloud":
+        return "cloud", None
+
     system = platform.system()
     machine = platform.machine()
 
@@ -37,9 +50,45 @@ def _init_whisper():
 ENGINE, MODEL = _init_whisper()
 
 
+def _transcribe_cloud(audio_path: str) -> str:
+    """Transcribe audio using OpenAI-compatible API"""
+    import requests
+
+    if not cloud_api_key:
+        raise ValueError("CLOUD_API_KEY must be set when using cloud transcription provider")
+
+    # Prepare the endpoint URL
+    base_url = cloud_api_base_url.rstrip("/")
+    if not base_url.endswith("/audio/transcriptions"):
+        endpoint = f"{base_url}/audio/transcriptions"
+    else:
+        endpoint = base_url
+
+    # Prepare the request
+    headers = {
+        "Authorization": f"Bearer {cloud_api_key}"
+    }
+
+    with open(audio_path, "rb") as audio_file:
+        files = {
+            "file": audio_file
+        }
+        data = {
+            "model": transcription_model_name
+        }
+
+        response = requests.post(endpoint, headers=headers, files=files, data=data, timeout=180)
+        response.raise_for_status()
+
+        result = response.json()
+        return result["text"]
+
+
 def transcribe_audio(audio_path: str) -> str:
     """Transcribe audio file to text"""
-    if ENGINE == "mlx":
+    if ENGINE == "cloud":
+        return _transcribe_cloud(audio_path)
+    elif ENGINE == "mlx":
         import mlx_whisper
 
         result = mlx_whisper.transcribe(
@@ -65,3 +114,5 @@ def transcribe_base64(audio_base64: str) -> str:
         return transcribe_audio(temp_path)
     finally:
         os.unlink(temp_path)
+
+

docs/API.md

@@ -813,16 +813,50 @@ OLLAMA_SERVER_PORT=11434
 LM_STUDIO_SERVER_URL=
 LM_STUDIO_SERVER_PORT=
 
-# LLM Provider selection
+# LLM Provider selection (ollama, lm_studio, or cloud)
 LLM_PROVIDER=ollama
 
 # Model name
 LLM_MODEL_NAME=llama2
 
+# Cloud API Configuration (for OpenAI-compatible APIs)
+# Only needed if using 'cloud' provider for LLM or transcription
+CLOUD_API_BASE_URL=https://api.openai.com/v1
+CLOUD_API_KEY=your_api_key_here
+
+# Transcription Provider (local or cloud)
+TRANSCRIPTION_PROVIDER=local
+# Model name for cloud transcription (whisper-large-v3 is default, OpenAI uses whisper-1)
+TRANSCRIPTION_MODEL_NAME=whisper-large-v3
+
 # Authorization token (generate with: openssl rand -hex 32)
 API_AUTH_TOKEN=your_secure_token_here
 ```
 
+### Configuration Options
+
+#### LLM Provider Options
+- **`ollama`**: Use local Ollama server for text cleaning
+- **`lm_studio`**: Use LM Studio server for text cleaning
+- **`cloud`**: Use any OpenAI-compatible API for text cleaning (OpenAI, Together AI, OpenRouter, Azure OpenAI, etc.)
+
+#### Transcription Provider Options
+- **`local`**: Use local Whisper models (mlx_whisper for Apple Silicon, faster-whisper for other systems)
+- **`cloud`**: Use OpenAI-compatible transcription API (OpenAI Whisper API, Groq, etc.)
+
+#### Cloud API Configuration
+When using `cloud` for either LLM_PROVIDER or TRANSCRIPTION_PROVIDER, you need to configure:
+- **`CLOUD_API_BASE_URL`**: The base URL of your OpenAI-compatible API (e.g., `https://api.openai.com/v1`)
+- **`CLOUD_API_KEY`**: Your API key for authentication
+
+**Supported Cloud Providers:**
+- OpenAI (https://api.openai.com/v1)
+- Together AI (https://api.together.xyz/v1)
+- OpenRouter (https://openrouter.ai/api/v1)
+- Azure OpenAI (https://YOUR_RESOURCE.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT)
+- Groq (https://api.groq.com/openai/v1)
+- Any other OpenAI-compatible API
+
 ## Using Different LLM Providers
 
 ### Using External Ollama Service
@@ -915,6 +949,133 @@ If you have Ollama or LM Studio running on a remote machine:
    LLM_PROVIDER=lm_studio
    ```
 
+### Using Cloud APIs (OpenAI-Compatible)
+
+OpenWhisper now supports using any OpenAI-compatible API for both transcription and text cleaning. This allows you to use cloud providers like OpenAI, Together AI, OpenRouter, Groq, and more.
+
+#### Using Cloud API for Text Cleaning (LLM)
+
+To use a cloud API for text cleaning/disfluency removal:
+
+1. Update your `.env` file:
+   ```dotenv
+   # Set LLM provider to cloud
+   LLM_PROVIDER=cloud
+
+   # Specify the model name (e.g., gpt-4, gpt-3.5-turbo, claude-3-opus, etc.)
+   LLM_MODEL_NAME=gpt-3.5-turbo
+
+   # Configure cloud API settings
+   CLOUD_API_BASE_URL=https://api.openai.com/v1
+   CLOUD_API_KEY=sk-your-api-key-here
+
+   # Keep transcription local if desired
+   TRANSCRIPTION_PROVIDER=local
+   ```
+
+2. Restart the backend service
+
+**Example configurations for different providers:**
+
+**OpenAI:**
+```dotenv
+CLOUD_API_BASE_URL=https://api.openai.com/v1
+CLOUD_API_KEY=sk-your-openai-key
+LLM_MODEL_NAME=gpt-3.5-turbo
+```
+
+**Together AI:**
+```dotenv
+CLOUD_API_BASE_URL=https://api.together.xyz/v1
+CLOUD_API_KEY=your-together-key
+LLM_MODEL_NAME=meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+```
+
+**OpenRouter:**
+```dotenv
+CLOUD_API_BASE_URL=https://openrouter.ai/api/v1
+CLOUD_API_KEY=sk-or-your-openrouter-key
+LLM_MODEL_NAME=anthropic/claude-3-opus
+```
+
+**Groq:**
+```dotenv
+CLOUD_API_BASE_URL=https://api.groq.com/openai/v1
+CLOUD_API_KEY=gsk_your-groq-key
+LLM_MODEL_NAME=llama-3.1-70b-versatile
+```
+
+#### Using Cloud API for Transcription
+
+To use a cloud API for transcription (instead of local Whisper):
+
+1. Update your `.env` file:
+   ```dotenv
+   # Set transcription provider to cloud
+   TRANSCRIPTION_PROVIDER=cloud
+
+   # Specify the transcription model (whisper-large-v3 is default, OpenAI uses whisper-1)
+   TRANSCRIPTION_MODEL_NAME=whisper-large-v3
+
+   # Configure cloud API settings
+   CLOUD_API_BASE_URL=https://api.openai.com/v1
+   CLOUD_API_KEY=sk-your-api-key-here
+
+   # Keep text cleaning local or cloud as desired
+   LLM_PROVIDER=ollama
+   LLM_MODEL_NAME=llama2
+   ```
+
+2. Restart the backend service
+
+**Example configurations for different transcription providers:**
+
+**OpenAI Whisper:**
+```dotenv
+CLOUD_API_BASE_URL=https://api.openai.com/v1
+CLOUD_API_KEY=sk-your-openai-key
+TRANSCRIPTION_MODEL_NAME=whisper-1
+```
+
+**Groq Whisper:**
+```dotenv
+CLOUD_API_BASE_URL=https://api.groq.com/openai/v1
+CLOUD_API_KEY=gsk_your-groq-key
+TRANSCRIPTION_MODEL_NAME=whisper-large-v3
+```
+
+#### Using Cloud for Both Services
+
+You can use cloud APIs for both transcription and text cleaning:
+
+```dotenv
+# Transcription settings
+TRANSCRIPTION_PROVIDER=cloud
+TRANSCRIPTION_MODEL_NAME=whisper-large-v3
+
+# Text cleaning settings
+LLM_PROVIDER=cloud
+LLM_MODEL_NAME=gpt-3.5-turbo
+
+# Shared cloud API settings
+CLOUD_API_BASE_URL=https://api.openai.com/v1
+CLOUD_API_KEY=sk-your-api-key-here
+```
+
+#### Benefits of Cloud APIs
+
+- **No local GPU required**: Run on any machine without powerful hardware
+- **Faster processing**: Cloud APIs often provide faster inference
+- **No model downloads**: No need to download large model files
+- **Scalability**: Easy to scale without managing infrastructure
+- **Cost-effective**: Pay only for what you use
+
+#### Considerations
+
+- **Privacy**: Audio and text data is sent to third-party services
+- **Costs**: Cloud APIs charge per request (check pricing for your provider)
+- **Internet dependency**: Requires stable internet connection
+- **Latency**: Network round-trip may add latency compared to local processing
 3. Ensure the remote server is accessible from your Docker network (firewall rules, network connectivity, etc.)
 
 ## Building and Running