Add optional TwelveLabs Pegasus transcription backend

README.md

@@ -7,6 +7,13 @@ Videos Transcription and Translation with Faster Whisper and ChatGPT
 
 This Notebook will guide you through the transcription and translation of video using [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) and ChatGPT. You'll be able to explore most inference parameters or use the Notebook as-is to store the transcript, translation and video audio in your Google Drive.
 
+## Transcription backends
+
+* [x] [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) (default, runs locally on GPU)
+* [x] [TwelveLabs Pegasus](https://twelvelabs.io) (optional, runs server-side, no GPU needed)
+
+The default transcription path uses Faster Whisper. There is also an **opt-in** cell that transcribes with TwelveLabs Pegasus, a video-understanding model that *watches* the video rather than only listening to the audio — useful for on-screen text, speaker context and noisy audio, and it requires no local GPU. Tick `use_twelvelabs` in that cell to use it; leave it unchecked to keep Whisper. Both paths feed the same merge/translate steps. Grab a free API key at [twelvelabs.io](https://twelvelabs.io) (generous free tier).
+
 ## Supported/Tested Platforms 
 
 * [x] Windows 11 Pro/RTX3060

autotranslate.ipynb

@@ -315,6 +315,108 @@
         "    fragments.append(dict(start=segment.start,end=segment.end,text=segment.text))\n"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tl-pegasus-md"
+      },
+      "source": [
+        "#@markdown # **Optional: Transcribe with TwelveLabs Pegasus** 🎬\n",
+        "#@markdown\n",
+        "#@markdown Instead of (or in addition to) Faster Whisper, you can transcribe with\n",
+        "#@markdown [TwelveLabs Pegasus](https://twelvelabs.io), a video understanding model.\n",
+        "#@markdown Because Pegasus *watches* the video (not just the audio), it tends to handle\n",
+        "#@markdown on-screen text, speaker context and noisy audio well, and it runs server-side\n",
+        "#@markdown so it needs no GPU.\n",
+        "#@markdown\n",
+        "#@markdown This cell is **opt-in and non-breaking**: leave `use_twelvelabs` unchecked to\n",
+        "#@markdown keep using the Whisper output from the cell above. When checked, it overwrites\n",
+        "#@markdown `fragments` with Pegasus segments using the same `{start, end, text}` shape, so\n",
+        "#@markdown the **Merge** and **Translate** cells below work unchanged.\n",
+        "#@markdown\n",
+        "#@markdown Get a free API key at https://twelvelabs.io (generous free tier).\n",
+        "#@markdown\n",
+        "#@markdown ---"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tl-pegasus-code"
+      },
+      "outputs": [],
+      "source": [
+        "#@title TwelveLabs Pegasus transcription (opt-in)\n",
+        "\n",
+        "#@markdown ### **Behavior control**\n",
+        "#@markdown #### Use TwelveLabs Pegasus instead of Whisper for this run\n",
+        "use_twelvelabs = False #@param {type:\"boolean\"}\n",
+        "#@markdown #### TwelveLabs API key (get one free at https://twelvelabs.io)\n",
+        "twelvelabs_api_key = \"\" #@param {type:\"string\"}\n",
+        "#@markdown #### Model\n",
+        "twelvelabs_model_name = \"pegasus1.5\" #@param [\"pegasus1.5\", \"pegasus1.2\"] {allow-input: true}\n",
+        "#@markdown #### Max tokens (raise for long videos)\n",
+        "twelvelabs_max_tokens = 4096 #@param {type:\"integer\"}\n",
+        "#@markdown ---\n",
+        "#@markdown **Pegasus reads the video directly, so it needs a publicly reachable video URL.**\n",
+        "#@markdown By default it reuses the `URL` you entered in the **Video selection** cell.\n",
+        "twelvelabs_video_url = \"\" #@param {type:\"string\"}\n",
+        "\n",
+        "if use_twelvelabs:\n",
+        "    # ! pip install twelvelabs>=1.2.8\n",
+        "    import json as _json\n",
+        "    import re as _re\n",
+        "    from twelvelabs import TwelveLabs\n",
+        "    from twelvelabs.types.video_context import VideoContext_Url\n",
+        "\n",
+        "    video_url = twelvelabs_video_url.strip() or URL\n",
+        "    if not twelvelabs_api_key.strip():\n",
+        "        raise ValueError(\"Please set twelvelabs_api_key (free key at https://twelvelabs.io).\")\n",
+        "\n",
+        "    client = TwelveLabs(api_key=twelvelabs_api_key.strip())\n",
+        "\n",
+        "    transcribe_prompt = (\n",
+        "        \"Transcribe the spoken audio of this video into time-stamped segments. \"\n",
+        "        \"Respond with ONLY a JSON array of objects, each having the keys \"\n",
+        "        '\"start\" (segment start in seconds, number), '\n",
+        "        '\"end\" (segment end in seconds, number) and '\n",
+        "        '\"text\" (the transcribed text, string). '\n",
+        "        \"Order the segments by start time and do not include any prose, \"\n",
+        "        \"markdown or code fences outside the JSON array.\"\n",
+        "    )\n",
+        "\n",
+        "    response = client.analyze(\n",
+        "        model_name=twelvelabs_model_name,\n",
+        "        video=VideoContext_Url(url=video_url),\n",
+        "        prompt=transcribe_prompt,\n",
+        "        max_tokens=twelvelabs_max_tokens,\n",
+        "    )\n",
+        "\n",
+        "    # Pegasus returns free-form text; pull the JSON array out of it.\n",
+        "    raw = response.data.strip()\n",
+        "    match = _re.search(r\"\\[.*\\]\", raw, _re.DOTALL)\n",
+        "    if not match:\n",
+        "        raise ValueError(f\"Could not find a JSON array in the Pegasus response:\\n{raw}\")\n",
+        "    segments = _json.loads(match.group(0))\n",
+        "\n",
+        "    fragments = [\n",
+        "        dict(start=float(s[\"start\"]), end=float(s[\"end\"]), text=str(s[\"text\"]))\n",
+        "        for s in segments\n",
+        "        if str(s.get(\"text\", \"\")).strip()\n",
+        "    ]\n",
+        "    # Drive the same downstream cells (merge + translate) as Whisper does.\n",
+        "    language_detected = \"auto\"\n",
+        "    word_level_timestamps = False\n",
+        "\n",
+        "    print(f\"TwelveLabs Pegasus produced {len(fragments)} segments.\")\n",
+        "    for fragment in fragments[:5]:\n",
+        "        print(f\"[{fragment['start']:.2f} --> {fragment['end']:.2f}] {fragment['text']}\")\n",
+        "else:\n",
+        "    print(\"use_twelvelabs is False; keeping the Faster Whisper transcript from the cell above.\")"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,