feat: add TwelveLabs Pegasus for full-video summaries (opt-in)

README.md

@@ -12,6 +12,7 @@ Transform lengthy YouTube videos into structured, digestible summaries. YouTube
 ## Features
 
 - **Smart Chapter Detection** - Automatically identifies chapters and topics with precise timestamps
+- **Full-Video Summaries (optional)** - Summarize the whole video (visual *and* audio) with [TwelveLabs Pegasus](https://twelvelabs.io), not just the caption transcript - great for demos, tutorials with on-screen text, or videos with sparse captions
 - **Multi-Language Support** - Generate summaries in English, German, French, Spanish, or Italian
 - **Clickable Timestamps** - Jump directly to any video position from the summary
 - **Visual Chapter Timeline** - See video structure at a glance with an interactive timeline
@@ -31,6 +32,9 @@ Transform lengthy YouTube videos into structured, digestible summaries. YouTube
 - **Z.AI Account** - For AI summarization (GLM-4.7)
   - Sign up at [z.ai](https://z.ai/subscribe?ic=D7NHC27OHD) (referral link)
   - Pricing: Coding Plan from $3/month or API on-demand
+- **TwelveLabs Account** (optional) - For full-video summaries with Pegasus
+  - Sign up at [twelvelabs.io](https://twelvelabs.io) - generous free tier
+  - Only needed if you want to summarize from the video itself instead of the transcript
 
 > **Disclosure**: The links above are referral links. Using them helps support this project at no extra cost to you.
 
@@ -142,6 +146,7 @@ Update your API keys anytime from the Settings page.
 - **Database**: [Prisma](https://www.prisma.io/) + SQLite
 - **Animations**: [Framer Motion](https://www.framer.com/motion/)
 - **LLM**: [Z.AI GLM-4.7](https://z.ai/subscribe?ic=D7NHC27OHD)
+- **Video Understanding** (optional): [TwelveLabs Pegasus](https://twelvelabs.io)
 - **Transcripts**: [Supadata API](https://supadata.ai/?ref=devrico003)
 
 ## Cloud Deployment

__tests__/lib/twelvelabs.test.ts

@@ -0,0 +1,144 @@
+import {
+  summarizeVideoWithPegasus,
+  isTwelveLabsConfigured,
+  clearTwelveLabsClient,
+  TwelveLabsError,
+} from '@/lib/twelvelabs';
+import { getUserApiKey } from '@/lib/userConfig';
+
+// Mock the user API key lookup so no database/encryption is required.
+jest.mock('@/lib/userConfig', () => ({
+  getUserApiKey: jest.fn(),
+}));
+
+// Mock the TwelveLabs SDK so the unit test makes no network calls.
+const mockIndexesList = jest.fn();
+const mockIndexesCreate = jest.fn();
+const mockTasksCreate = jest.fn();
+const mockTasksWaitForDone = jest.fn();
+const mockAnalyze = jest.fn();
+
+jest.mock('twelvelabs-js', () => ({
+  TwelveLabs: jest.fn().mockImplementation(() => ({
+    indexes: { list: mockIndexesList, create: mockIndexesCreate },
+    tasks: { create: mockTasksCreate, waitForDone: mockTasksWaitForDone },
+    analyze: mockAnalyze,
+  })),
+}));
+
+const mockedGetUserApiKey = getUserApiKey as jest.MockedFunction<
+  typeof getUserApiKey
+>;
+
+const USER_ID = 'test-user';
+const VIDEO_URL = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
+
+describe('lib/twelvelabs', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    clearTwelveLabsClient(USER_ID);
+    // Default: an existing index is available (async-iterable list).
+    mockIndexesList.mockReturnValue({
+      async *[Symbol.asyncIterator]() {
+        yield { id: 'index-123' };
+      },
+    });
+    mockTasksCreate.mockResolvedValue({ id: 'task-1' });
+    mockTasksWaitForDone.mockResolvedValue({
+      status: 'ready',
+      videoId: 'video-1',
+    });
+    mockAnalyze.mockResolvedValue({
+      data: '# Summary\n\nA full-video summary.',
+      usage: { outputTokens: 42 },
+    });
+  });
+
+  describe('isTwelveLabsConfigured', () => {
+    it('returns true when an API key is set', async () => {
+      mockedGetUserApiKey.mockResolvedValue('tlk_secret');
+      await expect(isTwelveLabsConfigured(USER_ID)).resolves.toBe(true);
+    });
+
+    it('returns false when no API key is set', async () => {
+      mockedGetUserApiKey.mockResolvedValue(null);
+      await expect(isTwelveLabsConfigured(USER_ID)).resolves.toBe(false);
+    });
+  });
+
+  describe('summarizeVideoWithPegasus', () => {
+    it('throws when TwelveLabs is not configured', async () => {
+      mockedGetUserApiKey.mockResolvedValue(null);
+      await expect(
+        summarizeVideoWithPegasus(VIDEO_URL, 'Summarize', USER_ID)
+      ).rejects.toThrow(TwelveLabsError);
+    });
+
+    it('indexes the video and returns the Pegasus summary', async () => {
+      mockedGetUserApiKey.mockResolvedValue('tlk_secret');
+
+      const result = await summarizeVideoWithPegasus(
+        VIDEO_URL,
+        'Summarize this video',
+        USER_ID
+      );
+
+      expect(result.summary).toContain('full-video summary');
+      expect(result.tokensUsed).toBe(42);
+
+      // Uploaded the video by URL into the existing index...
+      expect(mockTasksCreate).toHaveBeenCalledWith({
+        indexId: 'index-123',
+        videoUrl: VIDEO_URL,
+      });
+      // ...waited for indexing...
+      expect(mockTasksWaitForDone).toHaveBeenCalledWith(
+        'task-1',
+        expect.any(Object)
+      );
+      // ...and analyzed the indexed video with Pegasus.
+      expect(mockAnalyze).toHaveBeenCalledWith(
+        expect.objectContaining({
+          videoId: 'video-1',
+          modelName: 'pegasus1.2',
+          prompt: 'Summarize this video',
+        })
+      );
+      // No index was created since one already existed.
+      expect(mockIndexesCreate).not.toHaveBeenCalled();
+    });
+
+    it('creates an index when none exists', async () => {
+      mockedGetUserApiKey.mockResolvedValue('tlk_secret');
+      mockIndexesList.mockReturnValue({
+        async *[Symbol.asyncIterator]() {
+          // no existing indexes
+        },
+      });
+      mockIndexesCreate.mockResolvedValue({ id: 'new-index' });
+
+      await summarizeVideoWithPegasus(VIDEO_URL, 'Summarize', USER_ID);
+
+      expect(mockIndexesCreate).toHaveBeenCalledWith(
+        expect.objectContaining({
+          models: [
+            { modelName: 'pegasus1.2', modelOptions: ['visual', 'audio'] },
+          ],
+        })
+      );
+      expect(mockTasksCreate).toHaveBeenCalledWith({
+        indexId: 'new-index',
+        videoUrl: VIDEO_URL,
+      });
+    });
+
+    it('throws when indexing does not complete successfully', async () => {
+      mockedGetUserApiKey.mockResolvedValue('tlk_secret');
+      mockTasksWaitForDone.mockResolvedValue({ status: 'failed' });
+
+      await expect(
+        summarizeVideoWithPegasus(VIDEO_URL, 'Summarize', USER_ID)
+      ).rejects.toThrow(/indexing did not complete/i);
+    });
+  });
+});

app/api/settings/api-keys/route.ts

@@ -2,13 +2,15 @@ import { NextRequest, NextResponse } from "next/server";
 import { authenticateRequest } from "@/lib/apiAuth";
 import { getUserApiKeysWithMasked, deleteUserApiKey } from "@/lib/userConfig";
 import { clearGlmClient } from "@/lib/glm";
+import { clearTwelveLabsClient } from "@/lib/twelvelabs";
 
 const SERVICE_NAMES: Record<string, string> = {
   supadata: "Supadata",
   zai: "Z.AI (GLM-4.7)",
+  twelvelabs: "TwelveLabs (Pegasus)",
 };
 
-const VALID_SERVICES = ["supadata", "zai"];
+const VALID_SERVICES = ["supadata", "zai", "twelvelabs"];
 
 /**
  * GET - Retrieve status of all API keys (masked) for the user
@@ -76,6 +78,8 @@ export async function DELETE(request: NextRequest) {
     // Clear cached clients so they pick up the removal
     if (service === "zai") {
       clearGlmClient(auth.userId);
+    } else if (service === "twelvelabs") {
+      clearTwelveLabsClient(auth.userId);
     }
 
     return NextResponse.json({

app/api/setup/save-key/route.ts

@@ -4,9 +4,10 @@ import { authenticateRequest } from "@/lib/apiAuth";
 import { setUserApiKey } from "@/lib/userConfig";
 import { clearGlmClient } from "@/lib/glm";
 import { clearSupadataClient } from "@/lib/supadata";
+import { clearTwelveLabsClient } from "@/lib/twelvelabs";
 
 // Valid service names
-const VALID_SERVICES = ["supadata", "zai"];
+const VALID_SERVICES = ["supadata", "zai", "twelvelabs"];
 
 export async function POST(request: NextRequest) {
   try {
@@ -58,6 +59,8 @@ export async function POST(request: NextRequest) {
       clearSupadataClient(auth.userId);
     } else if (service === "zai") {
       clearGlmClient(auth.userId);
+    } else if (service === "twelvelabs") {
+      clearTwelveLabsClient(auth.userId);
     }
 
     return NextResponse.json({

app/api/setup/test-key/route.ts

@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 import { Supadata } from "@supadata/js";
 import OpenAI from "openai";
+import { TwelveLabs } from "twelvelabs-js";
 import { authenticateRequest } from "@/lib/apiAuth";
 
 export async function POST(request: NextRequest) {
@@ -93,6 +94,35 @@ export async function POST(request: NextRequest) {
       }
     }
 
+    if (service === "twelvelabs") {
+      // Test TwelveLabs API key by listing indexes (a lightweight authed call)
+      const client = new TwelveLabs({ apiKey });
+
+      try {
+        await client.indexes.list({ pageLimit: 1 });
+
+        // If we get here without error, the key is valid
+        return NextResponse.json({
+          success: true,
+          message: "TwelveLabs API key is valid",
+        });
+      } catch (apiError: unknown) {
+        const errorMessage = apiError instanceof Error ? apiError.message : "Unknown error";
+        // Check if it's an auth error
+        if (errorMessage.includes("401") || errorMessage.includes("unauthorized") || errorMessage.includes("invalid") || errorMessage.includes("api_key")) {
+          return NextResponse.json(
+            { success: false, error: "Invalid API key" },
+            { status: 401 }
+          );
+        }
+        // Other errors might be network issues but key could still be valid
+        return NextResponse.json(
+          { success: false, error: `API test failed: ${errorMessage}` },
+          { status: 400 }
+        );
+      }
+    }
+
     return NextResponse.json(
       { error: `Unsupported service: ${service}` },
       { status: 400 }