feat: introducing scenes workflow

docs/API.md

@@ -343,6 +343,65 @@ player.addChapters([
 ]);
 ```
 
+## `generateScenes(assetId, languageCode, options?)`
+
+Generates AI-powered scene boundaries by combining shot detection with timestamped transcript cues. Uses shot boundaries as candidate visual anchors, then groups adjacent shot windows into higher-level scenes with titles.
+
+**Parameters:**
+
+- `assetId` (string) - Mux video asset ID
+- `languageCode` (string) - Language code for captions (e.g., 'en', 'es', 'fr')
+- `options` (optional) - Configuration options
+
+**Options:**
+
+- `provider?: 'openai' | 'anthropic' | 'google'` - AI provider (default: 'openai')
+- `model?: string` - AI model to use (defaults: `gpt-5.1`, `claude-sonnet-4-5`, or `gemini-3-flash-preview`)
+- `promptOverrides?: object` - Override specific sections of the scenes prompt
+  - `task?: string` - Override the main scene segmentation task
+  - `outputFormat?: string` - Override the expected output format description
+  - `sceneGuidelines?: string` - Override scene count and formatting guidance
+  - `boundaryGuidelines?: string` - Override how shot anchors and transcript continuity should influence boundaries
+  - `titleGuidelines?: string` - Override scene title style guidance
+- `minScenesPerHour?: number` - Minimum scenes to generate per hour of content (default: 6)
+- `maxScenesPerHour?: number` - Maximum scenes to generate per hour of content (default: 20)
+- `outputLanguageCode?: string | 'auto'` - Language for generated scene titles (default: transcript language)
+- `pollIntervalMs?: number` - Poll interval passed to `waitForShotsForAsset()`
+- `maxAttempts?: number` - Max poll attempts passed to `waitForShotsForAsset()`
+- `createShotsIfMissing?: boolean` - Whether to request shots before polling (default: `true`)
+- `minShotWindowDurationSeconds?: number` - Minimum duration before short low-signal shot windows are merged (default: `2`)
+
+**Returns:**
+
+```typescript
+{
+  assetId: string;
+  languageCode: string;
+  scenes: Array<{
+    startTime: number; // Scene start time in seconds
+    endTime: number; // Scene end time in seconds
+    title: string; // Descriptive scene title
+  }>;
+  usage?: TokenUsage; // Token usage from the AI provider
+}
+```
+
+**Requirements:**
+
+- Asset must be a video asset
+- Asset must have a ready caption/transcript track in the specified language
+- Workflow requires completed shots and will request/poll for them by default
+
+**Example Output:**
+
+```typescript
+[
+  { startTime: 0, endTime: 18.4, title: "Opening Introduction" },
+  { startTime: 18.4, endTime: 47.2, title: "Product Overview" },
+  { startTime: 47.2, endTime: 91, title: "Hands-On Demonstration" }
+]
+```
+
 ## `translateAudio(assetId, toLanguageCode, options?)`
 
 Creates AI-dubbed audio tracks from existing media content using ElevenLabs voice cloning and translation. Uses the default audio track on your asset. Source language is auto-detected unless `fromLanguageCode` is provided.

docs/EXAMPLES.md

@@ -27,6 +27,10 @@ You can run examples directly from the project root without installing dependenc
 npm run example:chapters <asset-id> [language-code] [provider]
 npm run example:chapters:compare <asset-id> [language-code]
 
+# Scenes
+npm run example:scenes <asset-id> -- --language en --provider openai
+npm run example:scenes:compare <asset-id> -- --language en
+
 # Burned-in Caption Detection
 npm run example:burned-in <asset-id> [provider]
 npm run example:burned-in:compare <asset-id>
@@ -72,6 +76,12 @@ npm run example:ask-questions:multiple abc123 "Does this video contain people?"
 # Compare OpenAI vs Anthropic chapter generation
 npm run example:chapters:compare abc123 en
 
+# Generate scenes with broader segmentation
+npm run example:scenes abc123 -- --language en --provider openai --broad
+
+# Compare scene generation across providers
+npm run example:scenes:compare abc123 -- --language en
+
 # Run moderation analysis with Hive
 npm run example:moderation abc123 hive
 
@@ -183,6 +193,28 @@ npm run chapters:basic <your-asset-id> [language-code] [provider]
 npm run compare <your-asset-id> [language-code]
 ```
 
+## Scene Generation Examples
+
+- **Basic Scenes**: Generate shot-guided scene boundaries with titles
+- **Provider Comparison**: Compare OpenAI vs Anthropic vs Google scene generation
+
+```bash
+cd examples/scenes
+npm install
+npm run basic <your-asset-id> -- --language en --provider openai
+npm run compare <your-asset-id> -- --language en
+```
+
+The basic example also supports:
+
+```bash
+# Generate titles in another language
+npm run example:scenes abc123 -- --language en --output-language es
+
+# Ask for broader, less granular sceneing
+npm run example:scenes abc123 -- --language en --broad
+```
+
 ## Caption Translation Examples
 
 - **Basic Translation**: Translate captions and upload to Mux

docs/PROMPT-CUSTOMIZATION.md

@@ -41,6 +41,7 @@ const result = await getSummaryAndTags(assetId, {
 | --- | --- |
 | `getSummaryAndTags` | `task`, `title`, `description`, `keywords`, `qualityGuidelines` |
 | `generateChapters` | `task`, `titleGuidelines`, `chapterGuidelines` |
+| `generateScenes` | `task`, `outputFormat`, `sceneGuidelines`, `boundaryGuidelines`, `titleGuidelines` |
 | `hasBurnedInCaptions` | `task`, `positiveIndicators`, `negativeIndicators`, `confidenceGuidelines` |
 
 ## Presets: Common Override Patterns
@@ -110,6 +111,18 @@ const result = await generateChapters(assetId, "en", {
 });
 ```
 
+### Scene Boundary Style
+
+```ts
+const result = await generateScenes(assetId, "en", {
+  provider: "openai",
+  promptOverrides: {
+    boundaryGuidelines: "Prefer broader scenes unless there is a clear narrative or setting change.",
+    titleGuidelines: "Use short cinematic titles under 5 words.",
+  },
+});
+```
+
 ## Advanced: Full Section Objects
 
 For more control, you can pass a full `PromptSection` object instead of a string to customize the XML tag name and attributes:

docs/WORKFLOWS.md

@@ -246,6 +246,56 @@ player.addChapters(result.chapters);
 - Asset must have a ready caption/transcript track in the specified language
 - Uses existing auto-generated or uploaded captions/transcripts
 
+## Scene Generation
+
+Generate AI-powered scene boundaries by combining shot detection with timestamped transcript cues.
+
+Unlike chaptering, scene generation is grounded in visual structure first. The workflow waits for Mux shot detection, aligns each shot span to overlapping WebVTT cues, and asks the model to group those windows into coherent scenes. This helps avoid treating every camera cut as a new scene while still using visual anchors to detect meaningful transitions.
+
+```typescript
+import { generateScenes } from "@mux/ai/workflows";
+
+const result = await generateScenes("your-mux-asset-id", "en", {
+  provider: "openai",
+  promptOverrides: {
+    boundaryGuidelines: "Prefer broader scenes unless the narrative clearly shifts.",
+    titleGuidelines: "Use short, cinematic titles under 5 words."
+  }
+});
+
+console.log(result.scenes);
+// Array of {startTime: number, endTime: number, title: string}
+```
+
+### Requirements
+
+- Asset must be a video asset
+- Asset must have a ready caption/transcript track in the specified language
+- Asset must have available shots, or allow the workflow to create them
+
+### Prompt Customization
+
+`generateScenes` supports `promptOverrides` so consumers can steer segmentation behavior without replacing the whole workflow prompt.
+
+Available sections:
+
+- `task`
+- `outputFormat`
+- `sceneGuidelines`
+- `boundaryGuidelines`
+- `titleGuidelines`
+
+Example:
+
+```typescript
+const result = await generateScenes(assetId, "en", {
+  promptOverrides: {
+    sceneGuidelines: "Favor fewer, broader scenes for continuous talking-head content.",
+    titleGuidelines: "Use concise editorial titles with no more than 4 words."
+  }
+});
+```
+
 ## Embeddings
 
 Generate vector embeddings for semantic search over video or audio transcripts.

examples/scenes/basic-example.ts

@@ -0,0 +1,83 @@
+import { Command } from "commander";
+
+import { secondsToTimestamp } from "@mux/ai/primitives";
+import { generateScenes } from "@mux/ai/workflows";
+
+import "../env";
+
+type Provider = "openai" | "anthropic" | "google";
+
+const program = new Command();
+
+program
+  .name("scenes")
+  .description("Generate scene boundaries for a Mux video asset")
+  .argument("<asset-id>", "Mux asset ID to analyze")
+  .option("-l, --language <code>", "Language code for transcript/captions", "en")
+  .option("-p, --provider <provider>", "AI provider (openai, anthropic, google)", "openai")
+  .option("--output-language <code>", "Output language as BCP 47 code (e.g. 'fr', 'ja') or 'auto'")
+  .option("--broad", "Prefer broader scenes with fewer splits")
+  .action(async (assetId: string, options: {
+    language: string;
+    provider: Provider;
+    outputLanguage?: string;
+    broad?: boolean;
+  }) => {
+    if (!["openai", "anthropic", "google"].includes(options.provider)) {
+      console.error("Unsupported provider. Choose from: openai, anthropic, google");
+      process.exit(1);
+    }
+
+    console.log(`Generating scenes for asset: ${assetId}`);
+    console.log(`Language: ${options.language}`);
+    console.log(`Provider: ${options.provider}`);
+    if (options.outputLanguage)
+      console.log(`Output Language: ${options.outputLanguage}`);
+    if (options.broad)
+      console.log("Segmentation Style: broader scenes");
+    console.log();
+
+    try {
+      const start = Date.now();
+
+      console.log("Generating scenes...");
+      const result = await generateScenes(assetId, options.language, {
+        provider: options.provider,
+        outputLanguageCode: options.outputLanguage,
+        promptOverrides: options.broad ?
+            {
+              boundaryGuidelines: "Prefer broader scenes unless the narrative or setting clearly changes.",
+              titleGuidelines: "Use concise titles under 5 words.",
+            } :
+          {
+            titleGuidelines: "Use concise titles under 5 words.",
+          },
+      });
+
+      const duration = Date.now() - start;
+
+      console.log("Success!");
+      console.log(`Duration: ${duration}ms`);
+      console.log(`Generated ${result.scenes.length} scenes\n`);
+
+      console.log("Scene List:");
+      result.scenes.forEach((scene, index) => {
+        console.log(
+          `  ${index + 1}. ${secondsToTimestamp(scene.startTime)} - ${secondsToTimestamp(scene.endTime)} | ${scene.title}`,
+        );
+      });
+
+      if (result.usage) {
+        console.log("\nToken Usage:");
+        console.log(JSON.stringify(result.usage, null, 2));
+      }
+
+      console.log("\nStructured Output:");
+      console.log(JSON.stringify(result.scenes, null, 2));
+    } catch (error) {
+      console.error("Error:", error instanceof Error ? error.message : error);
+      process.exit(1);
+    }
+  });
+
+program.parse();

examples/scenes/provider-comparison.ts

@@ -0,0 +1,79 @@
+import { Command } from "commander";
+
+import { secondsToTimestamp } from "@mux/ai/primitives";
+import { generateScenes } from "@mux/ai/workflows";
+
+import "../env";
+
+interface ProviderConfig {
+  name: string;
+  provider: "openai" | "anthropic" | "google";
+}
+
+const program = new Command();
+
+program
+  .name("scenes-compare")
+  .description("Compare scene generation across multiple AI providers")
+  .argument("<asset-id>", "Mux asset ID to analyze")
+  .option("-l, --language <code>", "Language code for transcript/captions", "en")
+  .action(async (assetId: string, options: {
+    language: string;
+  }) => {
+    console.log(`Comparing scene generation for asset: ${assetId}`);
+    console.log(`Language: ${options.language}\n`);
+
+    try {
+      const providers: ProviderConfig[] = [
+        { name: "OpenAI", provider: "openai" },
+        { name: "Anthropic", provider: "anthropic" },
+        { name: "Google", provider: "google" },
+      ];
+
+      const results: Array<{
+        config: ProviderConfig;
+        result: Awaited<ReturnType<typeof generateScenes>>;
+        duration: number;
+      }> = [];
+
+      for (const config of providers) {
+        console.log(`Testing ${config.name} scene generation...`);
+        const start = Date.now();
+        const result = await generateScenes(assetId, options.language, { provider: config.provider });
+        const duration = Date.now() - start;
+
+        console.log("Results:");
+        console.log(`  Duration: ${duration}ms`);
+        console.log(`  Generated scenes: ${result.scenes.length}`);
+        console.log("  Scene breakdown:");
+        result.scenes.forEach((scene, index) => {
+          console.log(
+            `    ${index + 1}. ${secondsToTimestamp(scene.startTime)} - ${secondsToTimestamp(scene.endTime)} | ${scene.title}`,
+          );
+        });
+        if (result.usage) {
+          console.log("  Token usage:");
+          console.log(`    Input: ${result.usage.inputTokens ?? "n/a"}`);
+          console.log(`    Output: ${result.usage.outputTokens ?? "n/a"}`);
+          console.log(`    Total: ${result.usage.totalTokens ?? "n/a"}`);
+        }
+        console.log();
+
+        results.push({ config, result, duration });
+      }
+
+      console.log("\nProvider Comparison:");
+      results.forEach(({ config, result, duration }) => {
+        console.log(`${config.name}: ${result.scenes.length} scenes (${duration}ms)`);
+      });
+
+      const allFirstTitles = results
+        .map(({ config, result }) => `${config.name}: ${result.scenes[0]?.title ?? "No scenes returned"}`);
+      console.log(`\nFirst Scene Titles: ${allFirstTitles.join(" | ")}`);
+    } catch (error) {
+      console.error("Error:", error instanceof Error ? error.message : error);
+      process.exit(1);
+    }
+  });
+
+program.parse();

package.json

@@ -83,6 +83,8 @@
     "example:ask-questions:multiple": "npx tsx examples/ask-questions/multiple-questions-example.ts",
     "example:chapters": "npx tsx examples/chapters/basic-example.ts",
     "example:chapters:compare": "npx tsx examples/chapters/provider-comparison.ts",
+    "example:scenes": "npx tsx examples/scenes/basic-example.ts",
+    "example:scenes:compare": "npx tsx examples/scenes/provider-comparison.ts",
     "example:burned-in": "npx tsx examples/burned-in-captions/basic-example.ts",
     "example:burned-in:compare": "npx tsx examples/burned-in-captions/provider-comparison.ts",
     "example:summarization": "npx tsx examples/summarization/basic-example.ts",

src/workflows/index.ts

@@ -3,6 +3,7 @@ export * from "./burned-in-captions";
 export * from "./chapters";
 export * from "./embeddings";
 export * from "./moderation";
+export * from "./scenes";
 export * from "./summarization";
 export * from "./translate-audio";
 export * from "./translate-captions";