diff --git a/docs/API.md b/docs/API.md index 4daeff5..71acb7b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -52,7 +52,7 @@ interface SummaryAndTagsResult { Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API, Hive's Moderation API, or Google Cloud Vision SafeSearch. -- For **video assets**, this moderates **storyboard thumbnails** (image moderation). +- For **video assets**, this moderates **storyboard thumbnails** (image moderation). Optionally, set `includeTranscript: true` to **also** moderate the caption transcript text alongside the thumbnails. - For **audio-only assets**, this moderates the **underlying transcript text** (text moderation). Only `openai` supports this; `hive` and `google-vision-api` are image-only and will throw. **Parameters:** @@ -77,6 +77,13 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API, Hi - `retryDelay?: number` - Base delay between retries in milliseconds (default: 1000) - `maxRetryDelay?: number` - Maximum delay between retries in milliseconds (default: 10000) - `exponentialBackoff?: boolean` - Whether to use exponential backoff (default: true) +- `includeTranscript?: boolean` - When `true`, also moderate the caption transcript text for **video assets**, in addition to thumbnails (default: `false`). Only supported with provider `openai`; throws otherwise. Has no effect on audio-only assets, which always moderate transcript text. If set but no ready caption track exists (or the track has no parseable cues), transcript moderation is reported as `skipped` in `transcriptModeration` (`skipReason: "no_ready_text_track"` / `"no_transcript_content"` / `"no_cues"`) so it's auditable — the call itself still succeeds and thumbnails still moderate. Transcription is never triggered. Transcript scores are returned in the dedicated `transcriptScores` array, separate from `thumbnailScores`; they never enter the thumbnail `coverage` denominator. +- `transcriptWindowing?: object` - Optional tuning for transcript time-windowing (all fields optional; sensible defaults applied). Transcript moderation splits the caption track into **dynamic, overlapping** time windows whose size scales with the asset's duration: `windowSeconds = clamp(duration / targetWindowCount, minWindowSeconds, maxWindowSeconds)` and consecutive windows overlap by `max(minOverlapSeconds, windowSeconds * overlapFraction)`, so content straddling a window boundary is still scored intact. + - `targetWindowCount?: number` - Divisor used to derive the base window size from duration (default: `40`) + - `minWindowSeconds?: number` - Lower clamp on window size, in seconds (default: `20`) + - `maxWindowSeconds?: number` - Upper clamp on window size, in seconds (default: `120`) + - `overlapFraction?: number` - Fraction (0..1) of the window size used as overlap (default: `0.15`) + - `minOverlapSeconds?: number` - Lower clamp on the overlap, in seconds (default: `5`) **Hive note (audio-only):** transcript moderation submits `text_data` and requires a Hive **Text Moderation** project/API key. If you use a Visual Moderation key, Hive will reject the request (see [Hive Text Moderation docs](https://docs.thehive.ai/docs/classification-text)). @@ -87,9 +94,11 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API, Hi ```typescript { assetId: string; - mode: 'thumbnails' | 'transcript'; + // 'thumbnails' = images only (video); 'transcript' = transcript text only (audio-only); + // 'combined' = both images and transcript (video with includeTranscript that produced scores). + mode: 'thumbnails' | 'transcript' | 'combined'; isAudioOnly: boolean; - thumbnailScores: Array<{ // Individual thumbnail results + thumbnailScores: Array<{ // Image (thumbnail) results only; empty for audio-only assets url: string; time?: number; // Time in seconds of the thumbnail within the video sexual: number; // 0-1 score @@ -97,11 +106,33 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API, Hi error: boolean; errorMessage?: string; }>; - maxScores: { // Highest scores across all thumbnails (or transcript chunks for audio-only) + transcriptScores: Array<{ // Time-windowed transcript results; empty unless audio-only or includeTranscript produced scores + startTime: number; // Seconds — start of the moderated time window. Ranges may overlap between consecutive entries by design. + endTime: number; // Seconds — end of the moderated time window + sexual: number; // 0-1 score + violence: number; // 0-1 score + error: boolean; + errorMessage?: string; + }>; + transcriptModeration: { // Audit trail for transcript moderation: was it requested, did it complete, and if skipped, why? + requested: boolean; // true when the asset is audio-only OR includeTranscript was passed on a video + // 'completed' — moderation ran (at least one window was moderated; individual windows may still carry per-window `error`). + // 'skipped' — requested but no windows were moderated (see `skipReason` / `skipMessage`). + // 'not_requested' — video asset without `includeTranscript`. + status: 'completed' | 'skipped' | 'not_requested'; + // Present only when status === 'skipped'. Machine-readable reason: + // 'no_ready_text_track' — no ready caption/subtitle track for the asset (or none matching languageCode). + // 'no_transcript_content' — track exists but its transcript text was empty (missing track id, fetch failed, or blank VTT body). + // 'no_cues' — VTT body was non-empty but no cues could be parsed (or windowing produced zero windows). + skipReason?: 'no_ready_text_track' | 'no_transcript_content' | 'no_cues'; + // Present only when status === 'skipped'. Human-readable explanation. + skipMessage?: string; + }; + maxScores: { // Highest scores across thumbnails AND all transcript time windows sexual: number; violence: number; }; - coverage: { + coverage: { // Thumbnail sampling only (transcript windows are excluded) requestedSampleCount: number; successfulSampleCount: number; failedSampleCount: number; @@ -118,6 +149,16 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API, Hi } ``` +**Transcript moderation is auditable, not silent.** Whether the caller passed `includeTranscript: true` on a video asset or the asset was audio-only (implicit request), the result always carries a `transcriptModeration: TranscriptModerationStatus` field describing whether transcript moderation was `not_requested`, `completed`, or `skipped` — and, when skipped, a machine-readable `skipReason` (`no_ready_text_track` / `no_transcript_content` / `no_cues`) plus a human-readable `skipMessage`. This lets callers keep an audit trail for skipped moderations without having to distinguish "moderated cleanly with no findings" from "no caption track" by inspecting an empty `transcriptScores`. + +**Transcript moderation uses dynamic, overlapping time windows.** Each `transcriptScores` entry is a moderated **time window** carrying `startTime`/`endTime` (in seconds) — directly analogous to how a flagged thumbnail carries its `time` — so consumers can locate flagged speech on the timeline rather than receiving a single verdict for the whole transcript. Window **size scales with the asset's duration** (`windowSeconds = clamp(duration / 40, 20s, 120s)`), and consecutive windows **overlap** (~15% of the window size, at least 5 seconds) so content that straddles a window boundary is still scored intact in at least one window. Windows are aligned to caption-cue boundaries (a single cue is never split across windows) so the reported timecodes stay accurate. Because windows overlap by design, **consecutive `transcriptScores` entries' `[startTime, endTime]` ranges may overlap** by roughly the overlap amount. Windows are sent to OpenAI as **batched array requests** (multiple window texts per `/v1/moderations` call via the array `input`, with `results[]` mapped back index-aligned); if a batch is rejected as too large it is split in half and retried down to a single window. The windowing is tunable via the `transcriptWindowing` option (see above). When both thumbnails and transcript windows produce scores (a video asset with `includeTranscript`), `mode` is `'combined'`; `maxScores`/`exceedsThreshold` aggregate the highest `sexual`/`violence` across thumbnails **and** all transcript windows. + +**Completeness and coverage.** Before you rely on a result, understand exactly what was scored and how completely: + +- **All scored units are returned; scoring never stops at the first flag.** Every sampled thumbnail appears in `thumbnailScores` and every transcript window in `transcriptScores`, whether or not it crosses a threshold. `maxScores` and `exceedsThreshold` are a max-based rollup across all of them (`exceedsThreshold` is `true` if any single unit's `sexual`/`violence` exceeds its threshold). To find which region triggered a flag, scan the arrays for entries above your threshold. +- **Thumbnail (visual) coverage is sampled, not exhaustive.** Frames are sampled at `thumbnailInterval` (default every 10s), capped by `maxSamples`. Content between sampled frames is not inspected, so a result with no visual flag does not guarantee every frame is clean; lower `thumbnailInterval` or raise `maxSamples` for denser coverage. The `coverage` object (`requestedSampleCount`, `sampleCoverage`, `isLowConfidence`) describes **thumbnail sampling only** — transcript windows are excluded from its denominator, and `isLowConfidence` signals thin sampling without making thumbnail coverage exhaustive. +- **Transcript (text) coverage is exhaustive.** Every caption cue falls in at least one time window and windows overlap, so all spoken content is moderated. Because windows overlap, the same flagged utterance near a boundary can appear in two adjacent `transcriptScores` entries with overlapping `[startTime, endTime]` ranges; merge/deduplicate overlapping flagged ranges if you want one finding per utterance. + ## `hasBurnedInCaptions(assetId, options?)` Analyzes video frames to detect burned-in captions (hardcoded subtitles) that are permanently embedded in the video image. diff --git a/src/workflows/moderation.ts b/src/workflows/moderation.ts index a8e7b7a..90ecfe8 100644 --- a/src/workflows/moderation.ts +++ b/src/workflows/moderation.ts @@ -14,7 +14,8 @@ import { planSamplingTimestamps } from "../lib/sampling-plan.ts"; import { signUrl } from "../lib/url-signing.ts"; import { resolveMuxSigningContext } from "../lib/workflow-credentials.ts"; import { getThumbnailUrls } from "../primitives/thumbnails.ts"; -import { fetchTranscriptForAsset } from "../primitives/transcripts.ts"; +import type { VTTCue } from "../primitives/transcripts.ts"; +import { fetchTranscriptForAsset, parseVTTCues } from "../primitives/transcripts.ts"; import type { ImageSubmissionMode, MuxAIOptions, @@ -26,10 +27,10 @@ import type { // Types // ───────────────────────────────────────────────────────────────────────────── -/** Per-thumbnail moderation result returned from `getModerationScores`. */ +/** Per-thumbnail (image) moderation result returned from `getModerationScores`. */ export interface ThumbnailModerationScore { url: string; - /** Time in seconds of the thumbnail within the video. Absent for transcript moderation entries. */ + /** Time in seconds of the thumbnail within the video. */ time?: number; sexual: number; violence: number; @@ -37,14 +38,93 @@ export interface ThumbnailModerationScore { errorMessage?: string; } +/** Per-time-window transcript moderation result returned from `getModerationScores`. */ +export interface TranscriptModerationScore { + /** Seconds — start of the moderated time window (first cue's start time). */ + startTime: number; + /** Seconds — end of the moderated time window (last cue's end time). */ + endTime: number; + sexual: number; + violence: number; + error: boolean; + errorMessage?: string; +} + +/** + * Machine-readable reason transcript moderation was requested but skipped. + * + * - `"no_ready_text_track"` — the asset had no ready caption/subtitle track + * (or no track matching `languageCode`), so nothing could be fetched. + * - `"no_transcript_content"` — a track was found but its caption text was + * empty (missing track id, transcript URL fetch failed, or the returned VTT + * body was blank). + * - `"no_cues"` — the VTT body was non-empty but had no parseable cues (e.g. + * header-only, all cues had blank text, or windowing produced zero windows). + */ +export type TranscriptModerationSkipReason = + | "no_ready_text_track" | + "no_transcript_content" | + "no_cues"; + +/** + * Outcome of transcript moderation on a single asset. Attached to + * {@link ModerationResult.transcriptModeration} so callers can audit + * whether transcript moderation was requested, completed, or skipped + * (and why) rather than having to infer it from an empty `transcriptScores`. + */ +export interface TranscriptModerationStatus { + /** + * True if transcript moderation was requested for this call — either the + * asset is audio-only (implicit) or `includeTranscript: true` was passed on + * a video asset (explicit). + */ + requested: boolean; + /** + * - `"completed"` — transcript moderation ran successfully (at least one + * window was moderated; individual windows may still carry per-window + * `error` in {@link TranscriptModerationScore}). + * - `"skipped"` — moderation was requested but no windows were moderated; + * see {@link skipReason} / {@link skipMessage}. + * - `"not_requested"` — the caller did not request transcript moderation + * (video asset with `includeTranscript` unset/false). + */ + status: "completed" | "skipped" | "not_requested"; + /** Present only when `status === "skipped"`. Machine-readable reason. */ + skipReason?: TranscriptModerationSkipReason; + /** Present only when `status === "skipped"`. Human-readable explanation. */ + skipMessage?: string; +} + /** Aggregated moderation payload returned from `getModerationScores`. */ export interface ModerationResult { assetId: string; - /** Whether moderation ran on thumbnails (video) or transcript text (audio-only). */ - mode: "thumbnails" | "transcript"; - /** Convenience flag so callers can understand why `thumbnailScores` may contain a transcript entry. */ + /** + * What was moderated: + * - `"thumbnails"`: only image thumbnails (video without transcript moderation). + * - `"transcript"`: only transcript text (audio-only assets). + * - `"combined"`: both thumbnails and transcript text (video with `includeTranscript`). + */ + mode: "thumbnails" | "transcript" | "combined"; + /** Convenience flag indicating the asset has no video track (transcript-only moderation). */ isAudioOnly: boolean; + /** Image (thumbnail) moderation results. Empty for audio-only assets. */ thumbnailScores: ThumbnailModerationScore[]; + /** + * Transcript moderation results, one entry per moderated time window + * (each carries `startTime`/`endTime`). Empty unless audio-only or + * `includeTranscript` produced scores. + */ + transcriptScores: TranscriptModerationScore[]; + /** + * Audit trail for transcript moderation on this asset: whether it was + * requested, whether it completed or was skipped, and — when skipped — + * a machine-readable {@link TranscriptModerationSkipReason} plus a + * human-readable explanation. This exists so a caller who sets + * `includeTranscript: true` on a video asset can tell the difference + * between "no caption track" (skipped, auditable) and "moderated cleanly + * with no findings" (completed). + */ + transcriptModeration: TranscriptModerationStatus; /** Coverage metadata describing how many requested moderation samples actually succeeded. */ coverage: { requestedSampleCount: number; @@ -116,6 +196,38 @@ export interface ModerationOptions extends MuxAIOptions { imageSubmissionMode?: ImageSubmissionMode; /** Download tuning used when `imageSubmissionMode` === 'base64'. */ imageDownloadOptions?: ImageDownloadOptions; + /** + * When true, also moderate transcript text for video assets, in addition to thumbnails. + * No effect on audio-only assets, which always moderate transcript text. + * If set but no ready caption track exists (or the track has no parseable + * cues), transcript moderation is reported as skipped in + * {@link ModerationResult.transcriptModeration} (with a machine-readable + * `skipReason`) — the call itself still succeeds and thumbnails still moderate. + * Transcription is never triggered. Only supported with provider 'openai'. + * @default false + */ + includeTranscript?: boolean; + /** + * Tuning for transcript time-windowing. All optional; sensible defaults applied. + * + * Transcript moderation splits the caption track into overlapping time windows + * whose size scales with the asset's duration: + * `windowSeconds = clamp(duration / targetWindowCount, minWindowSeconds, maxWindowSeconds)` + * and consecutive windows overlap by `max(minOverlapSeconds, windowSeconds * overlapFraction)` + * so content straddling a window boundary is still scored intact in at least one window. + */ + transcriptWindowing?: { + /** Divisor used to derive the base window size from duration. @default 40 */ + targetWindowCount?: number; + /** Lower clamp on window size, in seconds. @default 20 */ + minWindowSeconds?: number; + /** Upper clamp on window size, in seconds. @default 120 */ + maxWindowSeconds?: number; + /** Fraction (0..1) of the window size used as overlap. @default 0.15 */ + overlapFraction?: number; + /** Lower clamp on the overlap, in seconds. @default 5 */ + minOverlapSeconds?: number; + }; } // ───────────────────────────────────────────────────────────────────────────── @@ -135,6 +247,47 @@ const OPENAI_MODERATION_MAX_DELAY_MS = 3000; const MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING = 0.5; const MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING = 3; +/** + * Default tuning for transcript time-windowing. Window SIZE scales with the + * asset's duration and consecutive windows OVERLAP so abuse straddling a + * boundary is still scored intact in at least one window: + * + * windowSeconds = clamp(duration / targetWindowCount, minWindowSeconds, maxWindowSeconds) + * overlapSeconds = max(minOverlapSeconds, windowSeconds * overlapFraction) + * stride = max(windowSeconds - overlapSeconds, 1) + * + * Callers may override any of these via `ModerationOptions.transcriptWindowing`. + */ +const DEFAULT_TRANSCRIPT_WINDOWING = { + targetWindowCount: 40, + minWindowSeconds: 20, + maxWindowSeconds: 120, + overlapFraction: 0.15, + minOverlapSeconds: 5, +} as const; + +/** Fully-resolved transcript windowing parameters (no optionals). */ +interface ResolvedTranscriptWindowingParams { + targetWindowCount: number; + minWindowSeconds: number; + maxWindowSeconds: number; + overlapFraction: number; + minOverlapSeconds: number; +} + +/** + * Maximum number of UTF-16 code units of concatenated cue text we send to + * OpenAI's moderation endpoint per BATCH request. OpenAI's moderation API + * accepts an array `input` and returns one `results[]` entry per element with + * no documented array-length cap — the real bound is the model's context + * window — so we batch multiple windows per request and cap the combined + * character budget conservatively. + */ +const TRANSCRIPT_BATCH_MAX_UTF16_CODE_UNITS = 100_000; + +/** Maximum number of window texts packed into a single batched moderation request. */ +const TRANSCRIPT_BATCH_MAX_ITEMS = 100; + const GOOGLE_VISION_ENDPOINT = "https://vision.googleapis.com/v1/images:annotate"; /** @@ -234,7 +387,10 @@ async function callOpenAIModerationApi({ credentials, }: { model: string; - input: string | Array<{ type: "image_url"; image_url: { url: string } }>; + input: + | string | + string[] | + Array<{ type: "image_url"; image_url: { url: string } }>; credentials?: WorkflowCredentialsInput; }): Promise { "use step"; @@ -371,80 +527,313 @@ async function requestOpenAIModeration( return processConcurrently(targetUrls, moderateImageWithOpenAI, maxConcurrent); } -async function requestOpenAITextModeration( - text: string, - model: string, - url: string, - credentials?: WorkflowCredentialsInput, -): Promise { - "use step"; - try { - const json: any = await callOpenAIModerationApi({ - model, - input: text, - credentials, +/** + * Hard ceiling on the UTF-16 code units of a SINGLE window's concatenated cue + * text. The dynamic, duration-driven windowing below is the primary driver of + * window size; this constant is only a rare safety guard so a single window + * built over a very dense stretch of speech can never exceed the moderation + * input budget. Such a window is split into sub-windows under the cap (cues + * stay atomic), each carrying its own cue span as `[startTime, endTime]`. + */ +const TRANSCRIPT_WINDOW_MAX_UTF16_CODE_UNITS = 10_000; + +/** A time-bounded window of transcript text built from caption cues. */ +interface TranscriptWindow { + startTime: number; + endTime: number; + text: string; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} + +/** + * Split a window's cues into sub-windows whose joined text stays under + * {@link TRANSCRIPT_WINDOW_MAX_UTF16_CODE_UNITS}. Cues are never split; each + * emitted sub-window carries its own cue span as `[startTime, endTime]`. + * + * A single cue whose text alone already exceeds the cap is emitted as its own + * sub-window (its text is sent as-is — OpenAI truncation is acceptable — so the + * timecodes stay accurate). + */ +function splitCuesUnderCharCeiling( + cues: VTTCue[], + maxUnits: number = TRANSCRIPT_WINDOW_MAX_UTF16_CODE_UNITS, +): TranscriptWindow[] { + const windows: TranscriptWindow[] = []; + let current: VTTCue[] = []; + let currentLength = 0; + + const flush = () => { + if (current.length === 0) { + return; + } + windows.push({ + startTime: Math.min(...current.map(cue => cue.startTime)), + endTime: Math.max(...current.map(cue => cue.endTime)), + text: current.map(cue => cue.text).join(" "), }); - const categoryScores = json.results?.[0]?.category_scores || {}; + current = []; + currentLength = 0; + }; - return { - url, - sexual: categoryScores.sexual || 0, - violence: categoryScores.violence || 0, - error: false, - }; - } catch (error) { - console.error("OpenAI text moderation failed:", error); - return { - url, - sexual: 0, - violence: 0, - error: true, - errorMessage: error instanceof Error ? error.message : String(error), - }; + for (const cue of cues) { + const cueText = cue.text; + // +1 accounts for the space joiner between cues. + const addedLength = currentLength === 0 ? cueText.length : currentLength + 1 + cueText.length; + if (current.length > 0 && addedLength > maxUnits) { + flush(); + } + current.push(cue); + currentLength = current.length === 1 ? cueText.length : currentLength + 1 + cueText.length; } + flush(); + + return windows; } -function chunkTextByUtf16CodeUnits(text: string, maxUnits: number): string[] { - if (!text.trim()) { +/** + * Build DYNAMIC, OVERLAPPING transcript windows from timestamped caption cues. + * + * Window SIZE scales with the asset's `duration` and consecutive windows + * OVERLAP so abuse straddling a window boundary is still scored intact in at + * least one window: + * + * windowSeconds = clamp(duration / targetWindowCount, minWindowSeconds, maxWindowSeconds) + * overlapSeconds = max(minOverlapSeconds, windowSeconds * overlapFraction) + * stride = max(windowSeconds - overlapSeconds, 1) // never <= 0 + * + * Window k covers the time interval `[k*stride, k*stride + windowSeconds]`. A + * cue belongs to window k when it intersects that interval, so boundary cues + * appear in both neighbouring windows. Empty windows (gaps of silence) are + * skipped, and two consecutive windows containing the exact same cue set are + * deduped to avoid a redundant request. A window whose joined text would exceed + * {@link TRANSCRIPT_WINDOW_MAX_UTF16_CODE_UNITS} is split into sub-windows under + * the cap as a rare safety guard. + * + * NOTE: because windows overlap by design, consecutive windows' reported + * `[startTime, endTime]` ranges may overlap by ~`overlapSeconds`. + * + * Exported for direct unit testing without network access. + */ +export function buildTranscriptWindows( + cues: VTTCue[], + duration: number, + params: ResolvedTranscriptWindowingParams = DEFAULT_TRANSCRIPT_WINDOWING, +): TranscriptWindow[] { + const usableCues = cues + .filter(cue => cue.text.trim().length > 0) + .slice() + .sort((a, b) => a.startTime - b.startTime); + if (usableCues.length === 0) { return []; } - if (text.length <= maxUnits) { - return [text]; + + const lastCueEnd = Math.max(...usableCues.map(cue => cue.endTime)); + // Fall back to the last cue's end time when the asset duration is missing/0. + const effectiveDuration = duration && duration > 0 ? duration : lastCueEnd; + + const windowSeconds = clamp( + effectiveDuration / params.targetWindowCount, + params.minWindowSeconds, + params.maxWindowSeconds, + ); + const overlapSeconds = Math.max( + params.minOverlapSeconds, + windowSeconds * params.overlapFraction, + ); + const stride = Math.max(windowSeconds - overlapSeconds, 1); + + const rawWindows: VTTCue[][] = []; + for (let k = 0; ; k++) { + const windowStart = k * stride; + if (windowStart > lastCueEnd) { + break; + } + const windowEnd = windowStart + windowSeconds; + // A cue intersects window k when it overlaps `[windowStart, windowEnd]`. + const windowCues = usableCues.filter( + cue => cue.startTime < windowEnd && cue.endTime > windowStart, + ); + if (windowCues.length > 0) { + rawWindows.push(windowCues); + } + // Guard against pathological non-advancing loops (stride is >= 1, so this + // is belt-and-suspenders only). + if (stride <= 0) { + break; + } } - const chunks: string[] = []; - for (let i = 0; i < text.length; i += maxUnits) { - const chunk = text.slice(i, i + maxUnits).trim(); - if (chunk) { - chunks.push(chunk); + + // Dedupe consecutive windows that contain the exact same cue set (possible + // with sparse speech + overlap) to avoid a redundant API call. + const dedupedCueGroups: VTTCue[][] = []; + const cueGroupKey = (group: VTTCue[]) => + group.map(cue => `${cue.startTime}:${cue.endTime}`).join("|"); + let previousKey: string | undefined; + for (const group of rawWindows) { + const key = cueGroupKey(group); + if (key === previousKey) { + continue; } + dedupedCueGroups.push(group); + previousKey = key; + } + + // Materialise windows, applying the rare hard-char-ceiling safety split. + const windows: TranscriptWindow[] = []; + for (const group of dedupedCueGroups) { + const joinedLength = group.reduce( + (sum, cue, index) => sum + cue.text.length + (index === 0 ? 0 : 1), + 0, + ); + if (joinedLength > TRANSCRIPT_WINDOW_MAX_UTF16_CODE_UNITS) { + windows.push(...splitCuesUnderCharCeiling(group)); + } else { + windows.push({ + startTime: Math.min(...group.map(cue => cue.startTime)), + endTime: Math.max(...group.map(cue => cue.endTime)), + text: group.map(cue => cue.text).join(" "), + }); + } + } + + return windows; +} + +/** A batch of transcript windows sent in a single array-`input` request. */ +interface TranscriptModerationBatch { + windows: TranscriptWindow[]; + model: string; + credentials?: WorkflowCredentialsInput; +} + +/** + * Pack windows into batches whose combined text stays under + * {@link TRANSCRIPT_BATCH_MAX_UTF16_CODE_UNITS} and whose item count stays + * under {@link TRANSCRIPT_BATCH_MAX_ITEMS}. A single window larger than the + * batch budget still occupies its own batch. + */ +function packWindowsIntoBatches(windows: TranscriptWindow[]): TranscriptWindow[][] { + const batches: TranscriptWindow[][] = []; + let current: TranscriptWindow[] = []; + let currentLength = 0; + + for (const window of windows) { + const wouldExceedChars = + current.length > 0 && currentLength + window.text.length > TRANSCRIPT_BATCH_MAX_UTF16_CODE_UNITS; + const wouldExceedItems = current.length >= TRANSCRIPT_BATCH_MAX_ITEMS; + if (wouldExceedChars || wouldExceedItems) { + batches.push(current); + current = []; + currentLength = 0; + } + current.push(window); + currentLength += window.text.length; + } + if (current.length > 0) { + batches.push(current); + } + + return batches; +} + +function transcriptErrorScore( + window: TranscriptWindow, + error: unknown, +): TranscriptModerationScore { + return { + startTime: window.startTime, + endTime: window.endTime, + sexual: 0, + violence: 0, + error: true, + errorMessage: error instanceof Error ? error.message : String(error), + }; +} + +/** + * Moderate a batch of transcript windows in a single array-`input` request, + * mapping each `results[i]` back to window `i`. + * + * Fallback: if the request fails with a 400 (too large) and the batch holds + * more than one window, split the batch in half and retry each half + * recursively (down to a single window). 429/5xx are handled by the existing + * retry/backoff inside {@link callOpenAIModerationApi}. A window that still + * fails yields an error `TranscriptModerationScore` carrying its timecodes. + */ +async function moderateTranscriptBatchWithOpenAI( + batch: TranscriptModerationBatch, +): Promise { + "use step"; + const { windows, model, credentials } = batch; + if (windows.length === 0) { + return []; + } + + try { + const json: any = await callOpenAIModerationApi({ + model, + input: windows.map(window => window.text), + credentials, + }); + const results: any[] = Array.isArray(json.results) ? json.results : []; + return windows.map((window, index) => { + const categoryScores = results[index]?.category_scores || {}; + return { + startTime: window.startTime, + endTime: window.endTime, + sexual: categoryScores.sexual || 0, + violence: categoryScores.violence || 0, + error: false, + }; + }); + } catch (error) { + const status = + error instanceof OpenAIModerationRequestError ? error.status : undefined; + // 400 typically means the batched input was too large: split and retry. + if (status === 400 && windows.length > 1) { + const mid = Math.ceil(windows.length / 2); + const [left, right] = await Promise.all([ + moderateTranscriptBatchWithOpenAI({ windows: windows.slice(0, mid), model, credentials }), + moderateTranscriptBatchWithOpenAI({ windows: windows.slice(mid), model, credentials }), + ]); + return [...left, ...right]; + } + console.error("OpenAI transcript moderation failed:", error); + return windows.map(window => transcriptErrorScore(window, error)); } - return chunks; } async function requestOpenAITranscriptModeration( - transcriptText: string, + cues: VTTCue[], + duration: number, model: string, maxConcurrent: number = 5, credentials?: WorkflowCredentialsInput, -): Promise { + windowingParams: ResolvedTranscriptWindowingParams = DEFAULT_TRANSCRIPT_WINDOWING, +): Promise { "use step"; - // OpenAI supports larger inputs, but chunking avoids pathological single-request sizes and - // mirrors our "max over segments" behavior used for thumbnail moderation. - const chunks = chunkTextByUtf16CodeUnits(transcriptText, 10_000); - if (!chunks.length) { - return [ - { url: "transcript:0", sexual: 0, violence: 0, error: true, errorMessage: "No transcript chunks to moderate" }, - ]; + // Build dynamic, overlapping time windows whose size scales with the asset's + // duration, then moderate them as array-batched requests. Each window maps to + // a score carrying its timecodes, mirroring the "max over segments" behavior + // used for thumbnail moderation. Consecutive windows' [startTime, endTime] + // ranges may overlap by design. + const windows = buildTranscriptWindows(cues, duration, windowingParams); + if (!windows.length) { + return []; } - const targets = chunks.map((chunk, idx) => ({ - chunk, - url: `transcript:${idx}`, - })); - return processConcurrently( - targets, - async entry => requestOpenAITextModeration(entry.chunk, model, entry.url, credentials), + const batches: TranscriptModerationBatch[] = packWindowsIntoBatches(windows).map( + windowBatch => ({ windows: windowBatch, model, credentials }), + ); + // Keep using the existing concurrency across BATCHES. + const batchResults = await processConcurrently( + batches, + moderateTranscriptBatchWithOpenAI, maxConcurrent, ); + return batchResults.flat(); } function getHiveCategoryScores( @@ -764,9 +1153,16 @@ export async function getModerationScores( maxConcurrent = 5, imageSubmissionMode = "url", imageDownloadOptions, + includeTranscript = false, + transcriptWindowing, credentials: providedCredentials, } = options; const credentials = providedCredentials; + // Merge any caller overrides over the module-level defaults. + const windowingParams: ResolvedTranscriptWindowingParams = { + ...DEFAULT_TRANSCRIPT_WINDOWING, + ...transcriptWindowing, + }; // Fetch asset data and playback ID from Mux via helper const { asset, playbackId, policy } = await getPlaybackIdForAsset(assetId, credentials); const videoTrackDurationSeconds = getVideoTrackDurationSecondsFromAsset(asset); @@ -789,26 +1185,44 @@ export async function getModerationScores( ); } - let thumbnailScores: ThumbnailModerationScore[]; + let thumbnailScores: ThumbnailModerationScore[] = []; + let transcriptScores: TranscriptModerationScore[] = []; let mode: ModerationResult["mode"] = "thumbnails"; let thumbnailCount: number | undefined; + // Transcript moderation audit trail. Populated at each branch below so the + // caller can tell "not requested" apart from "requested but skipped, and + // here's why". Overwritten later in the audio-only and includeTranscript + // branches. + let transcriptModeration: TranscriptModerationStatus = { + requested: false, + status: "not_requested", + }; if (isAudioOnly) { mode = "transcript"; + // Fetch the raw VTT (cleanTranscript: false) so we can parse per-cue + // timecodes and segment moderation by time window. `required: true` still + // throws when no usable caption track / VTT exists for an audio-only asset. const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, { languageCode, - cleanTranscript: true, + cleanTranscript: false, shouldSign: policy === "signed", credentials, required: true, }); if (provider === "openai") { - thumbnailScores = await requestOpenAITranscriptModeration( - transcriptResult.transcriptText, + // Audio-only implicitly requests transcript moderation. `required: true` + // above already threw for the "no transcript" case, so reaching here + // means we have transcript text — record as completed. + transcriptModeration = { requested: true, status: "completed" }; + transcriptScores = await requestOpenAITranscriptModeration( + parseVTTCues(transcriptResult.transcriptText), + duration, model || "omni-moderation-latest", maxConcurrent, credentials, + windowingParams, ); } else if (provider === "hive") { throw new Error("Hive does not support transcript moderation in this workflow. Use provider: 'openai' for audio-only assets."); @@ -881,42 +1295,134 @@ export async function getModerationScores( const exhaustiveCheck: never = provider; throw new Error(`Unsupported moderation provider: ${exhaustiveCheck}`); } + + if (includeTranscript) { + if (provider !== "openai") { + // Caller error — leave transcriptModeration in its "not_requested" + // default; the throw ends the call before it matters. + throw new Error("includeTranscript is only supported with provider 'openai'."); + } + const transcriptResult = await fetchTranscriptForAsset(asset, playbackId, { + languageCode, + cleanTranscript: false, + shouldSign: policy === "signed", + credentials, + required: false, + }); + // Classify why (if at all) we're skipping so callers get an auditable + // reason instead of silently seeing an empty `transcriptScores`. The + // three code paths are: + // 1. no track found by `fetchTranscriptForAsset` (track is undefined); + // 2. track found but its transcript text is empty (no id, fetch + // failure, or blank VTT body — all reported here as + // "no_transcript_content"); + // 3. transcript text exists but has no parseable cues, OR windowing + // produced zero windows — reported as "no_cues". + if (!transcriptResult.track) { + transcriptModeration = { + requested: true, + status: "skipped", + skipReason: "no_ready_text_track", + skipMessage: "No ready caption/subtitle track found for this asset.", + }; + } else if (!transcriptResult.transcriptText.trim()) { + transcriptModeration = { + requested: true, + status: "skipped", + skipReason: "no_transcript_content", + skipMessage: "Caption track was empty; nothing to moderate.", + }; + } else { + const cues = parseVTTCues(transcriptResult.transcriptText); + const windows = cues.length === 0 ? + [] : + buildTranscriptWindows(cues, duration, windowingParams); + if (windows.length === 0) { + transcriptModeration = { + requested: true, + status: "skipped", + skipReason: "no_cues", + skipMessage: "Caption track had no parseable cues.", + }; + } else { + transcriptModeration = { requested: true, status: "completed" }; + transcriptScores = await requestOpenAITranscriptModeration( + cues, + duration, + model || "omni-moderation-latest", + maxConcurrent, + credentials, + windowingParams, + ); + } + } + } } - const failed = thumbnailScores.filter(s => s.error); - const successful = thumbnailScores.filter(s => !s.error); + // A video asset that ran `includeTranscript` and produced transcript scores + // moderated both surfaces; reflect that in `mode`. + if (mode === "thumbnails" && transcriptScores.length > 0) { + mode = "combined"; + } + + // Aggregate across both surfaces (thumbnails + transcript) for the all-failed + // guard and for the max-score / threshold computation. + const allScores: Array<{ sexual: number; violence: number; error: boolean; errorMessage?: string; label: string }> = [ + ...thumbnailScores.map(s => ({ ...s, label: s.url })), + ...transcriptScores.map(s => ({ ...s, error: s.error ?? false, label: `transcript window ${s.startTime}-${s.endTime}s` })), + ]; + const failed = allScores.filter(s => s.error); + const successful = allScores.filter(s => !s.error); if (successful.length === 0) { - const details = failed.map(s => `${s.url}: ${s.errorMessage || "Unknown error"}`).join("; "); + const details = failed.map(s => `${s.label}: ${s.errorMessage || "Unknown error"}`).join("; "); throw new Error( - `Moderation failed for all ${thumbnailScores.length} sample(s): ${details}`, + `Moderation failed for all ${allScores.length} sample(s): ${details}`, ); } if (failed.length > 0) { console.warn( - `Moderation had partial failures (${failed.length}/${thumbnailScores.length}); continuing with successful samples.`, + `Moderation had partial failures (${failed.length}/${allScores.length}); continuing with successful samples.`, ); } - // Find highest scores across all thumbnails + // Find highest scores across both thumbnails and transcript time windows. const maxSexual = Math.max(...successful.map(s => s.sexual)); const maxViolence = Math.max(...successful.map(s => s.violence)); const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds }; - const requestedSampleCount = thumbnailScores.length; - const successfulSampleCount = successful.length; - const failedSampleCount = failed.length; - const sampleCoverage = successfulSampleCount / requestedSampleCount; - const hasEnoughSuccessfulSamples = mode === "transcript" || - successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING; - const isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || - !hasEnoughSuccessfulSamples; + // Coverage describes how well the *thumbnails* were sampled; transcript scores + // now live in their own array so they never enter this denominator. + const coverageScores = thumbnailScores; + const coverageFailed = coverageScores.filter(s => s.error); + const coverageSuccessful = coverageScores.filter(s => !s.error); + const requestedSampleCount = coverageScores.length; + const successfulSampleCount = coverageSuccessful.length; + const failedSampleCount = coverageFailed.length; + const sampleCoverage = requestedSampleCount > 0 ? successfulSampleCount / requestedSampleCount : 0; + // A transcript-only result (audio-only assets) has no thumbnails to sample, so + // it must not be penalized for "too few thumbnails" / zero thumbnail coverage. + // Treat it as confident as long as at least one transcript window succeeded. + const hasThumbnails = requestedSampleCount > 0; + const hasSuccessfulTranscript = transcriptScores.some(s => !s.error); + let isLowConfidence: boolean; + if (!hasThumbnails) { + // Transcript-only (audio-only) path: confidence is driven by transcript success. + isLowConfidence = !hasSuccessfulTranscript; + } else { + const hasEnoughSuccessfulSamples = + successfulSampleCount >= MIN_SUCCESSFUL_THUMBNAILS_FOR_CONFIDENT_THRESHOLDING; + isLowConfidence = sampleCoverage < MIN_SAMPLE_COVERAGE_FOR_CONFIDENT_THRESHOLDING || + !hasEnoughSuccessfulSamples; + } return { assetId, mode, isAudioOnly, thumbnailScores, + transcriptScores, + transcriptModeration, coverage: { requestedSampleCount, successfulSampleCount, diff --git a/tests/integration/moderation.test.ts b/tests/integration/moderation.test.ts index b70a152..0c08847 100644 --- a/tests/integration/moderation.test.ts +++ b/tests/integration/moderation.test.ts @@ -175,12 +175,15 @@ describe("moderation Integration Tests", () => { expect(result.mode).toBe("transcript"); expect(result.isAudioOnly).toBe(true); - expect(Array.isArray(result.thumbnailScores)).toBe(true); - expect(result.thumbnailScores.length).toBeGreaterThan(0); - expect(result.thumbnailScores.filter(s => !s.error).length).toBeGreaterThan(0); - expect(result.thumbnailScores[0].url.startsWith("transcript:")).toBe(true); - expect(typeof result.thumbnailScores[0].sexual).toBe("number"); - expect(typeof result.thumbnailScores[0].violence).toBe("number"); + expect(Array.isArray(result.transcriptScores)).toBe(true); + expect(result.transcriptScores.length).toBeGreaterThan(0); + expect(result.transcriptScores.filter(s => !s.error).length).toBeGreaterThan(0); + expect(result.thumbnailScores).toEqual([]); + expect(typeof result.transcriptScores[0].startTime).toBe("number"); + expect(typeof result.transcriptScores[0].endTime).toBe("number"); + expect(result.transcriptScores[0]).not.toHaveProperty("chunkIndex"); + expect(typeof result.transcriptScores[0].sexual).toBe("number"); + expect(typeof result.transcriptScores[0].violence).toBe("number"); }); it("should detect violent audio-only content for OpenAI", async () => { @@ -194,10 +197,13 @@ describe("moderation Integration Tests", () => { expect(result.mode).toBe("transcript"); expect(result.isAudioOnly).toBe(true); - expect(Array.isArray(result.thumbnailScores)).toBe(true); - expect(result.thumbnailScores.length).toBeGreaterThan(0); - expect(result.thumbnailScores.filter(s => !s.error).length).toBeGreaterThan(0); - expect(result.thumbnailScores[0].url.startsWith("transcript:")).toBe(true); + expect(Array.isArray(result.transcriptScores)).toBe(true); + expect(result.transcriptScores.length).toBeGreaterThan(0); + expect(result.transcriptScores.filter(s => !s.error).length).toBeGreaterThan(0); + expect(result.thumbnailScores).toEqual([]); + expect(typeof result.transcriptScores[0].startTime).toBe("number"); + expect(typeof result.transcriptScores[0].endTime).toBe("number"); + expect(result.transcriptScores[0]).not.toHaveProperty("chunkIndex"); // Assert violent content is detected expect(result.maxScores.violence).toBeGreaterThan(VIOLENCE_THRESHOLD); diff --git a/tests/unit/moderation-coverage.test.ts b/tests/unit/moderation-coverage.test.ts index 995d04b..5e818da 100644 --- a/tests/unit/moderation-coverage.test.ts +++ b/tests/unit/moderation-coverage.test.ts @@ -30,7 +30,7 @@ const { } = await import("../../src/lib/mux-assets"); const { resolveMuxSigningContext } = await import("../../src/lib/workflow-credentials"); const { getThumbnailUrls } = await import("../../src/primitives/thumbnails"); -const { getModerationScores } = await import("../../src/workflows/moderation"); +const { getModerationScores, buildTranscriptWindows } = await import("../../src/workflows/moderation"); const mockFetch = vi.fn(); @@ -124,6 +124,11 @@ describe("getModerationScores coverage metadata", () => { }); expect(result.exceedsThreshold).toBe(true); expect(result.maxScores.violence).toBe(0.9); + // Video, no includeTranscript → transcript moderation was not requested. + expect(result.transcriptModeration).toEqual({ + requested: false, + status: "not_requested", + }); }); it("keeps confidence normal when enough thumbnail samples succeed", async () => { @@ -169,3 +174,492 @@ describe("getModerationScores coverage metadata", () => { expect(result.exceedsThreshold).toBe(false); }); }); + +describe("getModerationScores includeTranscript (video assets)", () => { + const VTT_BODY = "WEBVTT\n\n00:00:01.000 --> 00:00:04.000\nsome flagged transcript line\n"; + + function videoAssetWithTextTrack() { + return { + asset: { + id: "asset-123", + tracks: [ + { id: "track-en", type: "text", status: "ready", text_type: "subtitles", language_code: "en" }, + ], + }, + playbackId: "playback-123", + policy: "public", + } as any; + } + + it("populates transcriptScores for a video asset and a high transcript score raises maxScores/exceedsThreshold", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + + const urls = [ + { url: "https://thumb.test/a.png", time: 0 }, + { url: "https://thumb.test/b.png", time: 10 }, + { url: "https://thumb.test/c.png", time: 20 }, + ]; + vi.mocked(getThumbnailUrls).mockResolvedValue(urls); + + mockFetch.mockImplementation(async (url, init) => { + // Transcript VTT fetch (GET to the .vtt URL). + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(VTT_BODY), + } as any; + } + + const body = JSON.parse(String(init?.body)); + // Text (transcript) moderation: `input` is an array of strings. Return a + // high sexual score for every window in the batch. + if (Array.isArray(body.input) && typeof body.input[0] === "string") { + return mockOpenAIModerationResponse({ + status: 200, + body: { + results: body.input.map(() => ({ category_scores: { sexual: 0.95, violence: 0.1 } })), + }, + }); + } + + // Image moderation: `input` is an array of image_url objects. Benign. + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.02, violence: 0.03 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + // Both surfaces moderated → combined mode. + expect(result.mode).toBe("combined"); + expect(result.isAudioOnly).toBe(false); + + // Transcript scores land in their own array as time windows carrying timecodes. + expect(result.transcriptScores.length).toBe(1); + expect(result.transcriptScores[0]).toMatchObject({ + startTime: 1, + endTime: 4, + error: false, + }); + expect(typeof result.transcriptScores[0].sexual).toBe("number"); + // No legacy chunk index on the new time-windowed entries. + expect(result.transcriptScores[0]).not.toHaveProperty("chunkIndex"); + + // thumbnailScores holds image entries only (each has a `time`). + expect(result.thumbnailScores.length).toBe(3); + expect(result.thumbnailScores.every(s => typeof s.time === "number")).toBe(true); + expect(result.thumbnailScores.every(s => "url" in s)).toBe(true); + + // The high transcript score drives maxScores and threshold. + expect(result.maxScores.sexual).toBe(0.95); + expect(result.exceedsThreshold).toBe(true); + + // Coverage is computed over thumbnails only. + expect(result.coverage.requestedSampleCount).toBe(3); + expect(result.coverage.successfulSampleCount).toBe(3); + + // Transcript moderation ran to completion for this asset. + expect(result.transcriptModeration).toEqual({ + requested: true, + status: "completed", + }); + }); + + // Builds a VTT whose cues each span [k*step, k*step + step - 1] seconds. + function buildEvenlySpacedVtt(cueCount: number, step: number): { vtt: string; cueTexts: string[] } { + const cueTexts: string[] = []; + let vtt = "WEBVTT\n\n"; + const fmt = (s: number) => { + const hh = String(Math.floor(s / 3600)).padStart(2, "0"); + const mm = String(Math.floor((s % 3600) / 60)).padStart(2, "0"); + const ss = String(s % 60).padStart(2, "0"); + return `${hh}:${mm}:${ss}.000`; + }; + for (let i = 0; i < cueCount; i++) { + const start = i * step; + const end = start + step - 1; + const text = `cue ${i} content`; + cueTexts.push(text); + vtt += `${fmt(start)} --> ${fmt(end)}\n${text}\n\n`; + } + return { vtt, cueTexts }; + } + + it("produces multiple overlapping windows whose time ranges overlap by design", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + // Short asset duration → minimum 20s windows, 5s overlap, 15s stride. + vi.mocked(getVideoTrackDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getAssetDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getThumbnailUrls).mockResolvedValue([ + { url: "https://thumb.test/a.png", time: 0 }, + ]); + + // 25 cues at 5s spacing → cues span 0..124s, far longer than one window. + const { vtt } = buildEvenlySpacedVtt(25, 5); + + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(vtt), + } as any; + } + const body = JSON.parse(String(init?.body)); + if (Array.isArray(body.input) && typeof body.input[0] === "string") { + return mockOpenAIModerationResponse({ + status: 200, + body: { results: body.input.map(() => ({ category_scores: { sexual: 0.01, violence: 0.02 } })) }, + }); + } + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.02, violence: 0.03 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + // Multiple windows, each carrying its own timecodes. + expect(result.transcriptScores.length).toBeGreaterThanOrEqual(2); + for (const score of result.transcriptScores) { + expect(typeof score.startTime).toBe("number"); + expect(typeof score.endTime).toBe("number"); + expect(score.endTime).toBeGreaterThanOrEqual(score.startTime); + expect(score).not.toHaveProperty("chunkIndex"); + } + // By design consecutive windows OVERLAP: at least one later window starts + // before the previous window ends. + const hasOverlap = result.transcriptScores.some( + (score, i) => i > 0 && score.startTime < result.transcriptScores[i - 1].endTime, + ); + expect(hasOverlap).toBe(true); + }); + + it("scales window size with asset duration (longer asset → fewer, larger windows)", () => { + // Same cue density (cues every 5s across ~10 minutes) but different durations. + const cues = Array.from({ length: 120 }, (_, i) => ({ + startTime: i * 5, + endTime: i * 5 + 4, + text: `cue ${i}`, + })); + + // Short asset → window clamps to the 20s floor (many small windows). + const shortWindows = buildTranscriptWindows(cues, 60); + // Long asset → window grows toward the 120s ceil (fewer, larger windows). + const longWindows = buildTranscriptWindows(cues, 4000); + + expect(shortWindows.length).toBeGreaterThan(longWindows.length); + const avgSpan = (windows: Array<{ startTime: number; endTime: number }>) => + windows.reduce((sum, w) => sum + (w.endTime - w.startTime), 0) / windows.length; + expect(avgSpan(longWindows)).toBeGreaterThan(avgSpan(shortWindows)); + }); + + it("unit: a cue straddling a window boundary appears in two consecutive windows", () => { + // duration 40 → windowSeconds 20, overlap 5, stride 15. Window 0 = [0,20], + // window 1 = [15,35]. A cue at [16,18] intersects both. + const cues = [ + { startTime: 1, endTime: 3, text: "alpha" }, + { startTime: 16, endTime: 18, text: "BOUNDARY" }, + { startTime: 30, endTime: 33, text: "omega" }, + ]; + const windows = buildTranscriptWindows(cues, 40); + const containing = windows.filter(w => w.text.includes("BOUNDARY")); + expect(containing.length).toBeGreaterThanOrEqual(2); + }); + + it("batches multiple windows into a single array `input` request", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + vi.mocked(getVideoTrackDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getAssetDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getThumbnailUrls).mockResolvedValue([ + { url: "https://thumb.test/a.png", time: 0 }, + ]); + + const { vtt } = buildEvenlySpacedVtt(25, 5); + + const transcriptInputSizes: number[] = []; + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(vtt), + } as any; + } + const body = JSON.parse(String(init?.body)); + if (Array.isArray(body.input) && typeof body.input[0] === "string") { + transcriptInputSizes.push(body.input.length); + return mockOpenAIModerationResponse({ + status: 200, + body: { results: body.input.map(() => ({ category_scores: { sexual: 0.01, violence: 0.02 } })) }, + }); + } + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.0, violence: 0.0 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + // The small windows fit in a single batched request whose `input` array + // carries more than one window text. + expect(transcriptInputSizes.length).toBe(1); + expect(transcriptInputSizes[0]).toBeGreaterThan(1); + // One score per window is returned, index-aligned to the batch. + expect(result.transcriptScores.length).toBe(transcriptInputSizes[0]); + }); + + it("splits a batch and retries when a batched request is rejected with 400", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + vi.mocked(getVideoTrackDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getAssetDurationSecondsFromAsset).mockReturnValue(40); + vi.mocked(getThumbnailUrls).mockResolvedValue([ + { url: "https://thumb.test/a.png", time: 0 }, + ]); + + const { vtt } = buildEvenlySpacedVtt(25, 5); + + let transcriptCallCount = 0; + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(vtt), + } as any; + } + const body = JSON.parse(String(init?.body)); + if (Array.isArray(body.input) && typeof body.input[0] === "string") { + transcriptCallCount++; + // Reject the first (full) batch as too large; accept the split halves. + if (body.input.length > 1 && transcriptCallCount === 1) { + return mockOpenAIModerationResponse({ + status: 400, + body: { error: { message: "input too large" } }, + }); + } + return mockOpenAIModerationResponse({ + status: 200, + body: { results: body.input.map(() => ({ category_scores: { sexual: 0.5, violence: 0.1 } })) }, + }); + } + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.0, violence: 0.0 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + // The initial 400 triggered a split-and-retry, producing per-window results + // with no errors. + expect(transcriptCallCount).toBeGreaterThan(1); + expect(result.transcriptScores.length).toBeGreaterThanOrEqual(2); + expect(result.transcriptScores.every(s => s.error === false)).toBe(true); + }); + + it("reports transcript moderation as skipped when no ready text track exists", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue({ + asset: { id: "asset-123", tracks: [] }, + playbackId: "playback-123", + policy: "public", + } as any); + + const urls = [ + { url: "https://thumb.test/a.png", time: 0 }, + { url: "https://thumb.test/b.png", time: 10 }, + { url: "https://thumb.test/c.png", time: 20 }, + ]; + vi.mocked(getThumbnailUrls).mockResolvedValue(urls); + + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + throw new Error("transcript fetch should not happen when no track exists"); + } + const body = JSON.parse(String(init?.body)); + expect(Array.isArray(body.input)).toBe(true); + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.02, violence: 0.03 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + expect(result.transcriptScores).toEqual([]); + expect(result.mode).toBe("thumbnails"); + expect(result.coverage.requestedSampleCount).toBe(3); + expect(result.exceedsThreshold).toBe(false); + // Skip is now surfaced through transcriptModeration for auditability. + expect(result.transcriptModeration).toEqual({ + requested: true, + status: "skipped", + skipReason: "no_ready_text_track", + skipMessage: "No ready caption/subtitle track found for this asset.", + }); + }); + + it("reports transcript moderation as skipped with no_cues when the caption track has no parseable cues", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + + const urls = [ + { url: "https://thumb.test/a.png", time: 0 }, + { url: "https://thumb.test/b.png", time: 10 }, + { url: "https://thumb.test/c.png", time: 20 }, + ]; + vi.mocked(getThumbnailUrls).mockResolvedValue(urls); + + // A VTT header with no cues at all — text is non-empty but parseVTTCues returns []. + const HEADER_ONLY_VTT = "WEBVTT\n\nNOTE this file has no cues\n"; + + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(HEADER_ONLY_VTT), + } as any; + } + const body = JSON.parse(String(init?.body)); + // No transcript batch should ever be sent when there are no cues. + expect(Array.isArray(body.input) && typeof body.input[0] === "string").toBe(false); + return mockOpenAIModerationResponse({ + status: 200, + body: { results: [{ category_scores: { sexual: 0.02, violence: 0.03 } }] }, + }); + }); + + const result = await getModerationScores("asset-123", { + provider: "openai", + model: "omni-moderation-latest", + includeTranscript: true, + }); + + expect(result.transcriptScores).toEqual([]); + expect(result.mode).toBe("thumbnails"); + expect(result.transcriptModeration).toEqual({ + requested: true, + status: "skipped", + skipReason: "no_cues", + skipMessage: "Caption track had no parseable cues.", + }); + }); + + it("places audio-only transcript results in transcriptScores and is not low-confidence", async () => { + vi.mocked(isAudioOnlyAsset).mockReturnValue(true); + vi.mocked(getPlaybackIdForAsset).mockResolvedValue({ + asset: { + id: "asset-audio", + tracks: [ + { id: "track-en", type: "text", status: "ready", text_type: "subtitles", language_code: "en" }, + ], + }, + playbackId: "playback-audio", + policy: "public", + } as any); + // Audio-only assets have no video track / thumbnails. + vi.mocked(getThumbnailUrls).mockResolvedValue([]); + + mockFetch.mockImplementation(async (url, init) => { + if (String(url).endsWith(".vtt")) { + return { + ok: true, + status: 200, + statusText: "OK", + text: vi.fn().mockResolvedValue(VTT_BODY), + } as any; + } + const body = JSON.parse(String(init?.body)); + // Should only ever be text (array-batched) moderation for audio-only. + expect(Array.isArray(body.input)).toBe(true); + expect(typeof body.input[0]).toBe("string"); + return mockOpenAIModerationResponse({ + status: 200, + body: { + results: body.input.map(() => ({ category_scores: { sexual: 0.04, violence: 0.06 } })), + }, + }); + }); + + const result = await getModerationScores("asset-audio", { + provider: "openai", + model: "omni-moderation-latest", + }); + + expect(result.mode).toBe("transcript"); + expect(result.isAudioOnly).toBe(true); + // Transcript scores live in their own array as time windows; thumbnailScores is empty. + expect(result.thumbnailScores).toEqual([]); + expect(result.transcriptScores.length).toBe(1); + expect(result.transcriptScores[0]).toMatchObject({ startTime: 1, endTime: 4, error: false }); + expect(result.transcriptScores[0]).not.toHaveProperty("chunkIndex"); + // Audio-only must not be penalized for having zero thumbnails. + expect(result.coverage.isLowConfidence).toBe(false); + expect(result.coverage.requestedSampleCount).toBe(0); + expect(result.maxScores.violence).toBe(0.06); + expect(result.exceedsThreshold).toBe(false); + // Audio-only implicitly requests transcript moderation. + expect(result.transcriptModeration).toEqual({ + requested: true, + status: "completed", + }); + }); + + it("throws when includeTranscript is used with a non-openai provider", async () => { + vi.mocked(getPlaybackIdForAsset).mockResolvedValue(videoAssetWithTextTrack()); + + const urls = [ + { url: "https://thumb.test/a.png", time: 0 }, + { url: "https://thumb.test/b.png", time: 10 }, + { url: "https://thumb.test/c.png", time: 20 }, + ]; + vi.mocked(getThumbnailUrls).mockResolvedValue(urls); + + mockFetch.mockImplementation(async () => + ({ + ok: true, + status: 200, + statusText: "OK", + json: vi.fn().mockResolvedValue({ + status: [{ response: { output: [{ classes: [] }] } }], + }), + text: vi.fn().mockResolvedValue("{}"), + }) as any); + + await expect( + getModerationScores("asset-123", { + provider: "hive", + includeTranscript: true, + }), + ).rejects.toThrow("includeTranscript is only supported with provider 'openai'."); + }); +});