diff --git a/docs/API.md b/docs/API.md index 831b380b..9cc79f99 100644 --- a/docs/API.md +++ b/docs/API.md @@ -64,7 +64,7 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or - `provider?: 'openai' | 'hive'` - Moderation provider (default: 'openai') - `model?: string` - OpenAI moderation model to use (default: `omni-moderation-latest`) - `languageCode?: string` - Transcript language code when moderating audio-only assets (optional) -- `thresholds?: { sexual?: number; violence?: number }` - Custom thresholds (default: {sexual: 0.7, violence: 0.8}) +- `thresholds?: { sexual?: number; violence?: number; hate?: number; selfHarm?: number; drugs?: number }` - Custom thresholds (default: 0.8 for all categories) - `thumbnailInterval?: number` - Seconds between thumbnails for long videos (default: 10) - `thumbnailWidth?: number` - Thumbnail width in pixels (default: 640) - `maxSamples?: number` - Maximum number of thumbnails to sample. Acts as a cap: if `thumbnailInterval` produces fewer samples than this limit the interval is respected; otherwise samples are evenly distributed with first and last frames pinned. (default: unlimited) @@ -91,12 +91,18 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or time?: number; // Time in seconds of the thumbnail within the video sexual: number; // 0-1 score violence: number; // 0-1 score + hate: number; // 0-1 score + selfHarm: number; // 0-1 score + drugs: number; // 0-1 score error: boolean; errorMessage?: string; }>; maxScores: { // Highest scores across all thumbnails (or transcript chunks for audio-only) sexual: number; violence: number; + hate: number; + selfHarm: number; + drugs: number; }; coverage: { requestedSampleCount: number; @@ -110,6 +116,9 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or thresholds: { // Threshold values used sexual: number; violence: number; + hate: number; + selfHarm: number; + drugs: number; }; usage?: TokenUsage; // Workflow usage metadata } diff --git a/src/workflows/moderation.ts b/src/workflows/moderation.ts index f924cb00..585cd2a8 100644 --- a/src/workflows/moderation.ts +++ b/src/workflows/moderation.ts @@ -33,6 +33,9 @@ export interface ThumbnailModerationScore { time?: number; sexual: number; violence: number; + hate: number; + selfHarm: number; + drugs: number; error: boolean; errorMessage?: string; } @@ -66,11 +69,17 @@ export interface ModerationResult { maxScores: { sexual: number; violence: number; + hate: number; + selfHarm: number; + drugs: number; }; exceedsThreshold: boolean; thresholds: { sexual: number; violence: number; + hate: number; + selfHarm: number; + drugs: number; }; } @@ -99,10 +108,13 @@ export interface ModerationOptions extends MuxAIOptions { * If omitted, the first ready text track will be used. */ languageCode?: string; - /** Override the default sexual/violence thresholds (0-1). */ + /** Override the default thresholds (0-1). */ thresholds?: { sexual?: number; violence?: number; + hate?: number; + selfHarm?: number; + drugs?: number; }; /** Interval between storyboard thumbnails in seconds (defaults to 10). */ thumbnailInterval?: number; @@ -125,6 +137,9 @@ export interface ModerationOptions extends MuxAIOptions { const DEFAULT_THRESHOLDS = { sexual: 0.8, violence: 0.8, + hate: 0.8, + selfHarm: 0.8, + drugs: 0.8, }; const DEFAULT_PROVIDER = "openai"; @@ -153,9 +168,24 @@ export const HIVE_VIOLENCE_CATEGORIES = [ "hanging", "noose", "human_corpse", - "yes_emaciated_body", +]; + +export const HIVE_HATE_CATEGORIES = [ + "yes_nazi", + "yes_terrorist", + "yes_kkk", + "yes_confederate", +]; + +export const HIVE_SELF_HARM_CATEGORIES = [ "yes_self_harm", - "garm_death_injury_or_military_conflict", + "yes_emaciated_body", +]; + +export const HIVE_ILLICIT_CATEGORIES = [ + "yes_pills", + "illicit_injectables", + "yes_marijuana", ]; class OpenAIModerationRequestError extends Error { @@ -319,6 +349,9 @@ async function moderateImageWithOpenAI(entry: { time: entry.time, sexual: categoryScores.sexual || 0, violence: categoryScores.violence || 0, + hate: categoryScores.hate || 0, + selfHarm: categoryScores["self-harm"] || 0, + drugs: categoryScores.illicit || 0, error: false, }; } catch (error) { @@ -328,6 +361,9 @@ async function moderateImageWithOpenAI(entry: { time: entry.time, sexual: 0, violence: 0, + hate: 0, + selfHarm: 0, + drugs: 0, error: true, errorMessage: error instanceof Error ? error.message : String(error), }; @@ -375,6 +411,9 @@ async function requestOpenAITextModeration( url, sexual: categoryScores.sexual || 0, violence: categoryScores.violence || 0, + hate: categoryScores.hate || 0, + selfHarm: categoryScores["self-harm"] || 0, + drugs: categoryScores.illicit || 0, error: false, }; } catch (error) { @@ -383,6 +422,9 @@ async function requestOpenAITextModeration( url, sexual: 0, violence: 0, + hate: 0, + selfHarm: 0, + drugs: 0, error: true, errorMessage: error instanceof Error ? error.message : String(error), }; @@ -418,7 +460,7 @@ async function requestOpenAITranscriptModeration( const chunks = chunkTextByUtf16CodeUnits(transcriptText, 10_000); if (!chunks.length) { return [ - { url: "transcript:0", sexual: 0, violence: 0, error: true, errorMessage: "No transcript chunks to moderate" }, + { url: "transcript:0", sexual: 0, violence: 0, hate: 0, selfHarm: 0, drugs: 0, error: true, errorMessage: "No transcript chunks to moderate" }, ]; } const targets = chunks.map((chunk, idx) => ({ @@ -517,12 +559,18 @@ async function moderateImageWithHive(entry: { const sexual = getHiveCategoryScores(classes, HIVE_SEXUAL_CATEGORIES); const violence = getHiveCategoryScores(classes, HIVE_VIOLENCE_CATEGORIES); + const hate = getHiveCategoryScores(classes, HIVE_HATE_CATEGORIES); + const selfHarm = getHiveCategoryScores(classes, HIVE_SELF_HARM_CATEGORIES); + const drugs = getHiveCategoryScores(classes, HIVE_ILLICIT_CATEGORIES); return { url: entry.url, time: entry.time, sexual, violence, + hate, + selfHarm, + drugs, error: false, }; } catch (error) { @@ -531,6 +579,9 @@ async function moderateImageWithHive(entry: { time: entry.time, sexual: 0, violence: 0, + hate: 0, + selfHarm: 0, + drugs: 0, error: true, errorMessage: error instanceof Error ? error.message : String(error), }; @@ -747,6 +798,9 @@ export async function getModerationScores( // Find highest scores across all thumbnails const maxSexual = Math.max(...successful.map(s => s.sexual)); const maxViolence = Math.max(...successful.map(s => s.violence)); + const maxHate = Math.max(...successful.map(s => s.hate)); + const maxSelfHarm = Math.max(...successful.map(s => s.selfHarm)); + const maxIllicit = Math.max(...successful.map(s => s.drugs)); const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds }; const requestedSampleCount = thumbnailScores.length; @@ -780,8 +834,16 @@ export async function getModerationScores( maxScores: { sexual: maxSexual, violence: maxViolence, + hate: maxHate, + selfHarm: maxSelfHarm, + drugs: maxIllicit, }, - exceedsThreshold: maxSexual > finalThresholds.sexual || maxViolence > finalThresholds.violence, + exceedsThreshold: + maxSexual > finalThresholds.sexual || + maxViolence > finalThresholds.violence || + maxHate > finalThresholds.hate || + maxSelfHarm > finalThresholds.selfHarm || + maxIllicit > finalThresholds.drugs, thresholds: finalThresholds, }; } diff --git a/tests/eval/moderation.eval.ts b/tests/eval/moderation.eval.ts index f45b9c70..b3bb534d 100644 --- a/tests/eval/moderation.eval.ts +++ b/tests/eval/moderation.eval.ts @@ -356,6 +356,15 @@ evalite("Moderation", { typeof output.maxScores.violence === "number" && output.maxScores.violence >= 0 && output.maxScores.violence <= 1, + typeof output.maxScores.hate === "number" && + output.maxScores.hate >= 0 && + output.maxScores.hate <= 1, + typeof output.maxScores.selfHarm === "number" && + output.maxScores.selfHarm >= 0 && + output.maxScores.selfHarm <= 1, + typeof output.maxScores.drugs === "number" && + output.maxScores.drugs >= 0 && + output.maxScores.drugs <= 1, // exceedsThreshold is boolean typeof output.exceedsThreshold === "boolean", // thresholds contain valid numbers in 0-1 range @@ -365,6 +374,15 @@ evalite("Moderation", { typeof output.thresholds.violence === "number" && output.thresholds.violence >= 0 && output.thresholds.violence <= 1, + typeof output.thresholds.hate === "number" && + output.thresholds.hate >= 0 && + output.thresholds.hate <= 1, + typeof output.thresholds.selfHarm === "number" && + output.thresholds.selfHarm >= 0 && + output.thresholds.selfHarm <= 1, + typeof output.thresholds.drugs === "number" && + output.thresholds.drugs >= 0 && + output.thresholds.drugs <= 1, ]; const passed = checks.filter(Boolean).length; @@ -485,6 +503,9 @@ evalite("Moderation", { { label: "Exceeds Threshold", value: String(output.exceedsThreshold) }, { label: "Max Sexual", value: output.maxScores.sexual.toFixed(4) }, { label: "Max Violence", value: output.maxScores.violence.toFixed(4) }, + { label: "Max Hate", value: output.maxScores.hate.toFixed(4) }, + { label: "Max Self-Harm", value: output.maxScores.selfHarm.toFixed(4) }, + { label: "Max Drugs", value: output.maxScores.drugs.toFixed(4) }, { label: "Samples", value: output.thumbnailScores.length }, { label: "Latency", value: `${Math.round(output.latencyMs)}ms` }, { label: "Duration", value: output.usage?.metadata?.assetDurationSeconds ? `${output.usage.metadata.assetDurationSeconds}s` : "n/a" }, diff --git a/tests/unit/moderation.test.ts b/tests/unit/moderation.test.ts index a576c735..0c1daa8d 100644 --- a/tests/unit/moderation.test.ts +++ b/tests/unit/moderation.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest"; -import { HIVE_SEXUAL_CATEGORIES, HIVE_VIOLENCE_CATEGORIES } from "../../src/workflows/moderation"; +import { + HIVE_HATE_CATEGORIES, + HIVE_ILLICIT_CATEGORIES, + HIVE_SELF_HARM_CATEGORIES, + HIVE_SEXUAL_CATEGORIES, + HIVE_VIOLENCE_CATEGORIES, +} from "../../src/workflows/moderation"; describe("hive moderation categories", () => { it("the HIVE_SEXUAL_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { @@ -23,9 +29,30 @@ describe("hive moderation categories", () => { "hanging", "noose", "human_corpse", - "yes_emaciated_body", + ]); + }); + + it("the HIVE_HATE_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { + expect(HIVE_HATE_CATEGORIES).toEqual([ + "yes_nazi", + "yes_terrorist", + "yes_kkk", + "yes_confederate", + ]); + }); + + it("the HIVE_SELF_HARM_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { + expect(HIVE_SELF_HARM_CATEGORIES).toEqual([ "yes_self_harm", - "garm_death_injury_or_military_conflict", + "yes_emaciated_body", + ]); + }); + + it("the HIVE_ILLICIT_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { + expect(HIVE_ILLICIT_CATEGORIES).toEqual([ + "yes_pills", + "illicit_injectables", + "yes_marijuana", ]); }); });