feat: expand moderation categories

docs/API.md

@@ -64,7 +64,7 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or
 - `provider?: 'openai' | 'hive'` - Moderation provider (default: 'openai')
 - `model?: string` - OpenAI moderation model to use (default: `omni-moderation-latest`)
 - `languageCode?: string` - Transcript language code when moderating audio-only assets (optional)
-- `thresholds?: { sexual?: number; violence?: number }` - Custom thresholds (default: {sexual: 0.7, violence: 0.8})
+- `thresholds?: { sexual?: number; violence?: number; hate?: number; selfHarm?: number; drugs?: number }` - Custom thresholds (default: 0.8 for all categories)
 - `thumbnailInterval?: number` - Seconds between thumbnails for long videos (default: 10)
 - `thumbnailWidth?: number` - Thumbnail width in pixels (default: 640)
 - `maxSamples?: number` - Maximum number of thumbnails to sample. Acts as a cap: if `thumbnailInterval` produces fewer samples than this limit the interval is respected; otherwise samples are evenly distributed with first and last frames pinned. (default: unlimited)
@@ -91,12 +91,18 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or
     time?: number; // Time in seconds of the thumbnail within the video
     sexual: number; // 0-1 score
     violence: number; // 0-1 score
+    hate: number; // 0-1 score
+    selfHarm: number; // 0-1 score
+    drugs: number; // 0-1 score
     error: boolean;
     errorMessage?: string;
   }>;
   maxScores: { // Highest scores across all thumbnails (or transcript chunks for audio-only)
     sexual: number;
     violence: number;
+    hate: number;
+    selfHarm: number;
+    drugs: number;
   };
   coverage: {
     requestedSampleCount: number;
@@ -110,6 +116,9 @@ Analyzes a Mux asset for inappropriate content using OpenAI's Moderation API or
   thresholds: { // Threshold values used
     sexual: number;
     violence: number;
+    hate: number;
+    selfHarm: number;
+    drugs: number;
   };
   usage?: TokenUsage; // Workflow usage metadata
 }

src/workflows/moderation.ts

@@ -33,6 +33,9 @@ export interface ThumbnailModerationScore {
   time?: number;
   sexual: number;
   violence: number;
+  hate: number;
+  selfHarm: number;
+  drugs: number;
   error: boolean;
   errorMessage?: string;
 }
@@ -66,11 +69,17 @@ export interface ModerationResult {
   maxScores: {
     sexual: number;
     violence: number;
+    hate: number;
+    selfHarm: number;
+    drugs: number;
   };
   exceedsThreshold: boolean;
   thresholds: {
     sexual: number;
     violence: number;
+    hate: number;
+    selfHarm: number;
+    drugs: number;
   };
 }
 
@@ -99,10 +108,13 @@ export interface ModerationOptions extends MuxAIOptions {
    * If omitted, the first ready text track will be used.
    */
   languageCode?: string;
-  /** Override the default sexual/violence thresholds (0-1). */
+  /** Override the default thresholds (0-1). */
   thresholds?: {
     sexual?: number;
     violence?: number;
+    hate?: number;
+    selfHarm?: number;
+    drugs?: number;
   };
   /** Interval between storyboard thumbnails in seconds (defaults to 10). */
   thumbnailInterval?: number;
@@ -125,6 +137,9 @@ export interface ModerationOptions extends MuxAIOptions {
 const DEFAULT_THRESHOLDS = {
   sexual: 0.8,
   violence: 0.8,
+  hate: 0.8,
+  selfHarm: 0.8,
+  drugs: 0.8,
 };
 
 const DEFAULT_PROVIDER = "openai";
@@ -153,9 +168,24 @@ export const HIVE_VIOLENCE_CATEGORIES = [
   "hanging",
   "noose",
   "human_corpse",
-  "yes_emaciated_body",
+];
+
+export const HIVE_HATE_CATEGORIES = [
+  "yes_nazi",
+  "yes_terrorist",
+  "yes_kkk",
+  "yes_confederate",
+];
+
+export const HIVE_SELF_HARM_CATEGORIES = [
   "yes_self_harm",
-  "garm_death_injury_or_military_conflict",
+  "yes_emaciated_body",
+];
+
+export const HIVE_ILLICIT_CATEGORIES = [
+  "yes_pills",
+  "illicit_injectables",
+  "yes_marijuana",
 ];
 
 class OpenAIModerationRequestError extends Error {
@@ -319,6 +349,9 @@ async function moderateImageWithOpenAI(entry: {
       time: entry.time,
       sexual: categoryScores.sexual || 0,
       violence: categoryScores.violence || 0,
+      hate: categoryScores.hate || 0,
+      selfHarm: categoryScores["self-harm"] || 0,
+      drugs: categoryScores.illicit || 0,
       error: false,
     };
   } catch (error) {
@@ -328,6 +361,9 @@ async function moderateImageWithOpenAI(entry: {
       time: entry.time,
       sexual: 0,
       violence: 0,
+      hate: 0,
+      selfHarm: 0,
+      drugs: 0,
       error: true,
       errorMessage: error instanceof Error ? error.message : String(error),
     };
@@ -375,6 +411,9 @@ async function requestOpenAITextModeration(
       url,
       sexual: categoryScores.sexual || 0,
       violence: categoryScores.violence || 0,
+      hate: categoryScores.hate || 0,
+      selfHarm: categoryScores["self-harm"] || 0,
+      drugs: categoryScores.illicit || 0,
       error: false,
     };
   } catch (error) {
@@ -383,6 +422,9 @@ async function requestOpenAITextModeration(
       url,
       sexual: 0,
       violence: 0,
+      hate: 0,
+      selfHarm: 0,
+      drugs: 0,
       error: true,
       errorMessage: error instanceof Error ? error.message : String(error),
     };
@@ -418,7 +460,7 @@ async function requestOpenAITranscriptModeration(
   const chunks = chunkTextByUtf16CodeUnits(transcriptText, 10_000);
   if (!chunks.length) {
     return [
-      { url: "transcript:0", sexual: 0, violence: 0, error: true, errorMessage: "No transcript chunks to moderate" },
+      { url: "transcript:0", sexual: 0, violence: 0, hate: 0, selfHarm: 0, drugs: 0, error: true, errorMessage: "No transcript chunks to moderate" },
     ];
   }
   const targets = chunks.map((chunk, idx) => ({
@@ -517,12 +559,18 @@ async function moderateImageWithHive(entry: {
 
     const sexual = getHiveCategoryScores(classes, HIVE_SEXUAL_CATEGORIES);
     const violence = getHiveCategoryScores(classes, HIVE_VIOLENCE_CATEGORIES);
+    const hate = getHiveCategoryScores(classes, HIVE_HATE_CATEGORIES);
+    const selfHarm = getHiveCategoryScores(classes, HIVE_SELF_HARM_CATEGORIES);
+    const drugs = getHiveCategoryScores(classes, HIVE_ILLICIT_CATEGORIES);
 
     return {
       url: entry.url,
       time: entry.time,
       sexual,
       violence,
+      hate,
+      selfHarm,
+      drugs,
       error: false,
     };
   } catch (error) {
@@ -531,6 +579,9 @@ async function moderateImageWithHive(entry: {
       time: entry.time,
       sexual: 0,
       violence: 0,
+      hate: 0,
+      selfHarm: 0,
+      drugs: 0,
       error: true,
       errorMessage: error instanceof Error ? error.message : String(error),
     };
@@ -747,6 +798,9 @@ export async function getModerationScores(
   // Find highest scores across all thumbnails
   const maxSexual = Math.max(...successful.map(s => s.sexual));
   const maxViolence = Math.max(...successful.map(s => s.violence));
+  const maxHate = Math.max(...successful.map(s => s.hate));
+  const maxSelfHarm = Math.max(...successful.map(s => s.selfHarm));
+  const maxIllicit = Math.max(...successful.map(s => s.drugs));
 
   const finalThresholds = { ...DEFAULT_THRESHOLDS, ...thresholds };
   const requestedSampleCount = thumbnailScores.length;
@@ -780,8 +834,16 @@ export async function getModerationScores(
     maxScores: {
       sexual: maxSexual,
       violence: maxViolence,
+      hate: maxHate,
+      selfHarm: maxSelfHarm,
+      drugs: maxIllicit,
     },
-    exceedsThreshold: maxSexual > finalThresholds.sexual || maxViolence > finalThresholds.violence,
+    exceedsThreshold:
+      maxSexual > finalThresholds.sexual ||
+      maxViolence > finalThresholds.violence ||
+      maxHate > finalThresholds.hate ||
+      maxSelfHarm > finalThresholds.selfHarm ||
+      maxIllicit > finalThresholds.drugs,
     thresholds: finalThresholds,
   };
 }

tests/eval/moderation.eval.ts

@@ -356,6 +356,15 @@ evalite("Moderation", {
           typeof output.maxScores.violence === "number" &&
           output.maxScores.violence >= 0 &&
           output.maxScores.violence <= 1,
+          typeof output.maxScores.hate === "number" &&
+          output.maxScores.hate >= 0 &&
+          output.maxScores.hate <= 1,
+          typeof output.maxScores.selfHarm === "number" &&
+          output.maxScores.selfHarm >= 0 &&
+          output.maxScores.selfHarm <= 1,
+          typeof output.maxScores.drugs === "number" &&
+          output.maxScores.drugs >= 0 &&
+          output.maxScores.drugs <= 1,
           // exceedsThreshold is boolean
           typeof output.exceedsThreshold === "boolean",
           // thresholds contain valid numbers in 0-1 range
@@ -365,6 +374,15 @@ evalite("Moderation", {
           typeof output.thresholds.violence === "number" &&
           output.thresholds.violence >= 0 &&
           output.thresholds.violence <= 1,
+          typeof output.thresholds.hate === "number" &&
+          output.thresholds.hate >= 0 &&
+          output.thresholds.hate <= 1,
+          typeof output.thresholds.selfHarm === "number" &&
+          output.thresholds.selfHarm >= 0 &&
+          output.thresholds.selfHarm <= 1,
+          typeof output.thresholds.drugs === "number" &&
+          output.thresholds.drugs >= 0 &&
+          output.thresholds.drugs <= 1,
         ];
 
         const passed = checks.filter(Boolean).length;
@@ -485,6 +503,9 @@ evalite("Moderation", {
       { label: "Exceeds Threshold", value: String(output.exceedsThreshold) },
       { label: "Max Sexual", value: output.maxScores.sexual.toFixed(4) },
       { label: "Max Violence", value: output.maxScores.violence.toFixed(4) },
+      { label: "Max Hate", value: output.maxScores.hate.toFixed(4) },
+      { label: "Max Self-Harm", value: output.maxScores.selfHarm.toFixed(4) },
+      { label: "Max Drugs", value: output.maxScores.drugs.toFixed(4) },
       { label: "Samples", value: output.thumbnailScores.length },
       { label: "Latency", value: `${Math.round(output.latencyMs)}ms` },
       { label: "Duration", value: output.usage?.metadata?.assetDurationSeconds ? `${output.usage.metadata.assetDurationSeconds}s` : "n/a" },

tests/unit/moderation.test.ts

@@ -1,6 +1,12 @@
 import { describe, expect, it } from "vitest";
 
-import { HIVE_SEXUAL_CATEGORIES, HIVE_VIOLENCE_CATEGORIES } from "../../src/workflows/moderation";
+import {
+  HIVE_HATE_CATEGORIES,
+  HIVE_ILLICIT_CATEGORIES,
+  HIVE_SELF_HARM_CATEGORIES,
+  HIVE_SEXUAL_CATEGORIES,
+  HIVE_VIOLENCE_CATEGORIES,
+} from "../../src/workflows/moderation";
 
 describe("hive moderation categories", () => {
   it("the HIVE_SEXUAL_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => {
@@ -23,9 +29,30 @@ describe("hive moderation categories", () => {
       "hanging",
       "noose",
       "human_corpse",
-      "yes_emaciated_body",
+    ]);
+  });
+
+  it("the HIVE_HATE_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => {
+    expect(HIVE_HATE_CATEGORIES).toEqual([
+      "yes_nazi",
+      "yes_terrorist",
+      "yes_kkk",
+      "yes_confederate",
+    ]);
+  });
+
+  it("the HIVE_SELF_HARM_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => {
+    expect(HIVE_SELF_HARM_CATEGORIES).toEqual([
       "yes_self_harm",
-      "garm_death_injury_or_military_conflict",
+      "yes_emaciated_body",
+    ]);
+  });
+
+  it("the HIVE_ILLICIT_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => {
+    expect(HIVE_ILLICIT_CATEGORIES).toEqual([
+      "yes_pills",
+      "illicit_injectables",
+      "yes_marijuana",
     ]);
   });
 });