From 89163220ae59f6d013d3129088a246b8ef5f54da Mon Sep 17 00:00:00 2001 From: bashybaranaba Date: Tue, 1 Sep 2026 18:50:27 +0300 Subject: [PATCH 1/2] Add unified Canvas media studio and provider billing --- apps/commons-api/.env.example | 12 + .../versioned/032_canvas_media_studio.sql | 112 ++ apps/commons-api/models/schema.ts | 210 +++ apps/commons-api/package.json | 1 + apps/commons-api/src/agent/agent.service.ts | 4 + apps/commons-api/src/app.module.ts | 2 + apps/commons-api/src/files/files.service.ts | 50 +- .../src/media/canvas.controller.ts | 95 ++ apps/commons-api/src/media/canvas.service.ts | 411 +++++ apps/commons-api/src/media/index.ts | 5 + .../src/media/media-model.registry.spec.ts | 84 + .../src/media/media-model.registry.ts | 220 +++ apps/commons-api/src/media/media.module.ts | 24 + apps/commons-api/src/media/media.service.ts | 541 +++++++ apps/commons-api/src/media/media.types.ts | 105 ++ .../providers/byteplus-media.provider.ts | 137 ++ .../media/providers/google-media.provider.ts | 195 +++ .../media/providers/kling-media.provider.ts | 159 ++ .../modules/model-provider/model-registry.ts | 22 +- .../providers/google.provider.ts | 3 +- .../src/modules/usage/usage.service.ts | 19 +- apps/commons-api/src/tool/tool.module.ts | 2 + .../src/tool/tools/common-tool.service.ts | 140 ++ .../app/api/canvas/[...path]/route.ts | 28 + .../app/studio/canvas/[artifactId]/page.tsx | 10 + .../components/artifacts/artifact-surface.tsx | 9 + .../components/canvas/canvas-studio.tsx | 1381 +++++++++++++++++ apps/commons-app/lib/canvas.ts | 179 +++ buildspec.aws.yml | 8 + docs/architecture/canvas-media-studio.md | 129 ++ infra/aws/ecs-express-service.yml | 61 + infra/aws/runtime-secret-keys.txt | 4 + package.json | 1 + pnpm-lock.yaml | 108 +- 34 files changed, 4460 insertions(+), 11 deletions(-) create mode 100644 apps/commons-api/migrations/versioned/032_canvas_media_studio.sql create mode 100644 apps/commons-api/src/media/canvas.controller.ts create mode 100644 apps/commons-api/src/media/canvas.service.ts create mode 100644 apps/commons-api/src/media/index.ts create mode 100644 apps/commons-api/src/media/media-model.registry.spec.ts create mode 100644 apps/commons-api/src/media/media-model.registry.ts create mode 100644 apps/commons-api/src/media/media.module.ts create mode 100644 apps/commons-api/src/media/media.service.ts create mode 100644 apps/commons-api/src/media/media.types.ts create mode 100644 apps/commons-api/src/media/providers/byteplus-media.provider.ts create mode 100644 apps/commons-api/src/media/providers/google-media.provider.ts create mode 100644 apps/commons-api/src/media/providers/kling-media.provider.ts create mode 100644 apps/commons-app/app/api/canvas/[...path]/route.ts create mode 100644 apps/commons-app/app/studio/canvas/[artifactId]/page.tsx create mode 100644 apps/commons-app/components/canvas/canvas-studio.tsx create mode 100644 apps/commons-app/lib/canvas.ts create mode 100644 docs/architecture/canvas-media-studio.md diff --git a/apps/commons-api/.env.example b/apps/commons-api/.env.example index 405b075a..f18548a9 100644 --- a/apps/commons-api/.env.example +++ b/apps/commons-api/.env.example @@ -10,6 +10,18 @@ POSTGRES_PASSWORD="" WALLET_PRIVATE_KEY="" OPENAI_API_KEY="" +# Unified Canvas providers. Model availability is derived from these keys at +# runtime; missing providers remain visible in the catalog but disabled. +GOOGLE_API_KEY="" +KLING_ACCESS_KEY="" +KLING_SECRET_KEY="" +BYTEPLUS_ARK_API_KEY="" +# Optional JSON maps keyed by Commons modelKey (preferred) or provider modelId. +# Required for any catalog entry whose public provider tariff is unavailable. +GOOGLE_MEDIA_PRICE_USD_JSON="" +KLING_MEDIA_PRICE_USD_JSON="" +BYTEPLUS_MEDIA_PRICE_USD_JSON="" +MEDIA_RESERVATION_TTL_SECONDS="7200" # Optional Knowledge Space retrieval overrides. Lexical + graph search remains # available when embeddings are disabled or the OpenAI key is absent. BRAIN_EMBEDDING_MODEL="text-embedding-3-small" diff --git a/apps/commons-api/migrations/versioned/032_canvas_media_studio.sql b/apps/commons-api/migrations/versioned/032_canvas_media_studio.sql new file mode 100644 index 00000000..80bdcc04 --- /dev/null +++ b/apps/commons-api/migrations/versioned/032_canvas_media_studio.sql @@ -0,0 +1,112 @@ +CREATE TABLE IF NOT EXISTS canvas_project ( + project_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + owner_user_id text NOT NULL, + workspace_id text, + name text NOT NULL, + description text, + root_item_id uuid NOT NULL REFERENCES library_item(item_id) ON DELETE RESTRICT, + active_item_id uuid NOT NULL REFERENCES library_item(item_id) ON DELETE RESTRICT, + settings jsonb DEFAULT '{}'::jsonb, + status text NOT NULL DEFAULT 'active', + deleted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + updated_at timestamptz NOT NULL DEFAULT timezone('utc', now()) +); + +CREATE INDEX IF NOT EXISTS idx_canvas_project_owner_updated + ON canvas_project(owner_user_id, updated_at); +CREATE INDEX IF NOT EXISTS idx_canvas_project_root + ON canvas_project(owner_user_id, root_item_id); + +CREATE TABLE IF NOT EXISTS canvas_revision ( + revision_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL REFERENCES canvas_project(project_id) ON DELETE CASCADE, + item_id uuid NOT NULL REFERENCES library_item(item_id) ON DELETE RESTRICT, + parent_revision_id uuid REFERENCES canvas_revision(revision_id) ON DELETE SET NULL, + operation text NOT NULL DEFAULT 'import', + provider text, + model_id text, + prompt_hash text, + inputs jsonb DEFAULT '[]'::jsonb, + settings jsonb DEFAULT '{}'::jsonb, + trace_id uuid, + created_by_type text NOT NULL DEFAULT 'human', + created_by_id text, + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()) +); + +CREATE INDEX IF NOT EXISTS idx_canvas_revision_project_created + ON canvas_revision(project_id, created_at); +CREATE UNIQUE INDEX IF NOT EXISTS uq_canvas_revision_project_item + ON canvas_revision(project_id, item_id); + +CREATE TABLE IF NOT EXISTS canvas_annotation ( + annotation_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid NOT NULL REFERENCES canvas_project(project_id) ON DELETE CASCADE, + revision_id uuid NOT NULL REFERENCES canvas_revision(revision_id) ON DELETE CASCADE, + parent_annotation_id uuid REFERENCES canvas_annotation(annotation_id) ON DELETE CASCADE, + kind text NOT NULL, + body text NOT NULL, + geometry jsonb, + start_ms integer, + end_ms integer, + status text NOT NULL DEFAULT 'open', + author_type text NOT NULL DEFAULT 'human', + author_id text NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb, + deleted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + updated_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + CONSTRAINT canvas_annotation_time_range_check CHECK ( + (start_ms IS NULL OR start_ms >= 0) AND + (end_ms IS NULL OR end_ms >= coalesce(start_ms, 0)) + ) +); + +CREATE INDEX IF NOT EXISTS idx_canvas_annotation_revision + ON canvas_annotation(project_id, revision_id, created_at); + +CREATE TABLE IF NOT EXISTS media_generation_job ( + job_id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + project_id uuid REFERENCES canvas_project(project_id) ON DELETE SET NULL, + owner_user_id text NOT NULL, + workspace_id text, + agent_id text REFERENCES agent(agent_id) ON DELETE SET NULL, + session_id uuid REFERENCES session(session_id) ON DELETE SET NULL, + trace_id uuid, + provider text NOT NULL, + model_id text NOT NULL, + media_kind text NOT NULL, + operation text NOT NULL, + prompt text NOT NULL, + input_item_ids jsonb DEFAULT '[]'::jsonb, + request jsonb DEFAULT '{}'::jsonb, + status text NOT NULL DEFAULT 'queued', + progress integer NOT NULL DEFAULT 0, + provider_operation_id text, + output_item_id uuid REFERENCES library_item(item_id) ON DELETE SET NULL, + error_code text, + error_message text, + estimated_cost_usd real, + actual_cost_usd real, + billing jsonb DEFAULT '{}'::jsonb, + started_at timestamptz, + completed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + updated_at timestamptz NOT NULL DEFAULT timezone('utc', now()) +); + +CREATE INDEX IF NOT EXISTS idx_media_job_owner_created + ON media_generation_job(owner_user_id, created_at); +CREATE INDEX IF NOT EXISTS idx_media_job_project_created + ON media_generation_job(project_id, created_at); +CREATE INDEX IF NOT EXISTS idx_media_job_queue + ON media_generation_job(status, created_at); + +-- Canvas links group existing private Library items without changing access. +ALTER TABLE library_link DROP CONSTRAINT IF EXISTS library_link_scope_type_check; +ALTER TABLE library_link ADD CONSTRAINT library_link_scope_type_check + CHECK (scope_type IN ( + 'session', 'code_project', 'agent', 'provenance_trace', + 'task', 'workflow', 'cli_run', 'sdk_run', 'canvas_project' + )); diff --git a/apps/commons-api/models/schema.ts b/apps/commons-api/models/schema.ts index 76e29dd1..a73e2708 100644 --- a/apps/commons-api/models/schema.ts +++ b/apps/commons-api/models/schema.ts @@ -918,6 +918,216 @@ export const libraryShareLink = pgTable( }), ); +/* ──────────────────────── CANVAS MEDIA STUDIO ──────────────────────── */ + +/** + * A private creative workspace. Canvas does not duplicate media bytes: every + * source and output remains a Library item while this table supplies project + * organization and an active revision pointer. + */ +export const canvasProject = pgTable( + 'canvas_project', + { + projectId: uuid('project_id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + ownerUserId: text('owner_user_id').notNull(), + workspaceId: text('workspace_id'), + name: text('name').notNull(), + description: text('description'), + rootItemId: uuid('root_item_id') + .notNull() + .references(() => libraryItem.itemId, { onDelete: 'restrict' }), + activeItemId: uuid('active_item_id') + .notNull() + .references(() => libraryItem.itemId, { onDelete: 'restrict' }), + settings: jsonb('settings').$type>().default({}), + status: text('status').notNull().default('active'), + deletedAt: timestamp('deleted_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + ownerUpdatedIdx: index('idx_canvas_project_owner_updated').on( + table.ownerUserId, + table.updatedAt, + ), + rootIdx: index('idx_canvas_project_root').on( + table.ownerUserId, + table.rootItemId, + ), + }), +); + +/** + * Immutable revision graph. A revision can have many input artifacts in + * `inputs`, while `parentRevisionId` supplies the primary history branch. + */ +export const canvasRevision = pgTable( + 'canvas_revision', + { + revisionId: uuid('revision_id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + projectId: uuid('project_id') + .notNull() + .references(() => canvasProject.projectId, { onDelete: 'cascade' }), + itemId: uuid('item_id') + .notNull() + .references(() => libraryItem.itemId, { onDelete: 'restrict' }), + parentRevisionId: uuid('parent_revision_id'), + operation: text('operation').notNull().default('import'), + provider: text('provider'), + modelId: text('model_id'), + promptHash: text('prompt_hash'), + inputs: jsonb('inputs').$type< + Array<{ itemId: string; role?: string; revisionId?: string }> + >().default([]), + settings: jsonb('settings').$type>().default({}), + // Soft reference because ProvenanceService persists runs asynchronously. + traceId: uuid('trace_id'), + createdByType: text('created_by_type').notNull().default('human'), + createdById: text('created_by_id'), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + projectCreatedIdx: index('idx_canvas_revision_project_created').on( + table.projectId, + table.createdAt, + ), + projectItemUnique: uniqueIndex('uq_canvas_revision_project_item').on( + table.projectId, + table.itemId, + ), + parentFk: foreignKey({ + columns: [table.parentRevisionId], + foreignColumns: [table.revisionId], + name: 'canvas_revision_parent_revision_fk', + }).onDelete('set null'), + }), +); + +/** + * Resolution-independent collaboration context. Geometry uses normalized + * coordinates (0..1); temporal ranges use integer milliseconds. + */ +export const canvasAnnotation = pgTable( + 'canvas_annotation', + { + annotationId: uuid('annotation_id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + projectId: uuid('project_id') + .notNull() + .references(() => canvasProject.projectId, { onDelete: 'cascade' }), + revisionId: uuid('revision_id') + .notNull() + .references(() => canvasRevision.revisionId, { onDelete: 'cascade' }), + parentAnnotationId: uuid('parent_annotation_id'), + kind: text('kind').notNull(), + body: text('body').notNull(), + geometry: jsonb('geometry').$type>(), + startMs: integer('start_ms'), + endMs: integer('end_ms'), + status: text('status').notNull().default('open'), + authorType: text('author_type').notNull().default('human'), + authorId: text('author_id').notNull(), + metadata: jsonb('metadata').$type>().default({}), + deletedAt: timestamp('deleted_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + projectRevisionIdx: index('idx_canvas_annotation_revision').on( + table.projectId, + table.revisionId, + table.createdAt, + ), + parentFk: foreignKey({ + columns: [table.parentAnnotationId], + foreignColumns: [table.annotationId], + name: 'canvas_annotation_parent_annotation_fk', + }).onDelete('cascade'), + timeRangeCheck: check( + 'canvas_annotation_time_range_check', + sql`(${table.startMs} is null or ${table.startMs} >= 0) and (${table.endMs} is null or ${table.endMs} >= coalesce(${table.startMs}, 0))`, + ), + }), +); + +/** Durable queue and audit envelope for user- or agent-initiated media work. */ +export const mediaGenerationJob = pgTable( + 'media_generation_job', + { + jobId: uuid('job_id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + projectId: uuid('project_id').references(() => canvasProject.projectId, { + onDelete: 'set null', + }), + ownerUserId: text('owner_user_id').notNull(), + workspaceId: text('workspace_id'), + agentId: text('agent_id').references(() => agent.agentId, { + onDelete: 'set null', + }), + sessionId: uuid('session_id').references(() => session.sessionId, { + onDelete: 'set null', + }), + // Soft reference because ProvenanceService persists runs asynchronously. + traceId: uuid('trace_id'), + provider: text('provider').notNull(), + modelId: text('model_id').notNull(), + mediaKind: text('media_kind').notNull(), + operation: text('operation').notNull(), + prompt: text('prompt').notNull(), + inputItemIds: jsonb('input_item_ids').$type().default([]), + request: jsonb('request').$type>().default({}), + status: text('status').notNull().default('queued'), + progress: integer('progress').notNull().default(0), + providerOperationId: text('provider_operation_id'), + outputItemId: uuid('output_item_id').references(() => libraryItem.itemId, { + onDelete: 'set null', + }), + errorCode: text('error_code'), + errorMessage: text('error_message'), + estimatedCostUsd: real('estimated_cost_usd'), + actualCostUsd: real('actual_cost_usd'), + billing: jsonb('billing').$type>().default({}), + startedAt: timestamp('started_at', { withTimezone: true }), + completedAt: timestamp('completed_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + ownerCreatedIdx: index('idx_media_job_owner_created').on( + table.ownerUserId, + table.createdAt, + ), + projectCreatedIdx: index('idx_media_job_project_created').on( + table.projectId, + table.createdAt, + ), + queueIdx: index('idx_media_job_queue').on( + table.status, + table.createdAt, + ), + }), +); + export const libraryAuditEvent = pgTable( 'library_audit_event', { diff --git a/apps/commons-api/package.json b/apps/commons-api/package.json index 6283b947..ccc888c4 100644 --- a/apps/commons-api/package.json +++ b/apps/commons-api/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@fontsource-variable/space-grotesk": "^5.3.0", + "@google/genai": "^2.20.0", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-dropdown-menu": "^2.1.15", "@radix-ui/react-select": "^2.2.6", diff --git a/apps/commons-api/src/agent/agent.service.ts b/apps/commons-api/src/agent/agent.service.ts index a764a4ce..5418aae3 100644 --- a/apps/commons-api/src/agent/agent.service.ts +++ b/apps/commons-api/src/agent/agent.service.ts @@ -499,6 +499,10 @@ export class AgentService implements OnModuleInit { - **searchLibraryArtifacts** — on-demand access to the calling user's Library. Omit query to list recent files, or search once by filename, type, or topic. Results are deliberately compact; do not probe with many generic queries. Use the returned fileId with readUploadedFile only when exact contents are needed. - **readUploadedFile** — read a selected library artifact in bounded chunks after permission checks. - **generateImage** — create an image with the stable GPT Image API. The result is stored in the owner's artifact library using their storage preference (Private S3 by default). + - **listMediaModels** — inspect exact creative model keys, provider availability, supported controls, and price basis before choosing a model. + - **getCanvasProject** — load a Canvas project's active artifact, revision history, annotations, and generation state before analysing or changing it. Read the active artifact with readUploadedFile when its actual media contents are needed. + - **annotateCanvas** — add precise normalized spatial notes or millisecond time-range notes that users and other agents can inspect and drag into chat. + - **generateMedia** — generate or transform images, video, speech, and music with a listed creative modelKey. Attach input Library item IDs for edits, and pass a Canvas project ID to create a recoverable revision with provenance. Media usage is pre-authorized in Commons credits and settled once from actual provider usage when available. - **uploadFileToIPFS** — explicit-only public IPFS publishing. Never call this for ordinary uploads, generated files, or library storage unless the user specifically asks for IPFS. ### Uploaded files diff --git a/apps/commons-api/src/app.module.ts b/apps/commons-api/src/app.module.ts index 3e3d6167..efad08b4 100644 --- a/apps/commons-api/src/app.module.ts +++ b/apps/commons-api/src/app.module.ts @@ -34,6 +34,7 @@ import { CapabilityProviderModule } from './provider'; import { UiPluginModule } from './ui-plugin'; import { ProvenanceModule } from './provenance'; import { BrainModule } from './brain'; +import { MediaModule } from './media'; @Module({ imports: [ @@ -45,6 +46,7 @@ import { BrainModule } from './brain'; PinataModule, ProvenanceModule, BrainModule, + MediaModule, // Feature modules AgentModule, diff --git a/apps/commons-api/src/files/files.service.ts b/apps/commons-api/src/files/files.service.ts index f58a4662..e43f5201 100644 --- a/apps/commons-api/src/files/files.service.ts +++ b/apps/commons-api/src/files/files.service.ts @@ -246,9 +246,11 @@ export class FilesService { buffer: Buffer; fileName: string; mimeType: string; - agentId: string; + agentId?: string; sessionId?: string; traceId?: string; + ownerId?: string; + workspaceId?: string | null; metadata?: Record; }) { return this.persistFile({ @@ -258,13 +260,55 @@ export class FilesService { agentId: input.agentId, sessionId: input.sessionId, traceId: input.traceId, - ownerId: input.agentId, - ownerType: 'agent', + ownerId: input.ownerId ?? input.agentId, + ownerType: input.ownerId ? 'user' : 'agent', + workspaceId: input.workspaceId, source: 'agent_generated', metadata: input.metadata, }); } + /** + * Internal media-processing access. The caller supplies the authenticated + * principal; this method reuses the same Library authorization path as agent + * reads and never returns a public URL. + */ + async loadOriginalForProcessing(input: { + fileId: string; + ownerId?: string; + workspaceId?: string | null; + agentId?: string; + sessionId?: string; + }) { + const file = await this.getFileOrThrow(input.fileId); + await this.assertCanAccess(file, { + ownerId: input.ownerId, + workspaceId: input.workspaceId ?? undefined, + agentId: input.agentId, + sessionId: input.sessionId, + }); + const original = (await this.getBlobs(file.itemId)).find( + (blob) => blob.role === 'original', + ); + if (!original) { + throw new NotFoundException(`The original bytes for ${file.name} are unavailable`); + } + return { + itemId: file.itemId, + name: file.name, + mimeType: file.mimeType, + kind: file.kind, + // The provider receives this only after the authenticated user selects + // the file as a generation input. It expires using the normal Library + // signed-URL policy and avoids embedding large private media in JSON. + url: await this.createSignedUrl( + original.storageBucket, + original.storagePath, + ), + buffer: await this.downloadBlobBuffer(original), + }; + } + async createSpreadsheetFile(input: { fileName: string; sheets: Array<{ diff --git a/apps/commons-api/src/media/canvas.controller.ts b/apps/commons-api/src/media/canvas.controller.ts new file mode 100644 index 00000000..6eacc14d --- /dev/null +++ b/apps/commons-api/src/media/canvas.controller.ts @@ -0,0 +1,95 @@ +import { Body, Controller, Get, Param, Patch, Post, Req } from '@nestjs/common'; +import type { Request } from 'express'; +import { RateLimit, resolveCallerId, type ApiKeyPrincipal } from '~/modules/auth'; +import { CanvasService } from './canvas.service'; +import { MediaService } from './media.service'; +import type { CreateMediaGenerationInput, MediaPrincipal } from './media.types'; + +@Controller({ version: '1', path: 'canvas' }) +export class CanvasController { + constructor( + private readonly canvas: CanvasService, + private readonly media: MediaService, + ) {} + + @Get('models') + models() { + return this.media.catalog(); + } + + @Post('projects/open') + open(@Req() request: Request, @Body() body: { artifactId: string }) { + return this.canvas.openArtifact(body.artifactId, requester(request)); + } + + @Get('projects/:projectId') + get(@Req() request: Request, @Param('projectId') projectId: string) { + return this.canvas.getProject(projectId, requester(request)); + } + + @Patch('projects/:projectId') + update( + @Req() request: Request, + @Param('projectId') projectId: string, + @Body() + body: { + name?: string; + description?: string; + activeRevisionId?: string; + settings?: Record; + }, + ) { + return this.canvas.updateProject(projectId, requester(request), body); + } + + @Post('projects/:projectId/annotations') + annotation( + @Req() request: Request, + @Param('projectId') projectId: string, + @Body() body: Parameters[2], + ) { + return this.canvas.createAnnotation(projectId, requester(request), body); + } + + @Patch('projects/:projectId/annotations/:annotationId') + updateAnnotation( + @Req() request: Request, + @Param('projectId') projectId: string, + @Param('annotationId') annotationId: string, + @Body() body: Parameters[3], + ) { + return this.canvas.updateAnnotation( + projectId, + annotationId, + requester(request), + body, + ); + } + + @Post('generations') + @RateLimit({ limit: 20, windowMs: 60_000, keyStrategy: 'user' }) + generate(@Req() request: Request, @Body() body: CreateMediaGenerationInput) { + return this.media.createGeneration(body, requester(request)); + } + + @Post('quote') + quote(@Req() request: Request, @Body() body: CreateMediaGenerationInput) { + return this.media.quote(body, requester(request)); + } + + @Get('generations/:jobId') + getGeneration(@Req() request: Request, @Param('jobId') jobId: string) { + return this.media.getGeneration(jobId, requester(request)); + } +} + +function requester(request: Request): MediaPrincipal { + const principal = (request as any).principal as ApiKeyPrincipal | undefined; + const principalId = resolveCallerId(request); + if (!principalId) throw new Error('Authenticated principal required'); + return { + principalId, + principalType: principal?.principalType ?? 'user', + workspaceId: principal?.workspaceId ?? null, + }; +} diff --git a/apps/commons-api/src/media/canvas.service.ts b/apps/commons-api/src/media/canvas.service.ts new file mode 100644 index 00000000..b9237ed7 --- /dev/null +++ b/apps/commons-api/src/media/canvas.service.ts @@ -0,0 +1,411 @@ +import { + BadRequestException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { and, asc, desc, eq, isNull, or, sql } from 'drizzle-orm'; +import { createHash } from 'node:crypto'; +import * as schema from '#/models/schema'; +import { DatabaseService } from '~/modules/database/database.service'; +import { LibraryService } from '~/files'; +import type { LibraryPrincipal } from '~/files/library.service'; +import type { MediaPrincipal } from './media.types'; + +type CreateAnnotationInput = { + revisionId: string; + parentAnnotationId?: string; + kind: 'comment' | 'point' | 'region' | 'time_range' | 'transcript' | 'freehand'; + body: string; + geometry?: Record; + startMs?: number; + endMs?: number; + metadata?: Record; +}; + +@Injectable() +export class CanvasService { + constructor( + private readonly db: DatabaseService, + private readonly library: LibraryService, + ) {} + + async openArtifact(artifactId: string, principal: MediaPrincipal) { + const libraryPrincipal = asLibraryPrincipal(principal); + const artifact = await this.library.get(artifactId, libraryPrincipal); + const existingRevision = await this.db.query.canvasRevision.findFirst({ + where: (table) => eq(table.itemId, artifactId), + orderBy: (table) => desc(table.createdAt), + }); + if (existingRevision) { + return this.getProject(existingRevision.projectId, principal); + } + const existingProject = await this.db.query.canvasProject.findFirst({ + where: (table) => + and( + eq(table.rootItemId, artifactId), + isNull(table.deletedAt), + sql`lower(${table.ownerUserId}) = lower(${principal.principalId})`, + ), + }); + if (existingProject) return this.getProject(existingProject.projectId, principal); + + const [project] = await this.db + .insert(schema.canvasProject) + .values({ + ownerUserId: principal.principalId, + workspaceId: principal.workspaceId ?? null, + name: artifact.name, + description: artifact.description, + rootItemId: artifactId, + activeItemId: artifactId, + settings: { schemaVersion: 1 }, + }) + .returning(); + await Promise.all([ + this.db.insert(schema.canvasRevision).values({ + projectId: project.projectId, + itemId: artifactId, + operation: 'import', + inputs: [], + createdByType: principal.principalType === 'agent' ? 'agent' : 'human', + createdById: principal.principalId, + }), + this.db + .insert(schema.libraryLink) + .values({ + itemId: artifactId, + scopeType: 'canvas_project', + scopeId: project.projectId, + }) + .onConflictDoNothing(), + ]); + return this.getProject(project.projectId, principal); + } + + async getProject(projectId: string, principal: MediaPrincipal) { + const project = await this.requireProject(projectId, principal, 'read'); + const [revisions, annotations, jobs] = await Promise.all([ + this.db.query.canvasRevision.findMany({ + where: (table) => eq(table.projectId, projectId), + orderBy: (table) => desc(table.createdAt), + }), + this.db.query.canvasAnnotation.findMany({ + where: (table) => + and(eq(table.projectId, projectId), isNull(table.deletedAt)), + orderBy: (table) => asc(table.createdAt), + }), + this.db.query.mediaGenerationJob.findMany({ + where: (table) => eq(table.projectId, projectId), + orderBy: (table) => desc(table.createdAt), + limit: 20, + }), + ]); + const itemIds = [...new Set(revisions.map((revision) => revision.itemId))]; + const items = itemIds.length + ? await this.db.query.libraryItem.findMany({ + where: (table) => + or(...itemIds.map((itemId) => eq(table.itemId, itemId))), + }) + : []; + const itemMap = new Map(items.map((item) => [item.itemId, item])); + return { + project, + revisions: revisions.map((revision) => ({ + ...revision, + artifact: publicArtifact(itemMap.get(revision.itemId)), + })), + annotations, + jobs: jobs.map(publicJob), + }; + } + + async updateProject( + projectId: string, + principal: MediaPrincipal, + input: { name?: string; description?: string; activeRevisionId?: string; settings?: Record }, + ) { + const project = await this.requireProject(projectId, principal, 'edit'); + let activeItemId = project.activeItemId; + if (input.activeRevisionId) { + const revision = await this.db.query.canvasRevision.findFirst({ + where: (table) => + and( + eq(table.revisionId, input.activeRevisionId!), + eq(table.projectId, projectId), + ), + }); + if (!revision) throw new BadRequestException('Revision does not belong to this project.'); + activeItemId = revision.itemId; + } + const [saved] = await this.db + .update(schema.canvasProject) + .set({ + ...(input.name !== undefined ? { name: cleanName(input.name) } : {}), + ...(input.description !== undefined + ? { description: input.description.trim().slice(0, 2_000) || null } + : {}), + ...(input.settings ? { settings: input.settings } : {}), + activeItemId, + updatedAt: new Date(), + }) + .where(eq(schema.canvasProject.projectId, projectId)) + .returning(); + return saved; + } + + async addRevision(input: { + projectId: string; + itemId: string; + parentItemId?: string; + operation: string; + provider?: string; + modelId?: string; + prompt?: string; + inputItemIds?: string[]; + settings?: Record; + traceId?: string; + createdByType: 'human' | 'agent' | 'service'; + createdById?: string; + }) { + const parent = input.parentItemId + ? await this.db.query.canvasRevision.findFirst({ + where: (table) => + and( + eq(table.projectId, input.projectId), + eq(table.itemId, input.parentItemId!), + ), + }) + : undefined; + const [revision] = await this.db + .insert(schema.canvasRevision) + .values({ + projectId: input.projectId, + itemId: input.itemId, + parentRevisionId: parent?.revisionId, + operation: input.operation, + provider: input.provider, + modelId: input.modelId, + promptHash: input.prompt ? sha256(input.prompt) : undefined, + inputs: (input.inputItemIds ?? []).map((itemId) => ({ itemId })), + settings: input.settings ?? {}, + traceId: input.traceId, + createdByType: input.createdByType, + createdById: input.createdById, + }) + .onConflictDoNothing() + .returning(); + await Promise.all([ + this.db + .update(schema.canvasProject) + .set({ activeItemId: input.itemId, updatedAt: new Date() }) + .where(eq(schema.canvasProject.projectId, input.projectId)), + this.db + .insert(schema.libraryLink) + .values({ + itemId: input.itemId, + scopeType: 'canvas_project', + scopeId: input.projectId, + }) + .onConflictDoNothing(), + ]); + return revision; + } + + async createAnnotation( + projectId: string, + principal: MediaPrincipal, + input: CreateAnnotationInput, + ) { + await this.requireProject(projectId, principal, 'edit'); + const revision = await this.db.query.canvasRevision.findFirst({ + where: (table) => + and( + eq(table.projectId, projectId), + eq(table.revisionId, input.revisionId), + ), + }); + if (!revision) throw new BadRequestException('Revision does not belong to this project.'); + const body = input.body?.trim(); + if (!body) throw new BadRequestException('Annotation text is required.'); + if (body.length > 8_000) throw new BadRequestException('Annotation is too long.'); + validateAnnotation(input); + if (input.parentAnnotationId) { + const parent = await this.db.query.canvasAnnotation.findFirst({ + where: (table) => + and( + eq(table.projectId, projectId), + eq(table.annotationId, input.parentAnnotationId!), + ), + }); + if (!parent) throw new BadRequestException('Parent annotation not found.'); + } + const [annotation] = await this.db + .insert(schema.canvasAnnotation) + .values({ + projectId, + revisionId: input.revisionId, + parentAnnotationId: input.parentAnnotationId, + kind: input.kind, + body, + geometry: input.geometry, + startMs: input.startMs, + endMs: input.endMs, + authorType: + principal.actorId || principal.principalType === 'agent' + ? 'agent' + : 'human', + authorId: principal.actorId ?? principal.principalId, + metadata: input.metadata ?? {}, + }) + .returning(); + return annotation; + } + + async updateAnnotation( + projectId: string, + annotationId: string, + principal: MediaPrincipal, + input: { body?: string; status?: 'open' | 'resolved'; deleted?: boolean }, + ) { + await this.requireProject(projectId, principal, 'edit'); + const annotation = await this.db.query.canvasAnnotation.findFirst({ + where: (table) => + and( + eq(table.projectId, projectId), + eq(table.annotationId, annotationId), + isNull(table.deletedAt), + ), + }); + if (!annotation) throw new NotFoundException('Annotation not found.'); + const [saved] = await this.db + .update(schema.canvasAnnotation) + .set({ + ...(input.body !== undefined + ? { body: input.body.trim().slice(0, 8_000) } + : {}), + ...(input.status ? { status: input.status } : {}), + ...(input.deleted ? { deletedAt: new Date() } : {}), + updatedAt: new Date(), + }) + .where(eq(schema.canvasAnnotation.annotationId, annotationId)) + .returning(); + return saved; + } + + async requireProject( + projectId: string, + principal: MediaPrincipal, + permission: 'read' | 'edit', + ) { + const project = await this.db.query.canvasProject.findFirst({ + where: (table) => + and(eq(table.projectId, projectId), isNull(table.deletedAt)), + }); + if (!project) throw new NotFoundException('Canvas project not found.'); + if (same(project.ownerUserId, principal.principalId)) return project; + if ( + project.workspaceId && + principal.workspaceId && + same(project.workspaceId, principal.workspaceId) + ) { + return project; + } + const grants = await this.db.query.libraryGrant.findMany({ + where: (table) => + and( + eq(table.itemId, project.rootItemId), + or( + and(eq(table.subjectType, 'user'), eq(table.subjectId, principal.principalId)), + principal.workspaceId + ? and( + eq(table.subjectType, 'workspace'), + eq(table.subjectId, principal.workspaceId), + ) + : undefined, + ), + or(isNull(table.expiresAt), sql`${table.expiresAt} > now()`), + ), + }); + const allowed = grants.some((grant) => + permission === 'read' + ? ['read', 'edit', 'manage'].includes(grant.permission) + : ['edit', 'manage'].includes(grant.permission), + ); + if (!allowed) throw new ForbiddenException('You do not have access to this Canvas project.'); + return project; + } +} + +function validateAnnotation(input: CreateAnnotationInput) { + if (input.startMs !== undefined && (!Number.isInteger(input.startMs) || input.startMs < 0)) { + throw new BadRequestException('startMs must be a positive integer.'); + } + if ( + input.endMs !== undefined && + (!Number.isInteger(input.endMs) || input.endMs < (input.startMs ?? 0)) + ) { + throw new BadRequestException('endMs must be after startMs.'); + } + if (input.kind === 'region' || input.kind === 'point' || input.kind === 'freehand') { + if (!input.geometry) throw new BadRequestException('Spatial annotations require geometry.'); + walkNumbers(input.geometry, (value) => { + if (value < 0 || value > 1) { + throw new BadRequestException('Annotation coordinates must be normalized from 0 to 1.'); + } + }); + } +} + +function walkNumbers(value: unknown, visit: (number: number) => void) { + if (typeof value === 'number') return visit(value); + if (Array.isArray(value)) return value.forEach((child) => walkNumbers(child, visit)); + if (value && typeof value === 'object') { + Object.values(value as Record).forEach((child) => + walkNumbers(child, visit), + ); + } +} + +function publicArtifact(item?: typeof schema.libraryItem.$inferSelect) { + if (!item) return undefined; + return { + itemId: item.itemId, + name: item.name, + description: item.description, + kind: item.kind, + mimeType: item.mimeType, + sizeBytes: item.sizeBytes, + source: item.source, + status: item.status, + metadata: item.metadata, + createdAt: item.createdAt, + }; +} + +function publicJob(job: typeof schema.mediaGenerationJob.$inferSelect) { + const { prompt: _prompt, request: _request, ...safe } = job; + return safe; +} + +function asLibraryPrincipal(principal: MediaPrincipal): LibraryPrincipal { + return { + principalId: principal.principalId, + principalType: principal.principalType, + workspaceId: principal.workspaceId, + }; +} + +function sha256(value: string) { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} + +function cleanName(value: string) { + const name = value.trim().slice(0, 160); + if (!name) throw new BadRequestException('Project name is required.'); + return name; +} + +function same(left?: string | null, right?: string | null) { + return Boolean(left && right && left.toLowerCase() === right.toLowerCase()); +} diff --git a/apps/commons-api/src/media/index.ts b/apps/commons-api/src/media/index.ts new file mode 100644 index 00000000..42418644 --- /dev/null +++ b/apps/commons-api/src/media/index.ts @@ -0,0 +1,5 @@ +export * from './media.module'; +export * from './media.service'; +export * from './canvas.service'; +export * from './media.types'; +export * from './media-model.registry'; diff --git a/apps/commons-api/src/media/media-model.registry.spec.ts b/apps/commons-api/src/media/media-model.registry.spec.ts new file mode 100644 index 00000000..b3f7d4b1 --- /dev/null +++ b/apps/commons-api/src/media/media-model.registry.spec.ts @@ -0,0 +1,84 @@ +import { + actualMediaCostFromUsage, + estimateMediaCost, + getMediaModel, + isMediaModelPriceConfigured, + MEDIA_MODEL_REGISTRY, + mediaUnitPrice, +} from './media-model.registry'; + +describe('media model registry', () => { + const originalKlingOverride = process.env.KLING_MEDIA_PRICE_USD_JSON; + + afterEach(() => { + if (originalKlingOverride === undefined) { + delete process.env.KLING_MEDIA_PRICE_USD_JSON; + } else { + process.env.KLING_MEDIA_PRICE_USD_JSON = originalKlingOverride; + } + }); + + it('uses stable model keys when an upstream model id spans media kinds', () => { + const image = getMediaModel('kling', 'kling:image:kling-v3-omni'); + const video = getMediaModel('kling', 'kling:video:kling-v3-omni'); + + expect(image.kind).toBe('image'); + expect(video.kind).toBe('video'); + expect(image.modelId).toBe(video.modelId); + expect(() => getMediaModel('kling', 'kling-v3-omni')).toThrow( + 'Unsupported or ambiguous', + ); + }); + + it('contains only unique Commons model keys', () => { + const keys = MEDIA_MODEL_REGISTRY.map((model) => model.modelKey); + expect(new Set(keys).size).toBe(keys.length); + }); + + it('prices Kling video by duration, resolution, audio, and video input', () => { + const model = getMediaModel('kling', 'kling:video:kling-v3-omni'); + + expect( + mediaUnitPrice( + model, + { resolution: '1080p', nativeAudio: false }, + ['video'], + ), + ).toBe(0.168); + expect( + estimateMediaCost( + model, + 'continue this shot', + { resolution: '1080p', nativeAudio: false, durationSeconds: 10 }, + ['video'], + ), + ).toBeCloseTo(1.68); + }); + + it('reconciles Seedance estimates against provider completion tokens', () => { + const model = getMediaModel( + 'byteplus', + 'byteplus:video:dreamina-seedance-2-0-260128', + ); + + expect( + actualMediaCostFromUsage( + model, + { completionTokens: 250_000, unitPriceUsd: 7 }, + 99, + ), + ).toBe(1.75); + }); + + it('keeps price-gated models unavailable until an explicit override exists', () => { + const model = getMediaModel('kling', 'kling:image:kling-image-o1'); + delete process.env.KLING_MEDIA_PRICE_USD_JSON; + expect(isMediaModelPriceConfigured(model)).toBe(false); + + process.env.KLING_MEDIA_PRICE_USD_JSON = JSON.stringify({ + [model.modelKey]: 0.08, + }); + expect(isMediaModelPriceConfigured(model)).toBe(true); + expect(estimateMediaCost(model, 'create an image', {}, [])).toBe(0.08); + }); +}); diff --git a/apps/commons-api/src/media/media-model.registry.ts b/apps/commons-api/src/media/media-model.registry.ts new file mode 100644 index 00000000..5a7819ea --- /dev/null +++ b/apps/commons-api/src/media/media-model.registry.ts @@ -0,0 +1,220 @@ +import { BadRequestException } from '@nestjs/common'; +import type { MediaModelDescriptor, MediaSettingField } from './media.types'; + +const GOOGLE_PRICING = 'https://ai.google.dev/gemini-api/docs/pricing'; +const KLING_PRICING = 'https://kling.ai/document-api/productBilling/billingMethod'; +const BYTEPLUS_PRICING = 'https://docs.byteplus.com/docs/ModelArk/1099320'; + +const options = (values: string[]) => values.map((value) => ({ label: value, value })); +const ASPECT_RATIOS = options(['auto', '1:1', '3:2', '2:3', '4:3', '3:4', '4:5', '5:4', '16:9', '9:16', '21:9']); +const aspect = (allowed = ASPECT_RATIOS): MediaSettingField => ({ key: 'aspectRatio', label: 'Aspect ratio', type: 'select', default: allowed[0]?.value ?? '1:1', options: allowed }); +const resolution = (values: string[], initial = values[0]): MediaSettingField => ({ key: 'resolution', label: 'Resolution', type: 'select', default: initial, options: options(values) }); +const duration = (values: number[], initial = values[0]): MediaSettingField => ({ key: 'durationSeconds', label: 'Duration', type: 'select', default: String(initial), options: values.map((value) => ({ label: `${value} seconds`, value: String(value) })) }); +const nativeAudio: MediaSettingField = { key: 'nativeAudio', label: 'Native audio', type: 'boolean', default: true, help: 'Generate synchronized audio with the video.' }; + +const GOOGLE_IMAGE_SETTINGS: MediaSettingField[] = [ + aspect(ASPECT_RATIOS.filter((value) => value.value !== 'auto')), + { key: 'imageSize', label: 'Resolution', type: 'select', default: '1K', options: options(['0.5K', '1K', '2K', '4K']) }, +]; +const GOOGLE_VIDEO_SETTINGS: MediaSettingField[] = [aspect(options(['16:9', '9:16'])), duration([4, 6, 8], 8), resolution(['720p', '1080p', '4k'], '720p')]; +const KLING_VIDEO_SETTINGS: MediaSettingField[] = [aspect(options(['16:9', '9:16', '1:1'])), duration([5, 10, 15], 5), resolution(['720p', '1080p', '4k'], '1080p'), nativeAudio]; +const SEEDANCE_SETTINGS: MediaSettingField[] = [ + aspect(options(['16:9', '9:16', '1:1', '4:3', '3:4', '21:9'])), duration([4, 5, 8, 10, 12], 5), resolution(['480p', '720p', '1080p', '4k'], '720p'), + { key: 'fps', label: 'Frame rate', type: 'select', default: '24', options: options(['24']) }, + { key: 'generateAudio', label: 'Generate audio', type: 'boolean', default: true }, + { key: 'cameraFixed', label: 'Fixed camera', type: 'boolean', default: false }, +]; + +const fixedPrice = (unit: MediaModelDescriptor['pricing']['unit'], usd: number, note: string, sourceUrl: string, extra: Partial = {}): MediaModelDescriptor['pricing'] => ({ unit, usd, note, sourceUrl, settlement: 'catalog', ...extra }); +const usagePrice = (unit: MediaModelDescriptor['pricing']['unit'], usd: number, note: string, sourceUrl: string, extra: Partial = {}): MediaModelDescriptor['pricing'] => ({ unit, usd, note, sourceUrl, settlement: 'provider_usage', ...extra }); + +const googleModels: MediaModelDescriptor[] = [ + { + modelKey: 'google:image:gemini-3.1-flash-lite-image', provider: 'google', modelId: 'gemini-3.1-flash-lite-image', displayName: 'Nano Banana 2 Lite', + description: 'Fast, cost-efficient image generation and editing.', kind: 'image', operations: ['generate', 'transform'], inputKinds: ['image'], maxInputs: 14, tier: 'fast', async: false, + settings: GOOGLE_IMAGE_SETTINGS.filter((field) => field.key !== 'imageSize'), pricing: fixedPrice('image', 0.0336, '1K image, standard tier', GOOGLE_PRICING), badges: ['SynthID', '1K'], + }, + { + modelKey: 'google:image:gemini-3.1-flash-image', provider: 'google', modelId: 'gemini-3.1-flash-image', displayName: 'Nano Banana 2', + description: 'High-quality image creation, references, and edits.', kind: 'image', operations: ['generate', 'transform'], inputKinds: ['image'], maxInputs: 14, tier: 'standard', async: false, + settings: GOOGLE_IMAGE_SETTINGS, pricing: fixedPrice('image', 0.067, '1K image, standard tier', GOOGLE_PRICING), badges: ['SynthID', 'up to 4K'], + }, + { + modelKey: 'google:image:gemini-3-pro-image', provider: 'google', modelId: 'gemini-3-pro-image', displayName: 'Nano Banana Pro', + description: 'Precision control, text rendering, and brand consistency.', kind: 'image', operations: ['generate', 'transform'], inputKinds: ['image'], maxInputs: 14, tier: 'frontier', async: false, + settings: GOOGLE_IMAGE_SETTINGS, pricing: fixedPrice('image', 0.134, '1K/2K image, standard tier', GOOGLE_PRICING), badges: ['SynthID', 'up to 4K'], + }, + ...[ + ['veo-3.1-lite-generate-preview', 'Veo 3.1 Lite', 'Fast generation and image-to-video with native audio.', 'fast', 0.05], + ['veo-3.1-fast-generate-preview', 'Veo 3.1 Fast', 'High-speed cinematic generation with native audio.', 'standard', 0.1], + ['veo-3.1-generate-preview', 'Veo 3.1', 'Premium cinematic generation with advanced controls.', 'frontier', 0.4], + ].map(([modelId, displayName, description, tier, usd]) => ({ + modelKey: `google:video:${modelId}`, provider: 'google', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'video' as const, + operations: ['generate', 'transform'] as ('generate' | 'transform')[], inputKinds: ['image'] as ('image')[], maxInputs: 2, tier: tier as MediaModelDescriptor['tier'], async: true, + settings: String(modelId).includes('lite') ? GOOGLE_VIDEO_SETTINGS.map((field) => field.key === 'resolution' ? resolution(['720p', '1080p'], '720p') : field) : GOOGLE_VIDEO_SETTINGS, + pricing: fixedPrice('second', Number(usd), 'per generated second with audio', GOOGLE_PRICING), badges: ['native audio', ...(String(modelId).includes('lite') ? [] : ['up to 4K'])], + })), + { + modelKey: 'google:audio:gemini-3.1-flash-tts-preview', provider: 'google', modelId: 'gemini-3.1-flash-tts-preview', displayName: 'Gemini 3.1 Speech', + description: 'Controllable speech with voice, style, pacing, and tone.', kind: 'audio', operations: ['generate'], inputKinds: [], maxInputs: 0, tier: 'standard', async: false, + settings: [{ key: 'voice', label: 'Voice', type: 'select', default: 'Kore', options: options(['Kore', 'Puck', 'Charon', 'Fenrir', 'Aoede', 'Leda', 'Orus', 'Zephyr']) }], + pricing: fixedPrice('audio_token', 0.00002, '$20 per 1M output audio tokens; about 25 tokens/second', GOOGLE_PRICING), + }, + ...[ + ['lyria-3-clip-preview', 'Lyria 3 Clip', 'Thirty-second musical clips, loops, and previews.', 'fast', 0.04], + ['lyria-3-pro-preview', 'Lyria 3 Pro', 'Full-length music with coherent structure and image guidance.', 'frontier', 0.08], + ].map(([modelId, displayName, description, tier, usd]) => ({ + modelKey: `google:music:${modelId}`, provider: 'google', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'music' as const, + operations: ['generate'] as ('generate')[], inputKinds: ['image'] as ('image')[], maxInputs: 1, tier: tier as MediaModelDescriptor['tier'], async: false, settings: [], + pricing: fixedPrice('request', Number(usd), 'per generation request', GOOGLE_PRICING), badges: ['48 kHz stereo'], + })), +]; + +const klingVideoModels: MediaModelDescriptor[] = [ + ['kling-v3-turbo', 'Kling 3.0 Turbo', 'Fast native-audio video generation.', 'fast', { '720p:audio': 0.112, '1080p:audio': 0.14 }], + ['kling-v3', 'Kling 3.0', 'Cinematic video, multi-shot narratives, and native audio.', 'frontier', { '720p:silent': 0.084, '1080p:silent': 0.112, '4k:silent': 0.42, '720p:audio': 0.126, '1080p:audio': 0.168, '4k:audio': 0.42 }], + ['kling-v3-omni', 'Kling 3.0 Omni', 'Multimodal image/video references, elements, storyboards, and audio.', 'frontier', { '720p:silent': 0.084, '1080p:silent': 0.112, '4k:silent': 0.42, '720p:audio': 0.112, '1080p:audio': 0.14, '4k:audio': 0.42, '720p:video:silent': 0.126, '1080p:video:silent': 0.168, '4k:video:silent': 0.42 }], + ['kling-video-o1', 'Kling Video O1', 'Unified instruction-based generation and video transformation.', 'frontier', { '720p:silent': 0.084, '1080p:silent': 0.112, '720p:video:silent': 0.126, '1080p:video:silent': 0.168 }], + ['kling-v2-6', 'Kling 2.6', 'Production video generation with optional native audio.', 'standard', { '720p:silent': 0.042, '1080p:silent': 0.07, '1080p:audio': 0.14 }], + ['kling-v2-5-turbo', 'Kling 2.5 Turbo', 'Efficient silent video generation.', 'fast', { '720p:silent': 0.042, '1080p:silent': 0.07 }], +].map(([modelId, displayName, description, tier, variants]) => ({ + modelKey: `kling:video:${modelId}`, provider: 'kling', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'video' as const, + operations: ['generate', 'transform'] as ('generate' | 'transform')[], inputKinds: ['image', 'video'] as ('image' | 'video')[], maxInputs: String(modelId).includes('omni') ? 10 : 2, + tier: tier as MediaModelDescriptor['tier'], async: true, + settings: String(modelId) === 'kling-v2-5-turbo' ? KLING_VIDEO_SETTINGS.filter((field) => field.key !== 'nativeAudio' && field.key !== 'resolution').concat(resolution(['720p', '1080p'], '1080p')) : KLING_VIDEO_SETTINGS, + pricing: fixedPrice('second', Math.max(...Object.values(variants as Record)), 'per generated second; rate varies by resolution, input, and audio', KLING_PRICING, { variants: variants as Record }), + badges: String(modelId).includes('v3') || String(modelId).includes('2-6') ? ['native audio'] : [], +})); + +const klingImageModels: MediaModelDescriptor[] = [ + ['kling-image-o1', 'Kling Image O1', 'Instruction-led image creation and precise multi-reference edits.', 'standard'], + ['kling-v3-omni', 'Kling Image 3.0 Omni', 'Native 4K image generation, consistent subjects, and series.', 'frontier'], +].map(([modelId, displayName, description, tier]) => ({ + modelKey: `kling:image:${modelId}`, provider: 'kling', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'image' as const, + operations: ['generate', 'transform'] as ('generate' | 'transform')[], inputKinds: ['image'] as ('image')[], maxInputs: 10, tier: tier as MediaModelDescriptor['tier'], async: true, + settings: [aspect(ASPECT_RATIOS), { key: 'imageSize', label: 'Resolution', type: 'select' as const, default: '2k', options: options(['1k', '2k', '4k']) }], + pricing: fixedPrice('image', 0, 'Provider tariff must be configured before use', KLING_PRICING, { requiresOverride: true }), badges: ['up to 4K'], +})); + +const seedreamModels: MediaModelDescriptor[] = [ + ['dola-seedream-5-0-pro-260628', 'Seedream 5.0 Pro', 'High-precision editing with spatial marks and coordinates.', 'frontier', 0.045, ['1K', '2K']], + ['seedream-5-0-lite-260128', 'Seedream 5.0 Lite', 'Fast generation, multi-reference fusion, and image sequences.', 'fast', 0.035, ['2K', '3K', '4K']], + ['seedream-4-5-251128', 'Seedream 4.5', 'High-quality text-to-image and multi-image reference generation.', 'standard', 0.04, ['2K', '4K']], + ['seedream-4-0-250828', 'Seedream 4.0', 'Flexible image creation and editing up to 4K.', 'standard', 0.03, ['1K', '2K', '4K']], +].map(([modelId, displayName, description, tier, usd, sizes]) => ({ + modelKey: `byteplus:image:${modelId}`, provider: 'byteplus', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'image' as const, + operations: ['generate', 'transform'] as ('generate' | 'transform')[], inputKinds: ['image'] as ('image')[], maxInputs: 14, tier: tier as MediaModelDescriptor['tier'], async: false, + settings: [aspect(ASPECT_RATIOS.filter((value) => value.value !== 'auto')), { key: 'imageSize', label: 'Resolution', type: 'select' as const, default: (sizes as string[])[0], options: options(sizes as string[]) }], + pricing: fixedPrice('image', Number(usd), 'per successfully generated image', BYTEPLUS_PRICING, String(modelId).includes('pro') ? { variants: { standard: 0.045, high_pixels: 0.09 } } : {}), + badges: [`up to ${(sizes as string[]).at(-1)}`], +})); + +const seedanceModels: MediaModelDescriptor[] = [ + ['dreamina-seedance-2-5-260628', 'Seedance 2.5', 'Latest multimodal video generation and transformation.', 'frontier', 0, true], + ['dreamina-seedance-2-0-260128', 'Seedance 2.0', 'Multimodal video with images, video, audio, and synchronized output.', 'frontier', 7, false], + ['dreamina-seedance-2-0-fast-260128', 'Seedance 2.0 Fast', 'Faster multimodal video generation.', 'fast', 5.6, false], + ['dreamina-seedance-2-0-mini-260615', 'Seedance 2.0 Mini', 'Cost-efficient multimodal video generation.', 'fast', 3.5, false], + ['seedance-1-5-pro-251215', 'Seedance 1.5 Pro', 'High-quality video with optional generated audio.', 'standard', 2.4, false], + ['seedance-1-0-pro-250528', 'Seedance 1.0 Pro', 'Professional silent video generation.', 'standard', 2.5, false], + ['seedance-1-0-pro-fast-251015', 'Seedance 1.0 Pro Fast', 'Fast, economical silent video generation.', 'fast', 1, false], +].map(([modelId, displayName, description, tier, usd, requiresOverride]) => ({ + modelKey: `byteplus:video:${modelId}`, provider: 'byteplus', modelId: String(modelId), displayName: String(displayName), description: String(description), kind: 'video' as const, + operations: ['generate', 'transform'] as ('generate' | 'transform')[], inputKinds: ['image', 'video', 'audio'] as ('image' | 'video' | 'audio')[], maxInputs: String(modelId).includes('2-') ? 12 : 1, + tier: tier as MediaModelDescriptor['tier'], async: true, settings: SEEDANCE_SETTINGS, + pricing: usagePrice('million_video_tokens', Number(usd), 'per 1M successfully generated video tokens; final charge uses provider completion_tokens', BYTEPLUS_PRICING, { + requiresOverride: Boolean(requiresOverride), variants: String(modelId).includes('2-0') ? { standard: Number(usd), video_input: Number(usd) * 0.61 } : undefined, + }), + badges: ['usage reconciled', ...(String(modelId).includes('2-') ? ['multimodal'] : [])], +})); + +/** A provider-neutral catalog drives Canvas, agent tools, workflows, pricing, and provenance. */ +export const MEDIA_MODEL_REGISTRY: MediaModelDescriptor[] = [...googleModels, ...klingVideoModels, ...klingImageModels, ...seedreamModels, ...seedanceModels]; + +export function getMediaModel(provider: string, selector: string, kind?: string) { + const byKey = MEDIA_MODEL_REGISTRY.find((entry) => entry.provider === provider && entry.modelKey === selector); + const matches = MEDIA_MODEL_REGISTRY.filter((entry) => entry.provider === provider && entry.modelId === selector && (!kind || entry.kind === kind)); + const model = byKey ?? (matches.length === 1 ? matches[0] : undefined); + if (!model) throw new BadRequestException(`Unsupported or ambiguous media model "${selector}"`); + return model; +} + +export function mediaPriceOverride(model: MediaModelDescriptor) { + const envName = `${model.provider.toUpperCase()}_MEDIA_PRICE_USD_JSON`; + const value = safePriceOverride(process.env[envName], model.modelKey, model.modelId); + return Number.isFinite(value) && value > 0 ? value : undefined; +} + +export function isMediaModelPriceConfigured(model: MediaModelDescriptor) { + return !model.pricing.requiresOverride || Boolean(mediaPriceOverride(model)); +} + +export function estimateMediaCost(model: MediaModelDescriptor, prompt: string, settings: Record, inputKinds: string[] = []) { + const override = mediaPriceOverride(model); + const unitPrice = override ?? selectUnitPrice(model, settings, inputKinds); + if (!Number.isFinite(unitPrice) || unitPrice <= 0) return NaN; + if (model.pricing.unit === 'second') return unitPrice * clampNumber(settings.durationSeconds, 1, 30, 5); + if (model.pricing.unit === 'million_video_tokens') { + const [width, height] = videoDimensions(String(settings.resolution ?? '720p'), String(settings.aspectRatio ?? '16:9')); + const outputTokens = (width * height * clampNumber(settings.fps, 1, 120, 24) * clampNumber(settings.durationSeconds, 1, 60, 5)) / 1024; + return (outputTokens / 1_000_000) * unitPrice; + } + if (model.pricing.unit === 'audio_token') { + const seconds = Math.max(1, prompt.length / 15); + return seconds * 25 * unitPrice + (Math.ceil(prompt.length / 4) / 1_000_000); + } + if (model.kind === 'image') { + const size = String(settings.imageSize ?? settings.resolution ?? '1K').toLowerCase(); + if (model.modelId === 'dola-seedream-5-0-pro-260628' && size === '2k') return (override ?? model.pricing.variants?.high_pixels ?? unitPrice) + Math.max(0, inputKinds.length - 1) * 0.003; + const multiplier = model.provider === 'google' ? size === '0.5k' ? 0.67 : size === '2k' ? 1.5 : size === '4k' ? 2.25 : 1 : 1; + return unitPrice * multiplier; + } + return unitPrice; +} + +export function actualMediaCostFromUsage(model: MediaModelDescriptor, providerUsage: Record | undefined, fallback: number) { + const completionTokens = Number(providerUsage?.completionTokens); + const unitPrice = Number(providerUsage?.unitPriceUsd ?? model.pricing.usd); + if (model.pricing.unit === 'million_video_tokens' && completionTokens > 0 && unitPrice > 0) return (completionTokens / 1_000_000) * unitPrice; + return fallback; +} + +export function mediaUnitPrice(model: MediaModelDescriptor, settings: Record, inputKinds: string[] = []) { + return mediaPriceOverride(model) ?? selectUnitPrice(model, settings, inputKinds); +} + +function selectUnitPrice(model: MediaModelDescriptor, settings: Record, inputKinds: string[]) { + const variants = model.pricing.variants; + if (!variants) return model.pricing.usd; + if (model.pricing.unit === 'million_video_tokens') return inputKinds.includes('video') ? variants.video_input ?? model.pricing.usd : variants.standard ?? model.pricing.usd; + if (model.pricing.unit === 'second') { + const resolution = String(settings.resolution ?? '1080p').toLowerCase(); + const audio = Boolean(settings.nativeAudio ?? false) ? 'audio' : 'silent'; + const video = inputKinds.includes('video') ? ':video' : ''; + return variants[`${resolution}${video}:${audio}`] ?? variants[`${resolution}:${audio}`] ?? model.pricing.usd; + } + return model.pricing.usd; +} + +function safePriceOverride(json: string | undefined, ...keys: string[]) { + if (!json) return NaN; + try { + const values = JSON.parse(json) as Record; + for (const key of keys) { + const value = Number(values[key]); + if (Number.isFinite(value) && value > 0) return value; + } + } catch { /* invalid policy is treated as unavailable */ } + return NaN; +} + +function videoDimensions(resolutionName: string, ratio: string): [number, number] { + const long = resolutionName === '4k' ? 3840 : resolutionName === '1080p' ? 1920 : resolutionName === '480p' ? 854 : 1280; + const short = resolutionName === '4k' ? 2160 : resolutionName === '1080p' ? 1080 : resolutionName === '480p' ? 480 : 720; + const [left, right] = ratio.split(':').map(Number); + return !Number.isFinite(left) || !Number.isFinite(right) || left >= right ? [long, short] : [short, long]; +} + +function clampNumber(value: unknown, min: number, max: number, fallback: number) { + const number = Number(value); + return Number.isFinite(number) ? Math.min(max, Math.max(min, number)) : fallback; +} diff --git a/apps/commons-api/src/media/media.module.ts b/apps/commons-api/src/media/media.module.ts new file mode 100644 index 00000000..044379d3 --- /dev/null +++ b/apps/commons-api/src/media/media.module.ts @@ -0,0 +1,24 @@ +import { Module } from '@nestjs/common'; +import { FilesModule } from '~/files'; +import { UsageModule } from '~/modules/usage'; +import { ProvenanceModule } from '~/provenance'; +import { CanvasController } from './canvas.controller'; +import { CanvasService } from './canvas.service'; +import { MediaService } from './media.service'; +import { GoogleMediaProvider } from './providers/google-media.provider'; +import { KlingMediaProvider } from './providers/kling-media.provider'; +import { BytePlusMediaProvider } from './providers/byteplus-media.provider'; + +@Module({ + imports: [FilesModule, UsageModule, ProvenanceModule], + controllers: [CanvasController], + providers: [ + CanvasService, + MediaService, + GoogleMediaProvider, + KlingMediaProvider, + BytePlusMediaProvider, + ], + exports: [CanvasService, MediaService], +}) +export class MediaModule {} diff --git a/apps/commons-api/src/media/media.service.ts b/apps/commons-api/src/media/media.service.ts new file mode 100644 index 00000000..ae3ac632 --- /dev/null +++ b/apps/commons-api/src/media/media.service.ts @@ -0,0 +1,541 @@ +import { + BadRequestException, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import { and, eq } from 'drizzle-orm'; +import { createHash, randomUUID } from 'node:crypto'; +import * as schema from '#/models/schema'; +import { DatabaseService } from '~/modules/database/database.service'; +import { FilesService, LibraryService } from '~/files'; +import { UsageService } from '~/modules/usage'; +import { ProvenanceService } from '~/provenance'; +import { CanvasService } from './canvas.service'; +import { + estimateMediaCost, + getMediaModel, + isMediaModelPriceConfigured, + MEDIA_MODEL_REGISTRY, +} from './media-model.registry'; +import { BytePlusMediaProvider } from './providers/byteplus-media.provider'; +import { GoogleMediaProvider } from './providers/google-media.provider'; +import { KlingMediaProvider } from './providers/kling-media.provider'; +import type { + CreateMediaGenerationInput, + MediaPrincipal, + MediaProviderAdapter, +} from './media.types'; + +@Injectable() +export class MediaService { + private readonly logger = new Logger(MediaService.name); + private readonly running = new Map>(); + + constructor( + private readonly db: DatabaseService, + private readonly files: FilesService, + private readonly library: LibraryService, + private readonly usage: UsageService, + private readonly provenance: ProvenanceService, + private readonly canvas: CanvasService, + private readonly google: GoogleMediaProvider, + private readonly kling: KlingMediaProvider, + private readonly byteplus: BytePlusMediaProvider, + ) {} + + catalog() { + const googleConfigured = Boolean( + process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY, + ); + const klingConfigured = Boolean(process.env.KLING_ACCESS_KEY && process.env.KLING_SECRET_KEY); + const byteplusConfigured = Boolean(process.env.BYTEPLUS_ARK_API_KEY ?? process.env.ARK_API_KEY); + const configured: Record = { + google: googleConfigured, + kling: klingConfigured, + byteplus: byteplusConfigured, + }; + return { + models: MEDIA_MODEL_REGISTRY.map((model) => ({ + ...model, + available: Boolean(configured[model.provider]) && isMediaModelPriceConfigured(model), + unavailableReason: !configured[model.provider] + ? 'provider_not_configured' + : !isMediaModelPriceConfigured(model) + ? 'price_not_configured' + : undefined, + })), + providers: [ + { + id: 'google', + displayName: 'Google', + configured: googleConfigured, + capabilities: ['image', 'video', 'audio', 'music'], + }, + { + id: 'kling', + displayName: 'Kling AI', + configured: klingConfigured, + capabilities: ['image', 'video'], + }, + { + id: 'byteplus', + displayName: 'BytePlus ModelArk', + configured: byteplusConfigured, + capabilities: ['image', 'video'], + }, + ], + billing: { + estimate: 'Credit authorization is shown before generation.', + settlement: 'Successful jobs settle once from catalog or provider-reported usage; failed jobs release the authorization.', + }, + }; + } + + async quote(input: CreateMediaGenerationInput, principal: MediaPrincipal) { + const provider = resolveProvider(input); + const selector = input.modelKey ?? input.modelId; + if (!selector) throw new BadRequestException('A media model is required.'); + const model = getMediaModel(provider, selector); + const settings = normalizeSettings(model.settings, input.settings ?? {}); + const inputKinds = await this.resolveInputKinds(input.inputItemIds ?? [], principal, model); + const estimatedCostUsd = estimateMediaCost(model, input.prompt?.trim() ?? '', settings, inputKinds); + const quote = this.usage.quoteCapability(`${model.kind}_generation`, estimatedCostUsd); + return { + ...quote, + modelKey: model.modelKey, + provider: model.provider, + modelId: model.modelId, + pricing: model.pricing, + settlement: model.pricing.settlement, + }; + } + + async createGeneration( + input: CreateMediaGenerationInput, + principal: MediaPrincipal, + options: { start?: boolean } = { start: true }, + ) { + const provider = resolveProvider(input); + const selector = input.modelKey ?? input.modelId; + if (!selector) throw new BadRequestException('A media model is required.'); + const model = getMediaModel(provider, selector); + const prompt = input.prompt?.trim(); + if (!prompt) throw new BadRequestException('A generation prompt is required.'); + if (prompt.length > 40_000) throw new BadRequestException('Prompt is too long.'); + const operation = input.operation ?? (input.inputItemIds?.length ? 'transform' : 'generate'); + if (!model.operations.includes(operation)) { + throw new BadRequestException(`${model.displayName} does not support ${operation}.`); + } + const settings = normalizeSettings(model.settings, input.settings ?? {}); + let inputItemIds = [...new Set((input.inputItemIds ?? []).filter(Boolean))]; + let project: typeof schema.canvasProject.$inferSelect | undefined; + if (input.projectId) { + project = await this.canvas.requireProject(input.projectId, principal, 'edit'); + if (operation === 'transform' && !inputItemIds.length) { + inputItemIds = [project.activeItemId]; + } + } + if (inputItemIds.length > model.maxInputs) { + throw new BadRequestException( + `${model.displayName} accepts at most ${model.maxInputs} input artifact(s).`, + ); + } + const inputKinds = await this.resolveInputKinds(inputItemIds, principal, model); + const traceId = randomUUID(); + const estimatedCostUsd = estimateMediaCost(model, prompt, settings, inputKinds); + const quote = this.usage.quoteCapability(`${model.kind}_generation`, estimatedCostUsd); + const [job] = await this.db + .insert(schema.mediaGenerationJob) + .values({ + projectId: project?.projectId, + ownerUserId: principal.principalId, + workspaceId: principal.workspaceId ?? null, + agentId: input.agentId, + sessionId: input.sessionId, + traceId, + provider, + modelId: model.modelId, + mediaKind: model.kind, + operation, + prompt, + inputItemIds, + request: { settings, toolCallId: input.toolCallId, modelKey: model.modelKey, inputKinds, quote }, + estimatedCostUsd, + billing: { quote, pricing: model.pricing, status: 'authorized_pending' }, + }) + .returning(); + if (options.start !== false) this.kick(job.jobId); + return this.publicJob(job); + } + + private async resolveInputKinds( + itemIds: string[], + principal: MediaPrincipal, + model: ReturnType, + ) { + const kinds: string[] = []; + for (const itemId of itemIds) { + const item = await this.library.get(itemId, { + principalId: principal.principalId, + principalType: principal.principalType, + workspaceId: principal.workspaceId, + }); + if (!model.inputKinds.includes(item.kind as any)) { + throw new BadRequestException(`${model.displayName} cannot use ${item.kind} input ${item.name}.`); + } + kinds.push(item.kind); + } + return kinds; + } + + async getGeneration(jobId: string, principal: MediaPrincipal) { + const job = await this.requireJob(jobId, principal); + if (job.status === 'queued') this.kick(job.jobId); + return this.publicJob(job); + } + + async generateAndWait( + input: CreateMediaGenerationInput, + principal: MediaPrincipal, + ) { + const job = await this.createGeneration(input, principal, { start: false }); + await this.run(job.jobId); + const completed = await this.requireJob(job.jobId, principal); + if (completed.status !== 'completed' || !completed.outputItemId) { + throw new BadRequestException( + completed.errorMessage || 'Media generation did not complete.', + ); + } + const preview = await this.library.preview(completed.outputItemId, { + principalId: principal.principalId, + principalType: principal.principalType, + workspaceId: principal.workspaceId, + }); + return { + job: this.publicJob(completed), + artifact: { + itemId: preview.itemId, + name: preview.name, + kind: preview.kind, + mimeType: preview.mimeType, + url: preview.inline?.url ?? preview.download?.url, + }, + }; + } + + private kick(jobId: string) { + queueMicrotask(() => { + void this.run(jobId).catch((error) => + this.logger.error( + `Media job ${jobId} crashed: ${error instanceof Error ? error.stack : String(error)}`, + ), + ); + }); + } + + private run(jobId: string) { + const existing = this.running.get(jobId); + if (existing) return existing; + const promise = this.execute(jobId).finally(() => this.running.delete(jobId)); + this.running.set(jobId, promise); + return promise; + } + + private async execute(jobId: string) { + const [job] = await this.db + .update(schema.mediaGenerationJob) + .set({ status: 'running', progress: 2, startedAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(schema.mediaGenerationJob.jobId, jobId), + eq(schema.mediaGenerationJob.status, 'queued'), + ), + ) + .returning(); + if (!job) return; + const request = (job.request ?? {}) as Record; + const settings = (request.settings ?? {}) as Record; + const inputKinds = Array.isArray(request.inputKinds) ? request.inputKinds.map(String) : []; + const model = getMediaModel(job.provider, String(request.modelKey ?? job.modelId), job.mediaKind); + const provider = this.provider(job.provider, job.modelId); + const traceStarted = this.provenance.startRun({ + traceId: job.traceId!, + sessionId: job.sessionId ?? undefined, + agentId: job.agentId ?? undefined, + scopeType: 'canvas_project', + scopeId: job.projectId ?? job.jobId, + initiator: job.ownerUserId, + workspaceId: job.workspaceId ?? undefined, + provider: job.provider, + modelId: job.modelId, + input: { + prompt: job.prompt, + inputItemIds: job.inputItemIds, + settings, + }, + metadata: { + schema: 'https://provenancekit.com/context/v2', + mediaKind: job.mediaKind, + operation: job.operation, + projectId: job.projectId, + inputItemIds: job.inputItemIds, + }, + }); + let reservation: { reservationId?: string } | null = null; + let costCaptured = false; + let settledCostUsd = 0; + const startedAt = Date.now(); + try { + reservation = await this.usage.authorizeCapability({ + principalId: job.ownerUserId, + capability: `${job.mediaKind}_generation`, + estimatedCostUsd: job.estimatedCostUsd ?? estimateMediaCost(model, job.prompt, settings, inputKinds), + idempotencyKey: `capability:media:${job.jobId}`, + agentId: job.agentId ?? undefined, + sessionId: job.sessionId ?? undefined, + metadata: { provider: job.provider, modelId: job.modelId, projectId: job.projectId }, + }); + const assets = await Promise.all( + (job.inputItemIds ?? []).map((fileId) => + this.files.loadOriginalForProcessing({ + fileId, + ownerId: job.ownerUserId, + workspaceId: job.workspaceId, + agentId: job.agentId ?? undefined, + sessionId: job.sessionId ?? undefined, + }), + ), + ); + this.provenance.recordEvent(job.traceId!, { + category: 'model', + eventType: 'media.generate', + name: model.displayName, + phase: 'generation', + status: 'running', + summary: `${job.operation} ${job.mediaKind} with ${model.displayName}`, + payload: { inputItemIds: job.inputItemIds, settings }, + performedBy: { + type: job.agentId ? 'agent' : 'human', + id: job.agentId ?? job.ownerUserId, + }, + }); + const output = await provider.generate({ + model, + prompt: job.prompt, + operation: job.operation as any, + inputs: assets, + settings, + onProgress: async (progress, providerOperationId) => { + await this.db + .update(schema.mediaGenerationJob) + .set({ progress, providerOperationId, updatedAt: new Date() }) + .where(eq(schema.mediaGenerationJob.jobId, job.jobId)); + }, + }); + const actualCostUsd = output.billing?.actualCostUsd ?? job.estimatedCostUsd + ?? estimateMediaCost(model, job.prompt, settings, inputKinds); + settledCostUsd = actualCostUsd; + await this.usage.settleCapability({ + reservationId: reservation?.reservationId, + capability: `${job.mediaKind}_generation`, + actualCostUsd, + idempotencyKey: `capability:media:${job.jobId}:capture`, + agentId: job.agentId ?? undefined, + sessionId: job.sessionId ?? undefined, + metadata: { + provider: job.provider, + modelId: job.modelId, + modelKey: model.modelKey, + providerOperationId: output.providerOperationId, + billing: output.billing, + }, + }); + costCaptured = true; + const created = await this.files.createGeneratedFile({ + buffer: output.buffer, + fileName: `${slug(model.displayName)}-${Date.now()}.${output.extension}`, + mimeType: output.mimeType, + agentId: job.agentId ?? undefined, + sessionId: job.sessionId ?? undefined, + traceId: job.traceId ?? undefined, + ownerId: job.ownerUserId, + workspaceId: job.workspaceId, + metadata: { + provider: job.provider, + model: job.modelId, + canvasProjectId: job.projectId, + sourceFileId: job.inputItemIds?.[0], + inputFileIds: job.inputItemIds, + operation: job.operation, + generationSettings: settings, + promptHash: sha256(job.prompt), + providerOutput: output.metadata, + billing: output.billing, + }, + }); + if (job.projectId) { + const project = await this.db.query.canvasProject.findFirst({ + where: (table) => eq(table.projectId, job.projectId!), + }); + await this.canvas.addRevision({ + projectId: job.projectId, + itemId: created.fileId, + parentItemId: project?.activeItemId, + operation: job.operation, + provider: job.provider, + modelId: job.modelId, + prompt: job.prompt, + inputItemIds: job.inputItemIds ?? [], + settings, + traceId: job.traceId ?? undefined, + createdByType: job.agentId ? 'agent' : 'human', + createdById: job.agentId ?? job.ownerUserId, + }); + } + await this.db + .update(schema.mediaGenerationJob) + .set({ + status: 'completed', + progress: 100, + outputItemId: created.fileId, + providerOperationId: output.providerOperationId, + actualCostUsd, + billing: { + quote: request.quote, + pricing: model.pricing, + settlement: output.billing ?? { + actualCostUsd, + quantity: 1, + unit: model.pricing.unit, + unitPriceUsd: actualCostUsd, + source: 'catalog', + }, + status: 'settled', + }, + completedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(schema.mediaGenerationJob.jobId, job.jobId)); + this.provenance.recordEvent(job.traceId!, { + category: 'output', + eventType: 'artifact.revision.created', + name: created.name, + phase: 'persistence', + status: 'completed', + summary: `Created ${job.mediaKind} artifact revision`, + result: { itemId: created.fileId, mimeType: output.mimeType }, + durationMs: Date.now() - startedAt, + }); + this.provenance.finishRun(job.traceId!, { + status: 'completed', + output: { itemId: created.fileId, projectId: job.projectId }, + durationMs: Date.now() - startedAt, + costUsd: actualCostUsd, + }); + } catch (error) { + if (!costCaptured) await this.usage.releaseCapability(reservation?.reservationId); + const message = safeError(error); + await this.db + .update(schema.mediaGenerationJob) + .set({ + status: 'failed', + errorCode: providerErrorCode(error), + errorMessage: message, + actualCostUsd: costCaptured ? settledCostUsd : null, + billing: { + ...((job.billing ?? {}) as Record), + status: costCaptured ? 'settled_provider_succeeded_artifact_failed' : 'released', + ...(costCaptured ? { actualCostUsd: settledCostUsd } : {}), + }, + completedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(schema.mediaGenerationJob.jobId, job.jobId)); + if (traceStarted) { + this.provenance.finishRun(job.traceId!, { + status: 'failed', + error: message, + output: { error: message }, + durationMs: Date.now() - startedAt, + costUsd: costCaptured ? settledCostUsd : 0, + }); + } + this.logger.warn(`Media job ${job.jobId} failed: ${message}`); + } + } + + private provider(providerId: string, modelId: string): MediaProviderAdapter { + const providers: MediaProviderAdapter[] = [this.google, this.kling, this.byteplus]; + const provider = providers.find( + (entry) => entry.id === providerId && entry.supports(modelId), + ); + if (!provider) throw new BadRequestException(`Provider ${providerId} is unavailable.`); + return provider; + } + + private async requireJob(jobId: string, principal: MediaPrincipal) { + const job = await this.db.query.mediaGenerationJob.findFirst({ + where: (table) => eq(table.jobId, jobId), + }); + if (!job) throw new NotFoundException('Media generation not found.'); + if (same(job.ownerUserId, principal.principalId)) return job; + if (job.projectId) { + await this.canvas.requireProject(job.projectId, principal, 'read'); + return job; + } + throw new NotFoundException('Media generation not found.'); + } + + private publicJob(job: typeof schema.mediaGenerationJob.$inferSelect) { + const { prompt: _prompt, request: _request, ...safe } = job; + return safe; + } +} + +function normalizeSettings( + fields: Array<{ key: string; default?: string | number | boolean; options?: Array<{ value: string }> }>, + provided: Record, +) { + const allowed = new Set(fields.map((field) => field.key)); + const unknown = Object.keys(provided).filter((key) => !allowed.has(key)); + if (unknown.length) throw new BadRequestException(`Unsupported setting(s): ${unknown.join(', ')}`); + const output: Record = {}; + for (const field of fields) { + const value = provided[field.key] ?? field.default; + if (value === undefined) continue; + if (field.options?.length && !field.options.some((option) => option.value === String(value))) { + throw new BadRequestException(`Invalid ${field.key} setting.`); + } + output[field.key] = value; + } + return output; +} + +function safeError(error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return message.replace(/(api[-_ ]?key|authorization|token)\s*[:=]\s*\S+/gi, '$1=[redacted]').slice(0, 2_000); +} + +function providerErrorCode(error: unknown) { + const status = Number((error as any)?.status ?? (error as any)?.statusCode); + return Number.isFinite(status) ? `provider_${status}` : 'generation_failed'; +} + +function slug(value: string) { + return value.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); +} + +function sha256(value: string) { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} + +function same(left?: string | null, right?: string | null) { + return Boolean(left && right && left.toLowerCase() === right.toLowerCase()); +} + +function resolveProvider(input: CreateMediaGenerationInput) { + if (input.provider) return input.provider; + const prefixed = input.modelKey?.split(':')[0]; + return prefixed || 'google'; +} diff --git a/apps/commons-api/src/media/media.types.ts b/apps/commons-api/src/media/media.types.ts new file mode 100644 index 00000000..a46686df --- /dev/null +++ b/apps/commons-api/src/media/media.types.ts @@ -0,0 +1,105 @@ +export type MediaKind = 'image' | 'video' | 'audio' | 'music'; +export type MediaOperation = 'generate' | 'transform'; + +export type MediaSettingField = { + key: string; + label: string; + type: 'select' | 'number' | 'boolean' | 'text'; + default?: string | number | boolean; + options?: Array<{ label: string; value: string }>; + min?: number; + max?: number; + step?: number; + help?: string; +}; + +export type MediaModelDescriptor = { + /** Stable Commons identifier. Provider model names are not always unique by modality. */ + modelKey: string; + provider: string; + /** Exact model identifier sent to the upstream provider. */ + modelId: string; + displayName: string; + description: string; + kind: MediaKind; + operations: MediaOperation[]; + inputKinds: MediaKind[]; + maxInputs: number; + tier: 'fast' | 'standard' | 'frontier'; + async: boolean; + settings: MediaSettingField[]; + pricing: { + unit: 'image' | 'second' | 'request' | 'audio_token' | 'million_video_tokens'; + usd: number; + note: string; + sourceUrl: string; + settlement: 'catalog' | 'provider_usage'; + variants?: Record; + requiresOverride?: boolean; + }; + badges?: string[]; +}; + +export type MediaInputAsset = { + itemId: string; + name: string; + kind: string; + mimeType: string; + /** Short-lived, provider-readable URL for large video/audio references. */ + url?: string; + buffer: Buffer; +}; + +export type MediaGenerateRequest = { + model: MediaModelDescriptor; + prompt: string; + operation: MediaOperation; + inputs: MediaInputAsset[]; + settings: Record; + onProgress?: (progress: number, providerOperationId?: string) => Promise; +}; + +export type MediaProviderOutput = { + buffer: Buffer; + mimeType: string; + extension: string; + providerOperationId?: string; + metadata?: Record; + billing?: { + actualCostUsd: number; + quantity: number; + unit: string; + unitPriceUsd: number; + source: 'catalog' | 'provider_usage'; + providerUsage?: Record; + }; +}; + +export interface MediaProviderAdapter { + readonly id: string; + supports(modelId: string): boolean; + generate(input: MediaGenerateRequest): Promise; +} + +export type CreateMediaGenerationInput = { + projectId?: string; + provider?: string; + /** Preferred stable catalog identifier; modelId remains accepted for older clients. */ + modelKey?: string; + modelId?: string; + prompt: string; + operation?: MediaOperation; + inputItemIds?: string[]; + settings?: Record; + agentId?: string; + sessionId?: string; + toolCallId?: string; +}; + +export type MediaPrincipal = { + principalId: string; + principalType: 'user' | 'agent' | 'service'; + workspaceId?: string | null; + /** Optional agent acting on behalf of the authorized principal. */ + actorId?: string; +}; diff --git a/apps/commons-api/src/media/providers/byteplus-media.provider.ts b/apps/commons-api/src/media/providers/byteplus-media.provider.ts new file mode 100644 index 00000000..a26e8aaf --- /dev/null +++ b/apps/commons-api/src/media/providers/byteplus-media.provider.ts @@ -0,0 +1,137 @@ +import { BadGatewayException, BadRequestException, Injectable, ServiceUnavailableException } from '@nestjs/common'; +import { safeFetch } from '~/utils/safe-fetch'; +import { actualMediaCostFromUsage, estimateMediaCost, mediaUnitPrice } from '../media-model.registry'; +import type { MediaGenerateRequest, MediaProviderAdapter, MediaProviderOutput } from '../media.types'; + +const DEFAULT_BASE_URL = 'https://ark.ap-southeast.bytepluses.com/api/v3'; + +@Injectable() +export class BytePlusMediaProvider implements MediaProviderAdapter { + readonly id = 'byteplus'; + + supports(modelId: string) { + return /seedream|seedance/i.test(modelId); + } + + async generate(input: MediaGenerateRequest): Promise { + if (!process.env.ARK_API_KEY && !process.env.BYTEPLUS_ARK_API_KEY) { + throw new ServiceUnavailableException('BytePlus ModelArk media generation is not configured on this environment.'); + } + return input.model.kind === 'image' ? this.image(input) : this.video(input); + } + + private async image(input: MediaGenerateRequest): Promise { + const payload = await this.request('/images/generations', { + method: 'POST', + body: JSON.stringify({ + model: input.model.modelId, + prompt: input.prompt, + ...(input.inputs.length ? { image: input.inputs.map(dataUrl) } : {}), + size: String(input.settings.imageSize ?? '2K'), + output_format: input.model.modelId.includes('5-0') ? 'png' : 'jpeg', + response_format: 'b64_json', + watermark: false, + }), + }); + const encoded = payload?.data?.[0]?.b64_json; + if (!encoded) throw new BadGatewayException('ModelArk returned no generated image.'); + const mimeType = input.model.modelId.includes('5-0') ? 'image/png' : 'image/jpeg'; + const inputKinds = input.inputs.map((asset) => asset.kind); + const actualCostUsd = estimateMediaCost(input.model, input.prompt, input.settings, inputKinds); + return { + buffer: Buffer.from(encoded, 'base64'), mimeType, extension: mimeType === 'image/png' ? 'png' : 'jpg', + providerOperationId: payload.id ?? payload.created?.toString(), + metadata: { requestId: payload.id, usage: payload.usage }, + billing: { + actualCostUsd, quantity: 1, unit: 'image', unitPriceUsd: actualCostUsd, source: 'catalog', + providerUsage: { ...safeUsage(payload.usage), generatedImages: 1 }, + }, + }; + } + + private async video(input: MediaGenerateRequest): Promise { + const flags = [ + `--resolution ${String(input.settings.resolution ?? '720p').replace('p', '')}`, + `--duration ${Number(input.settings.durationSeconds ?? 5)}`, + `--ratio ${String(input.settings.aspectRatio ?? '16:9')}`, + `--camerafixed ${Boolean(input.settings.cameraFixed)}`, + `--generate_audio ${Boolean(input.settings.generateAudio)}`, + ].join(' '); + const created = await this.request('/contents/generations/tasks', { + method: 'POST', + body: JSON.stringify({ + model: input.model.modelId, + content: [ + { type: 'text', text: `${input.prompt} ${flags}` }, + ...input.inputs.map((asset) => { + if (asset.mimeType.startsWith('image/')) return { type: 'image_url', image_url: { url: dataUrl(asset) } }; + if (asset.mimeType.startsWith('video/')) return { type: 'video_url', video_url: { url: dataUrl(asset) } }; + return { type: 'audio_url', audio_url: { url: dataUrl(asset) } }; + }), + ], + }), + }); + const taskId = String(created.id ?? created.task_id ?? ''); + if (!taskId) throw new BadGatewayException('ModelArk returned no task identifier.'); + let result: any; + for (let poll = 0; poll < 240; poll += 1) { + if (poll) await delay(5_000); + result = await this.request(`/contents/generations/tasks/${encodeURIComponent(taskId)}`); + const status = String(result.status ?? '').toLowerCase(); + await input.onProgress?.(Math.min(92, 8 + poll * 2), taskId); + if (status === 'succeeded') break; + if (status === 'failed' || status === 'cancelled') throw new BadGatewayException(result.error?.message || `Seedance generation ${status}.`); + } + if (String(result?.status).toLowerCase() !== 'succeeded') throw new BadGatewayException('Seedance generation timed out.'); + const url = result.content?.video_url ?? result.content?.[0]?.video_url ?? result.video_url; + if (!url) throw new BadGatewayException('ModelArk returned no generated video.'); + const downloaded = await downloadProviderOutput(String(url), 500 * 1024 * 1024); + const completionTokens = Number(result.usage?.completion_tokens ?? 0); + const inputKinds = input.inputs.map((asset) => asset.kind); + const unitPriceUsd = mediaUnitPrice(input.model, input.settings, inputKinds); + const estimated = estimateMediaCost(input.model, input.prompt, input.settings, inputKinds); + const providerUsage = { completionTokens, unitPriceUsd, totalTokens: result.usage?.total_tokens, durationSeconds: result.duration, resolution: result.resolution, fps: result.fps }; + const actualCostUsd = actualMediaCostFromUsage(input.model, providerUsage, estimated); + return { + buffer: downloaded.buffer, mimeType: 'video/mp4', extension: 'mp4', providerOperationId: taskId, + metadata: { taskId, usage: providerUsage, durationSeconds: result.duration, resolution: result.resolution, ratio: result.ratio, fps: result.fps, audio: result.audio }, + billing: { actualCostUsd, quantity: completionTokens || 0, unit: 'video_token', unitPriceUsd: unitPriceUsd / 1_000_000, source: completionTokens > 0 ? 'provider_usage' : 'catalog', providerUsage }, + }; + } + + private async request(path: string, init: RequestInit = {}) { + const baseUrl = (process.env.BYTEPLUS_ARK_API_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/$/, ''); + const apiKey = process.env.BYTEPLUS_ARK_API_KEY ?? process.env.ARK_API_KEY!; + const response = await safeFetch(`${baseUrl}${path}`, { + ...init, + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json', ...(init.headers ?? {}) }, + }); + const payload = await response.json().catch(() => null) as any; + if (!response.ok) throw new BadGatewayException(payload?.error?.message || payload?.message || `ModelArk API returned ${response.status}.`); + return payload; + } +} + +function dataUrl(asset: MediaGenerateRequest['inputs'][number]) { + if (asset.url) return asset.url; + if (asset.buffer.byteLength > 50 * 1024 * 1024) throw new BadRequestException(`${asset.name} is too large for an inline provider reference.`); + return `data:${asset.mimeType};base64,${asset.buffer.toString('base64')}`; +} + +function safeUsage(value: unknown) { + return value && typeof value === 'object' ? value as Record : {}; +} + +async function downloadProviderOutput(url: string, maxBytes: number) { + const response = await safeFetch(url); + if (!response.ok) throw new BadGatewayException(`Could not download provider output (${response.status}).`); + const declared = Number(response.headers.get('content-length') ?? 0); + if (declared > maxBytes) throw new BadRequestException('Provider output exceeds the supported size.'); + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > maxBytes) throw new BadRequestException('Provider output exceeds the supported size.'); + return { buffer }; +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/apps/commons-api/src/media/providers/google-media.provider.ts b/apps/commons-api/src/media/providers/google-media.provider.ts new file mode 100644 index 00000000..a890a4b5 --- /dev/null +++ b/apps/commons-api/src/media/providers/google-media.provider.ts @@ -0,0 +1,195 @@ +import { + BadGatewayException, + BadRequestException, + Injectable, + ServiceUnavailableException, +} from '@nestjs/common'; +import { GoogleGenAI } from '@google/genai'; +import { WaveFile } from 'wavefile'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { + MediaGenerateRequest, + MediaProviderAdapter, + MediaProviderOutput, +} from '../media.types'; + +@Injectable() +export class GoogleMediaProvider implements MediaProviderAdapter { + readonly id = 'google'; + + supports(modelId: string) { + return /^(gemini-|veo-|lyria-)/.test(modelId); + } + + async generate(input: MediaGenerateRequest): Promise { + const client = this.client(); + if (input.model.kind === 'video') return this.video(client, input); + if (input.model.kind === 'image') return this.image(client, input); + if (input.model.kind === 'audio') return this.speech(client, input); + if (input.model.kind === 'music') return this.music(client, input); + throw new BadRequestException(`Unsupported media kind ${input.model.kind}`); + } + + private client() { + const apiKey = process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY; + if (!apiKey) { + throw new ServiceUnavailableException( + 'Google media generation is not configured on this environment.', + ); + } + return new GoogleGenAI({ apiKey }); + } + + private async image(client: GoogleGenAI, request: MediaGenerateRequest) { + const input: any[] = [{ type: 'text', text: request.prompt }]; + for (const asset of request.inputs) { + if (!asset.mimeType.startsWith('image/')) { + throw new BadRequestException('Nano Banana inputs must be images.'); + } + input.push({ + type: 'image', + mime_type: asset.mimeType, + data: asset.buffer.toString('base64'), + }); + } + const interaction: any = await (client.interactions as any).create({ + model: request.model.modelId, + input, + response_format: { + type: 'image', + mime_type: 'image/png', + aspect_ratio: String(request.settings.aspectRatio ?? '1:1'), + ...(request.model.modelId !== 'gemini-3.1-flash-lite-image' + ? { image_size: String(request.settings.imageSize ?? '1K') } + : {}), + }, + }); + const output = interaction.output_image; + if (!output?.data) throw new BadGatewayException('Google returned no image.'); + return { + buffer: Buffer.from(output.data, 'base64'), + mimeType: output.mime_type ?? 'image/png', + extension: output.mime_type === 'image/jpeg' ? 'jpg' : 'png', + providerOperationId: interaction.id, + metadata: { interactionId: interaction.id, synthId: true }, + } satisfies MediaProviderOutput; + } + + private async speech(client: GoogleGenAI, request: MediaGenerateRequest) { + const voice = String(request.settings.voice ?? 'Kore'); + const interaction: any = await (client.interactions as any).create({ + model: request.model.modelId, + input: request.prompt, + response_format: { type: 'audio' }, + generation_config: { speech_config: [{ voice }] }, + }); + const output = interaction.output_audio; + if (!output?.data) throw new BadGatewayException('Google returned no speech audio.'); + const pcm = Buffer.from(output.data, 'base64'); + const wave = new WaveFile(); + wave.fromScratch(1, 24_000, '16', new Int16Array( + pcm.buffer, + pcm.byteOffset, + Math.floor(pcm.byteLength / 2), + )); + return { + buffer: Buffer.from(wave.toBuffer()), + mimeType: 'audio/wav', + extension: 'wav', + providerOperationId: interaction.id, + metadata: { interactionId: interaction.id, voice, sourceEncoding: 'pcm_s16le' }, + } satisfies MediaProviderOutput; + } + + private async music(client: GoogleGenAI, request: MediaGenerateRequest) { + const input: any[] = [{ type: 'text', text: request.prompt }]; + const image = request.inputs[0]; + if (image) { + if (!image.mimeType.startsWith('image/')) { + throw new BadRequestException('Lyria reference input must be an image.'); + } + input.push({ + type: 'image', + mime_type: image.mimeType, + data: image.buffer.toString('base64'), + }); + } + const interaction: any = await (client.interactions as any).create({ + model: request.model.modelId, + input, + response_format: { type: 'audio' }, + }); + const output = interaction.output_audio; + if (!output?.data) throw new BadGatewayException('Google returned no music audio.'); + return { + buffer: Buffer.from(output.data, 'base64'), + mimeType: output.mime_type ?? 'audio/mpeg', + extension: 'mp3', + providerOperationId: interaction.id, + metadata: { interactionId: interaction.id, lyrics: interaction.output_text }, + } satisfies MediaProviderOutput; + } + + private async video(client: GoogleGenAI, request: MediaGenerateRequest) { + const images = request.inputs.filter((input) => input.mimeType.startsWith('image/')); + if (images.length !== request.inputs.length) { + throw new BadRequestException('Veo reference inputs must be images.'); + } + let operation: any = await (client.models as any).generateVideos({ + model: request.model.modelId, + prompt: request.prompt, + ...(images[0] + ? { + image: { + imageBytes: images[0].buffer.toString('base64'), + mimeType: images[0].mimeType, + }, + } + : {}), + config: { + aspectRatio: String(request.settings.aspectRatio ?? '16:9'), + durationSeconds: Number(request.settings.durationSeconds ?? 8), + resolution: String(request.settings.resolution ?? '720p'), + ...(images[1] + ? { + lastFrame: { + imageBytes: images[1].buffer.toString('base64'), + mimeType: images[1].mimeType, + }, + } + : {}), + }, + }); + await request.onProgress?.(10, operation.name); + let polls = 0; + while (!operation.done) { + await delay(10_000); + operation = await (client.operations as any).getVideosOperation({ operation }); + polls += 1; + await request.onProgress?.(Math.min(90, 15 + polls * 5), operation.name); + if (polls > 180) throw new BadGatewayException('Veo generation timed out.'); + } + const video = operation.response?.generatedVideos?.[0]?.video; + if (!video) throw new BadGatewayException('Google returned no video.'); + const directory = await mkdtemp(join(tmpdir(), 'agent-commons-veo-')); + const downloadPath = join(directory, 'generated.mp4'); + try { + await (client.files as any).download({ file: video, downloadPath }); + return { + buffer: await readFile(downloadPath), + mimeType: 'video/mp4', + extension: 'mp4', + providerOperationId: operation.name, + metadata: { operationName: operation.name, nativeAudio: true }, + } satisfies MediaProviderOutput; + } finally { + await rm(directory, { recursive: true, force: true }).catch(() => undefined); + } + } +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/apps/commons-api/src/media/providers/kling-media.provider.ts b/apps/commons-api/src/media/providers/kling-media.provider.ts new file mode 100644 index 00000000..0758104b --- /dev/null +++ b/apps/commons-api/src/media/providers/kling-media.provider.ts @@ -0,0 +1,159 @@ +import { BadGatewayException, BadRequestException, Injectable, ServiceUnavailableException } from '@nestjs/common'; +import { createHmac } from 'node:crypto'; +import { safeFetch } from '~/utils/safe-fetch'; +import { estimateMediaCost, mediaUnitPrice } from '../media-model.registry'; +import type { MediaGenerateRequest, MediaProviderAdapter, MediaProviderOutput } from '../media.types'; + +const DEFAULT_BASE_URL = 'https://api-singapore.klingai.com'; + +@Injectable() +export class KlingMediaProvider implements MediaProviderAdapter { + readonly id = 'kling'; + + supports(modelId: string) { + return /^kling-/.test(modelId); + } + + async generate(input: MediaGenerateRequest): Promise { + const accessKey = process.env.KLING_ACCESS_KEY; + const secretKey = process.env.KLING_SECRET_KEY; + if (!accessKey || !secretKey) throw new ServiceUnavailableException('Kling media generation is not configured on this environment.'); + const token = signKlingJwt(accessKey, secretKey); + return input.model.kind === 'image' ? this.image(input, token) : this.video(input, token); + } + + private async image(input: MediaGenerateRequest, token: string): Promise { + const path = '/v1/images/omni-image'; + const created = await this.request(path, token, { + method: 'POST', + body: JSON.stringify({ + model_name: input.model.modelId, + prompt: input.prompt, + image_list: input.inputs.map((asset) => ({ image: providerReference(asset) })), + resolution: String(input.settings.imageSize ?? '2k').toLowerCase(), + aspect_ratio: String(input.settings.aspectRatio ?? 'auto'), + n: 1, + watermark_info: { enabled: false }, + }), + }); + const taskId = taskIdOf(created); + const result = await this.poll(`${path}/${encodeURIComponent(taskId)}`, token, taskId, input); + const url = result.data?.task_result?.images?.[0]?.url ?? result.data?.task_result?.series_images?.[0]?.url; + if (!url) throw new BadGatewayException('Kling returned no generated image.'); + const downloaded = await downloadProviderOutput(url, 25 * 1024 * 1024); + const actualCostUsd = estimateMediaCost(input.model, input.prompt, input.settings, input.inputs.map((asset) => asset.kind)); + return { + buffer: downloaded.buffer, + mimeType: downloaded.mimeType.startsWith('image/') ? downloaded.mimeType : 'image/png', + extension: downloaded.mimeType.includes('jpeg') ? 'jpg' : 'png', + providerOperationId: taskId, + metadata: { taskId, requestId: result.request_id, finalUnitDeduction: result.data?.final_unit_deduction, finalBalanceDeduction: result.data?.final_balance_deduction }, + billing: { + actualCostUsd, quantity: 1, unit: 'image', unitPriceUsd: actualCostUsd, source: 'catalog', + providerUsage: { finalUnitDeduction: result.data?.final_unit_deduction, finalBalanceDeduction: result.data?.final_balance_deduction }, + }, + }; + } + + private async video(input: MediaGenerateRequest, token: string): Promise { + const omni = input.model.modelId.includes('omni') || input.model.modelId.includes('o1') || input.inputs.some((asset) => asset.kind === 'video'); + const path = omni ? '/v1/videos/omni-video' : input.inputs.length ? '/v1/videos/image2video' : '/v1/videos/text2video'; + const images = input.inputs.filter((asset) => asset.mimeType.startsWith('image/')); + const videos = input.inputs.filter((asset) => asset.mimeType.startsWith('video/')); + const common = { + model_name: input.model.modelId, + prompt: input.prompt, + duration: String(input.settings.durationSeconds ?? 5), + aspect_ratio: String(input.settings.aspectRatio ?? '16:9'), + resolution: String(input.settings.resolution ?? '1080p').replace('p', ''), + sound: Boolean(input.settings.nativeAudio) ? 'on' : 'off', + watermark_info: { enabled: false }, + }; + const body = omni + ? { + ...common, + image_list: images.map((asset) => ({ image: providerReference(asset) })), + video_list: videos.map((asset) => ({ video: providerReference(asset) })), + } + : images[0] + ? { ...common, image: providerReference(images[0]), tail_image: images[1] ? providerReference(images[1]) : undefined } + : common; + const created = await this.request(path, token, { method: 'POST', body: JSON.stringify(body) }); + const taskId = taskIdOf(created); + const result = await this.poll(`${path}/${encodeURIComponent(taskId)}`, token, taskId, input); + const video = result.data?.task_result?.videos?.[0]; + if (!video?.url) throw new BadGatewayException('Kling returned no generated video.'); + const downloaded = await downloadProviderOutput(video.url, 500 * 1024 * 1024); + const inputKinds = input.inputs.map((asset) => asset.kind); + const seconds = Number(video.duration ?? input.settings.durationSeconds ?? 5); + const unitPriceUsd = mediaUnitPrice(input.model, input.settings, inputKinds); + const actualCostUsd = unitPriceUsd * seconds; + return { + buffer: downloaded.buffer, mimeType: 'video/mp4', extension: 'mp4', providerOperationId: taskId, + metadata: { taskId, requestId: result.request_id, durationSeconds: seconds, finalUnitDeduction: result.data?.final_unit_deduction, finalBalanceDeduction: result.data?.final_balance_deduction }, + billing: { + actualCostUsd, quantity: seconds, unit: 'second', unitPriceUsd, source: 'catalog', + providerUsage: { durationSeconds: seconds, finalUnitDeduction: result.data?.final_unit_deduction, finalBalanceDeduction: result.data?.final_balance_deduction }, + }, + }; + } + + private async poll(path: string, token: string, taskId: string, input: MediaGenerateRequest) { + for (let poll = 0; poll < 180; poll += 1) { + if (poll) await delay(5_000); + const result = await this.request(path, token); + const status = String(result.data?.task_status ?? '').toLowerCase(); + await input.onProgress?.(Math.min(92, 8 + poll * 3), taskId); + if (status === 'succeed') return result; + if (status === 'failed') throw new BadGatewayException(result.data?.task_status_msg || 'Kling generation failed.'); + } + throw new BadGatewayException('Kling generation timed out.'); + } + + private async request(path: string, token: string, init: RequestInit = {}) { + const baseUrl = (process.env.KLING_API_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/$/, ''); + const response = await safeFetch(`${baseUrl}${path}`, { + ...init, + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', ...(init.headers ?? {}) }, + }); + const payload = await response.json().catch(() => null) as any; + if (!response.ok || payload?.code !== 0) throw new BadGatewayException(payload?.message || `Kling API returned ${response.status}.`); + return payload; + } +} + +function taskIdOf(payload: any) { + const taskId = payload?.data?.task_id; + if (!taskId) throw new BadGatewayException('Kling returned no task identifier.'); + return String(taskId); +} + +async function downloadProviderOutput(url: string, maxBytes: number) { + const response = await safeFetch(url); + if (!response.ok) throw new BadGatewayException(`Could not download provider output (${response.status}).`); + const declared = Number(response.headers.get('content-length') ?? 0); + if (declared > maxBytes) throw new BadRequestException('Provider output exceeds the supported size.'); + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > maxBytes) throw new BadRequestException('Provider output exceeds the supported size.'); + return { buffer, mimeType: response.headers.get('content-type')?.split(';')[0] ?? 'application/octet-stream' }; +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function providerReference(asset: MediaGenerateRequest['inputs'][number]) { + return asset.url ?? asset.buffer.toString('base64'); +} + +function signKlingJwt(accessKey: string, secretKey: string) { + const now = Math.floor(Date.now() / 1000); + const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from( + JSON.stringify({ iss: accessKey, nbf: now - 5, exp: now + 1800 }), + ).toString('base64url'); + const signature = createHmac('sha256', secretKey) + .update(`${header}.${payload}`) + .digest('base64url'); + return `${header}.${payload}.${signature}`; +} diff --git a/apps/commons-api/src/modules/model-provider/model-registry.ts b/apps/commons-api/src/modules/model-provider/model-registry.ts index ea9174c0..236e68c7 100644 --- a/apps/commons-api/src/modules/model-provider/model-registry.ts +++ b/apps/commons-api/src/modules/model-provider/model-registry.ts @@ -134,14 +134,28 @@ export const MODEL_REGISTRY: ModelRegistryEntry[] = [ // ── Google ───────────────────────────────────────────────────────────────── { provider: 'google', - modelId: 'gemini-2.0-flash', - displayName: 'Gemini 2.0 Flash', + modelId: 'gemini-3.6-flash', + displayName: 'Gemini 3.6 Flash', contextWindow: 1000000, supportsTools: true, supportsStreaming: true, supportsVision: true, - inputPricePer1kTokens: 0.0001, - outputPricePer1kTokens: 0.0004, + inputPricePer1kTokens: 0.0015, + cachedInputPricePer1kTokens: 0.00015, + outputPricePer1kTokens: 0.009, + tier: 'frontier', + }, + { + provider: 'google', + modelId: 'gemini-3.5-flash-lite', + displayName: 'Gemini 3.5 Flash-Lite', + contextWindow: 1000000, + supportsTools: true, + supportsStreaming: true, + supportsVision: true, + inputPricePer1kTokens: 0.0003, + cachedInputPricePer1kTokens: 0.00003, + outputPricePer1kTokens: 0.0025, tier: 'fast', }, { diff --git a/apps/commons-api/src/modules/model-provider/providers/google.provider.ts b/apps/commons-api/src/modules/model-provider/providers/google.provider.ts index 919f4feb..4abfc2ee 100644 --- a/apps/commons-api/src/modules/model-provider/providers/google.provider.ts +++ b/apps/commons-api/src/modules/model-provider/providers/google.provider.ts @@ -42,7 +42,8 @@ function thinkingConfigFor( export function buildGoogleModel(config: ModelConfig): ChatGoogleGenerativeAI { const logger = new Logger('GoogleProvider'); - const apiKey = config.apiKey ?? process.env.GOOGLE_API_KEY; + const apiKey = + config.apiKey ?? process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY; if (!apiKey) { logger.warn('No Google API key found — requests will fail'); } diff --git a/apps/commons-api/src/modules/usage/usage.service.ts b/apps/commons-api/src/modules/usage/usage.service.ts index 3b1f51bd..c61f7290 100644 --- a/apps/commons-api/src/modules/usage/usage.service.ts +++ b/apps/commons-api/src/modules/usage/usage.service.ts @@ -155,7 +155,9 @@ export class UsageService { idempotencyKey: input.idempotencyKey, agentId: input.agentId, sessionId: input.sessionId, - ttlSeconds: 600, + // Media providers commonly queue long-running video jobs. Keep the + // authorization alive through polling and settle it exactly once. + ttlSeconds: Number(process.env.MEDIA_RESERVATION_TTL_SECONDS || 7200), metadata: { capability: input.capability, estimatedCostUsd: input.estimatedCostUsd, @@ -164,6 +166,17 @@ export class UsageService { }); } + /** Public-safe quote used by creative controls before a reservation is made. */ + quoteCapability(capability: string, estimatedCostUsd: number) { + return { + capability, + estimatedCostUsd: roundMoney(estimatedCostUsd), + estimatedCredits: this.capabilityCredits(capability, estimatedCostUsd), + currency: 'credits' as const, + pricingPolicy: 'provider_cost_plus_platform_margin' as const, + }; + } + async settleCapability(input: { reservationId?: string | null; capability: string; @@ -336,3 +349,7 @@ export class UsageService { return { ...agg, events }; } } + +function roundMoney(value: number) { + return Math.round(value * 1_000_000) / 1_000_000; +} diff --git a/apps/commons-api/src/tool/tool.module.ts b/apps/commons-api/src/tool/tool.module.ts index c62e04e4..c0a111b6 100644 --- a/apps/commons-api/src/tool/tool.module.ts +++ b/apps/commons-api/src/tool/tool.module.ts @@ -28,6 +28,7 @@ import { UsageModule } from '~/modules/usage'; import { CapabilityProviderModule } from '~/provider'; import { UiPluginModule } from '~/ui-plugin'; import { BrainModule } from '~/brain'; +import { MediaModule } from '~/media'; @Module({ imports: [ @@ -46,6 +47,7 @@ import { BrainModule } from '~/brain'; CapabilityProviderModule, UiPluginModule, BrainModule, + MediaModule, ], controllers: [ ToolController, diff --git a/apps/commons-api/src/tool/tools/common-tool.service.ts b/apps/commons-api/src/tool/tools/common-tool.service.ts index f89c1b6c..ed8ac785 100644 --- a/apps/commons-api/src/tool/tools/common-tool.service.ts +++ b/apps/commons-api/src/tool/tools/common-tool.service.ts @@ -48,6 +48,7 @@ import { type UiPluginSurface, } from '~/ui-plugin'; import { BrainService } from '~/brain'; +import { CanvasService, MediaService } from '~/media'; type ToolExecutionMetadata = { agentId?: string; @@ -402,6 +403,56 @@ export interface CommonTool { }[] >; + /** + * Generate or transform an image, video, speech clip, or music artifact with + * a model from the shared Canvas media catalog. Outputs are private Library + * artifacts and, when projectId is supplied, immutable Canvas revisions. + * Use modelKey values returned by listMediaModels; never invent model names. + */ + generateMedia(props: { + projectId?: string; + modelKey: string; + prompt: string; + operation?: 'generate' | 'transform'; + inputItemIds?: string[]; + settings?: Record; + agentId: string; + sessionId?: string; + }): Promise<{ + job: Record; + artifact: { + itemId: string; + name: string; + kind: string; + mimeType: string; + url?: string; + }; + }>; + + /** List exact creative model keys, capabilities, settings, and credit pricing. */ + listMediaModels(): Promise>; + + /** + * Read a Canvas project's active artifact, immutable revision graph, + * annotations, and recent generation jobs before analysing or editing it. + */ + getCanvasProject(props: { + projectId: string; + agentId: string; + }): Promise>; + + /** Add a normalized spatial or temporal annotation to a Canvas revision. */ + annotateCanvas(props: { + projectId: string; + revisionId: string; + kind: 'comment' | 'point' | 'region' | 'time_range' | 'transcript' | 'freehand'; + body: string; + geometry?: Record; + startMs?: number; + endMs?: number; + agentId: string; + }): Promise>; + /** * Upload a file directly to IPFS via Pinata. */ @@ -940,6 +991,8 @@ export class CommonToolService { private capabilityProviders: CapabilityProviderService, private uiPlugins: UiPluginService, private brains: BrainService, + private media: MediaService, + private canvas: CanvasService, ) {} private async capabilityOwner(agentId: string) { @@ -1520,6 +1573,93 @@ export class CommonToolService { return results; } + async listMediaModels() { + return this.media.catalog(); + } + + async getCanvasProject( + props: { projectId: string; agentId: string }, + metadata?: ToolExecutionMetadata, + ) { + const agentId = this.requireToolAgentId(props.agentId, metadata); + const owner = await this.capabilityOwner(agentId); + return this.canvas.getProject(props.projectId, { + principalId: owner.principalId, + principalType: 'user', + workspaceId: owner.workspaceId, + actorId: agentId, + }); + } + + async annotateCanvas( + props: { + projectId: string; + revisionId: string; + kind: 'comment' | 'point' | 'region' | 'time_range' | 'transcript' | 'freehand'; + body: string; + geometry?: Record; + startMs?: number; + endMs?: number; + agentId: string; + }, + metadata?: ToolExecutionMetadata, + ) { + const agentId = this.requireToolAgentId(props.agentId, metadata); + const owner = await this.capabilityOwner(agentId); + return this.canvas.createAnnotation( + props.projectId, + { + principalId: owner.principalId, + principalType: 'user', + workspaceId: owner.workspaceId, + actorId: agentId, + }, + { + revisionId: props.revisionId, + kind: props.kind, + body: props.body, + geometry: props.geometry, + startMs: props.startMs, + endMs: props.endMs, + }, + ); + } + + async generateMedia( + props: { + projectId?: string; + modelKey: string; + prompt: string; + operation?: 'generate' | 'transform'; + inputItemIds?: string[]; + settings?: Record; + agentId: string; + sessionId?: string; + }, + metadata?: ToolExecutionMetadata, + ) { + const agentId = this.requireToolAgentId(props.agentId, metadata); + const owner = await this.capabilityOwner(agentId); + return this.media.generateAndWait( + { + projectId: props.projectId, + modelKey: props.modelKey, + prompt: props.prompt, + operation: props.operation, + inputItemIds: props.inputItemIds, + settings: props.settings, + agentId, + sessionId: metadata?.sessionId ?? props.sessionId, + toolCallId: metadata?.toolCallId, + }, + { + principalId: owner.principalId, + principalType: 'user', + workspaceId: owner.workspaceId, + }, + ); + } + /** * Upload a file directly to IPFS using Pinata. * - `props.base64String` is your file’s data encoded in base64. diff --git a/apps/commons-app/app/api/canvas/[...path]/route.ts b/apps/commons-app/app/api/canvas/[...path]/route.ts new file mode 100644 index 00000000..64e8caf9 --- /dev/null +++ b/apps/commons-app/app/api/canvas/[...path]/route.ts @@ -0,0 +1,28 @@ +import { NextRequest } from "next/server"; +import { proxyBackend } from "@/lib/backend-proxy"; +import { requireCurrentCommonsUser } from "@/lib/current-user"; + +type Context = { params: Promise<{ path: string[] }> }; + +async function forward(request: NextRequest, context: Context) { + const { user, response } = await requireCurrentCommonsUser(); + if (!user) return response; + const { path } = await context.params; + const query = request.nextUrl.searchParams.toString(); + let body: unknown; + if (!["GET", "DELETE"].includes(request.method)) { + body = await request.json().catch(() => ({})); + } + return proxyBackend( + `/v1/canvas/${path.map(encodeURIComponent).join("/")}${query ? `?${query}` : ""}`, + { + method: request.method as "GET" | "POST" | "PATCH" | "DELETE", + body, + }, + ); +} + +export const GET = forward; +export const POST = forward; +export const PATCH = forward; +export const DELETE = forward; diff --git a/apps/commons-app/app/studio/canvas/[artifactId]/page.tsx b/apps/commons-app/app/studio/canvas/[artifactId]/page.tsx new file mode 100644 index 00000000..3ad111c2 --- /dev/null +++ b/apps/commons-app/app/studio/canvas/[artifactId]/page.tsx @@ -0,0 +1,10 @@ +import { CanvasStudio } from "@/components/canvas/canvas-studio"; + +export default async function CanvasArtifactPage({ + params, +}: { + params: Promise<{ artifactId: string }>; +}) { + const { artifactId } = await params; + return ; +} diff --git a/apps/commons-app/components/artifacts/artifact-surface.tsx b/apps/commons-app/components/artifacts/artifact-surface.tsx index 9c0f3540..d1ba79e5 100644 --- a/apps/commons-app/components/artifacts/artifact-surface.tsx +++ b/apps/commons-app/components/artifacts/artifact-surface.tsx @@ -1,5 +1,6 @@ "use client"; +import Link from "next/link"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { BadgeCheck, @@ -226,6 +227,14 @@ export function ArtifactSurface({

+ + + Open in Canvas + { diff --git a/apps/commons-app/components/canvas/canvas-studio.tsx b/apps/commons-app/components/canvas/canvas-studio.tsx new file mode 100644 index 00000000..18bb0e2b --- /dev/null +++ b/apps/commons-app/components/canvas/canvas-studio.tsx @@ -0,0 +1,1381 @@ +"use client"; + +import Link from "next/link"; +import { + AlertCircle, + AudioLines, + Bot, + Check, + CheckCircle2, + ChevronLeft, + ChevronRight, + Clock3, + Download, + GitBranch, + History, + ImageIcon, + Layers3, + LibraryBig, + Loader2, + MapPin, + MessageSquareText, + Music2, + PanelLeftClose, + PanelLeftOpen, + PanelRightClose, + PanelRightOpen, + Pause, + Play, + Plus, + Scan, + Settings2, + Share2, + Sparkles, + Video, + WandSparkles, + X, + ZoomIn, + ZoomOut, +} from "lucide-react"; +import { + type PointerEvent as ReactPointerEvent, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { + LibraryPickerDialog, + type LibraryPickerItem, +} from "@/components/sessions/chat/library-picker-dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { openCommonsCopilotPrompt } from "@/lib/commons-copilot-events"; +import { + type CanvasAnnotation, + type CanvasAnnotationKind, + type CanvasPreview, + type CanvasProjectBundle, + type CanvasRevision, + type MediaCatalog, + type MediaJob, + type MediaKind, + type MediaModel, + type MediaOperation, + type MediaQuote, + formatCanvasTime, + unwrapCanvasPayload, +} from "@/lib/canvas"; +import { cn } from "@/lib/utils"; + +type RightPanel = "project" | "notes" | "history"; +type AnnotationTool = "select" | "region" | "point" | "time_range"; +type Rect = { x: number; y: number; width: number; height: number }; + +const KIND_OPTIONS: Array<{ + kind: MediaKind; + label: string; + icon: typeof ImageIcon; +}> = [ + { kind: "image", label: "Image", icon: ImageIcon }, + { kind: "video", label: "Video", icon: Video }, + { kind: "audio", label: "Speech", icon: AudioLines }, + { kind: "music", label: "Music", icon: Music2 }, +]; + +export function CanvasStudio({ artifactId }: { artifactId: string }) { + const [catalog, setCatalog] = useState(null); + const [bundle, setBundle] = useState(null); + const [preview, setPreview] = useState(null); + const [loading, setLoading] = useState(true); + const [previewLoading, setPreviewLoading] = useState(false); + const [error, setError] = useState(""); + const [kind, setKind] = useState("image"); + const [modelId, setModelId] = useState(""); + const [quote, setQuote] = useState(null); + const [operation, setOperation] = useState("transform"); + const [prompt, setPrompt] = useState(""); + const [settings, setSettings] = useState>({}); + const [references, setReferences] = useState([]); + const [pickerOpen, setPickerOpen] = useState(false); + const [leftOpen, setLeftOpen] = useState(true); + const [rightOpen, setRightOpen] = useState(true); + const [rightPanel, setRightPanel] = useState("project"); + const [currentJob, setCurrentJob] = useState(null); + const [lastJob, setLastJob] = useState(null); + const [generating, setGenerating] = useState(false); + const [annotationTool, setAnnotationTool] = + useState("select"); + const [draftRect, setDraftRect] = useState(null); + const [draftPoint, setDraftPoint] = useState<{ x: number; y: number } | null>( + null, + ); + const [annotationBody, setAnnotationBody] = useState(""); + const [annotationSaving, setAnnotationSaving] = useState(false); + const [zoom, setZoom] = useState(1); + const [currentTimeMs, setCurrentTimeMs] = useState(0); + const [durationMs, setDurationMs] = useState(0); + const [playing, setPlaying] = useState(false); + const stageRef = useRef(null); + const dragStartRef = useRef<{ x: number; y: number } | null>(null); + const mediaRef = useRef(null); + + const loadProject = useCallback( + async (projectId?: string) => { + const response = projectId + ? await fetch(`/api/canvas/projects/${encodeURIComponent(projectId)}`, { + cache: "no-store", + }) + : await fetch("/api/canvas/projects/open", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ artifactId }), + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + throw new Error( + payload?.message || payload?.error || "Could not open this project", + ); + } + const next = unwrapCanvasPayload(payload); + setBundle(next); + return next; + }, + [artifactId], + ); + + useEffect(() => { + let cancelled = false; + void Promise.all([ + fetch("/api/canvas/models", { cache: "no-store" }).then(async (response) => { + const payload = await response.json().catch(() => null); + if (!response.ok) throw new Error("Could not load creative models"); + return unwrapCanvasPayload(payload); + }), + loadProject(), + ]) + .then(([nextCatalog, nextBundle]) => { + if (cancelled) return; + setCatalog(nextCatalog); + setBundle(nextBundle); + }) + .catch((cause) => { + if (!cancelled) { + setError(cause instanceof Error ? cause.message : "Canvas could not open"); + } + }) + .finally(() => { + if (!cancelled) setLoading(false); + }); + return () => { + cancelled = true; + }; + }, [loadProject]); + + useEffect(() => { + if (!bundle?.project.activeItemId) return; + const controller = new AbortController(); + setPreviewLoading(true); + fetch( + `/api/library/${encodeURIComponent(bundle.project.activeItemId)}/preview`, + { cache: "no-store", signal: controller.signal }, + ) + .then(async (response) => { + const payload = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(payload?.message || "Could not preview this revision"); + } + return unwrapCanvasPayload(payload); + }) + .then((next) => { + setPreview(next); + const previewKind = mediaKindFor(next.kind, next.mimeType); + if (previewKind) setKind(previewKind); + }) + .catch((cause) => { + if (!controller.signal.aborted) { + setError(cause instanceof Error ? cause.message : "Preview failed"); + } + }) + .finally(() => { + if (!controller.signal.aborted) setPreviewLoading(false); + }); + return () => controller.abort(); + }, [bundle?.project.activeItemId]); + + const models = useMemo( + () => catalog?.models.filter((model) => model.kind === kind) ?? [], + [catalog, kind], + ); + const model = useMemo( + () => models.find((entry) => entry.modelKey === modelId) ?? models[0], + [modelId, models], + ); + + useEffect(() => { + if (!model) return; + setModelId(model.modelKey); + setSettings( + Object.fromEntries( + model.settings + .filter((field) => field.default !== undefined) + .map((field) => [field.key, field.default]), + ), + ); + if (!model.operations.includes(operation)) { + setOperation(model.operations[0] ?? "generate"); + } + }, [model?.modelKey]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!model || !bundle) return; + const controller = new AbortController(); + const timeout = window.setTimeout(() => { + const inputItemIds = references.map((item) => item.itemId); + if (operation === "transform" && bundle.project.activeItemId) inputItemIds.unshift(bundle.project.activeItemId); + void fetch("/api/canvas/quote", { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: controller.signal, + body: JSON.stringify({ + modelKey: model.modelKey, + provider: model.provider, + prompt, + inputItemIds: [...new Set(inputItemIds)].slice(0, model.maxInputs), + settings, + }), + }).then(async (response) => { + const payload = await response.json().catch(() => null); + if (!response.ok) throw new Error(payload?.message || "Quote unavailable"); + setQuote(unwrapCanvasPayload(payload)); + }).catch(() => { + if (!controller.signal.aborted) setQuote(null); + }); + }, 350); + return () => { + window.clearTimeout(timeout); + controller.abort(); + }; + }, [bundle, model, operation, prompt, references, settings]); + + useEffect(() => { + if (!currentJob || !["queued", "running"].includes(currentJob.status)) { + return; + } + let stopped = false; + let timeout: number | undefined; + const poll = async () => { + try { + const response = await fetch( + `/api/canvas/generations/${encodeURIComponent(currentJob.jobId)}`, + { cache: "no-store" }, + ); + const payload = await response.json().catch(() => null); + if (!response.ok) throw new Error(payload?.message || "Generation failed"); + const next = unwrapCanvasPayload(payload); + if (stopped) return; + setCurrentJob(next); + setLastJob(next); + if (next.status === "completed") { + await loadProject(bundle?.project.projectId); + setGenerating(false); + return; + } + if (next.status === "failed" || next.status === "cancelled") { + setGenerating(false); + return; + } + timeout = window.setTimeout(poll, 1_500); + } catch (cause) { + if (!stopped) { + setError(cause instanceof Error ? cause.message : "Generation failed"); + setGenerating(false); + } + } + }; + timeout = window.setTimeout(poll, 800); + return () => { + stopped = true; + if (timeout) window.clearTimeout(timeout); + }; + }, [currentJob?.jobId, currentJob?.status, bundle?.project.projectId, loadProject]); + + const activeRevision = useMemo( + () => + bundle?.revisions.find( + (revision) => revision.itemId === bundle.project.activeItemId, + ) ?? bundle?.revisions[0], + [bundle], + ); + const activeAnnotations = useMemo( + () => + bundle?.annotations.filter( + (annotation) => annotation.revisionId === activeRevision?.revisionId, + ) ?? [], + [activeRevision?.revisionId, bundle?.annotations], + ); + const annotationDraftOpen = Boolean(draftRect || draftPoint) || annotationTool === "time_range"; + + async function generate() { + if (!model || !bundle || !prompt.trim()) return; + setGenerating(true); + setError(""); + const inputIds = references.map((item) => item.itemId); + if (operation === "transform" && bundle.project.activeItemId) { + inputIds.unshift(bundle.project.activeItemId); + } + const response = await fetch("/api/canvas/generations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + projectId: bundle.project.projectId, + provider: model.provider, + modelKey: model.modelKey, + operation, + prompt: prompt.trim(), + inputItemIds: [...new Set(inputIds)].slice(0, model.maxInputs), + settings, + }), + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + setGenerating(false); + setError(payload?.message || payload?.error || "Could not start generation"); + return; + } + const job = unwrapCanvasPayload(payload); + setCurrentJob(job); + setLastJob(job); + } + + async function activateRevision(revision: CanvasRevision) { + if (!bundle || revision.itemId === bundle.project.activeItemId) return; + const response = await fetch( + `/api/canvas/projects/${encodeURIComponent(bundle.project.projectId)}`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ activeRevisionId: revision.revisionId }), + }, + ); + const payload = await response.json().catch(() => null); + if (!response.ok) { + setError(payload?.message || "Could not switch revision"); + return; + } + await loadProject(bundle.project.projectId); + } + + async function renameProject(name: string) { + if (!bundle || !name.trim() || name.trim() === bundle.project.name) return; + const response = await fetch( + `/api/canvas/projects/${encodeURIComponent(bundle.project.projectId)}`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: name.trim() }), + }, + ); + if (response.ok) await loadProject(bundle.project.projectId); + } + + function stagePoint(event: ReactPointerEvent) { + const bounds = event.currentTarget.getBoundingClientRect(); + return { + x: clamp((event.clientX - bounds.left) / bounds.width), + y: clamp((event.clientY - bounds.top) / bounds.height), + }; + } + + function onStagePointerDown(event: ReactPointerEvent) { + if (annotationTool === "select" || annotationTool === "time_range") return; + event.currentTarget.setPointerCapture(event.pointerId); + const point = stagePoint(event); + if (annotationTool === "point") { + setDraftPoint(point); + setDraftRect(null); + setRightPanel("notes"); + setRightOpen(true); + return; + } + dragStartRef.current = point; + setDraftPoint(null); + setDraftRect({ x: point.x, y: point.y, width: 0, height: 0 }); + } + + function onStagePointerMove(event: ReactPointerEvent) { + if (annotationTool !== "region" || !dragStartRef.current) return; + const point = stagePoint(event); + const start = dragStartRef.current; + setDraftRect({ + x: Math.min(start.x, point.x), + y: Math.min(start.y, point.y), + width: Math.abs(point.x - start.x), + height: Math.abs(point.y - start.y), + }); + } + + function onStagePointerUp() { + dragStartRef.current = null; + setDraftRect((current) => { + if (!current || current.width < 0.01 || current.height < 0.01) return null; + setRightPanel("notes"); + setRightOpen(true); + return current; + }); + } + + async function saveAnnotation() { + if (!bundle || !activeRevision || !annotationBody.trim()) return; + let annotationKind: CanvasAnnotationKind = "comment"; + let geometry: Record | undefined; + let startMs: number | undefined; + let endMs: number | undefined; + if (draftRect) { + annotationKind = "region"; + geometry = { ...draftRect, coordinateSpace: "rendered_viewport" }; + } else if (draftPoint) { + annotationKind = "point"; + geometry = { ...draftPoint, coordinateSpace: "rendered_viewport" }; + } else if (annotationTool === "time_range") { + annotationKind = "time_range"; + startMs = Math.round(currentTimeMs); + endMs = Math.round(Math.min(durationMs || currentTimeMs + 5_000, currentTimeMs + 5_000)); + } + setAnnotationSaving(true); + const response = await fetch( + `/api/canvas/projects/${encodeURIComponent(bundle.project.projectId)}/annotations`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + revisionId: activeRevision.revisionId, + kind: annotationKind, + body: annotationBody.trim(), + geometry, + startMs, + endMs, + metadata: { schemaVersion: 1 }, + }), + }, + ); + const payload = await response.json().catch(() => null); + setAnnotationSaving(false); + if (!response.ok) { + setError(payload?.message || "Could not save annotation"); + return; + } + setAnnotationBody(""); + setDraftRect(null); + setDraftPoint(null); + setAnnotationTool("select"); + await loadProject(bundle.project.projectId); + } + + async function resolveAnnotation(annotation: CanvasAnnotation) { + if (!bundle) return; + const response = await fetch( + `/api/canvas/projects/${encodeURIComponent(bundle.project.projectId)}/annotations/${encodeURIComponent(annotation.annotationId)}`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: annotation.status === "resolved" ? "open" : "resolved", + }), + }, + ); + if (response.ok) await loadProject(bundle.project.projectId); + } + + function askCopilot(annotation?: CanvasAnnotation) { + if (!bundle || !activeRevision) return; + const location = annotationLocation(annotation); + openCommonsCopilotPrompt({ + mode: "draft", + intentId: `canvas-${bundle.project.projectId}-${annotation?.annotationId ?? "project"}`, + text: [ + `Help me work on Canvas project “${bundle.project.name}”.`, + `Project ID: ${bundle.project.projectId}`, + `Current artifact ID: ${bundle.project.activeItemId}`, + `Revision ID: ${activeRevision.revisionId}`, + annotation ? `Annotation ID: ${annotation.annotationId}` : "", + annotation ? `Annotation: ${annotation.body}` : "", + location ? `Target: ${location}` : "", + "Use the Canvas media tools when I ask you to analyze, transform, annotate, or create a new revision. Ask before any irreversible or costly action.", + ] + .filter(Boolean) + .join("\n"), + }); + } + + function togglePlayback() { + const media = mediaRef.current; + if (!media) return; + if (media.paused) void media.play(); + else media.pause(); + } + + if (loading) { + return ; + } + + if (!bundle) { + return ( +
+
+ +

+ Canvas could not open +

+

{error}

+ +
+
+ ); + } + + const displayedJob = currentJob ?? lastJob ?? bundle.jobs[0] ?? null; + const canGenerate = Boolean(model?.available && prompt.trim() && !generating); + const estimatedCost = quote?.estimatedCostUsd ?? (model ? estimateCost(model, settings, prompt) : 0); + const temporal = preview?.mimeType.startsWith("video/") || preview?.mimeType.startsWith("audio/"); + + return ( +
+
+ +
+ void renameProject(event.currentTarget.value)} + className="min-w-0 max-w-[340px] flex-1 truncate rounded-md border border-transparent bg-transparent px-2 py-1 text-sm font-semibold outline-none transition hover:border-stone-200 focus:border-stone-300 focus:bg-white" + /> + + + Saved + +
+ setLeftOpen((value) => !value)} + > + {leftOpen ? : } + + setRightOpen((value) => !value)} + > + {rightOpen ? : } + +
+ + + + + {preview?.download?.url ? ( + + + Export + + ) : null} +
+
+ + {error ? ( +
+ + {error} + +
+ ) : null} + +
+ {leftOpen ? ( +