Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions src/ai-sdk/generate-video.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,16 @@ export type GenerateVideoPrompt =
video?: DataContent | Array<DataContent>;
};

/**
* A video model can be passed as a string ID (resolved via the default
* provider set on `globalThis.AI_SDK_DEFAULT_PROVIDER`) or as a full
* `VideoModelV3` object. This mirrors how the Vercel AI SDK's
* `generateImage` accepts string model IDs.
*/
export type VideoModel = VideoModelV3 | string;

export interface GenerateVideoOptions {
model: VideoModelV3;
model: VideoModel;
prompt: GenerateVideoPrompt;
n?: number;
resolution?: `${number}x${number}`;
Expand Down Expand Up @@ -122,11 +130,37 @@ function normalizePrompt(prompt: GenerateVideoPrompt): {
};
}

/**
* Resolve a `VideoModel` (which may be a string ID) into a `VideoModelV3`
* object. String IDs are resolved through `globalThis.AI_SDK_DEFAULT_PROVIDER`
* (set by the render service before calling `render()`), mirroring how the
* Vercel AI SDK's `generateImage` resolves string image model IDs.
*/
function resolveVideoModel(model: VideoModel): VideoModelV3 {
if (typeof model !== "string") {
return model;
}

const provider = globalThis.AI_SDK_DEFAULT_PROVIDER as
| { videoModel?: (id: string) => VideoModelV3 }
| undefined;

if (!provider?.videoModel) {
throw new Error(
`Cannot resolve video model "${model}" — no default provider with videoModel() is set. ` +
'Either pass a VideoModelV3 object (e.g. varg.videoModel("seedance_2_5")) ' +
"or ensure globalThis.AI_SDK_DEFAULT_PROVIDER is configured.",
);
}

return provider.videoModel(model);
}

export async function generateVideo(
options: GenerateVideoOptions,
): Promise<GenerateVideoResult> {
const {
model,
model: modelArg,
prompt: promptArg,
n = 1,
resolution,
Expand All @@ -141,6 +175,8 @@ export async function generateVideo(

const { prompt, files } = normalizePrompt(promptArg);

const model = resolveVideoModel(modelArg);

const result = await model.doGenerate({
prompt: prompt ?? "",
n,
Expand Down
1 change: 1 addition & 0 deletions src/ai-sdk/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export {
type GenerateVideoPrompt,
type GenerateVideoResult,
generateVideo,
type VideoModel,
} from "./generate-video";
export {
generatePlaceholder,
Expand Down
30 changes: 28 additions & 2 deletions src/react/renderers/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import type {
Layer,
VideoLayer,
} from "../../ai-sdk/providers/editly/types";
import type { VideoModelV3 } from "../../ai-sdk/video-model";
import { ResolvedElement } from "../resolved-element";
import type {
ClipProps,
Expand Down Expand Up @@ -90,6 +91,29 @@ function toImageModelV3(
};
}

function toVideoModelV3(
model: Parameters<typeof generateVideo>[0]["model"],
): VideoModelV3 {
if (typeof model === "object" && model.specificationVersion === "v3") {
return model;
}
// for string IDs, create a shell that satisfies the type.
// in preview mode the middleware intercepts before doGenerate is called;
// in strict mode generateVideo() resolves the string via the default provider.
const modelId = typeof model === "string" ? model : model.modelId;
return {
specificationVersion: "v3",
provider: "placeholder",
modelId,
maxVideosPerCall: 1,
doGenerate: async () => {
throw new Error(
`toVideoModelV3 shell: doGenerate should not be called in preview mode (model: ${modelId})`,
);
},
};
}

export async function renderRoot(
element: VargElement<"render">,
options: RenderOptions,
Expand Down Expand Up @@ -179,7 +203,7 @@ export async function renderRoot(
return cachedGenerateVideo({
...opts,
model: wrapVideoModel({
model: opts.model,
model: toVideoModelV3(opts.model),
middleware: placeholderFallbackMiddleware({
mode: "preview",
onFallback: () => {},
Expand All @@ -190,7 +214,9 @@ export async function renderRoot(
}

const result = await cachedGenerateVideo(opts);
emitPricing("video", opts.model.modelId, result);
const videoModelId =
typeof opts.model === "string" ? opts.model : opts.model.modelId;
emitPricing("video", videoModelId, result);
return result;
};

Expand Down
9 changes: 5 additions & 4 deletions src/react/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type {
} from "@ai-sdk/provider";
import type { CacheStorage } from "../ai-sdk/cache";
import type { File } from "../ai-sdk/file";
import type { VideoModel } from "../ai-sdk/generate-video";
import type { MusicModelV3 } from "../ai-sdk/music-model";
import type { FFmpegBackend } from "../ai-sdk/providers/editly/backends";
import type {
Expand Down Expand Up @@ -164,7 +165,7 @@ export type VideoProps = BaseProps &
TrimProps & {
prompt?: VideoPrompt;
src?: string;
model?: VideoModelV3;
model?: VideoModel;
resize?: ResizeMode;
cropPosition?: CropPosition;
aspectRatio?: `${number}:${number}`;
Expand Down Expand Up @@ -206,9 +207,9 @@ export interface TalkingHeadProps extends BaseProps {
/** Pre-resolved or lazy speech element to use as the audio track. */
audio?: VargElement<"speech">;
/** Lipsync video model (e.g. fal.videoModel("sync-v2-pro")). */
model?: VideoModelV3;
model?: VideoModel;
/** Separate lipsync model override (defaults to `model`). */
lipsyncModel?: VideoModelV3;
lipsyncModel?: VideoModel;
/** Video resolution for lipsync generation (default: "720p") */
resolution?: "480p" | "720p" | "1080p";
position?:
Expand Down Expand Up @@ -355,7 +356,7 @@ export type RenderMode = "strict" | "preview";

export interface DefaultModels {
image?: ImageModelV3;
video?: VideoModelV3;
video?: VideoModel;
speech?: SpeechModelV3;
music?: MusicModelV3;
transcription?: TranscriptionModelV3;
Expand Down
Loading