From 10ef81f9fafe9b5ec0abdca33e7f8712712c1f7f Mon Sep 17 00:00:00 2001 From: Maksim Smatrou Date: Wed, 15 Jul 2026 11:41:51 +0200 Subject: [PATCH 1/2] feat[api]: add Ideogram 4 diffusion support to SDK --- packages/sdk/contract/schema.json | 88 ++++++++++- packages/sdk/examples/diffusion-ideogram.ts | 118 ++++++++++++++ packages/sdk/package.json | 2 +- packages/sdk/schemas/sdcpp-config.ts | 17 +- .../bare/plugins/sdcpp-generation/plugin.ts | 23 +++ .../test/bare/sdcpp-ideogram-plugin.test.ts | 148 ++++++++++++++++++ 6 files changed, 387 insertions(+), 9 deletions(-) create mode 100644 packages/sdk/examples/diffusion-ideogram.ts create mode 100644 packages/sdk/test/bare/sdcpp-ideogram-plugin.test.ts diff --git a/packages/sdk/contract/schema.json b/packages/sdk/contract/schema.json index c6faee75ff..baa2e00361 100644 --- a/packages/sdk/contract/schema.json +++ b/packages/sdk/contract/schema.json @@ -6540,7 +6540,7 @@ "properties": { "mode": { "default": "diffusion", - "description": "Operation mode for the diffusion plugin. `'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from the primary model plus optional auxiliary text encoders, VAE, and ESRGAN upscaler, and exposes diffusion({ ... }). `'upscale'` builds a standalone ESRGAN upscaler from the primary model file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). `'video'` builds a `VideoStableDiffusion` pipeline and exposes video({ ... }). The video layout is selected from the auxiliary sources: supplying `embeddingsConnectorsModelSrc` loads the LTX-2 layout (Gemma text encoder via `llmModelSrc` + video VAE + connectors, optional `audioVaeModelSrc` for synchronized audio); otherwise the Wan layout is used (UMT5 text encoder via `t5XxlModelSrc` + VAE). On React Native, loading the video model on-device will likely fail because the video diffusion models currently shipped by the SDK are too large to load on typical mobile devices; pass a `delegate` to `loadModel(...)` to run generation on a desktop peer instead.", + "description": "Operation mode for the diffusion plugin. `'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from the primary model plus optional auxiliary text encoders, VAE, unconditional diffusion model, and ESRGAN upscaler, and exposes diffusion({ ... }). `'upscale'` builds a standalone ESRGAN upscaler from the primary model file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). `'video'` builds a `VideoStableDiffusion` pipeline and exposes video({ ... }). The video layout is selected from the auxiliary sources: supplying `embeddingsConnectorsModelSrc` loads the LTX-2 layout (Gemma text encoder via `llmModelSrc` + video VAE + connectors, optional `audioVaeModelSrc` for synchronized audio); otherwise the Wan layout is used (UMT5 text encoder via `t5XxlModelSrc` + VAE). On React Native, loading the video model on-device will likely fail because the video diffusion models currently shipped by the SDK are too large to load on typical mobile devices; pass a `delegate` to `loadModel(...)` to run generation on a desktop peer instead.", "type": "string", "enum": ["diffusion", "upscale", "video"], "title": "LoadModelSrcRequestSdcppGenerationModelConfigMode" @@ -6885,7 +6885,7 @@ ] }, "llmModelSrc": { - "description": "LLM text encoder model — required for FLUX.2 [klein] (Qwen3) and for LTX-2 video (Gemma).", + "description": "LLM text encoder model — required for FLUX.2 [klein] (Qwen3), Ideogram 4 (Qwen3-VL), and LTX-2 video (Gemma).", "anyOf": [ { "type": "string" @@ -6967,7 +6967,7 @@ ] }, "vaeModelSrc": { - "description": "VAE decoder model — required for FLUX.2 [klein] and LTX-2 video (video VAE), optional for SDXL.", + "description": "VAE decoder model — required for FLUX.2 [klein], Ideogram 4, and LTX-2 video (video VAE); optional for SDXL.", "anyOf": [ { "type": "string" @@ -7130,6 +7130,88 @@ } ] }, + "uncondModelSrc": { + "description": "Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, llmModelSrc (Qwen3-VL), and vaeModelSrc.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "src": { + "type": "string" + }, + "name": { + "type": "string" + }, + "modelId": { + "type": "string" + }, + "registryPath": { + "type": "string" + }, + "registrySource": { + "type": "string" + }, + "blobCoreKey": { + "type": "string" + }, + "blobIndex": { + "type": "number" + }, + "engine": { + "type": "string" + }, + "expectedSize": { + "type": "number" + }, + "sha256Checksum": { + "type": "string" + }, + "addon": { + "anyOf": [ + { + "type": "string", + "enum": [ + "llamacpp-completion", + "whispercpp-transcription", + "bci-whispercpp-transcription", + "llamacpp-embedding", + "nmtcpp-translation", + "onnx-tts", + "tts-ggml", + "parakeet-transcription", + "ggml-ocr", + "sdcpp-generation", + "ggml-vla", + "ggml-classification", + "llm", + "whisper", + "bci", + "embeddings", + "nmt", + "parakeet", + "tts", + "ocr", + "diffusion", + "vla", + "classification" + ], + "title": "LoadModelSrcRequestSdcppGenerationModelConfigUncondModelSrcAddon" + }, + { + "type": "string", + "const": "vad" + } + ] + } + }, + "required": ["src"], + "title": "LoadModelSrcRequestSdcppGenerationModelConfigUncondModelSrc" + } + ] + }, "clipVisionModelSrc": { "description": "OpenCLIP ViT-H/14 weights (`clip_vision_h.safetensors`). Required for Wan image-to-video (`img2vid`); omit for text-to-video-only pipelines. Not used by LTX-2 (its img2vid path needs no CLIP-vision projection).", "anyOf": [ diff --git a/packages/sdk/examples/diffusion-ideogram.ts b/packages/sdk/examples/diffusion-ideogram.ts new file mode 100644 index 0000000000..3b6ccd5283 --- /dev/null +++ b/packages/sdk/examples/diffusion-ideogram.ts @@ -0,0 +1,118 @@ +import { diffusion, loadModel, unloadModel } from '@qvac/sdk' +import fs from 'fs' +import path from 'path' + +const diffusionModelSrc = + process.argv[2] || + 'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4-Q4_0.gguf' +const uncondModelSrc = + process.argv[3] || + 'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4_uncond-Q4_0.gguf' +const llmModelSrc = + process.argv[4] || + 'https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q4_K_M.gguf' +const vaeModelSrc = + process.argv[5] || + 'https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors' +const outputDir = process.argv[6] || '.' + +function formatMb(bytes: number) { + return (bytes / 1e6).toFixed(1) +} + +const structuredPrompt = { + high_level_description: + 'A bright product photo of one yellow coffee mug on a clean office desk with a tiny readable label that says IDEO.', + style_description: { + aesthetics: 'simple product photography, clean bright office desk, minimal composition', + lighting: 'soft studio light with gentle shadows', + photo: 'sharp focus, crisp readable label text', + medium: 'product photo', + color_palette: ['#FFD54A', '#FFFFFF', '#111111', '#7EC8E3'] + }, + compositional_deconstruction: { + canvas: 'Square canvas, upright orientation. All text is horizontal and readable.', + background: 'Clean white office desk with a soft blue background gradient.', + elements: [ + { + type: 'obj', + bbox: [230, 260, 800, 740], + desc: 'Exactly one matte yellow ceramic coffee mug centered on the desk.' + }, + { + type: 'text', + bbox: [510, 360, 620, 640], + text: 'IDEO', + desc: 'Small black label printed horizontally on the mug.' + } + ] + } +} + +if (process.argv.includes('--help')) { + console.log('Usage: tsx examples/diffusion-ideogram.ts [model] [uncond] [llm] [vae] [output-dir]') + console.log('▸ Defaults use the four Hugging Face model URLs supported by the diffusion addon.') + process.exit(0) +} + +let modelId: string | undefined + +try { + console.log('▸ Loading Ideogram 4 split-layout model...') + modelId = await loadModel({ + modelSrc: diffusionModelSrc, + modelType: 'sdcpp-generation', + modelConfig: { + device: 'gpu', + threads: 4, + diffusion_fa: true, + offload_to_cpu: true, + llmModelSrc, + vaeModelSrc, + uncondModelSrc + }, + onProgress: (progress) => { + const line = `▸ Downloading ${progress.percentage.toFixed(0)}% (${formatMb(progress.downloaded)}/${formatMb(progress.total)} MB)` + process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`) + if (progress.percentage >= 100) process.stderr.write('\n') + } + }) + console.log(`▸ Model loaded: ${modelId}`) + + const { progressStream, outputs, stats } = diffusion({ + modelId, + prompt: JSON.stringify(structuredPrompt), + width: 768, + height: 768, + steps: 16, + cfg_scale: 7, + seed: 42 + }) + + for await (const { step, totalSteps } of progressStream) { + console.log(`▸ step ${step}/${totalSteps}`) + } + + const buffers = await outputs + for (let i = 0; i < buffers.length; i++) { + const outputPath = path.join(outputDir, `ideogram_${i}.png`) + fs.writeFileSync(outputPath, buffers[i]!) + console.log(`▸ Saved ${outputPath}`) + } + + console.log('▸ Stats:', await stats) + await unloadModel({ modelId, clearStorage: false }) + modelId = undefined + console.log('▸ Done') + process.exit(0) +} catch (error) { + if (modelId) { + try { + await unloadModel({ modelId, clearStorage: false }) + } catch { + // Preserve the original error. + } + } + console.error('✖', error) + process.exit(1) +} diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 8f84c28435..a1d2d69f25 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -205,7 +205,7 @@ "@qvac/bci-whispercpp": "^0.4.1", "@qvac/classification-ggml": "^0.10.2", "@qvac/decoder-audio": "^0.5.0", - "@qvac/diffusion-cpp": "^0.13.3", + "@qvac/diffusion-cpp": "^0.14.0", "@qvac/embed-llamacpp": "^0.26.2", "@qvac/error": "^0.1.1", "@qvac/langdetect-text": "^0.1.2", diff --git a/packages/sdk/schemas/sdcpp-config.ts b/packages/sdk/schemas/sdcpp-config.ts index 7bf87fe4a4..5b119bde6a 100644 --- a/packages/sdk/schemas/sdcpp-config.ts +++ b/packages/sdk/schemas/sdcpp-config.ts @@ -54,7 +54,8 @@ export const sdcppConfigSchema = z.object({ .describe( 'Operation mode for the diffusion plugin. ' + "`'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from " + - 'the primary model plus optional auxiliary text encoders, VAE, and ESRGAN ' + + 'the primary model plus optional auxiliary text encoders, VAE, unconditional ' + + 'diffusion model, and ESRGAN ' + 'upscaler, and exposes diffusion({ ... }). ' + "`'upscale'` builds a standalone ESRGAN upscaler from the primary model " + 'file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). ' + @@ -147,18 +148,24 @@ export const sdcppConfigSchema = z.object({ llmModelSrc: modelSrcInputSchema .optional() .describe( - 'LLM text encoder model — required for FLUX.2 [klein] (Qwen3) and for ' + - 'LTX-2 video (Gemma).' + 'LLM text encoder model — required for FLUX.2 [klein] (Qwen3), ' + + 'Ideogram 4 (Qwen3-VL), and LTX-2 video (Gemma).' ), vaeModelSrc: modelSrcInputSchema .optional() .describe( - 'VAE decoder model — required for FLUX.2 [klein] and LTX-2 video (video VAE), ' + - 'optional for SDXL.' + 'VAE decoder model — required for FLUX.2 [klein], Ideogram 4, and ' + + 'LTX-2 video (video VAE); optional for SDXL.' ), highNoiseDiffusionModelSrc: modelSrcInputSchema .optional() .describe('High-noise diffusion expert — required for Wan 2.2 mixture-of-experts video models'), + uncondModelSrc: modelSrcInputSchema + .optional() + .describe( + 'Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, ' + + 'llmModelSrc (Qwen3-VL), and vaeModelSrc.' + ), clipVisionModelSrc: modelSrcInputSchema .optional() .describe( diff --git a/packages/sdk/server/bare/plugins/sdcpp-generation/plugin.ts b/packages/sdk/server/bare/plugins/sdcpp-generation/plugin.ts index 75994f432a..0b27a0db83 100644 --- a/packages/sdk/server/bare/plugins/sdcpp-generation/plugin.ts +++ b/packages/sdk/server/bare/plugins/sdcpp-generation/plugin.ts @@ -41,6 +41,7 @@ type DiffusionArtifactKey = | 'llmModelPath' | 'vaeModelPath' | 'highNoiseDiffusionModelPath' + | 'uncondModelPath' | 'audioVaeModelPath' | 'embeddingsConnectorsModelPath' | 'esrganModelPath' @@ -96,6 +97,19 @@ export const diffusionPlugin = definePlugin({ cfg: SdcppConfig, ctx: ResolveContext ): Promise> { + if (cfg.uncondModelSrc && cfg.mode !== 'diffusion') { + throw new ModelLoadFailedError( + 'modelConfig.uncondModelSrc is Ideogram 4 diffusion only. ' + + "Use mode: 'diffusion' or remove uncondModelSrc." + ) + } + if (cfg.uncondModelSrc && (!cfg.llmModelSrc || !cfg.vaeModelSrc)) { + throw new ModelLoadFailedError( + 'modelConfig.uncondModelSrc selects the Ideogram 4 layout and requires ' + + 'modelConfig.llmModelSrc (Qwen3-VL) and modelConfig.vaeModelSrc.' + ) + } + // Standalone-upscaler mode never references auxiliary models: the primary // modelSrc IS the ESRGAN file. Skip resolution to avoid downloading // unused encoders/VAEs and to keep load fast. @@ -111,6 +125,7 @@ export const diffusionPlugin = definePlugin({ llmModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, + uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, upscaler, @@ -153,6 +168,7 @@ export const diffusionPlugin = definePlugin({ llmModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, + uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, esrganModelSrc @@ -172,6 +188,7 @@ export const diffusionPlugin = definePlugin({ llmModelPath, vaeModelPath, highNoiseDiffusionModelPath, + uncondModelPath, audioVaeModelPath, embeddingsConnectorsModelPath, esrganModelPath @@ -183,6 +200,7 @@ export const diffusionPlugin = definePlugin({ llmModelSrc ? resolve(llmModelSrc) : undefined, vaeModelSrc ? resolve(vaeModelSrc) : undefined, highNoiseDiffusionModelSrc ? resolve(highNoiseDiffusionModelSrc) : undefined, + uncondModelSrc ? resolve(uncondModelSrc) : undefined, audioVaeModelSrc ? resolve(audioVaeModelSrc) : undefined, embeddingsConnectorsModelSrc ? resolve(embeddingsConnectorsModelSrc) : undefined, esrganModelSrc ? resolve(esrganModelSrc) : undefined @@ -198,6 +216,7 @@ export const diffusionPlugin = definePlugin({ ...(llmModelPath && { llmModelPath }), ...(vaeModelPath && { vaeModelPath }), ...(highNoiseDiffusionModelPath && { highNoiseDiffusionModelPath }), + ...(uncondModelPath && { uncondModelPath }), ...(audioVaeModelPath && { audioVaeModelPath }), ...(embeddingsConnectorsModelPath && { embeddingsConnectorsModelPath }), ...(esrganModelPath && { esrganModelPath }) @@ -314,6 +333,7 @@ export const diffusionPlugin = definePlugin({ llmModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, + uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, upscaler, @@ -339,6 +359,9 @@ export const diffusionPlugin = definePlugin({ ...(artifacts?.['t5XxlModelPath'] && { t5Xxl: artifacts['t5XxlModelPath'] }), ...(artifacts?.['llmModelPath'] && { llm: artifacts['llmModelPath'] }), ...(artifacts?.['vaeModelPath'] && { vae: artifacts['vaeModelPath'] }), + ...(artifacts?.['uncondModelPath'] && { + uncondModel: artifacts['uncondModelPath'] + }), ...(artifacts?.['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] }) } diff --git a/packages/sdk/test/bare/sdcpp-ideogram-plugin.test.ts b/packages/sdk/test/bare/sdcpp-ideogram-plugin.test.ts new file mode 100644 index 0000000000..d30e3be2c4 --- /dev/null +++ b/packages/sdk/test/bare/sdcpp-ideogram-plugin.test.ts @@ -0,0 +1,148 @@ +import test from 'brittle' +import { sdcppConfigSchema } from '@/schemas' + +type DiffusionDebugModel = { + _files?: Record +} + +const IDEOGRAM_MODEL = + 'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4-Q4_0.gguf' +const IDEOGRAM_UNCOND_MODEL = + 'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4_uncond-Q4_0.gguf' +const IDEOGRAM_LLM = + 'https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q4_K_M.gguf' +const IDEOGRAM_VAE = + 'https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors' + +test('sdcppConfigSchema: accepts the Ideogram 4 companion layout', (t) => { + const result = sdcppConfigSchema.safeParse({ + mode: 'diffusion', + device: 'gpu', + diffusion_fa: true, + offload_to_cpu: true, + llmModelSrc: IDEOGRAM_LLM, + vaeModelSrc: IDEOGRAM_VAE, + uncondModelSrc: IDEOGRAM_UNCOND_MODEL + }) + + t.is(result.success, true) +}) + +test('sdcpp plugin resolveConfig: resolves Ideogram companions and strips source fields', async (t) => { + const { diffusionPlugin } = await import('@/server/bare/plugins/sdcpp-generation/plugin') + + const resolved = await diffusionPlugin.resolveConfig!( + { + mode: 'diffusion', + diffusion_fa: true, + offload_to_cpu: true, + llmModelSrc: IDEOGRAM_LLM, + vaeModelSrc: IDEOGRAM_VAE, + uncondModelSrc: IDEOGRAM_UNCOND_MODEL + }, + { + resolveModelPath: async (src: unknown) => `/cache/${String(src).split('/').pop()}`, + modelSrc: IDEOGRAM_MODEL, + modelType: 'sdcpp-generation' + } + ) + + t.alike(resolved.artifacts, { + llmModelPath: '/cache/Qwen3-VL-8B-Instruct-Q4_K_M.gguf', + vaeModelPath: '/cache/flux2-vae.safetensors', + uncondModelPath: '/cache/ideogram4_uncond-Q4_0.gguf' + }) + t.is('llmModelSrc' in resolved.config, false) + t.is('vaeModelSrc' in resolved.config, false) + t.is('uncondModelSrc' in resolved.config, false) + t.is(resolved.config.mode, 'diffusion') + t.is(resolved.config.diffusion_fa, true) + t.is(resolved.config.offload_to_cpu, true) +}) + +test('sdcpp plugin resolveConfig: rejects Ideogram unconditional model outside diffusion mode', async (t) => { + const [{ diffusionPlugin }, { ModelLoadFailedError }] = await Promise.all([ + import('@/server/bare/plugins/sdcpp-generation/plugin'), + import('@/utils/errors-server') + ]) + let resolveCalls = 0 + + try { + await diffusionPlugin.resolveConfig!( + { + mode: 'video', + llmModelSrc: IDEOGRAM_LLM, + vaeModelSrc: IDEOGRAM_VAE, + uncondModelSrc: IDEOGRAM_UNCOND_MODEL + }, + { + resolveModelPath: async () => { + resolveCalls++ + return '/cache/model' + }, + modelSrc: IDEOGRAM_MODEL, + modelType: 'sdcpp-generation' + } + ) + t.fail('expected ModelLoadFailedError for Ideogram unconditional model in video mode') + } catch (error) { + t.ok(error instanceof ModelLoadFailedError) + t.is(resolveCalls, 0, 'guard fires before any companion download') + } +}) + +test('sdcpp plugin resolveConfig: requires Ideogram LLM and VAE companions', async (t) => { + const [{ diffusionPlugin }, { ModelLoadFailedError }] = await Promise.all([ + import('@/server/bare/plugins/sdcpp-generation/plugin'), + import('@/utils/errors-server') + ]) + + for (const config of [ + { + mode: 'diffusion' as const, + vaeModelSrc: IDEOGRAM_VAE, + uncondModelSrc: IDEOGRAM_UNCOND_MODEL + }, + { + mode: 'diffusion' as const, + llmModelSrc: IDEOGRAM_LLM, + uncondModelSrc: IDEOGRAM_UNCOND_MODEL + } + ]) { + try { + await diffusionPlugin.resolveConfig!(config, { + resolveModelPath: async () => '/cache/model', + modelSrc: IDEOGRAM_MODEL, + modelType: 'sdcpp-generation' + }) + t.fail('expected ModelLoadFailedError for an incomplete Ideogram layout') + } catch (error) { + t.ok(error instanceof ModelLoadFailedError) + } + } +}) + +test('sdcpp plugin createModel: wires Ideogram unconditional model into the addon', async (t) => { + const { diffusionPlugin } = await import('@/server/bare/plugins/sdcpp-generation/plugin') + + const result = diffusionPlugin.createModel({ + modelId: 'ideogram-test', + modelPath: '/tmp/ideogram4-Q4_0.gguf', + modelConfig: { + mode: 'diffusion', + diffusion_fa: true, + offload_to_cpu: true + }, + artifacts: { + llmModelPath: '/tmp/Qwen3-VL-8B-Instruct-Q4_K_M.gguf', + vaeModelPath: '/tmp/flux2-vae.safetensors', + uncondModelPath: '/tmp/ideogram4_uncond-Q4_0.gguf' + } + }) + + const files = (result.model as unknown as DiffusionDebugModel)._files ?? {} + t.is(files['model'], '/tmp/ideogram4-Q4_0.gguf') + t.is(files['llm'], '/tmp/Qwen3-VL-8B-Instruct-Q4_K_M.gguf') + t.is(files['vae'], '/tmp/flux2-vae.safetensors') + t.is(files['uncondModel'], '/tmp/ideogram4_uncond-Q4_0.gguf') +}) From 6bc6ac81f4da5392b7ceb6aa5ca9ac842afc5a4c Mon Sep 17 00:00:00 2001 From: Maksim Smatrou Date: Wed, 15 Jul 2026 15:06:02 +0200 Subject: [PATCH 2/2] doc: document Ideogram structured prompt requirement --- packages/sdk/contract/schema.json | 2 +- packages/sdk/examples/diffusion-ideogram.ts | 2 ++ packages/sdk/schemas/sdcpp-config.ts | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/sdk/contract/schema.json b/packages/sdk/contract/schema.json index baa2e00361..3d61bae38c 100644 --- a/packages/sdk/contract/schema.json +++ b/packages/sdk/contract/schema.json @@ -7131,7 +7131,7 @@ ] }, "uncondModelSrc": { - "description": "Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, llmModelSrc (Qwen3-VL), and vaeModelSrc.", + "description": "Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, llmModelSrc (Qwen3-VL), vaeModelSrc, and a JSON-serialized structured caption with explicit bounding boxes as the generation prompt. Plain-text prompts produce degenerate output or the model's placeholder response.", "anyOf": [ { "type": "string" diff --git a/packages/sdk/examples/diffusion-ideogram.ts b/packages/sdk/examples/diffusion-ideogram.ts index 3b6ccd5283..e7b049f60c 100644 --- a/packages/sdk/examples/diffusion-ideogram.ts +++ b/packages/sdk/examples/diffusion-ideogram.ts @@ -20,6 +20,8 @@ function formatMb(bytes: number) { return (bytes / 1e6).toFixed(1) } +// Ideogram 4 expects a JSON-serialized structured caption with explicit +// bounding boxes. Plain-text prompts produce degenerate or placeholder output. const structuredPrompt = { high_level_description: 'A bright product photo of one yellow coffee mug on a clean office desk with a tiny readable label that says IDEO.', diff --git a/packages/sdk/schemas/sdcpp-config.ts b/packages/sdk/schemas/sdcpp-config.ts index 5b119bde6a..d2e705170f 100644 --- a/packages/sdk/schemas/sdcpp-config.ts +++ b/packages/sdk/schemas/sdcpp-config.ts @@ -164,7 +164,9 @@ export const sdcppConfigSchema = z.object({ .optional() .describe( 'Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, ' + - 'llmModelSrc (Qwen3-VL), and vaeModelSrc.' + 'llmModelSrc (Qwen3-VL), vaeModelSrc, and a JSON-serialized structured ' + + 'caption with explicit bounding boxes as the generation prompt. Plain-text ' + + "prompts produce degenerate output or the model's placeholder response." ), clipVisionModelSrc: modelSrcInputSchema .optional()