QVAC-22218 feat[api]: add Ideogram 4 diffusion support

packages/sdk/contract/schema.json

@@ -6540,7 +6540,7 @@
                   "properties": {
                     "mode": {
                       "default": "diffusion",
-                      "description": "Operation mode for the diffusion plugin. `'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from the primary model plus optional auxiliary text encoders, VAE, and ESRGAN upscaler, and exposes diffusion({ ... }). `'upscale'` builds a standalone ESRGAN upscaler from the primary model file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). `'video'` builds a `VideoStableDiffusion` pipeline and exposes video({ ... }). The video layout is selected from the auxiliary sources: supplying `embeddingsConnectorsModelSrc` loads the LTX-2 layout (Gemma text encoder via `llmModelSrc` + video VAE + connectors, optional `audioVaeModelSrc` for synchronized audio); otherwise the Wan layout is used (UMT5 text encoder via `t5XxlModelSrc` + VAE). On React Native, loading the video model on-device will likely fail because the video diffusion models currently shipped by the SDK are too large to load on typical mobile devices; pass a `delegate` to `loadModel(...)` to run generation on a desktop peer instead.",
+                      "description": "Operation mode for the diffusion plugin. `'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from the primary model plus optional auxiliary text encoders, VAE, unconditional diffusion model, and ESRGAN upscaler, and exposes diffusion({ ... }). `'upscale'` builds a standalone ESRGAN upscaler from the primary model file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). `'video'` builds a `VideoStableDiffusion` pipeline and exposes video({ ... }). The video layout is selected from the auxiliary sources: supplying `embeddingsConnectorsModelSrc` loads the LTX-2 layout (Gemma text encoder via `llmModelSrc` + video VAE + connectors, optional `audioVaeModelSrc` for synchronized audio); otherwise the Wan layout is used (UMT5 text encoder via `t5XxlModelSrc` + VAE). On React Native, loading the video model on-device will likely fail because the video diffusion models currently shipped by the SDK are too large to load on typical mobile devices; pass a `delegate` to `loadModel(...)` to run generation on a desktop peer instead.",
                       "type": "string",
                       "enum": ["diffusion", "upscale", "video"],
                       "title": "LoadModelSrcRequestSdcppGenerationModelConfigMode"
@@ -6885,7 +6885,7 @@
                       ]
                     },
                     "llmModelSrc": {
-                      "description": "LLM text encoder model — required for FLUX.2 [klein] (Qwen3) and for LTX-2 video (Gemma).",
+                      "description": "LLM text encoder model — required for FLUX.2 [klein] (Qwen3), Ideogram 4 (Qwen3-VL), and LTX-2 video (Gemma).",
                       "anyOf": [
                         {
                           "type": "string"
@@ -6967,7 +6967,7 @@
                       ]
                     },
                     "vaeModelSrc": {
-                      "description": "VAE decoder model — required for FLUX.2 [klein] and LTX-2 video (video VAE), optional for SDXL.",
+                      "description": "VAE decoder model — required for FLUX.2 [klein], Ideogram 4, and LTX-2 video (video VAE); optional for SDXL.",
                       "anyOf": [
                         {
                           "type": "string"
@@ -7130,6 +7130,88 @@
                         }
                       ]
                     },
+                    "uncondModelSrc": {
+                      "description": "Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, llmModelSrc (Qwen3-VL), vaeModelSrc, and a JSON-serialized structured caption with explicit bounding boxes as the generation prompt. Plain-text prompts produce degenerate output or the model's placeholder response.",
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "object",
+                          "properties": {
+                            "src": {
+                              "type": "string"
+                            },
+                            "name": {
+                              "type": "string"
+                            },
+                            "modelId": {
+                              "type": "string"
+                            },
+                            "registryPath": {
+                              "type": "string"
+                            },
+                            "registrySource": {
+                              "type": "string"
+                            },
+                            "blobCoreKey": {
+                              "type": "string"
+                            },
+                            "blobIndex": {
+                              "type": "number"
+                            },
+                            "engine": {
+                              "type": "string"
+                            },
+                            "expectedSize": {
+                              "type": "number"
+                            },
+                            "sha256Checksum": {
+                              "type": "string"
+                            },
+                            "addon": {
+                              "anyOf": [
+                                {
+                                  "type": "string",
+                                  "enum": [
+                                    "llamacpp-completion",
+                                    "whispercpp-transcription",
+                                    "bci-whispercpp-transcription",
+                                    "llamacpp-embedding",
+                                    "nmtcpp-translation",
+                                    "onnx-tts",
+                                    "tts-ggml",
+                                    "parakeet-transcription",
+                                    "ggml-ocr",
+                                    "sdcpp-generation",
+                                    "ggml-vla",
+                                    "ggml-classification",
+                                    "llm",
+                                    "whisper",
+                                    "bci",
+                                    "embeddings",
+                                    "nmt",
+                                    "parakeet",
+                                    "tts",
+                                    "ocr",
+                                    "diffusion",
+                                    "vla",
+                                    "classification"
+                                  ],
+                                  "title": "LoadModelSrcRequestSdcppGenerationModelConfigUncondModelSrcAddon"
+                                },
+                                {
+                                  "type": "string",
+                                  "const": "vad"
+                                }
+                              ]
+                            }
+                          },
+                          "required": ["src"],
+                          "title": "LoadModelSrcRequestSdcppGenerationModelConfigUncondModelSrc"
+                        }
+                      ]
+                    },
                     "clipVisionModelSrc": {
                       "description": "OpenCLIP ViT-H/14 weights (`clip_vision_h.safetensors`). Required for Wan image-to-video (`img2vid`); omit for text-to-video-only pipelines. Not used by LTX-2 (its img2vid path needs no CLIP-vision projection).",
                       "anyOf": [

packages/sdk/examples/diffusion-ideogram.ts

@@ -0,0 +1,120 @@
+import { diffusion, loadModel, unloadModel } from '@qvac/sdk'
+import fs from 'fs'
+import path from 'path'
+
+const diffusionModelSrc =
+  process.argv[2] ||
+  'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4-Q4_0.gguf'
+const uncondModelSrc =
+  process.argv[3] ||
+  'https://huggingface.co/leejet/ideogram-4-GGUF/resolve/main/ideogram4_uncond-Q4_0.gguf'
+const llmModelSrc =
+  process.argv[4] ||
+  'https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q4_K_M.gguf'
+const vaeModelSrc =
+  process.argv[5] ||
+  'https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors'
+const outputDir = process.argv[6] || '.'
+
+function formatMb(bytes: number) {
+  return (bytes / 1e6).toFixed(1)
+}
+
+// Ideogram 4 expects a JSON-serialized structured caption with explicit
+// bounding boxes. Plain-text prompts produce degenerate or placeholder output.
+const structuredPrompt = {
+  high_level_description:
+    'A bright product photo of one yellow coffee mug on a clean office desk with a tiny readable label that says IDEO.',
+  style_description: {
+    aesthetics: 'simple product photography, clean bright office desk, minimal composition',
+    lighting: 'soft studio light with gentle shadows',
+    photo: 'sharp focus, crisp readable label text',
+    medium: 'product photo',
+    color_palette: ['#FFD54A', '#FFFFFF', '#111111', '#7EC8E3']
+  },
+  compositional_deconstruction: {
+    canvas: 'Square canvas, upright orientation. All text is horizontal and readable.',
+    background: 'Clean white office desk with a soft blue background gradient.',
+    elements: [
+      {
+        type: 'obj',
+        bbox: [230, 260, 800, 740],
+        desc: 'Exactly one matte yellow ceramic coffee mug centered on the desk.'
+      },
+      {
+        type: 'text',
+        bbox: [510, 360, 620, 640],
+        text: 'IDEO',
+        desc: 'Small black label printed horizontally on the mug.'
+      }
+    ]
+  }
+}
+
+if (process.argv.includes('--help')) {
+  console.log('Usage: tsx examples/diffusion-ideogram.ts [model] [uncond] [llm] [vae] [output-dir]')
+  console.log('▸ Defaults use the four Hugging Face model URLs supported by the diffusion addon.')
+  process.exit(0)
+}
+
+let modelId: string | undefined
+
+try {
+  console.log('▸ Loading Ideogram 4 split-layout model...')
+  modelId = await loadModel({
+    modelSrc: diffusionModelSrc,
+    modelType: 'sdcpp-generation',
+    modelConfig: {
+      device: 'gpu',
+      threads: 4,
+      diffusion_fa: true,
+      offload_to_cpu: true,
+      llmModelSrc,
+      vaeModelSrc,
+      uncondModelSrc
+    },
+    onProgress: (progress) => {
+      const line = `▸ Downloading ${progress.percentage.toFixed(0)}% (${formatMb(progress.downloaded)}/${formatMb(progress.total)} MB)`
+      process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`)
+      if (progress.percentage >= 100) process.stderr.write('\n')
+    }
+  })
+  console.log(`▸ Model loaded: ${modelId}`)
+
+  const { progressStream, outputs, stats } = diffusion({
+    modelId,
+    prompt: JSON.stringify(structuredPrompt),
+    width: 768,
+    height: 768,
+    steps: 16,
+    cfg_scale: 7,
+    seed: 42
+  })
+
+  for await (const { step, totalSteps } of progressStream) {
+    console.log(`▸ step ${step}/${totalSteps}`)
+  }
+
+  const buffers = await outputs
+  for (let i = 0; i < buffers.length; i++) {
+    const outputPath = path.join(outputDir, `ideogram_${i}.png`)
+    fs.writeFileSync(outputPath, buffers[i]!)
+    console.log(`▸ Saved ${outputPath}`)
+  }
+
+  console.log('▸ Stats:', await stats)
+  await unloadModel({ modelId, clearStorage: false })
+  modelId = undefined
+  console.log('▸ Done')
+  process.exit(0)
+} catch (error) {
+  if (modelId) {
+    try {
+      await unloadModel({ modelId, clearStorage: false })
+    } catch {
+      // Preserve the original error.
+    }
+  }
+  console.error('✖', error)
+  process.exit(1)
+}

packages/sdk/package.json

@@ -205,7 +205,7 @@
     "@qvac/bci-whispercpp": "^0.4.1",
     "@qvac/classification-ggml": "^0.10.2",
     "@qvac/decoder-audio": "^0.5.0",
-    "@qvac/diffusion-cpp": "^0.13.3",
+    "@qvac/diffusion-cpp": "^0.14.0",
     "@qvac/embed-llamacpp": "^0.26.2",
     "@qvac/error": "^0.1.1",
     "@qvac/langdetect-text": "^0.1.2",

packages/sdk/schemas/sdcpp-config.ts

@@ -54,7 +54,8 @@ export const sdcppConfigSchema = z.object({
     .describe(
       'Operation mode for the diffusion plugin. ' +
         "`'diffusion'` (default) builds a full SD / SDXL / SD3 / FLUX pipeline from " +
-        'the primary model plus optional auxiliary text encoders, VAE, and ESRGAN ' +
+        'the primary model plus optional auxiliary text encoders, VAE, unconditional ' +
+        'diffusion model, and ESRGAN ' +
         'upscaler, and exposes diffusion({ ... }). ' +
         "`'upscale'` builds a standalone ESRGAN upscaler from the primary model " +
         'file alone (auxiliary model sources are ignored) and exposes upscale({ ... }). ' +
@@ -147,18 +148,26 @@ export const sdcppConfigSchema = z.object({
   llmModelSrc: modelSrcInputSchema
     .optional()
     .describe(
-      'LLM text encoder model — required for FLUX.2 [klein] (Qwen3) and for ' +
-        'LTX-2 video (Gemma).'
+      'LLM text encoder model — required for FLUX.2 [klein] (Qwen3), ' +
+        'Ideogram 4 (Qwen3-VL), and LTX-2 video (Gemma).'
     ),
   vaeModelSrc: modelSrcInputSchema
     .optional()
     .describe(
-      'VAE decoder model — required for FLUX.2 [klein] and LTX-2 video (video VAE), ' +
-        'optional for SDXL.'
+      'VAE decoder model — required for FLUX.2 [klein], Ideogram 4, and ' +
+        'LTX-2 video (video VAE); optional for SDXL.'
     ),
   highNoiseDiffusionModelSrc: modelSrcInputSchema
     .optional()
     .describe('High-noise diffusion expert — required for Wan 2.2 mixture-of-experts video models'),
+  uncondModelSrc: modelSrcInputSchema
+    .optional()
+    .describe(
+      'Unconditional diffusion model — Ideogram 4 only. Requires diffusion mode, ' +
+        'llmModelSrc (Qwen3-VL), vaeModelSrc, and a JSON-serialized structured ' +
+        'caption with explicit bounding boxes as the generation prompt. Plain-text ' +
+        "prompts produce degenerate output or the model's placeholder response."
+    ),
   clipVisionModelSrc: modelSrcInputSchema
     .optional()
     .describe(

packages/sdk/server/bare/plugins/sdcpp-generation/plugin.ts

@@ -41,6 +41,7 @@ type DiffusionArtifactKey =
   | 'llmModelPath'
   | 'vaeModelPath'
   | 'highNoiseDiffusionModelPath'
+  | 'uncondModelPath'
   | 'audioVaeModelPath'
   | 'embeddingsConnectorsModelPath'
   | 'esrganModelPath'
@@ -96,6 +97,19 @@ export const diffusionPlugin = definePlugin({
     cfg: SdcppConfig,
     ctx: ResolveContext
   ): Promise<ResolveResult<SdcppConfig, DiffusionArtifactKey>> {
+    if (cfg.uncondModelSrc && cfg.mode !== 'diffusion') {
+      throw new ModelLoadFailedError(
+        'modelConfig.uncondModelSrc is Ideogram 4 diffusion only. ' +
+          "Use mode: 'diffusion' or remove uncondModelSrc."
+      )
+    }
+    if (cfg.uncondModelSrc && (!cfg.llmModelSrc || !cfg.vaeModelSrc)) {
+      throw new ModelLoadFailedError(
+        'modelConfig.uncondModelSrc selects the Ideogram 4 layout and requires ' +
+          'modelConfig.llmModelSrc (Qwen3-VL) and modelConfig.vaeModelSrc.'
+      )
+    }
+
     // Standalone-upscaler mode never references auxiliary models: the primary
     // modelSrc IS the ESRGAN file. Skip resolution to avoid downloading
     // unused encoders/VAEs and to keep load fast.
@@ -111,6 +125,7 @@ export const diffusionPlugin = definePlugin({
       llmModelSrc,
       vaeModelSrc,
       highNoiseDiffusionModelSrc,
+      uncondModelSrc,
       audioVaeModelSrc,
       embeddingsConnectorsModelSrc,
       upscaler,
@@ -153,6 +168,7 @@ export const diffusionPlugin = definePlugin({
       llmModelSrc,
       vaeModelSrc,
       highNoiseDiffusionModelSrc,
+      uncondModelSrc,
       audioVaeModelSrc,
       embeddingsConnectorsModelSrc,
       esrganModelSrc
@@ -172,6 +188,7 @@ export const diffusionPlugin = definePlugin({
       llmModelPath,
       vaeModelPath,
       highNoiseDiffusionModelPath,
+      uncondModelPath,
       audioVaeModelPath,
       embeddingsConnectorsModelPath,
       esrganModelPath
@@ -183,6 +200,7 @@ export const diffusionPlugin = definePlugin({
       llmModelSrc ? resolve(llmModelSrc) : undefined,
       vaeModelSrc ? resolve(vaeModelSrc) : undefined,
       highNoiseDiffusionModelSrc ? resolve(highNoiseDiffusionModelSrc) : undefined,
+      uncondModelSrc ? resolve(uncondModelSrc) : undefined,
       audioVaeModelSrc ? resolve(audioVaeModelSrc) : undefined,
       embeddingsConnectorsModelSrc ? resolve(embeddingsConnectorsModelSrc) : undefined,
       esrganModelSrc ? resolve(esrganModelSrc) : undefined
@@ -198,6 +216,7 @@ export const diffusionPlugin = definePlugin({
         ...(llmModelPath && { llmModelPath }),
         ...(vaeModelPath && { vaeModelPath }),
         ...(highNoiseDiffusionModelPath && { highNoiseDiffusionModelPath }),
+        ...(uncondModelPath && { uncondModelPath }),
         ...(audioVaeModelPath && { audioVaeModelPath }),
         ...(embeddingsConnectorsModelPath && { embeddingsConnectorsModelPath }),
         ...(esrganModelPath && { esrganModelPath })
@@ -314,6 +333,7 @@ export const diffusionPlugin = definePlugin({
         llmModelSrc,
         vaeModelSrc,
         highNoiseDiffusionModelSrc,
+        uncondModelSrc,
         audioVaeModelSrc,
         embeddingsConnectorsModelSrc,
         upscaler,
@@ -339,6 +359,9 @@ export const diffusionPlugin = definePlugin({
       ...(artifacts?.['t5XxlModelPath'] && { t5Xxl: artifacts['t5XxlModelPath'] }),
       ...(artifacts?.['llmModelPath'] && { llm: artifacts['llmModelPath'] }),
       ...(artifacts?.['vaeModelPath'] && { vae: artifacts['vaeModelPath'] }),
+      ...(artifacts?.['uncondModelPath'] && {
+        uncondModel: artifacts['uncondModelPath']
+      }),
       ...(artifacts?.['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
     }