diff --git a/packages/llm-llamacpp/test/integration/continuous-batching.test.js b/packages/llm-llamacpp/test/integration/continuous-batching.test.js index 6c01cbb675..a2d1b12de0 100644 --- a/packages/llm-llamacpp/test/integration/continuous-batching.test.js +++ b/packages/llm-llamacpp/test/integration/continuous-batching.test.js @@ -460,6 +460,79 @@ safeTest( } ) +// The MTMD batch tests below are desktop-only. This smoke test is the one that +// runs on the Device Farm pools: it is the minimum shape that proves the batch +// scheduler drives a vision slot on device (parallel: 2, one image), keeping +// per-slot context and device time within mobile budgets. +safeTest( + 'continuous batching MTMD: mobile smoke — image and text slots decode together', + { timeout: 900_000, skip: isDarwin }, + async (t) => { + // parallel: 2 against the default ctx_size 4096 leaves each slot a 2048-token + // window — comfortably above SmolVLM2-500M's ~256 vision tokens plus prompt + // and output, which matters because oversized prompts are rejected outright + // rather than truncated. + const model = await setupMultimodalBatchModel(t, { parallel: '2' }) + + const imageCase = IMAGE_CASES[0] + const textCase = CASES[0] + t.ok( + fs.existsSync(getMediaPath(imageCase.imageFile)), + `media file ${imageCase.imageFile} exists` + ) + + // One image slot beside one text slot. Vision encode is serialized across + // slots, so this pairing is what would expose an encode barrier starving the + // text slot: the text sequence must keep decoding while the image encodes. + const batchInput = [ + buildBatchItem(imageCase), + { + id: textCase.id, + prompt: [ + { role: 'system', content: 'Answer with one word only.' }, + { role: 'user', content: textCase.user } + ], + runOptions: { generationParams: { predict: 16 } } + } + ] + + const batchResponse = await model.run(batchInput) + const streamingProgress = logStreamingProgress(batchResponse, 'cb-mtmd-mobile') + const batchResults = await batchResponse.await() + streamingProgress.flush() + + const avgConcurrentSeq = toNumber(batchResponse.stats.avgConcurrentSeq) + t.comment(`native TPS: ${toNumber(batchResponse.stats.TPS)}`) + t.comment(`avgConcurrentSeq: ${avgConcurrentSeq}`) + + t.alike( + batchResults.map((r) => r.id), + [imageCase.id, textCase.id], + 'both ids reported in input order' + ) + + const resultsById = new Map(batchResults.map((r) => [r.id, r.output])) + for (const item of [imageCase, textCase]) { + const output = resultsById.get(item.id) || '' + console.log(`[cb-mtmd-mobile result] ${item.id}: ${output.trim()}`) + t.comment(`${item.id}: ${output.trim()}`) + t.ok( + containsExpectedWord(output, item.expected), + `${item.id} output includes one of [${item.expected.join(', ')}]. Full output: "${output.trim()}"` + ) + } + + // Both sequences are admitted in the same wave, so they overlap for at least + // the steps before the shorter text slot finishes. A mean pinned at exactly + // 1.0 means the slots never decoded together and batching collapsed to + // serial execution. + t.ok( + avgConcurrentSeq > 1.0, + `avgConcurrentSeq (${avgConcurrentSeq}) > 1.0 confirms the image and text slots decoded together` + ) + } +) + test( 'continuous batching MTMD: image-only batch returns correct descriptions', { timeout: 900_000, skip: skipHeavyPlatform }, @@ -559,6 +632,80 @@ test( } ) +test( + 'continuous batching MTMD: image prompts outnumber slots and roll through freed slots', + { timeout: 900_000, skip: skipHeavyPlatform }, + async (t) => { + // The other MTMD batch tests submit 4 image prompts into 4 slots, so every + // sequence is admitted in the initial wave and no slot is ever recycled. + // Halving the slots forces the second half of IMAGE_CASES to wait in + // pending_ and be admitted into slots freed by finished sequences, which is + // the rolling-admission path: a per-slot MTMD driver must reset its media + // and vision-encode a new image while the other slot keeps decoding. + const model = await setupMultimodalBatchModel(t, { parallel: '2' }) + const slotCount = 2 + + for (const item of IMAGE_CASES) { + t.ok(fs.existsSync(getMediaPath(item.imageFile)), `media file ${item.imageFile} exists`) + } + + t.ok( + IMAGE_CASES.length > slotCount, + `${IMAGE_CASES.length} image prompts exceed ${slotCount} slots, forcing rolling admission` + ) + + const batchInput = IMAGE_CASES.map(buildBatchItem) + const batchStartedAt = Date.now() + const batchResponse = await model.run(batchInput) + const streamingProgress = logStreamingProgress(batchResponse, 'cb-mtmd-rolling') + const batchResults = await batchResponse.await() + streamingProgress.flush() + + const avgConcurrentSeq = toNumber(batchResponse.stats.avgConcurrentSeq) + t.comment(`elapsed: ${Date.now() - batchStartedAt}ms`) + t.comment(`native TPS: ${toNumber(batchResponse.stats.TPS)}`) + t.comment(`avgConcurrentSeq: ${avgConcurrentSeq}`) + + t.alike( + batchResults.map((r) => r.id), + IMAGE_CASES.map((item) => item.id), + 'all ids reported in input order despite being admitted in separate waves' + ) + t.alike( + streamingProgress.ids().sort(), + IMAGE_CASES.map((item) => item.id).sort(), + 'all ids emitted streaming chunks' + ) + + // IMAGE_CASES pairs two questions per image (elephant.jpg, news-paper.jpg). + // A slot that fails to reset media on recycle answers a late prompt from the + // image the previous sequence loaded, so a wrong-image answer surfaces here. + const resultsById = new Map(batchResults.map((r) => [r.id, r.output])) + for (const item of IMAGE_CASES) { + const output = resultsById.get(item.id) || '' + console.log(`[cb-mtmd-rolling result] ${item.id}: ${output.trim()}`) + t.comment(`${item.id}: ${output.trim()}`) + t.ok( + containsExpectedWord(output, item.expected), + `${item.id} output includes one of [${item.expected.join(', ')}]. Full output: "${output.trim()}"` + ) + } + + // Sequences admitted later still overlap the ones already decoding. The bar + // is below the 4-slot tests' 1.5: only 2 slots are ever live, and the tail + // of each wave drains to a single sequence, which pulls the mean down. + t.ok( + avgConcurrentSeq > 1.2, + `avgConcurrentSeq (${avgConcurrentSeq}) > 1.2 confirms rolled-in sequences decode alongside active ones` + ) + // Admission must respect n_seq_max: more pending work must not oversubscribe. + t.ok( + avgConcurrentSeq <= slotCount, + `avgConcurrentSeq (${avgConcurrentSeq}) never exceeds the ${slotCount} configured slots` + ) + } +) + test( 'continuous batching MTMD: mixed image+text batch processes all slot types correctly', { timeout: 1_200_000, skip: skipHeavyPlatform }, diff --git a/packages/llm-llamacpp/test/mobile/model-manifest.json b/packages/llm-llamacpp/test/mobile/model-manifest.json index d18b76f12e..8210440b7e 100644 --- a/packages/llm-llamacpp/test/mobile/model-manifest.json +++ b/packages/llm-llamacpp/test/mobile/model-manifest.json @@ -27,6 +27,14 @@ { "name": "Llama-3.2-1B-Instruct-Q4_0.gguf", "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" + }, + { + "name": "SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF/resolve/main/SmolVLM2-500M-Video-Instruct-Q8_0.gguf" + }, + { + "name": "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF/resolve/main/mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf" } ], "runFinetuningPauseResumeTest": [