From c32fc119e8c29d75e40330508f417f71326cbac5 Mon Sep 17 00:00:00 2001 From: Ramaz Tskhadadze Date: Wed, 15 Jul 2026 10:53:40 +0400 Subject: [PATCH] test: cover VLM continuous batching on mobile and rolling slot admission The continuousBatching group is already registered for both ios and android in test-groups.json, but every test in continuous-batching.test.js gates on skipHeavyPlatform (isMobile || isDarwin), so both Device Farm runs import the module and execute zero tests. VLM continuous batching has no mobile coverage. Add a mobile smoke test at parallel: 2 with one image slot beside one text slot, skipped only on darwin so it runs on the Device Farm pools and on Linux. Vision encode is serialized across slots, so pairing an image with a text prompt is what catches an encode barrier starving the text slot. It asserts both outputs are correct and that the slots decoded together. Add a desktop test for rolling admission: the existing MTMD tests submit 4 image prompts into 4 slots, so every sequence is admitted in the initial wave and no slot is ever recycled. Halving the slots forces the second half of IMAGE_CASES to wait in pending_ and be admitted into freed slots, exercising a per-slot MTMD driver resetting its media and vision-encoding a new image while the other slot keeps decoding. Stage the SmolVLM2 model and mmproj for runContinuousBatchingTest in model-manifest.json. The entry listed only Llama-3.2-1B, so the Android pre-stage would have missed the vision models and downloaded them on-device from HuggingFace. Entries are added by hand rather than regenerating, since the generator scrapes through the _image-common.js require and would also stage Qwen3VL-2B and its mmproj, which this file never loads. --- .../integration/continuous-batching.test.js | 147 ++++++++++++++++++ .../test/mobile/model-manifest.json | 8 + 2 files changed, 155 insertions(+) diff --git a/packages/llm-llamacpp/test/integration/continuous-batching.test.js b/packages/llm-llamacpp/test/integration/continuous-batching.test.js index 6c01cbb675..a2d1b12de0 100644 --- a/packages/llm-llamacpp/test/integration/continuous-batching.test.js +++ b/packages/llm-llamacpp/test/integration/continuous-batching.test.js @@ -460,6 +460,79 @@ safeTest( } ) +// The MTMD batch tests below are desktop-only. This smoke test is the one that +// runs on the Device Farm pools: it is the minimum shape that proves the batch +// scheduler drives a vision slot on device (parallel: 2, one image), keeping +// per-slot context and device time within mobile budgets. +safeTest( + 'continuous batching MTMD: mobile smoke — image and text slots decode together', + { timeout: 900_000, skip: isDarwin }, + async (t) => { + // parallel: 2 against the default ctx_size 4096 leaves each slot a 2048-token + // window — comfortably above SmolVLM2-500M's ~256 vision tokens plus prompt + // and output, which matters because oversized prompts are rejected outright + // rather than truncated. + const model = await setupMultimodalBatchModel(t, { parallel: '2' }) + + const imageCase = IMAGE_CASES[0] + const textCase = CASES[0] + t.ok( + fs.existsSync(getMediaPath(imageCase.imageFile)), + `media file ${imageCase.imageFile} exists` + ) + + // One image slot beside one text slot. Vision encode is serialized across + // slots, so this pairing is what would expose an encode barrier starving the + // text slot: the text sequence must keep decoding while the image encodes. + const batchInput = [ + buildBatchItem(imageCase), + { + id: textCase.id, + prompt: [ + { role: 'system', content: 'Answer with one word only.' }, + { role: 'user', content: textCase.user } + ], + runOptions: { generationParams: { predict: 16 } } + } + ] + + const batchResponse = await model.run(batchInput) + const streamingProgress = logStreamingProgress(batchResponse, 'cb-mtmd-mobile') + const batchResults = await batchResponse.await() + streamingProgress.flush() + + const avgConcurrentSeq = toNumber(batchResponse.stats.avgConcurrentSeq) + t.comment(`native TPS: ${toNumber(batchResponse.stats.TPS)}`) + t.comment(`avgConcurrentSeq: ${avgConcurrentSeq}`) + + t.alike( + batchResults.map((r) => r.id), + [imageCase.id, textCase.id], + 'both ids reported in input order' + ) + + const resultsById = new Map(batchResults.map((r) => [r.id, r.output])) + for (const item of [imageCase, textCase]) { + const output = resultsById.get(item.id) || '' + console.log(`[cb-mtmd-mobile result] ${item.id}: ${output.trim()}`) + t.comment(`${item.id}: ${output.trim()}`) + t.ok( + containsExpectedWord(output, item.expected), + `${item.id} output includes one of [${item.expected.join(', ')}]. Full output: "${output.trim()}"` + ) + } + + // Both sequences are admitted in the same wave, so they overlap for at least + // the steps before the shorter text slot finishes. A mean pinned at exactly + // 1.0 means the slots never decoded together and batching collapsed to + // serial execution. + t.ok( + avgConcurrentSeq > 1.0, + `avgConcurrentSeq (${avgConcurrentSeq}) > 1.0 confirms the image and text slots decoded together` + ) + } +) + test( 'continuous batching MTMD: image-only batch returns correct descriptions', { timeout: 900_000, skip: skipHeavyPlatform }, @@ -559,6 +632,80 @@ test( } ) +test( + 'continuous batching MTMD: image prompts outnumber slots and roll through freed slots', + { timeout: 900_000, skip: skipHeavyPlatform }, + async (t) => { + // The other MTMD batch tests submit 4 image prompts into 4 slots, so every + // sequence is admitted in the initial wave and no slot is ever recycled. + // Halving the slots forces the second half of IMAGE_CASES to wait in + // pending_ and be admitted into slots freed by finished sequences, which is + // the rolling-admission path: a per-slot MTMD driver must reset its media + // and vision-encode a new image while the other slot keeps decoding. + const model = await setupMultimodalBatchModel(t, { parallel: '2' }) + const slotCount = 2 + + for (const item of IMAGE_CASES) { + t.ok(fs.existsSync(getMediaPath(item.imageFile)), `media file ${item.imageFile} exists`) + } + + t.ok( + IMAGE_CASES.length > slotCount, + `${IMAGE_CASES.length} image prompts exceed ${slotCount} slots, forcing rolling admission` + ) + + const batchInput = IMAGE_CASES.map(buildBatchItem) + const batchStartedAt = Date.now() + const batchResponse = await model.run(batchInput) + const streamingProgress = logStreamingProgress(batchResponse, 'cb-mtmd-rolling') + const batchResults = await batchResponse.await() + streamingProgress.flush() + + const avgConcurrentSeq = toNumber(batchResponse.stats.avgConcurrentSeq) + t.comment(`elapsed: ${Date.now() - batchStartedAt}ms`) + t.comment(`native TPS: ${toNumber(batchResponse.stats.TPS)}`) + t.comment(`avgConcurrentSeq: ${avgConcurrentSeq}`) + + t.alike( + batchResults.map((r) => r.id), + IMAGE_CASES.map((item) => item.id), + 'all ids reported in input order despite being admitted in separate waves' + ) + t.alike( + streamingProgress.ids().sort(), + IMAGE_CASES.map((item) => item.id).sort(), + 'all ids emitted streaming chunks' + ) + + // IMAGE_CASES pairs two questions per image (elephant.jpg, news-paper.jpg). + // A slot that fails to reset media on recycle answers a late prompt from the + // image the previous sequence loaded, so a wrong-image answer surfaces here. + const resultsById = new Map(batchResults.map((r) => [r.id, r.output])) + for (const item of IMAGE_CASES) { + const output = resultsById.get(item.id) || '' + console.log(`[cb-mtmd-rolling result] ${item.id}: ${output.trim()}`) + t.comment(`${item.id}: ${output.trim()}`) + t.ok( + containsExpectedWord(output, item.expected), + `${item.id} output includes one of [${item.expected.join(', ')}]. Full output: "${output.trim()}"` + ) + } + + // Sequences admitted later still overlap the ones already decoding. The bar + // is below the 4-slot tests' 1.5: only 2 slots are ever live, and the tail + // of each wave drains to a single sequence, which pulls the mean down. + t.ok( + avgConcurrentSeq > 1.2, + `avgConcurrentSeq (${avgConcurrentSeq}) > 1.2 confirms rolled-in sequences decode alongside active ones` + ) + // Admission must respect n_seq_max: more pending work must not oversubscribe. + t.ok( + avgConcurrentSeq <= slotCount, + `avgConcurrentSeq (${avgConcurrentSeq}) never exceeds the ${slotCount} configured slots` + ) + } +) + test( 'continuous batching MTMD: mixed image+text batch processes all slot types correctly', { timeout: 1_200_000, skip: skipHeavyPlatform }, diff --git a/packages/llm-llamacpp/test/mobile/model-manifest.json b/packages/llm-llamacpp/test/mobile/model-manifest.json index d18b76f12e..8210440b7e 100644 --- a/packages/llm-llamacpp/test/mobile/model-manifest.json +++ b/packages/llm-llamacpp/test/mobile/model-manifest.json @@ -27,6 +27,14 @@ { "name": "Llama-3.2-1B-Instruct-Q4_0.gguf", "url": "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" + }, + { + "name": "SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF/resolve/main/SmolVLM2-500M-Video-Instruct-Q8_0.gguf" + }, + { + "name": "mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM2-500M-Video-Instruct-GGUF/resolve/main/mmproj-SmolVLM2-500M-Video-Instruct-Q8_0.gguf" } ], "runFinetuningPauseResumeTest": [