@@ -319,7 +319,7 @@ index 9069463fe..b7fa1e534 100644
319319 }
320320
321321 auto result = common_speculative_get_output_limits(
322- @@ -211 ,6 +213 ,30 @@ struct server_slot {
322+ @@ -212 ,6 +214 ,30 @@ struct server_slot {
323323 mtmd_context * mctx = nullptr;
324324 mtmd::batch_ptr mbatch = nullptr;
325325
@@ -350,7 +350,7 @@ index 9069463fe..b7fa1e534 100644
350350 // speculative decoding
351351 common_speculative * spec;
352352
353- @@ -400 ,6 +426 ,8 @@ struct server_slot {
353+ @@ -391 ,6 +417 ,8 @@ struct server_slot {
354354
355355 // clear multimodal state
356356 mbatch.reset();
@@ -359,7 +359,7 @@ index 9069463fe..b7fa1e534 100644
359359 }
360360
361361 void init_sampler() const {
362- @@ -941 ,6 +969 ,14 @@ public:
362+ @@ -829 ,6 +857 ,14 @@ public:
363363 mtmd_context * mctx = nullptr;
364364 const llama_vocab * vocab = nullptr;
365365
@@ -374,7 +374,7 @@ index 9069463fe..b7fa1e534 100644
374374 server_queue queue_tasks;
375375 server_response queue_results;
376376
377- @@ -1394 ,6 +1430 ,10 @@ private:
377+ @@ -1288 ,6 +1324 ,10 @@ private:
378378 slot.mctx = mctx;
379379 slot.prompt.tokens.has_mtmd = mctx != nullptr;
380380
@@ -385,7 +385,7 @@ index 9069463fe..b7fa1e534 100644
385385 SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
386386
387387 slot.callback_on_release = [this](int id_slot) {
388- @@ -1847 ,6 +1887 ,28 @@ private:
388+ @@ -1748 ,6 +1788 ,28 @@ private:
389389
390390 SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
391391
@@ -414,7 +414,7 @@ index 9069463fe..b7fa1e534 100644
414414 // initialize samplers
415415 if (task.need_sampling()) {
416416 try {
417- @@ -1864 ,6 +1926 ,9 @@ private:
417+ @@ -1765 ,6 +1827 ,9 @@ private:
418418 // TODO: getting pre sampling logits is not yet supported with backend sampling
419419 use_backend_sampling &= !need_pre_sample_logits;
420420
@@ -424,7 +424,7 @@ index 9069463fe..b7fa1e534 100644
424424 // TODO: tmp until backend sampling is fully implemented
425425 if (use_backend_sampling) {
426426 llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
427- @@ -1879 ,9 +1944 ,13 @@ private:
427+ @@ -1783 ,9 +1848 ,13 @@ private:
428428
429429 slot.task = std::make_unique<const server_task>(std::move(task));
430430
@@ -441,7 +441,7 @@ index 9069463fe..b7fa1e534 100644
441441
442442 // reset server kill-switch counter
443443 n_empty_consecutive = 0;
444- @@ -2158 ,6 +2227 ,18 @@ private:
444+ @@ -2050 ,6 +2119 ,18 @@ private:
445445 queue_results.send(std::move(res));
446446 }
447447
@@ -460,22 +460,15 @@ index 9069463fe..b7fa1e534 100644
460460 void send_final_response(server_slot & slot) {
461461 auto res = std::make_unique<server_task_result_cmpl_final>();
462462
463- @@ -2657 ,6 +2738 ,7 @@ private:
463+ @@ -2556 ,6 +2637 ,7 @@ private:
464464 case SERVER_TASK_TYPE_EMBEDDING:
465465 case SERVER_TASK_TYPE_RERANK:
466466 case SERVER_TASK_TYPE_SCORE:
467467+ case SERVER_TASK_TYPE_TTS:
468468 {
469469 // special case: if input is provided via CLI, tokenize it first
470470 // otherwise, no need to tokenize as it's already done inside the HTTP thread
471- @@ -3082,10 +3164,18 @@ private:
472- SRV_ERR("pre_decode() failed: %s\n", e.what());
473- abort_all_slots("pre_decode() failed: " + std::string(e.what()));
474-
475- // the batch is half-built and not rendered, skip now to avoid UB
476- return;
477- }
478-
471+ @@ -3007,1 +3089,9 @@ private:
479472+ // note: TTS slots bypass the shared batch entirely
480473+ try {
481474+ process_tts_slots();
@@ -485,9 +478,7 @@ index 9069463fe..b7fa1e534 100644
485478+ }
486479+
487480 GGML_ASSERT(batch.slot_batched || batch.size() == 0);
488-
489- if (batch.slot_batched) {
490- @@ -3162,10 +3252,77 @@ private:
481+ @@ -3074,10 +3164,77 @@ private:
491482 }
492483 }
493484
@@ -565,7 +556,7 @@ index 9069463fe..b7fa1e534 100644
565556 if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
566557 if (!params_base.ctx_shift) {
567558 // this check is redundant (for good)
568- @@ -3238 ,7 +3395 ,7 @@ private:
559+ @@ -3150 ,7 +3307 ,7 @@ private:
569560
570561 // determine which slots are generating and drafting
571562 iterate(slots, [&](server_slot & slot) {
@@ -574,7 +565,7 @@ index 9069463fe..b7fa1e534 100644
574565 return;
575566 }
576567
577- @@ -3370 ,7 +3527 ,7 @@ private:
568+ @@ -3284 ,7 +3441 ,7 @@ private:
578569 return; // batch is full, skip remaining slots
579570 }
580571
@@ -583,7 +574,7 @@ index 9069463fe..b7fa1e534 100644
583574 return;
584575 }
585576
586- @@ -4379 ,6 +4536 ,8 @@ server_context_meta server_context::get_meta() const {
577+ @@ -4433 ,6 +4590 ,8 @@ server_context_meta server_context::get_meta() const {
587578 /* has_inp_image */ impl->chat_params.allow_image,
588579 /* has_inp_audio */ impl->chat_params.allow_audio,
589580 /* has_inp_video */ impl->chat_params.allow_video,
@@ -592,7 +583,7 @@ index 9069463fe..b7fa1e534 100644
592583 /* json_ui_settings */ impl->json_ui_settings,
593584 /* slot_n_ctx */ impl->get_slot_n_ctx(),
594585 /* pooling_type */ llama_pooling_type(impl->ctx_tgt),
595- @@ -4458 ,6 +4617 ,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
586+ @@ -4512 ,6 +4671 ,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
596587
597588 res->set_req(&req); // will also set spipe if needed
598589
@@ -604,7 +595,7 @@ index 9069463fe..b7fa1e534 100644
604595 int32_t sse_ping_interval = params.sse_ping_interval;
605596
606597 try {
607- @@ -5435 ,6 +5599 ,150 @@ void server_routes::init_routes() {
598+ @@ -5399 ,6 +5563 ,150 @@ void server_routes::init_routes() {
608599 return res;
609600 };
610601
@@ -780,7 +771,7 @@ diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
780771index 1ee677553..939630b8b 100644
781772--- a/tools/server/server-task.cpp
782773+++ b/tools/server/server-task.cpp
783- @@ -1523 ,6 +1523 ,17 @@ json server_task_result_rerank::to_json() {
774+ @@ -1497 ,6 +1497 ,17 @@ json server_task_result_rerank::to_json() {
784775 };
785776 }
786777
@@ -836,7 +827,7 @@ index 5bedf1987..e6ca67a65 100644
836827 return true;
837828 default:
838829 return false;
839- @@ -514 ,5 +520 ,15 @@ struct server_task_result_embd : server_task_result {
830+ @@ -494 ,5 +500 ,15 @@ struct server_task_result_embd : server_task_result {
840831 json to_json_oaicompat();
841832 };
842833
0 commit comments