diff --git a/doc/source/getting_started/installation.rst b/doc/source/getting_started/installation.rst index 3dc4d6ad05..378de9cbc5 100644 --- a/doc/source/getting_started/installation.rst +++ b/doc/source/getting_started/installation.rst @@ -65,7 +65,7 @@ Currently, supported models include: .. vllm_start -- ``code-llama``, ``code-llama-instruct``, ``code-llama-python``, ``deepseek``, ``deepseek-chat``, ``deepseek-coder``, ``deepseek-coder-instruct``, ``deepseek-r1-distill-llama``, ``gorilla-openfunctions-v2``, ``HuatuoGPT-o1-LLaMA-3.1``, ``llama-2``, ``llama-2-chat``, ``llama-3``, ``llama-3-instruct``, ``llama-3.1``, ``llama-3.1-instruct``, ``llama-3.3-instruct``, ``tiny-llama``, ``wizardcoder-python-v1.0``, ``wizardmath-v1.0``, ``Yi``, ``Yi-1.5``, ``Yi-1.5-chat``, ``Yi-1.5-chat-16k``, ``Yi-200k``, ``Yi-chat`` +- ``code-llama``, ``code-llama-instruct``, ``code-llama-python``, ``deepseek``, ``deepseek-chat``, ``deepseek-coder``, ``deepseek-coder-instruct``, ``deepseek-r1-distill-llama``, ``gorilla-openfunctions-v2``, ``HuatuoGPT-o1-LLaMA-3.1``, ``llama-2``, ``llama-2-chat``, ``llama-3``, ``llama-3-instruct``, ``llama-3.1``, ``llama-3.1-instruct``, ``llama-3.3-instruct``, ``minicpm5-1b``, ``tiny-llama``, ``wizardcoder-python-v1.0``, ``wizardmath-v1.0``, ``Yi``, ``Yi-1.5``, ``Yi-1.5-chat``, ``Yi-1.5-chat-16k``, ``Yi-200k``, ``Yi-chat`` - ``codestral-v0.1``, ``mistral-instruct-v0.1``, ``mistral-instruct-v0.2``, ``mistral-instruct-v0.3``, ``mistral-large-instruct``, ``mistral-nemo-instruct``, ``mistral-v0.1``, ``openhermes-2.5``, ``seallm_v2`` - ``Baichuan-M2``, ``codeqwen1.5``, ``codeqwen1.5-chat``, ``deepseek-r1-distill-qwen``, ``DianJin-R1``, ``fin-r1``, ``HuatuoGPT-o1-Qwen2.5``, ``KAT-V1``, ``marco-o1``, ``qwen1.5-chat``, ``qwen2-instruct``, ``qwen2.5``, ``qwen2.5-coder``, ``qwen2.5-coder-instruct``, ``qwen2.5-instruct``, ``qwen2.5-instruct-1m``, ``qwenLong-l1``, ``QwQ-32B``, ``QwQ-32B-Preview``, ``seallms-v3``, ``skywork-or1``, ``skywork-or1-preview``, ``XiYanSQL-QwenCoder-2504`` - ``llama-3.2-vision``, ``llama-3.2-vision-instruct`` @@ -98,6 +98,7 @@ Currently, supported models include: - ``MiniMax-M2``, ``MiniMax-M2.5``, ``MiniMax-M2.7`` - ``GLM-4.7-Flash`` - ``glm-5``, ``glm-5.1`` +- ``DeepSeek-V4-Flash``, ``DeepSeek-V4-Pro`` .. vllm_end To install Xinference and vLLM:: diff --git a/doc/source/models/builtin/llm/deepseek-v4-flash.rst b/doc/source/models/builtin/llm/deepseek-v4-flash.rst index a48d59e102..2cc0eae0b6 100644 --- a/doc/source/models/builtin/llm/deepseek-v4-flash.rst +++ b/doc/source/models/builtin/llm/deepseek-v4-flash.rst @@ -20,7 +20,7 @@ Model Spec 1 (pytorch, 284 Billion) - **Model Format:** pytorch - **Model Size (in billions):** 284 - **Quantizations:** none -- **Engines**: Transformers +- **Engines**: vLLM, Transformers - **Model ID:** deepseek-ai/DeepSeek-V4-Flash - **Model Hubs**: `Hugging Face `__, `ModelScope `__ diff --git a/doc/source/models/builtin/llm/deepseek-v4-pro.rst b/doc/source/models/builtin/llm/deepseek-v4-pro.rst index e3c77bcb4f..cb8fd561a5 100644 --- a/doc/source/models/builtin/llm/deepseek-v4-pro.rst +++ b/doc/source/models/builtin/llm/deepseek-v4-pro.rst @@ -20,7 +20,7 @@ Model Spec 1 (pytorch, 1600 Billion) - **Model Format:** pytorch - **Model Size (in billions):** 1600 - **Quantizations:** none -- **Engines**: Transformers +- **Engines**: vLLM, Transformers - **Model ID:** deepseek-ai/DeepSeek-V4-Pro - **Model Hubs**: `Hugging Face `__, `ModelScope `__ diff --git a/doc/source/models/builtin/llm/dianjin-r1.rst b/doc/source/models/builtin/llm/dianjin-r1.rst index a7d653f807..5b9d3ae4cb 100644 --- a/doc/source/models/builtin/llm/dianjin-r1.rst +++ b/doc/source/models/builtin/llm/dianjin-r1.rst @@ -7,7 +7,7 @@ DianJin-R1 - **Context Length:** 32768 - **Model Name:** DianJin-R1 - **Languages:** en, zh -- **Abilities:** chat, tools +- **Abilities:** chat, reasoning, hybrid, tools - **Description:** Tongyi DianJin is a financial intelligence solution platform built by Alibaba Cloud, dedicated to providing financial business developers with a convenient artificial intelligence application development environment. Specifications diff --git a/doc/source/models/builtin/llm/index.rst b/doc/source/models/builtin/llm/index.rst index 82fe6df085..1da275fc27 100644 --- a/doc/source/models/builtin/llm/index.rst +++ b/doc/source/models/builtin/llm/index.rst @@ -187,7 +187,7 @@ The following is a list of built-in LLM in Xinference: - DeepSeek-VL2, an advanced series of large Mixture-of-Experts (MoE) Vision-Language Models that significantly improves upon its predecessor, DeepSeek-VL. DeepSeek-VL2 demonstrates superior capabilities across various tasks, including but not limited to visual question answering, optical character recognition, document/table/chart understanding, and visual grounding. * - :ref:`dianjin-r1 ` - - chat, tools + - chat, reasoning, hybrid, tools - 32768 - Tongyi DianJin is a financial intelligence solution platform built by Alibaba Cloud, dedicated to providing financial business developers with a convenient artificial intelligence application development environment. @@ -426,6 +426,11 @@ The following is a list of built-in LLM in Xinference: - 32768 - MiniCPM4 series are highly efficient large language models (LLMs) designed explicitly for end-side devices, which achieves this efficiency through systematic innovation in four key dimensions: model architecture, training data, training algorithms, and inference systems. + * - :ref:`minicpm5-1b ` + - chat, reasoning, hybrid, tools + - 131072 + - MiniCPM5-1B is the first model in the MiniCPM5 series. It is a dense 1B Transformer built for on-device, local deployment, and resource-constrained scenarios, reaching 1B-class open-source SOTA. Supports hybrid thinking via enable_thinking and native XML-style tool calling (MCP-compatible). + * - :ref:`minimax-m2 ` - chat, tools, reasoning - 196608 @@ -785,7 +790,7 @@ The following is a list of built-in LLM in Xinference: .. toctree:: :maxdepth: 3 - + baichuan-2 baichuan-2-chat @@ -950,6 +955,8 @@ The following is a list of built-in LLM in Xinference: minicpm4 + minicpm5-1b + minimax-m2 minimax-m2.5 @@ -1092,4 +1099,3 @@ The following is a list of built-in LLM in Xinference: yi-chat - diff --git a/doc/source/models/builtin/llm/minicpm5-1b.rst b/doc/source/models/builtin/llm/minicpm5-1b.rst new file mode 100644 index 0000000000..a933652fc6 --- /dev/null +++ b/doc/source/models/builtin/llm/minicpm5-1b.rst @@ -0,0 +1,62 @@ +.. _models_llm_minicpm5-1b: + +======================================== +minicpm5-1b +======================================== + +- **Context Length:** 131072 +- **Model Name:** minicpm5-1b +- **Languages:** en, zh +- **Abilities:** chat, reasoning, hybrid, tools +- **Description:** MiniCPM5-1B is the first model in the MiniCPM5 series. It is a dense 1B Transformer built for on-device, local deployment, and resource-constrained scenarios, reaching 1B-class open-source SOTA. Supports hybrid thinking via enable_thinking and native XML-style tool calling (MCP-compatible). + +Specifications +^^^^^^^^^^^^^^ + + +Model Spec 1 (pytorch, 1 Billion) +++++++++++++++++++++++++++++++++++++++++ + +- **Model Format:** pytorch +- **Model Size (in billions):** 1 +- **Quantizations:** none +- **Engines**: vLLM, Transformers, SGLang +- **Model ID:** openbmb/MiniCPM5-1B +- **Model Hubs**: `Hugging Face `__, `ModelScope `__ + +Execute the following command to launch the model, remember to replace ``${quantization}`` with your +chosen quantization method from the options listed above:: + + xinference launch --model-engine ${engine} --model-name minicpm5-1b --size-in-billions 1 --model-format pytorch --quantization ${quantization} + + +Model Spec 2 (ggufv2, 1 Billion) +++++++++++++++++++++++++++++++++++++++++ + +- **Model Format:** ggufv2 +- **Model Size (in billions):** 1 +- **Quantizations:** F16, Q4_K_M, Q8_0 +- **Engines**: vLLM, llama.cpp +- **Model ID:** openbmb/MiniCPM5-1B-GGUF +- **Model Hubs**: `Hugging Face `__, `ModelScope `__ + +Execute the following command to launch the model, remember to replace ``${quantization}`` with your +chosen quantization method from the options listed above:: + + xinference launch --model-engine ${engine} --model-name minicpm5-1b --size-in-billions 1 --model-format ggufv2 --quantization ${quantization} + + +Model Spec 3 (mlx, 1 Billion) +++++++++++++++++++++++++++++++++++++++++ + +- **Model Format:** mlx +- **Model Size (in billions):** 1 +- **Quantizations:** 4bit +- **Engines**: MLX +- **Model ID:** openbmb/MiniCPM5-1B-MLX +- **Model Hubs**: `Hugging Face `__, `ModelScope `__ + +Execute the following command to launch the model, remember to replace ``${quantization}`` with your +chosen quantization method from the options listed above:: + + xinference launch --model-engine ${engine} --model-name minicpm5-1b --size-in-billions 1 --model-format mlx --quantization ${quantization} diff --git a/doc/source/models/builtin/llm/qwen3.6.rst b/doc/source/models/builtin/llm/qwen3.6.rst index cbbb0274b3..30cad84cf4 100644 --- a/doc/source/models/builtin/llm/qwen3.6.rst +++ b/doc/source/models/builtin/llm/qwen3.6.rst @@ -46,7 +46,23 @@ chosen quantization method from the options listed above:: xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 27 --model-format fp8 --quantization ${quantization} -Model Spec 3 (pytorch, 35 Billion) +Model Spec 3 (awq, 27 Billion) +++++++++++++++++++++++++++++++++++++++++ + +- **Model Format:** awq +- **Model Size (in billions):** 27 +- **Quantizations:** Int4 +- **Engines**: vLLM, Transformers, SGLang +- **Model ID:** QuantTrio/Qwen3.6-27B-AWQ +- **Model Hubs**: `Hugging Face `__, `ModelScope `__ + +Execute the following command to launch the model, remember to replace ``${quantization}`` with your +chosen quantization method from the options listed above:: + + xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 27 --model-format awq --quantization ${quantization} + + +Model Spec 4 (pytorch, 35 Billion) ++++++++++++++++++++++++++++++++++++++++ - **Model Format:** pytorch @@ -62,7 +78,7 @@ chosen quantization method from the options listed above:: xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 35 --model-format pytorch --quantization ${quantization} -Model Spec 4 (fp8, 35 Billion) +Model Spec 5 (fp8, 35 Billion) ++++++++++++++++++++++++++++++++++++++++ - **Model Format:** fp8 @@ -78,7 +94,7 @@ chosen quantization method from the options listed above:: xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 35 --model-format fp8 --quantization ${quantization} -Model Spec 5 (awq, 35 Billion) +Model Spec 6 (awq, 35 Billion) ++++++++++++++++++++++++++++++++++++++++ - **Model Format:** awq @@ -94,7 +110,7 @@ chosen quantization method from the options listed above:: xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 35 --model-format awq --quantization ${quantization} -Model Spec 6 (ggufv2, 35 Billion) +Model Spec 7 (ggufv2, 35 Billion) ++++++++++++++++++++++++++++++++++++++++ - **Model Format:** ggufv2 @@ -110,7 +126,7 @@ chosen quantization method from the options listed above:: xinference launch --model-engine ${engine} --model-name qwen3.6 --size-in-billions 35 --model-format ggufv2 --quantization ${quantization} -Model Spec 7 (mlx, 35 Billion) +Model Spec 8 (mlx, 35 Billion) ++++++++++++++++++++++++++++++++++++++++ - **Model Format:** mlx diff --git a/doc/source/user_guide/backends.rst b/doc/source/user_guide/backends.rst index a29e91dda9..f86f83f9e0 100644 --- a/doc/source/user_guide/backends.rst +++ b/doc/source/user_guide/backends.rst @@ -128,7 +128,7 @@ Currently, supported model includes: .. vllm_start -- ``code-llama``, ``code-llama-instruct``, ``code-llama-python``, ``deepseek``, ``deepseek-chat``, ``deepseek-coder``, ``deepseek-coder-instruct``, ``deepseek-r1-distill-llama``, ``gorilla-openfunctions-v2``, ``HuatuoGPT-o1-LLaMA-3.1``, ``llama-2``, ``llama-2-chat``, ``llama-3``, ``llama-3-instruct``, ``llama-3.1``, ``llama-3.1-instruct``, ``llama-3.3-instruct``, ``tiny-llama``, ``wizardcoder-python-v1.0``, ``wizardmath-v1.0``, ``Yi``, ``Yi-1.5``, ``Yi-1.5-chat``, ``Yi-1.5-chat-16k``, ``Yi-200k``, ``Yi-chat`` +- ``code-llama``, ``code-llama-instruct``, ``code-llama-python``, ``deepseek``, ``deepseek-chat``, ``deepseek-coder``, ``deepseek-coder-instruct``, ``deepseek-r1-distill-llama``, ``gorilla-openfunctions-v2``, ``HuatuoGPT-o1-LLaMA-3.1``, ``llama-2``, ``llama-2-chat``, ``llama-3``, ``llama-3-instruct``, ``llama-3.1``, ``llama-3.1-instruct``, ``llama-3.3-instruct``, ``minicpm5-1b``, ``tiny-llama``, ``wizardcoder-python-v1.0``, ``wizardmath-v1.0``, ``Yi``, ``Yi-1.5``, ``Yi-1.5-chat``, ``Yi-1.5-chat-16k``, ``Yi-200k``, ``Yi-chat`` - ``codestral-v0.1``, ``mistral-instruct-v0.1``, ``mistral-instruct-v0.2``, ``mistral-instruct-v0.3``, ``mistral-large-instruct``, ``mistral-nemo-instruct``, ``mistral-v0.1``, ``openhermes-2.5``, ``seallm_v2`` - ``Baichuan-M2``, ``codeqwen1.5``, ``codeqwen1.5-chat``, ``deepseek-r1-distill-qwen``, ``DianJin-R1``, ``fin-r1``, ``HuatuoGPT-o1-Qwen2.5``, ``KAT-V1``, ``marco-o1``, ``qwen1.5-chat``, ``qwen2-instruct``, ``qwen2.5``, ``qwen2.5-coder``, ``qwen2.5-coder-instruct``, ``qwen2.5-instruct``, ``qwen2.5-instruct-1m``, ``qwenLong-l1``, ``QwQ-32B``, ``QwQ-32B-Preview``, ``seallms-v3``, ``skywork-or1``, ``skywork-or1-preview``, ``XiYanSQL-QwenCoder-2504`` - ``llama-3.2-vision``, ``llama-3.2-vision-instruct`` @@ -161,6 +161,7 @@ Currently, supported model includes: - ``MiniMax-M2``, ``MiniMax-M2.5``, ``MiniMax-M2.7`` - ``GLM-4.7-Flash`` - ``glm-5``, ``glm-5.1`` +- ``DeepSeek-V4-Flash``, ``DeepSeek-V4-Pro`` .. vllm_end .. _sglang_backend: diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json index 1a7b1d7ecc..5eec85316c 100644 --- a/xinference/model/llm/llm_family.json +++ b/xinference/model/llm/llm_family.json @@ -992,6 +992,8 @@ ], "model_ability": [ "chat", + "reasoning", + "hybrid", "tools" ], "model_description": "Tongyi DianJin is a financial intelligence solution platform built by Alibaba Cloud, dedicated to providing financial business developers with a convenient artificial intelligence application development environment.", @@ -10920,6 +10922,111 @@ ] } }, + { + "version": 2, + "context_length": 131072, + "model_name": "minicpm5-1b", + "model_lang": [ + "en", + "zh" + ], + "model_ability": [ + "chat", + "reasoning", + "hybrid", + "tools" + ], + "model_description": "MiniCPM5-1B is the first model in the MiniCPM5 series. It is a dense 1B Transformer built for on-device, local deployment, and resource-constrained scenarios, reaching 1B-class open-source SOTA. Supports hybrid thinking via enable_thinking and native XML-style tool calling (MCP-compatible).", + "model_specs": [ + { + "model_format": "pytorch", + "model_size_in_billions": 1, + "model_src": { + "huggingface": { + "quantizations": [ + "none" + ], + "model_id": "openbmb/MiniCPM5-1B" + }, + "modelscope": { + "quantizations": [ + "none" + ], + "model_id": "OpenBMB/MiniCPM5-1B" + } + } + }, + { + "model_format": "ggufv2", + "model_size_in_billions": 1, + "model_src": { + "huggingface": { + "quantizations": [ + "F16", + "Q4_K_M", + "Q8_0" + ], + "model_id": "openbmb/MiniCPM5-1B-GGUF", + "model_file_name_template": "MiniCPM5-1B-{quantization}.gguf" + }, + "modelscope": { + "quantizations": [ + "F16", + "Q4_K_M", + "Q8_0" + ], + "model_id": "OpenBMB/MiniCPM5-1B-GGUF", + "model_file_name_template": "MiniCPM5-1B-{quantization}.gguf" + } + } + }, + { + "model_format": "mlx", + "model_size_in_billions": 1, + "model_src": { + "huggingface": { + "quantizations": [ + "4bit" + ], + "model_id": "openbmb/MiniCPM5-1B-MLX" + }, + "modelscope": { + "quantizations": [ + "4bit" + ], + "model_id": "OpenBMB/MiniCPM5-1B-MLX" + } + } + } + ], + "chat_template": "{{- bos_token }}{%- if tools %}\n {%- set tool_definitions %}\n {{- \"# Tools\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson(ensure_ascii=False) }}\n {%- endfor %}\n {{- '\\n\\n\\nTool usage guidelines:\\n- You may call zero or more functions. If no function calls are needed, just answer normally and do not include any .\\n- When calling a function, return an XML object within using:\\nparam-value\\n- param-value may be multi-line. If it contains <, & or newline characters, wrap it in a CDATA block: ' }}\n {%- endset %}\n \n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if '' in messages[0].content %}\n {{- messages[0].content.replace('', tool_definitions) }}\n {%- else %}\n {{- messages[0].content + '\\n\\n' + tool_definitions }}\n {%- endif %}\n {%- else %}\n {{- tool_definitions.lstrip() }}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n \n {%- if message.tool_calls %}\n {%- set content_parts = content.split('') %}\n {%- set processed_content = content_parts[0] %}\n {%- set tool_calls_count = message.tool_calls|length %}\n {%- set tool_sep_count = content_parts|length - 1 %}\n {%- set min_count = [tool_calls_count, tool_sep_count]|min %}\n \n {%- for i in range(1, content_parts|length) %}\n {%- set tool_index = i - 1 %}\n {%- if tool_index < tool_calls_count %}\n {%- set tool_call = message.tool_calls[tool_index] %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- set single_tool_xml %}\n {{- '' }}\n {%- if tool_call.arguments %}\n {%- set args_dict = tool_call.arguments %}\n {%- for param_name, param_value in args_dict.items() %}\n {{- '' }}\n {%- if param_value is string and ('<' in param_value or '&' in param_value or '\\n' in param_value) %}\n {{- '' }}\n {%- else %}\n {{- param_value }}\n {%- endif %}\n {{- '' }}\n {%- endfor %}\n {%- endif %}\n {{- '' }}\n {%- endset %}\n {%- set processed_content = processed_content + single_tool_xml + content_parts[i] %}\n {%- else %}\n {%- set processed_content = processed_content + content_parts[i] %}\n {%- endif %}\n {%- endfor %}\n \n {%- if tool_calls_count > tool_sep_count %}\n {%- for remaining_index in range(tool_sep_count, tool_calls_count) %}\n {%- set tool_call = message.tool_calls[remaining_index] %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- set remaining_tool_xml %}\n {{- '' }}\n {%- if tool_call.arguments %}\n {%- set args_dict = tool_call.arguments %}\n {%- for param_name, param_value in args_dict.items() %}\n {{- '' }}\n {%- if param_value is string and ('<' in param_value or '&' in param_value or '\\n' in param_value) %}\n {{- '' }}\n {%- else %}\n {{- param_value }}\n {%- endif %}\n {{- '' }}\n {%- endfor %}\n {%- endif %}\n {{- '' }}\n {%- endset %}\n {%- set processed_content = processed_content + remaining_tool_xml %}\n {%- endfor %}\n {%- endif %}\n \n {%- set content = processed_content %}\n {%- endif %}\n \n {%- if loop.index0 > ns.last_query_index %}\n {%- if reasoning_content %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n \n {%- if message.tool_calls and not has_tool_sep %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '' }}\n {%- if tool_call.arguments %}\n {%- set args_dict = tool_call.arguments %}\n {%- for param_name, param_value in args_dict.items() %}\n {{- '' }}\n {%- if param_value is string and ('<' in param_value or '&' in param_value or '\\n' in param_value) %}\n {{- '' }}\n {%- else %}\n {{- param_value }}\n {%- endif %}\n {{- '' }}\n {%- endfor %}\n {%- endif %}\n {{- '' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- if message.content is string %}\n {{- content }}\n {%- else %}\n {{- message.content | tojson(ensure_ascii=False) }}\n {%- endif %}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined %}\n {%- if enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- elif enable_thinking is true %}\n {{- '\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endif %}\n", + "stop_token_ids": [ + 1, + 130073 + ], + "stop": [ + "", + "<|im_end|>" + ], + "reasoning_start_tag": "", + "reasoning_end_tag": "", + "updated_at": 1780925102, + "featured": false, + "architectures": [ + "LlamaForCausalLM" + ], + "model_type": "llama", + "virtualenv": { + "packages": [ + "transformers>=5.6 ; #engine# == \"Transformers\"", + "#mlx_dependencies# ; #engine# == \"MLX\"", + "#vllm_dependencies# ; #engine# == \"vllm\"", + "#system_numpy# ; #engine# == \"vllm\"", + "#llama_cpp_dependencies# ; #engine# == \"llama.cpp\"", + "#sglang_dependencies# ; #engine# == \"sglang\"" + ] + } + }, { "version": 2, "context_length": 8192, diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index b4bcf621e1..fadbb06685 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -574,7 +574,40 @@ def _load_model(self, **kwargs): ) if stop_token_ids := self.model_family.stop_token_ids: for stop_token_id in stop_token_ids: - tokenizer.add_eos_token(stop_token_id) + # mlx_lm's TokenizerWrapper used to expose ``add_eos_token`` + # as a method; newer versions removed it and HF tokenizers in + # transformers>=5 expose ``add_eos_token`` as a bool attribute. + # Fall back to extending the wrapper's eos token id set. + add_eos = getattr(tokenizer, "add_eos_token", None) + if callable(add_eos): + add_eos(stop_token_id) + else: + eos_ids = getattr(tokenizer, "_eos_token_ids", None) + if eos_ids is None: + eos_ids = set() + try: + setattr(tokenizer, "_eos_token_ids", eos_ids) + except Exception: + pass + try: + if isinstance(eos_ids, set): + eos_ids.add(stop_token_id) + elif isinstance(eos_ids, list): + if stop_token_id not in eos_ids: + eos_ids.append(stop_token_id) + else: + logger.warning( + "Unsupported type %s for tokenizer._eos_token_ids; " + "stop token %s was not registered.", + type(eos_ids).__name__, + stop_token_id, + ) + except Exception: + logger.warning( + "Failed to register stop token %s on tokenizer.", + stop_token_id, + exc_info=True, + ) # Store model and tokenizer for batch model creation later self._model = model diff --git a/xinference/ui/gradio/chat_interface.py b/xinference/ui/gradio/chat_interface.py index 7de804efb9..32fa0ea3ef 100644 --- a/xinference/ui/gradio/chat_interface.py +++ b/xinference/ui/gradio/chat_interface.py @@ -18,11 +18,10 @@ import os import tempfile from io import BytesIO -from typing import Generator, List, Optional +from typing import Any, Dict, Generator, List, Optional import gradio as gr import PIL.Image -from gradio import ChatMessage from gradio.components import Markdown, Textbox from gradio.layouts import Accordion, Column, Row @@ -96,20 +95,53 @@ def build(self) -> "gr.Blocks": def build_chat_interface(self) -> "gr.Blocks": def generate_wrapper( message: str, - history: List[ChatMessage], + history: List, max_tokens: int, temperature: float, lora_name: str, stream: bool, + enable_thinking: bool, ) -> Generator: + """Single-shot chat handler. + + Takes the textbox value and the full history, appends the user + turn, calls the model, and yields updated (textbox_value, history) + tuples so Gradio renders both the user bubble and the assistant + reply in one event chain (avoids the previous double-step lambda + + .then() chain which caused state desync / no-reply bugs). + """ + history = list(history or []) + + # Ignore empty / whitespace-only sends (Gradio fires submit on + # blur as well, which previously triggered a 400). + user_text = (message or "").strip() + if not user_text: + yield "", history + return + + # Show the user message immediately. + history.append({"role": "user", "content": user_text}) + yield "", history + from ...client import RESTfulClient - client = RESTfulClient(self.endpoint) - client._set_token(self._access_token) - model = client.get_model(self.model_uid) - assert isinstance(model, RESTfulChatModelHandle) + try: + client = RESTfulClient(self.endpoint) + client._set_token(self._access_token) + model = client.get_model(self.model_uid) + assert isinstance(model, RESTfulChatModelHandle) + except Exception as e: # pragma: no cover - surface to UI + logger.exception("Failed to get model handle") + history.append( + { + "role": "assistant", + "content": f"⚠️ Failed to connect to model: {e}", + } + ) + yield "", history + return - generate_config = { + generate_config: dict = { "temperature": temperature, "stream": stream, "lora_name": lora_name, @@ -117,98 +149,103 @@ def generate_wrapper( if max_tokens > 0: generate_config["max_tokens"] = max_tokens - # Convert history to messages format - messages = [] + # Convert history (assistant + user messages, ignoring any + # ``metadata`` thinking bubbles) into OpenAI-style messages. + messages: List[Dict[str, Any]] = [] for msg in history: - # ignore thinking content - if msg["metadata"]: + if isinstance(msg, dict): + metadata = msg.get("metadata") + role = msg.get("role") + content = msg.get("content") + else: + metadata = getattr(msg, "metadata", None) + role = getattr(msg, "role", None) + content = getattr(msg, "content", None) + if metadata: continue - messages.append({"role": msg["role"], "content": msg["content"]}) - - if stream: - response_content = "" - reasoning_content = "" - - for chunk in model.chat( - messages=messages, - generate_config=generate_config, # type: ignore - ): - assert isinstance(chunk, dict) - if not chunk["choices"]: - continue - delta = chunk["choices"][0]["delta"] + if not role or content is None: + continue + messages.append({"role": role, "content": content}) - # Process reasoning content - if ( - "reasoning_content" in delta - and delta["reasoning_content"] is not None - ): - reasoning_content += html.escape(delta["reasoning_content"]) - - # Process actual content - if "content" in delta and delta["content"] is not None: - response_content += html.escape(delta["content"]) - - # Build message content based on what we have - if reasoning_content and not response_content: - # Only reasoning so far - message_content = reasoning_content - message_metadata = {"title": "💭 Thinking Process"} - elif reasoning_content and response_content: - # Both reasoning and content, combine them - message_content = f"💭 **Thinking Process**\n```\n{reasoning_content}\n```\n\n{response_content}" - message_metadata = None - else: - # Only content - message_content = response_content - message_metadata = None - - current_message = ChatMessage( - role="assistant", - content=message_content, - metadata=message_metadata, + chat_kwargs: Dict[str, Any] = { + "messages": messages, + "generate_config": generate_config, # type: ignore + } + if "reasoning" in self.model_ability: + chat_kwargs["enable_thinking"] = bool(enable_thinking) + + try: + if stream: + response_content = "" + reasoning_content = "" + # Placeholder assistant bubble we keep updating. + history.append({"role": "assistant", "content": ""}) + for chunk in model.chat(**chat_kwargs): + if not isinstance(chunk, dict): + continue + choices = chunk.get("choices") or [] + if not choices: + continue + delta = choices[0].get("delta") or {} + rc = delta.get("reasoning_content") + if rc: + reasoning_content += str(rc) + c = delta.get("content") + if c: + response_content += str(c) + if reasoning_content and not response_content: + bubble = ( + "💭 **Thinking**\n```\n" + reasoning_content + "\n```" + ) + elif reasoning_content and response_content: + bubble = ( + "💭 **Thinking**\n```\n" + + reasoning_content + + "\n```\n\n" + + response_content + ) + else: + bubble = response_content + history[-1] = {"role": "assistant", "content": bubble} + yield "", history + else: + result = model.chat(**chat_kwargs) + if not isinstance(result, dict): + raise RuntimeError(f"Unexpected response: {result!r}") + result_choices = result.get("choices") or [] + if not result_choices: + raise RuntimeError(f"No choices in response: {result!r}") + first_choice = result_choices[0] + raw_message: Any = ( + first_choice.get("message") + if isinstance(first_choice, dict) + else None ) - - if history: - last_msg = history[-1] - last_role = ( - last_msg.get("role") - if isinstance(last_msg, dict) - else last_msg.role + mg: Dict[str, Any] = ( + raw_message if isinstance(raw_message, dict) else {} + ) + reasoning_content = mg.get("reasoning_content") or "" + content = mg.get("content") or "" + if reasoning_content: + bubble = ( + "💭 **Thinking**\n```\n" + + str(reasoning_content) + + "\n```\n\n" + + str(content) ) - if last_role == "assistant": - history[-1] = current_message - else: - history.append(current_message) else: - history.append(current_message) - - yield history - else: - result = model.chat( - messages=messages, - generate_config=generate_config, # type: ignore - ) - assert isinstance(result, dict) - mg = result["choices"][0]["message"] - if "reasoning_content" in mg: - reasoning_content = mg["reasoning_content"] - if reasoning_content is not None: - reasoning_content = html.escape(str(reasoning_content)) - history.append( - ChatMessage( - role="assistant", - content=reasoning_content, - metadata={"title": "💭 Thinking Process"}, - ) - ) - - content = mg["content"] - response_content = ( - html.escape(str(content)) if content is not None else "" + bubble = str(content) + history.append({"role": "assistant", "content": bubble}) + yield "", history + except Exception as e: # pragma: no cover - surface to UI + logger.exception("Chat completion failed") + history.append( + { + "role": "assistant", + "content": f"⚠️ Chat failed: {e}", + } ) - history.append(ChatMessage(role="assistant", content=response_content)) - yield history + yield "", history with gr.Blocks( title=f"🚀 Xinference Chat Bot : {self.model_name} 🚀", @@ -310,27 +347,28 @@ def generate_wrapper( ) stream = gr.Checkbox(label="Stream", value=True) lora_name = gr.Text(label="LoRA Name") + enable_thinking = gr.Checkbox( + label="Enable Thinking (hybrid reasoning)", + value=True, + visible="reasoning" in self.model_ability, + ) - # deal with message submit - textbox.submit( - lambda m, h: ("", h + [ChatMessage(role="user", content=m)]), - [textbox, chatbot], - [textbox, chatbot], - ).then( - generate_wrapper, - [textbox, chatbot, max_tokens, temperature, lora_name, stream], - chatbot, - ) - - submit_btn.click( - lambda m, h: ("", h + [ChatMessage(role="user", content=m)]), - [textbox, chatbot], - [textbox, chatbot], - ).then( - generate_wrapper, - [textbox, chatbot, max_tokens, temperature, lora_name, stream], + # deal with message submit — single-step: generate_wrapper itself + # appends the user turn, streams the assistant turn, and clears + # the textbox via the tuple ("", history) it yields each step. + submit_inputs = [ + textbox, chatbot, - ) + max_tokens, + temperature, + lora_name, + stream, + enable_thinking, + ] + submit_outputs = [textbox, chatbot] + + textbox.submit(generate_wrapper, submit_inputs, submit_outputs) + submit_btn.click(generate_wrapper, submit_inputs, submit_outputs) return chat_interface