From 5e776f0791d9a1d654736c23467de06cd6d18f14 Mon Sep 17 00:00:00 2001 From: ranzhejiang Date: Mon, 6 Jul 2026 07:59:19 +0000 Subject: [PATCH] add hy-v3 fp8 support for new version Signed-off-by: ranzhejiang --- ...216\250\347\220\206\346\211\213\345\206\214.md" | 2 +- vllm/model_executor/models/hy_v3.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git "a/scripts/docs/vllm\346\216\250\347\220\206\346\211\213\345\206\214.md" "b/scripts/docs/vllm\346\216\250\347\220\206\346\211\213\345\206\214.md" index e6d043492d32..32a24ec72158 100644 --- "a/scripts/docs/vllm\346\216\250\347\220\206\346\211\213\345\206\214.md" +++ "b/scripts/docs/vllm\346\216\250\347\220\206\346\211\213\345\206\214.md" @@ -1772,7 +1772,7 @@ bash ./start_gaudi_vllm_server.sh -w /data/hf_models/HY3.0-FP8-Testing-G2 \ -b 128 \ -n 16 \ -u 0.9 \ --e "--enable-prefix-caching \ +-e "--enable-prefix-caching" \ -c /recipe_cache_HY3.0-FP8-Testing-G2/ ``` diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py index f56112816dd4..8563b138f550 100644 --- a/vllm/model_executor/models/hy_v3.py +++ b/vllm/model_executor/models/hy_v3.py @@ -597,6 +597,20 @@ def load_weights(self, weights: Iterable[tuple[str, loaded_params: set[str] = set() for name, loaded_weight in weights: + # Minimal fix: skip next-n predict/spec layer weights + # (e.g. model.layers.80.* when num_hidden_layers=80 and + # num_nextn_predict_layers=1), because this modeling only + # instantiates the base decoder layers. + skip_spec_layer = False + if (hasattr(self.config, "num_nextn_predict_layers") + and self.config.num_nextn_predict_layers > 0): + layer_idx = self.config.num_hidden_layers + for i in range(self.config.num_nextn_predict_layers): + if name.startswith(f"model.layers.{layer_idx + i}."): + skip_spec_layer = True + break + if skip_spec_layer: + continue if "rotary_emb.inv_freq" in name: continue if (bool(_cfg(self.config, "tie_word_embeddings", False))