From fab36acbb4ea789ea7c9c14991925450d42130ba Mon Sep 17 00:00:00 2001 From: tlkahn Date: Mon, 20 Jul 2026 16:55:39 +0700 Subject: [PATCH 1/2] feat(model): E2 DeepSeek family split + config fields (#109) Correct A1 interim debt that collapsed every DeepSeek HF model_type onto MODEL_DEEPSEEK_V4. Land the model_config_t surface E2 forward/sanitize needs, without enabling generate. - model_family_t: MODEL_DEEPSEEK_V3, MODEL_DEEPSEEK_V32, MODEL_DEEPSEEK_V4 - model_family_from_type: deepseek_v3 / deepseek_v3_2+deepseek_v32 / deepseek_v4 - Parse MLA, MoE extras, rope mscale_all_dim, V3.2 indexer, norm_topk_prob - n_routed_experts aliases into num_experts only when num_experts == 0 - weights_load rejects all three with honest family labels (no GGUF) - Gate still rejects all DeepSeek families as unsupported --- src/model/model.c | 49 ++++- src/model/model.h | 25 +++ src/model/weights.c | 12 +- .../config.json | 12 ++ .../model_config_deepseek_v3/config.json | 32 +++ .../model_config_deepseek_v32/config.json | 35 ++++ .../config.json | 10 + .../config.json | 11 ++ .../model_config_deepseek_v4/config.json | 16 +- tests/test_emodel_gate.c | 3 +- tests/test_model_config.c | 183 +++++++++++++++++- tests/test_weights.c | 3 +- tests/test_weights_gpu.c | 27 ++- 13 files changed, 400 insertions(+), 18 deletions(-) create mode 100644 tests/fixtures/model_config_deepseek_num_experts_both/config.json create mode 100644 tests/fixtures/model_config_deepseek_v3/config.json create mode 100644 tests/fixtures/model_config_deepseek_v32/config.json create mode 100644 tests/fixtures/model_config_deepseek_v3_2_alias/config.json create mode 100644 tests/fixtures/model_config_deepseek_v3_defaults/config.json diff --git a/src/model/model.c b/src/model/model.c index 46422d3..b4fcb25 100644 --- a/src/model/model.c +++ b/src/model/model.c @@ -204,10 +204,12 @@ model_family_t model_family_from_type(const char *model_type) { return MODEL_LFM2; if (strcmp(model_type, "nemotron_h") == 0) return MODEL_NEMOTRON_H; - if (strcmp(model_type, "deepseek_v3") == 0 || - strcmp(model_type, "deepseek_v3_2") == 0 || - strcmp(model_type, "deepseek_v32") == 0 || - strcmp(model_type, "deepseek_v4") == 0) + if (strcmp(model_type, "deepseek_v3") == 0) + return MODEL_DEEPSEEK_V3; + if (strcmp(model_type, "deepseek_v3_2") == 0 || + strcmp(model_type, "deepseek_v32") == 0) + return MODEL_DEEPSEEK_V32; + if (strcmp(model_type, "deepseek_v4") == 0) return MODEL_DEEPSEEK_V4; if (strcmp(model_type, "bert") == 0) return MODEL_BERT; @@ -534,6 +536,8 @@ static int apply_family_defaults(model_config_t *cfg, yyjson_val *cfg_obj, } break; + case MODEL_DEEPSEEK_V3: + case MODEL_DEEPSEEK_V32: case MODEL_DEEPSEEK_V4: if (set_llama_style_defaults(cfg, cfg_obj)) return -1; @@ -681,6 +685,9 @@ int model_config_load(model_config_t *cfg, const char *model_dir) { cfg->mamba_mlp_act = HIDDEN_ACT_RELU_SQ; cfg->layer_norm_eps = 1e-12f; cfg->weight_prefix = "model"; + /* DeepSeek mlx-lm defaults (honored when keys absent; explicit values win). */ + cfg->routed_scaling_factor = 1.0f; + cfg->norm_topk_prob = true; char *path = path_join(model_dir, "config.json"); if (!path) @@ -751,6 +758,32 @@ int model_config_load(model_config_t *cfg, const char *model_dir) { &cfg->moe_intermediate_size, 0) || get_int_nonneg(cfg_obj, "shared_expert_intermediate_size", &cfg->shared_expert_intermediate_size, 0) || + /* DeepSeek MLA */ + get_int_nonneg(cfg_obj, "q_lora_rank", &cfg->q_lora_rank, 0) || + get_int_nonneg(cfg_obj, "kv_lora_rank", &cfg->kv_lora_rank, 0) || + get_int_nonneg(cfg_obj, "qk_rope_head_dim", &cfg->qk_rope_head_dim, + 0) || + get_int_nonneg(cfg_obj, "qk_nope_head_dim", &cfg->qk_nope_head_dim, + 0) || + get_int_nonneg(cfg_obj, "v_head_dim", &cfg->v_head_dim, 0) || + /* DeepSeek MoE extras */ + get_int_nonneg(cfg_obj, "n_routed_experts", &cfg->n_routed_experts, + 0) || + get_int_nonneg(cfg_obj, "n_shared_experts", &cfg->n_shared_experts, + 0) || + get_f32(cfg_obj, "routed_scaling_factor", &cfg->routed_scaling_factor, + cfg->routed_scaling_factor) || + get_int_nonneg(cfg_obj, "moe_layer_freq", &cfg->moe_layer_freq, 0) || + get_int_nonneg(cfg_obj, "first_k_dense_replace", + &cfg->first_k_dense_replace, 0) || + get_int_nonneg(cfg_obj, "n_group", &cfg->n_group, 0) || + get_int_nonneg(cfg_obj, "topk_group", &cfg->topk_group, 0) || + get_bool(cfg_obj, "norm_topk_prob", &cfg->norm_topk_prob, + cfg->norm_topk_prob) || + /* DeepSeek V3.2 / Flash indexer */ + get_int_nonneg(cfg_obj, "index_head_dim", &cfg->index_head_dim, 0) || + get_int_nonneg(cfg_obj, "index_n_heads", &cfg->index_n_heads, 0) || + get_int_nonneg(cfg_obj, "index_topk", &cfg->index_topk, 0) || get_int_nonneg(cfg_obj, "linear_num_key_heads", &cfg->linear_num_key_heads, 0) || get_int_nonneg(cfg_obj, "linear_num_value_heads", @@ -787,6 +820,12 @@ int model_config_load(model_config_t *cfg, const char *model_dir) { } } + /* DeepSeek HF publishes n_routed_experts, not num_experts. One-way alias + so existing MoE detectors (num_experts > 0) keep working. Do not + overwrite an explicit num_experts. */ + if (cfg->n_routed_experts > 0 && cfg->num_experts == 0) + cfg->num_experts = cfg->n_routed_experts; + if (cfg->num_hidden_layers > MLXD_MAX_LAYERS) goto fail; @@ -829,6 +868,8 @@ int model_config_load(model_config_t *cfg, const char *model_dir) { cfg->rope_low_freq_factor) || get_f32(rs, "high_freq_factor", &cfg->rope_high_freq_factor, cfg->rope_high_freq_factor) || + get_f32(rs, "mscale_all_dim", &cfg->rope_scaling_mscale_all_dim, + cfg->rope_scaling_mscale_all_dim) || get_int_nonneg(rs, "original_max_position_embeddings", &cfg->rope_original_max_position_embeddings, 0)) goto fail; diff --git a/src/model/model.h b/src/model/model.h index 8ecc2b9..775e1bf 100644 --- a/src/model/model.h +++ b/src/model/model.h @@ -19,6 +19,8 @@ typedef enum { MODEL_MISTRAL, MODEL_LFM2, MODEL_NEMOTRON_H, + MODEL_DEEPSEEK_V3, + MODEL_DEEPSEEK_V32, MODEL_DEEPSEEK_V4, MODEL_BERT, } model_family_t; @@ -69,6 +71,7 @@ typedef struct { float rope_scaling_factor; float rope_low_freq_factor; float rope_high_freq_factor; + float rope_scaling_mscale_all_dim; /* 0 = absent */ int rope_original_max_position_embeddings; bool rope_proportional; /* gemma4: rope_parameters.full_attention.rope_type == "proportional" */ float rope_proportional_factor; /* gemma4: rope_parameters.full_attention.factor */ @@ -106,6 +109,28 @@ typedef struct { int moe_intermediate_size; int shared_expert_intermediate_size; + /* DeepSeek MoE extras (E2 / decision 11) */ + int n_routed_experts; + int n_shared_experts; + float routed_scaling_factor; + int moe_layer_freq; + int first_k_dense_replace; + int n_group; /* MoE group count; NOT mamba_n_groups */ + int topk_group; + bool norm_topk_prob; + + /* DeepSeek MLA */ + int q_lora_rank; + int kv_lora_rank; + int qk_rope_head_dim; + int qk_nope_head_dim; + int v_head_dim; + + /* DeepSeek V3.2 / Flash indexer (parse-only until #114/#115) */ + int index_head_dim; + int index_n_heads; + int index_topk; + /* Linear attention (qwen3_5) */ int linear_num_key_heads; int linear_num_value_heads; diff --git a/src/model/weights.c b/src/model/weights.c index 7446d64..7208bed 100644 --- a/src/model/weights.c +++ b/src/model/weights.c @@ -944,9 +944,17 @@ int weights_load(weights_t *w, const char *model_dir, if (!w || !model_dir || !cfg) return -1; memset(w, 0, sizeof(*w)); - if (cfg->family == MODEL_DEEPSEEK_V4) { + if (cfg->family == MODEL_DEEPSEEK_V3 || + cfg->family == MODEL_DEEPSEEK_V32 || + cfg->family == MODEL_DEEPSEEK_V4) { + const char *label = + cfg->family == MODEL_DEEPSEEK_V3 ? "deepseek_v3" : + cfg->family == MODEL_DEEPSEEK_V32 ? "deepseek_v32" : + "deepseek_v4"; if (err && errlen > 0) - snprintf(err, errlen, "deepseek_v4 MLA attention not yet implemented (Stage E)"); + snprintf(err, errlen, + "%s MLA/MoE forward not yet implemented (Stage E2)", + label); return -1; } diff --git a/tests/fixtures/model_config_deepseek_num_experts_both/config.json b/tests/fixtures/model_config_deepseek_num_experts_both/config.json new file mode 100644 index 0000000..ace14e0 --- /dev/null +++ b/tests/fixtures/model_config_deepseek_num_experts_both/config.json @@ -0,0 +1,12 @@ +{ + "model_type": "deepseek_v3", + "vocab_size": 102400, + "hidden_size": 512, + "num_hidden_layers": 2, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "intermediate_size": 1024, + "max_position_embeddings": 4096, + "num_experts": 8, + "n_routed_experts": 16 +} diff --git a/tests/fixtures/model_config_deepseek_v3/config.json b/tests/fixtures/model_config_deepseek_v3/config.json new file mode 100644 index 0000000..63bd9d2 --- /dev/null +++ b/tests/fixtures/model_config_deepseek_v3/config.json @@ -0,0 +1,32 @@ +{ + "model_type": "deepseek_v3", + "vocab_size": 102400, + "hidden_size": 512, + "num_hidden_layers": 4, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "intermediate_size": 1024, + "max_position_embeddings": 4096, + "q_lora_rank": 128, + "kv_lora_rank": 64, + "qk_rope_head_dim": 32, + "qk_nope_head_dim": 64, + "v_head_dim": 64, + "n_routed_experts": 16, + "n_shared_experts": 1, + "num_experts_per_tok": 4, + "routed_scaling_factor": 2.5, + "moe_layer_freq": 1, + "first_k_dense_replace": 1, + "n_group": 4, + "topk_group": 2, + "moe_intermediate_size": 256, + "norm_topk_prob": true, + "rope_scaling": { + "type": "yarn", + "factor": 40.0, + "mscale_all_dim": 1.0, + "beta_fast": 32, + "beta_slow": 1 + } +} diff --git a/tests/fixtures/model_config_deepseek_v32/config.json b/tests/fixtures/model_config_deepseek_v32/config.json new file mode 100644 index 0000000..e69d98a --- /dev/null +++ b/tests/fixtures/model_config_deepseek_v32/config.json @@ -0,0 +1,35 @@ +{ + "model_type": "deepseek_v32", + "vocab_size": 102400, + "hidden_size": 512, + "num_hidden_layers": 4, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "intermediate_size": 1024, + "max_position_embeddings": 4096, + "q_lora_rank": 128, + "kv_lora_rank": 64, + "qk_rope_head_dim": 32, + "qk_nope_head_dim": 64, + "v_head_dim": 64, + "n_routed_experts": 16, + "n_shared_experts": 1, + "num_experts_per_tok": 4, + "routed_scaling_factor": 2.5, + "moe_layer_freq": 1, + "first_k_dense_replace": 1, + "n_group": 4, + "topk_group": 2, + "moe_intermediate_size": 256, + "norm_topk_prob": true, + "index_head_dim": 128, + "index_n_heads": 64, + "index_topk": 2048, + "rope_scaling": { + "type": "yarn", + "factor": 40.0, + "mscale_all_dim": 1.0, + "beta_fast": 32, + "beta_slow": 1 + } +} diff --git a/tests/fixtures/model_config_deepseek_v3_2_alias/config.json b/tests/fixtures/model_config_deepseek_v3_2_alias/config.json new file mode 100644 index 0000000..fe23843 --- /dev/null +++ b/tests/fixtures/model_config_deepseek_v3_2_alias/config.json @@ -0,0 +1,10 @@ +{ + "model_type": "deepseek_v3_2", + "vocab_size": 102400, + "hidden_size": 512, + "num_hidden_layers": 2, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "intermediate_size": 1024, + "max_position_embeddings": 4096 +} diff --git a/tests/fixtures/model_config_deepseek_v3_defaults/config.json b/tests/fixtures/model_config_deepseek_v3_defaults/config.json new file mode 100644 index 0000000..7260781 --- /dev/null +++ b/tests/fixtures/model_config_deepseek_v3_defaults/config.json @@ -0,0 +1,11 @@ +{ + "model_type": "deepseek_v3", + "vocab_size": 102400, + "hidden_size": 512, + "num_hidden_layers": 2, + "num_attention_heads": 8, + "num_key_value_heads": 8, + "intermediate_size": 1024, + "max_position_embeddings": 4096, + "n_routed_experts": 8 +} diff --git a/tests/fixtures/model_config_deepseek_v4/config.json b/tests/fixtures/model_config_deepseek_v4/config.json index 3a674cd..4bc6e33 100644 --- a/tests/fixtures/model_config_deepseek_v4/config.json +++ b/tests/fixtures/model_config_deepseek_v4/config.json @@ -6,5 +6,19 @@ "num_attention_heads": 16, "num_key_value_heads": 16, "intermediate_size": 10944, - "max_position_embeddings": 163840 + "max_position_embeddings": 163840, + "q_lora_rank": 1024, + "qk_rope_head_dim": 64, + "n_routed_experts": 256, + "n_shared_experts": 1, + "num_experts_per_tok": 6, + "routed_scaling_factor": 1.5, + "moe_intermediate_size": 2048, + "index_head_dim": 128, + "index_n_heads": 64, + "index_topk": 512, + "rope_scaling": { + "type": "yarn", + "factor": 16.0 + } } diff --git a/tests/test_emodel_gate.c b/tests/test_emodel_gate.c index d31842e..1c8aaec 100644 --- a/tests/test_emodel_gate.c +++ b/tests/test_emodel_gate.c @@ -127,7 +127,8 @@ static void test_reject_all_other_families(void) { MODEL_FAMILY_UNKNOWN, MODEL_GEMMA3, MODEL_QWEN2, MODEL_QWEN3_5_MOE, MODEL_LFM2, - MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V4, MODEL_BERT, + MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V3, MODEL_DEEPSEEK_V32, + MODEL_DEEPSEEK_V4, MODEL_BERT, }; for (size_t i = 0; i < sizeof(others) / sizeof(others[0]); i++) { model_config_t cfg = make_supported(); diff --git a/tests/test_model_config.c b/tests/test_model_config.c index 8faca37..0768ad8 100644 --- a/tests/test_model_config.c +++ b/tests/test_model_config.c @@ -279,12 +279,16 @@ static void test_family_from_type(void) { assert(model_family_from_type("lfm2_moe") == MODEL_LFM2); assert(model_family_from_type("lfm2_audio") == MODEL_LFM2); assert(model_family_from_type("nemotron_h") == MODEL_NEMOTRON_H); - assert(model_family_from_type("deepseek_v3") == MODEL_DEEPSEEK_V4); - assert(model_family_from_type("deepseek_v3_2") == MODEL_DEEPSEEK_V4); - assert(model_family_from_type("deepseek_v32") == MODEL_DEEPSEEK_V4); + assert(model_family_from_type("deepseek_v3") == MODEL_DEEPSEEK_V3); + assert(model_family_from_type("deepseek_v3_2") == MODEL_DEEPSEEK_V32); + assert(model_family_from_type("deepseek_v32") == MODEL_DEEPSEEK_V32); assert(model_family_from_type("deepseek_v4") == MODEL_DEEPSEEK_V4); assert(model_family_from_type("bert") == MODEL_BERT); + /* negative: do not prefix-match arbitrary deepseek* */ + assert(model_family_from_type("deepseek_v5") == MODEL_FAMILY_UNKNOWN); + assert(model_family_from_type("deepseek") == MODEL_FAMILY_UNKNOWN); + assert(model_family_from_type("qwen3_moe") == MODEL_FAMILY_UNKNOWN); assert(model_family_from_type("qwen3_next") == MODEL_FAMILY_UNKNOWN); assert(model_family_from_type("diffusion_gemma") == MODEL_FAMILY_UNKNOWN); @@ -388,10 +392,35 @@ static void test_family_defaults(void) { assert(cfg.head_dim == 768 / 12); model_config_free(&cfg); + /* deepseek_v3 */ + rc = model_config_load(&cfg, MLXD_FIXTURES_DIR "/model_config_deepseek_v3"); + assert(rc == 0); + assert(cfg.family == MODEL_DEEPSEEK_V3); + assert(cfg.hidden_act == HIDDEN_ACT_SILU); + assert(cfg.weight_prefix != NULL && strcmp(cfg.weight_prefix, "model") == 0); + model_config_free(&cfg); + + /* deepseek_v32 */ + rc = model_config_load(&cfg, MLXD_FIXTURES_DIR "/model_config_deepseek_v32"); + assert(rc == 0); + assert(cfg.family == MODEL_DEEPSEEK_V32); + assert(cfg.hidden_act == HIDDEN_ACT_SILU); + assert(cfg.weight_prefix != NULL && strcmp(cfg.weight_prefix, "model") == 0); + model_config_free(&cfg); + + /* deepseek_v3_2 alias through full load path */ + rc = model_config_load(&cfg, + MLXD_FIXTURES_DIR "/model_config_deepseek_v3_2_alias"); + assert(rc == 0); + assert(cfg.family == MODEL_DEEPSEEK_V32); + model_config_free(&cfg); + /* deepseek_v4 */ rc = model_config_load(&cfg, MLXD_FIXTURES_DIR "/model_config_deepseek_v4"); assert(rc == 0); assert(cfg.family == MODEL_DEEPSEEK_V4); + assert(cfg.hidden_act == HIDDEN_ACT_SILU); + assert(cfg.weight_prefix != NULL && strcmp(cfg.weight_prefix, "model") == 0); model_config_free(&cfg); /* qwen3 reuse */ @@ -1223,6 +1252,147 @@ static void test_gemma4_hidden_act_parsed(void) { model_config_free(&cfg); } +/* --- E2 / #109: DeepSeek family split + config fields -------------------- */ + +static void test_deepseek_config_fields_zero_init(void) { + model_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + assert(cfg.q_lora_rank == 0); + assert(cfg.kv_lora_rank == 0); + assert(cfg.qk_rope_head_dim == 0); + assert(cfg.qk_nope_head_dim == 0); + assert(cfg.v_head_dim == 0); + assert(cfg.n_routed_experts == 0); + assert(cfg.n_shared_experts == 0); + assert(cfg.routed_scaling_factor == 0.0f); + assert(cfg.moe_layer_freq == 0); + assert(cfg.first_k_dense_replace == 0); + assert(cfg.n_group == 0); + assert(cfg.topk_group == 0); + assert(cfg.rope_scaling_mscale_all_dim == 0.0f); + assert(cfg.index_head_dim == 0); + assert(cfg.index_n_heads == 0); + assert(cfg.index_topk == 0); + assert(cfg.norm_topk_prob == false); /* pre-default; load sets true */ +} + +static void test_deepseek_v3_config_fields(void) { + model_config_t cfg; + assert(model_config_load(&cfg, + MLXD_FIXTURES_DIR "/model_config_deepseek_v3") == 0); + assert(cfg.family == MODEL_DEEPSEEK_V3); + assert(cfg.q_lora_rank == 128); + assert(cfg.kv_lora_rank == 64); + assert(cfg.qk_rope_head_dim == 32); + assert(cfg.qk_nope_head_dim == 64); + assert(cfg.v_head_dim == 64); + assert(cfg.n_routed_experts == 16); + assert(cfg.n_shared_experts == 1); + assert(cfg.num_experts == 16); /* R4 alias from n_routed_experts */ + assert(cfg.num_experts_per_tok == 4); + assert(cfg.routed_scaling_factor == 2.5f); + assert(cfg.moe_layer_freq == 1); + assert(cfg.first_k_dense_replace == 1); + assert(cfg.n_group == 4); + assert(cfg.topk_group == 2); + assert(cfg.moe_intermediate_size == 256); + assert(cfg.norm_topk_prob == true); + assert(cfg.rope_scaling_type && strcmp(cfg.rope_scaling_type, "yarn") == 0); + assert(cfg.rope_scaling_factor == 40.0f); + assert(cfg.rope_scaling_mscale_all_dim == 1.0f); + assert(cfg.index_head_dim == 0); /* V3 has no indexer */ + assert(cfg.index_n_heads == 0); + assert(cfg.index_topk == 0); + model_config_free(&cfg); +} + +static void test_deepseek_v32_config_fields(void) { + model_config_t cfg; + assert(model_config_load( + &cfg, MLXD_FIXTURES_DIR "/model_config_deepseek_v32") == 0); + assert(cfg.family == MODEL_DEEPSEEK_V32); + assert(cfg.q_lora_rank == 128); + assert(cfg.kv_lora_rank == 64); + assert(cfg.qk_rope_head_dim == 32); + assert(cfg.qk_nope_head_dim == 64); + assert(cfg.v_head_dim == 64); + assert(cfg.n_routed_experts == 16); + assert(cfg.num_experts == 16); + assert(cfg.n_shared_experts == 1); + assert(cfg.num_experts_per_tok == 4); + assert(cfg.routed_scaling_factor == 2.5f); + assert(cfg.moe_layer_freq == 1); + assert(cfg.first_k_dense_replace == 1); + assert(cfg.n_group == 4); + assert(cfg.topk_group == 2); + assert(cfg.moe_intermediate_size == 256); + assert(cfg.norm_topk_prob == true); + assert(cfg.rope_scaling_type && strcmp(cfg.rope_scaling_type, "yarn") == 0); + assert(cfg.rope_scaling_factor == 40.0f); + assert(cfg.rope_scaling_mscale_all_dim == 1.0f); + assert(cfg.index_head_dim == 128); + assert(cfg.index_n_heads == 64); + assert(cfg.index_topk == 2048); + model_config_free(&cfg); +} + +static void test_deepseek_v4_config_intersection(void) { + model_config_t cfg; + assert(model_config_load(&cfg, + MLXD_FIXTURES_DIR "/model_config_deepseek_v4") == 0); + assert(cfg.family == MODEL_DEEPSEEK_V4); + assert(cfg.q_lora_rank == 1024); + assert(cfg.qk_rope_head_dim == 64); + /* Flash omits these V3 MLA fields - stay 0 */ + assert(cfg.kv_lora_rank == 0); + assert(cfg.qk_nope_head_dim == 0); + assert(cfg.v_head_dim == 0); + assert(cfg.n_group == 0); + assert(cfg.topk_group == 0); + assert(cfg.moe_layer_freq == 0); + assert(cfg.first_k_dense_replace == 0); + assert(cfg.n_routed_experts == 256); + assert(cfg.num_experts == 256); /* R4 alias */ + assert(cfg.n_shared_experts == 1); + assert(cfg.num_experts_per_tok == 6); + assert(cfg.routed_scaling_factor == 1.5f); + assert(cfg.moe_intermediate_size == 2048); + assert(cfg.index_head_dim == 128); + assert(cfg.index_n_heads == 64); + assert(cfg.index_topk == 512); + assert(cfg.rope_scaling_type && strcmp(cfg.rope_scaling_type, "yarn") == 0); + assert(cfg.rope_scaling_factor == 16.0f); + assert(cfg.rope_scaling_mscale_all_dim == 0.0f); /* absent on Flash */ + model_config_free(&cfg); +} + +static void test_deepseek_defaults_absent_keys(void) { + model_config_t cfg; + assert(model_config_load( + &cfg, + MLXD_FIXTURES_DIR "/model_config_deepseek_v3_defaults") == 0); + assert(cfg.family == MODEL_DEEPSEEK_V3); + /* mlx-lm defaults when keys absent */ + assert(cfg.norm_topk_prob == true); + assert(cfg.routed_scaling_factor == 1.0f); + /* only n_routed_experts present -> alias into num_experts */ + assert(cfg.n_routed_experts == 8); + assert(cfg.num_experts == 8); + model_config_free(&cfg); +} + +static void test_deepseek_num_experts_not_clobbered(void) { + model_config_t cfg; + assert(model_config_load( + &cfg, + MLXD_FIXTURES_DIR "/model_config_deepseek_num_experts_both") == + 0); + assert(cfg.family == MODEL_DEEPSEEK_V3); + assert(cfg.num_experts == 8); /* explicit wins */ + assert(cfg.n_routed_experts == 16); + model_config_free(&cfg); +} + int main(void) { test_happy_path(); test_kv_heads_default(); @@ -1265,6 +1435,13 @@ int main(void) { test_gemma4_hidden_act_parsed(); test_gemma4_null_defaults(); + test_deepseek_config_fields_zero_init(); + test_deepseek_v3_config_fields(); + test_deepseek_v32_config_fields(); + test_deepseek_v4_config_intersection(); + test_deepseek_defaults_absent_keys(); + test_deepseek_num_experts_not_clobbered(); + printf("test_model_config: all passed\n"); return 0; } diff --git a/tests/test_weights.c b/tests/test_weights.c index c4d6a86..061645f 100644 --- a/tests/test_weights.c +++ b/tests/test_weights.c @@ -732,7 +732,8 @@ static void test_expected_names_other_families_zero(void) { MODEL_FAMILY_UNKNOWN, MODEL_GEMMA3, MODEL_QWEN2, MODEL_QWEN3_5_MOE, MODEL_LFM2, - MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V4, MODEL_BERT, + MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V3, MODEL_DEEPSEEK_V32, + MODEL_DEEPSEEK_V4, MODEL_BERT, }; for (size_t i = 0; i < sizeof(others) / sizeof(others[0]); i++) { diff --git a/tests/test_weights_gpu.c b/tests/test_weights_gpu.c index a0b0c5c..ee9c00a 100644 --- a/tests/test_weights_gpu.c +++ b/tests/test_weights_gpu.c @@ -120,15 +120,30 @@ static void test_validation_missing_shard(void) { rmdir(tmpdir); } -static void test_validation_deepseek_v4_rejected(void) { +static void assert_deepseek_weights_rejected(model_family_t fam, + const char *label) { model_config_t cfg = {0}; - cfg.family = MODEL_DEEPSEEK_V4; - + cfg.family = fam; weights_t w; char err[256] = {0}; + char expect_prefix[64]; int rc = weights_load(&w, FIXTURES "/tiny_qwen3", &cfg, err, sizeof(err)); assert(rc == -1); - assert(strstr(err, "GGUF") != NULL); + /* Message is "