feat(model): E2 DeepSeek family split + config fields (#109)

src/model/model.c

@@ -204,10 +204,12 @@ model_family_t model_family_from_type(const char *model_type) {
         return MODEL_LFM2;
     if (strcmp(model_type, "nemotron_h") == 0)
         return MODEL_NEMOTRON_H;
-    if (strcmp(model_type, "deepseek_v3") == 0 ||
-        strcmp(model_type, "deepseek_v3_2") == 0 ||
-        strcmp(model_type, "deepseek_v32") == 0 ||
-        strcmp(model_type, "deepseek_v4") == 0)
+    if (strcmp(model_type, "deepseek_v3") == 0)
+        return MODEL_DEEPSEEK_V3;
+    if (strcmp(model_type, "deepseek_v3_2") == 0 ||
+        strcmp(model_type, "deepseek_v32") == 0)
+        return MODEL_DEEPSEEK_V32;
+    if (strcmp(model_type, "deepseek_v4") == 0)
         return MODEL_DEEPSEEK_V4;
     if (strcmp(model_type, "bert") == 0)
         return MODEL_BERT;
@@ -534,10 +536,24 @@ static int apply_family_defaults(model_config_t *cfg, yyjson_val *cfg_obj,
         }
         break;
 
+    case MODEL_DEEPSEEK_V3:
+    case MODEL_DEEPSEEK_V32:
     case MODEL_DEEPSEEK_V4:
         if (set_llama_style_defaults(cfg, cfg_obj))
             return -1;
         cfg->hidden_act = HIDDEN_ACT_SILU;
+        /* mlx-lm defaults when keys absent; explicit values already parsed win.
+           Mirrors qwen3_5 attn_output_gate key-absence pattern. */
+        {
+            yyjson_val *v = yyjson_obj_get(cfg_obj, "norm_topk_prob");
+            if (!v || yyjson_is_null(v))
+                cfg->norm_topk_prob = true;
+        }
+        {
+            yyjson_val *v = yyjson_obj_get(cfg_obj, "routed_scaling_factor");
+            if (!v || yyjson_is_null(v))
+                cfg->routed_scaling_factor = 1.0f;
+        }
         break;
 
     case MODEL_BERT:
@@ -751,6 +767,32 @@ int model_config_load(model_config_t *cfg, const char *model_dir) {
                        &cfg->moe_intermediate_size, 0) ||
         get_int_nonneg(cfg_obj, "shared_expert_intermediate_size",
                        &cfg->shared_expert_intermediate_size, 0) ||
+        /* DeepSeek MLA */
+        get_int_nonneg(cfg_obj, "q_lora_rank", &cfg->q_lora_rank, 0) ||
+        get_int_nonneg(cfg_obj, "kv_lora_rank", &cfg->kv_lora_rank, 0) ||
+        get_int_nonneg(cfg_obj, "qk_rope_head_dim", &cfg->qk_rope_head_dim,
+                       0) ||
+        get_int_nonneg(cfg_obj, "qk_nope_head_dim", &cfg->qk_nope_head_dim,
+                       0) ||
+        get_int_nonneg(cfg_obj, "v_head_dim", &cfg->v_head_dim, 0) ||
+        /* DeepSeek MoE extras */
+        get_int_nonneg(cfg_obj, "n_routed_experts", &cfg->n_routed_experts,
+                       0) ||
+        get_int_nonneg(cfg_obj, "n_shared_experts", &cfg->n_shared_experts,
+                       0) ||
+        get_f32(cfg_obj, "routed_scaling_factor", &cfg->routed_scaling_factor,
+                cfg->routed_scaling_factor) ||
+        get_int_nonneg(cfg_obj, "moe_layer_freq", &cfg->moe_layer_freq, 0) ||
+        get_int_nonneg(cfg_obj, "first_k_dense_replace",
+                       &cfg->first_k_dense_replace, 0) ||
+        get_int_nonneg(cfg_obj, "n_group", &cfg->n_group, 0) ||
+        get_int_nonneg(cfg_obj, "topk_group", &cfg->topk_group, 0) ||
+        get_bool(cfg_obj, "norm_topk_prob", &cfg->norm_topk_prob,
+                 cfg->norm_topk_prob) ||
+        /* DeepSeek V3.2 / Flash indexer */
+        get_int_nonneg(cfg_obj, "index_head_dim", &cfg->index_head_dim, 0) ||
+        get_int_nonneg(cfg_obj, "index_n_heads", &cfg->index_n_heads, 0) ||
+        get_int_nonneg(cfg_obj, "index_topk", &cfg->index_topk, 0) ||
         get_int_nonneg(cfg_obj, "linear_num_key_heads",
                        &cfg->linear_num_key_heads, 0) ||
         get_int_nonneg(cfg_obj, "linear_num_value_heads",
@@ -787,6 +829,12 @@ int model_config_load(model_config_t *cfg, const char *model_dir) {
         }
     }
 
+    /* DeepSeek HF publishes n_routed_experts, not num_experts. One-way alias
+       so existing MoE detectors (num_experts > 0) keep working. Do not
+       overwrite an explicit num_experts. */
+    if (cfg->n_routed_experts > 0 && cfg->num_experts == 0)
+        cfg->num_experts = cfg->n_routed_experts;
+
     if (cfg->num_hidden_layers > MLXD_MAX_LAYERS)
         goto fail;
 
@@ -829,6 +877,8 @@ int model_config_load(model_config_t *cfg, const char *model_dir) {
                     cfg->rope_low_freq_factor) ||
             get_f32(rs, "high_freq_factor", &cfg->rope_high_freq_factor,
                     cfg->rope_high_freq_factor) ||
+            get_f32(rs, "mscale_all_dim", &cfg->rope_scaling_mscale_all_dim,
+                    cfg->rope_scaling_mscale_all_dim) ||
             get_int_nonneg(rs, "original_max_position_embeddings",
                            &cfg->rope_original_max_position_embeddings, 0))
             goto fail;

src/model/model.h

@@ -19,6 +19,8 @@ typedef enum {
     MODEL_MISTRAL,
     MODEL_LFM2,
     MODEL_NEMOTRON_H,
+    MODEL_DEEPSEEK_V3,
+    MODEL_DEEPSEEK_V32,
     MODEL_DEEPSEEK_V4,
     MODEL_BERT,
 } model_family_t;
@@ -69,6 +71,7 @@ typedef struct {
     float  rope_scaling_factor;
     float  rope_low_freq_factor;
     float  rope_high_freq_factor;
+    float  rope_scaling_mscale_all_dim; /* 0 = absent */
     int    rope_original_max_position_embeddings;
     bool   rope_proportional;        /* gemma4: rope_parameters.full_attention.rope_type == "proportional" */
     float  rope_proportional_factor; /* gemma4: rope_parameters.full_attention.factor */
@@ -106,6 +109,32 @@ typedef struct {
     int moe_intermediate_size;
     int shared_expert_intermediate_size;
 
+    /* DeepSeek MoE extras (E2 / decision 11)
+       Integer 0 = key absent / unset (R7). This intentionally diverges from
+       mlx-lm ModelArgs defaults (n_group/topk_group/moe_layer_freq default 1)
+       so callers can distinguish "not published" from "explicit 1".
+       #113 route wiring must not treat 0 as a ready-to-route group count. */
+    int   n_routed_experts;
+    int   n_shared_experts;
+    float routed_scaling_factor; /* mlx-lm default 1.0 when absent on DeepSeek */
+    int   moe_layer_freq;        /* 0 = absent; mlx-lm default 1 */
+    int   first_k_dense_replace; /* 0 = absent; mlx-lm default 0 */
+    int   n_group;               /* 0 = absent; mlx-lm default 1. NOT mamba_n_groups */
+    int   topk_group;            /* 0 = absent; mlx-lm default 1 */
+    bool  norm_topk_prob;        /* mlx-lm default true when absent on DeepSeek */
+
+    /* DeepSeek MLA */
+    int q_lora_rank;
+    int kv_lora_rank;
+    int qk_rope_head_dim;
+    int qk_nope_head_dim;
+    int v_head_dim;
+
+    /* DeepSeek V3.2 / Flash indexer (parse-only until #114/#115) */
+    int index_head_dim;
+    int index_n_heads;
+    int index_topk;
+
     /* Linear attention (qwen3_5) */
     int  linear_num_key_heads;
     int  linear_num_value_heads;

src/model/weights.c

@@ -944,9 +944,17 @@ int weights_load(weights_t *w, const char *model_dir,
     if (!w || !model_dir || !cfg) return -1;
     memset(w, 0, sizeof(*w));
 
-    if (cfg->family == MODEL_DEEPSEEK_V4) {
+    if (cfg->family == MODEL_DEEPSEEK_V3 ||
+        cfg->family == MODEL_DEEPSEEK_V32 ||
+        cfg->family == MODEL_DEEPSEEK_V4) {
+        const char *label =
+            cfg->family == MODEL_DEEPSEEK_V3  ? "deepseek_v3"  :
+            cfg->family == MODEL_DEEPSEEK_V32 ? "deepseek_v32" :
+                                               "deepseek_v4";
         if (err && errlen > 0)
-            snprintf(err, errlen, "deepseek_v4 MLA attention not yet implemented (Stage E)");
+            snprintf(err, errlen,
+                     "%s MLA/MoE forward not yet implemented (Stage E2)",
+                     label);
         return -1;
     }
 

tests/fixtures/model_config_deepseek_num_experts_both/config.json

@@ -0,0 +1,12 @@
+{
+  "model_type": "deepseek_v3",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "num_experts": 8,
+  "n_routed_experts": 16
+}

tests/fixtures/model_config_deepseek_v3/config.json

@@ -0,0 +1,32 @@
+{
+  "model_type": "deepseek_v3",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "q_lora_rank": 128,
+  "kv_lora_rank": 64,
+  "qk_rope_head_dim": 32,
+  "qk_nope_head_dim": 64,
+  "v_head_dim": 64,
+  "n_routed_experts": 16,
+  "n_shared_experts": 1,
+  "num_experts_per_tok": 4,
+  "routed_scaling_factor": 2.5,
+  "moe_layer_freq": 1,
+  "first_k_dense_replace": 1,
+  "n_group": 4,
+  "topk_group": 2,
+  "moe_intermediate_size": 256,
+  "norm_topk_prob": true,
+  "rope_scaling": {
+    "type": "yarn",
+    "factor": 40.0,
+    "mscale_all_dim": 1.0,
+    "beta_fast": 32,
+    "beta_slow": 1
+  }
+}

tests/fixtures/model_config_deepseek_v32/config.json

@@ -0,0 +1,35 @@
+{
+  "model_type": "deepseek_v32",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "q_lora_rank": 128,
+  "kv_lora_rank": 64,
+  "qk_rope_head_dim": 32,
+  "qk_nope_head_dim": 64,
+  "v_head_dim": 64,
+  "n_routed_experts": 16,
+  "n_shared_experts": 1,
+  "num_experts_per_tok": 4,
+  "routed_scaling_factor": 2.5,
+  "moe_layer_freq": 1,
+  "first_k_dense_replace": 1,
+  "n_group": 4,
+  "topk_group": 2,
+  "moe_intermediate_size": 256,
+  "norm_topk_prob": true,
+  "index_head_dim": 128,
+  "index_n_heads": 64,
+  "index_topk": 2048,
+  "rope_scaling": {
+    "type": "yarn",
+    "factor": 40.0,
+    "mscale_all_dim": 1.0,
+    "beta_fast": 32,
+    "beta_slow": 1
+  }
+}

tests/fixtures/model_config_deepseek_v3_2_alias/config.json

@@ -0,0 +1,10 @@
+{
+  "model_type": "deepseek_v3_2",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096
+}

tests/fixtures/model_config_deepseek_v3_defaults/config.json

@@ -0,0 +1,11 @@
+{
+  "model_type": "deepseek_v3",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "n_routed_experts": 8
+}

tests/fixtures/model_config_deepseek_v3_norm_topk_false/config.json

@@ -0,0 +1,12 @@
+{
+  "model_type": "deepseek_v3",
+  "vocab_size": 102400,
+  "hidden_size": 512,
+  "num_hidden_layers": 2,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 4096,
+  "n_routed_experts": 8,
+  "norm_topk_prob": false
+}

tests/fixtures/model_config_deepseek_v4/config.json

@@ -6,5 +6,19 @@
   "num_attention_heads": 16,
   "num_key_value_heads": 16,
   "intermediate_size": 10944,
-  "max_position_embeddings": 163840
+  "max_position_embeddings": 163840,
+  "q_lora_rank": 1024,
+  "qk_rope_head_dim": 64,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "num_experts_per_tok": 6,
+  "routed_scaling_factor": 1.5,
+  "moe_intermediate_size": 2048,
+  "index_head_dim": 128,
+  "index_n_heads": 64,
+  "index_topk": 512,
+  "rope_scaling": {
+    "type": "yarn",
+    "factor": 16.0
+  }
 }

tests/test_emodel_gate.c

@@ -127,7 +127,8 @@ static void test_reject_all_other_families(void) {
         MODEL_FAMILY_UNKNOWN, MODEL_GEMMA3,
         MODEL_QWEN2, MODEL_QWEN3_5_MOE,
         MODEL_LFM2,
-        MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V4, MODEL_BERT,
+        MODEL_NEMOTRON_H, MODEL_DEEPSEEK_V3, MODEL_DEEPSEEK_V32,
+        MODEL_DEEPSEEK_V4, MODEL_BERT,
     };
     for (size_t i = 0; i < sizeof(others) / sizeof(others[0]); i++) {
         model_config_t cfg = make_supported();