pipenetwork commited on
Commit
3fb8d28
·
verified ·
1 Parent(s): b8a0f3e

Bundle the fixed glm_moe_dsa runtime (shared indexer layers) via model_file; weights unchanged

Browse files
Files changed (3) hide show
  1. README.md +20 -0
  2. config.json +232 -231
  3. glm_moe_dsa.py +703 -0
README.md CHANGED
@@ -13,6 +13,26 @@ tags:
13
 
14
  # GLM-5.2-MLX-6bit
15
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  MLX (Apple Silicon) conversion of [zai-org/GLM-5.2](https://huggingface.co/zai-org/GLM-5.2) — a `glm_moe_dsa` MoE (256 experts, DeepSeek-V3.2-style sparse attention) — quantized to **6-bit**.
17
 
18
  ## Quantizations
 
13
 
14
  # GLM-5.2-MLX-6bit
15
 
16
+ ## Runtime — updated 2026-08-28: load with `--trust-remote-code`
17
+
18
+ This repository now bundles `glm_moe_dsa.py` (declared via `model_file` in `config.json`), a fixed runtime
19
+ for this architecture, and needs it:
20
+
21
+ ```bash
22
+ mlx_lm.generate --model pipenetwork/GLM-5.2-MLX-6bit --trust-remote-code --prompt "..." --max-tokens 300
23
+ ```
24
+
25
+ mlx-lm's own `glm_moe_dsa` builds a lightning indexer on all 78 layers, but GLM-5.2 ships indexer weights
26
+ on 21 (`indexer_types`: the other 57 "shared" layers reuse the previous full layer's top-k selection).
27
+ `mlx_lm.load` loads leniently and left those 57 indexers at random initialisation. Prompts up to 2048
28
+ tokens were unaffected (the indexer is bypassed below `index_topk`); beyond that, 57 of 78 layers attended
29
+ to keys chosen by random projections. The bundled runtime implements the schedule as the reference does
30
+ (plus fp32 indexer scores and router logits and the indexer LayerNorm epsilon); tiny-config parity against
31
+ `transformers` 5.16 is 4e-7 with the sparse path live, and a strict load of this checkpoint reports zero
32
+ missing and zero unexpected tensors. Details, tests and the GLM-5.3 builds made with it:
33
+ [github.com/PipeNetwork/glm53-mlx](https://github.com/PipeNetwork/glm53-mlx). The weights are unchanged.
34
+
35
+
36
  MLX (Apple Silicon) conversion of [zai-org/GLM-5.2](https://huggingface.co/zai-org/GLM-5.2) — a `glm_moe_dsa` MoE (256 experts, DeepSeek-V3.2-style sparse attention) — quantized to **6-bit**.
37
 
38
  ## Quantizations
config.json CHANGED
@@ -1,233 +1,234 @@
1
  {
2
- "architectures": [
3
- "GlmMoeDsaForCausalLM"
4
- ],
5
- "attention_bias": false,
6
- "attention_dropout": 0.0,
7
- "dtype": "bfloat16",
8
- "eos_token_id": [
9
- 154820,
10
- 154827,
11
- 154829
12
- ],
13
- "ep_size": 1,
14
- "first_k_dense_replace": 3,
15
- "head_dim": 192,
16
- "hidden_act": "silu",
17
- "hidden_size": 6144,
18
- "index_head_dim": 128,
19
- "index_n_heads": 32,
20
- "index_share_for_mtp_iteration": true,
21
- "index_skip_topk_offset": 3,
22
- "index_topk": 2048,
23
- "index_topk_freq": 4,
24
- "index_topk_pattern": null,
25
- "indexer_rope_interleave": true,
26
- "indexer_types": [
27
- "full",
28
- "full",
29
- "full",
30
- "shared",
31
- "shared",
32
- "shared",
33
- "full",
34
- "shared",
35
- "shared",
36
- "shared",
37
- "full",
38
- "shared",
39
- "shared",
40
- "shared",
41
- "full",
42
- "shared",
43
- "shared",
44
- "shared",
45
- "full",
46
- "shared",
47
- "shared",
48
- "shared",
49
- "full",
50
- "shared",
51
- "shared",
52
- "shared",
53
- "full",
54
- "shared",
55
- "shared",
56
- "shared",
57
- "full",
58
- "shared",
59
- "shared",
60
- "shared",
61
- "full",
62
- "shared",
63
- "shared",
64
- "shared",
65
- "full",
66
- "shared",
67
- "shared",
68
- "shared",
69
- "full",
70
- "shared",
71
- "shared",
72
- "shared",
73
- "full",
74
- "shared",
75
- "shared",
76
- "shared",
77
- "full",
78
- "shared",
79
- "shared",
80
- "shared",
81
- "full",
82
- "shared",
83
- "shared",
84
- "shared",
85
- "full",
86
- "shared",
87
- "shared",
88
- "shared",
89
- "full",
90
- "shared",
91
- "shared",
92
- "shared",
93
- "full",
94
- "shared",
95
- "shared",
96
- "shared",
97
- "full",
98
- "shared",
99
- "shared",
100
- "shared",
101
- "full",
102
- "shared",
103
- "shared",
104
- "shared"
105
- ],
106
- "initializer_range": 0.02,
107
- "intermediate_size": 12288,
108
- "kv_lora_rank": 512,
109
- "max_position_embeddings": 1048576,
110
- "mlp_layer_types": [
111
- "dense",
112
- "dense",
113
- "dense",
114
- "sparse",
115
- "sparse",
116
- "sparse",
117
- "sparse",
118
- "sparse",
119
- "sparse",
120
- "sparse",
121
- "sparse",
122
- "sparse",
123
- "sparse",
124
- "sparse",
125
- "sparse",
126
- "sparse",
127
- "sparse",
128
- "sparse",
129
- "sparse",
130
- "sparse",
131
- "sparse",
132
- "sparse",
133
- "sparse",
134
- "sparse",
135
- "sparse",
136
- "sparse",
137
- "sparse",
138
- "sparse",
139
- "sparse",
140
- "sparse",
141
- "sparse",
142
- "sparse",
143
- "sparse",
144
- "sparse",
145
- "sparse",
146
- "sparse",
147
- "sparse",
148
- "sparse",
149
- "sparse",
150
- "sparse",
151
- "sparse",
152
- "sparse",
153
- "sparse",
154
- "sparse",
155
- "sparse",
156
- "sparse",
157
- "sparse",
158
- "sparse",
159
- "sparse",
160
- "sparse",
161
- "sparse",
162
- "sparse",
163
- "sparse",
164
- "sparse",
165
- "sparse",
166
- "sparse",
167
- "sparse",
168
- "sparse",
169
- "sparse",
170
- "sparse",
171
- "sparse",
172
- "sparse",
173
- "sparse",
174
- "sparse",
175
- "sparse",
176
- "sparse",
177
- "sparse",
178
- "sparse",
179
- "sparse",
180
- "sparse",
181
- "sparse",
182
- "sparse",
183
- "sparse",
184
- "sparse",
185
- "sparse",
186
- "sparse",
187
- "sparse",
188
- "sparse"
189
- ],
190
- "model_type": "glm_moe_dsa",
191
- "moe_intermediate_size": 2048,
192
- "moe_layer_freq": 1,
193
- "n_group": 1,
194
- "n_routed_experts": 256,
195
- "n_shared_experts": 1,
196
- "norm_topk_prob": true,
197
- "num_attention_heads": 64,
198
- "num_experts_per_tok": 8,
199
- "num_hidden_layers": 78,
200
- "num_key_value_heads": 64,
201
- "num_nextn_predict_layers": 1,
202
- "pad_token_id": 154820,
203
- "pretraining_tp": 1,
204
- "q_lora_rank": 2048,
205
- "qk_head_dim": 256,
206
- "qk_nope_head_dim": 192,
207
- "qk_rope_head_dim": 64,
208
- "quantization": {
209
- "group_size": 64,
210
- "bits": 6,
211
- "mode": "affine"
212
- },
213
- "quantization_config": {
214
- "group_size": 64,
215
- "bits": 6,
216
- "mode": "affine"
217
- },
218
- "rms_norm_eps": 1e-05,
219
- "rope_interleave": true,
220
- "rope_parameters": {
221
- "rope_theta": 8000000,
222
- "rope_type": "default"
223
- },
224
- "routed_scaling_factor": 2.5,
225
- "scoring_func": "sigmoid",
226
- "tie_word_embeddings": false,
227
- "topk_group": 1,
228
- "topk_method": "noaux_tc",
229
- "transformers_version": "5.12.0",
230
- "use_cache": true,
231
- "v_head_dim": 256,
232
- "vocab_size": 154880
 
233
  }
 
1
  {
2
+ "architectures": [
3
+ "GlmMoeDsaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": [
9
+ 154820,
10
+ 154827,
11
+ 154829
12
+ ],
13
+ "ep_size": 1,
14
+ "first_k_dense_replace": 3,
15
+ "head_dim": 192,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 6144,
18
+ "index_head_dim": 128,
19
+ "index_n_heads": 32,
20
+ "index_share_for_mtp_iteration": true,
21
+ "index_skip_topk_offset": 3,
22
+ "index_topk": 2048,
23
+ "index_topk_freq": 4,
24
+ "index_topk_pattern": null,
25
+ "indexer_rope_interleave": true,
26
+ "indexer_types": [
27
+ "full",
28
+ "full",
29
+ "full",
30
+ "shared",
31
+ "shared",
32
+ "shared",
33
+ "full",
34
+ "shared",
35
+ "shared",
36
+ "shared",
37
+ "full",
38
+ "shared",
39
+ "shared",
40
+ "shared",
41
+ "full",
42
+ "shared",
43
+ "shared",
44
+ "shared",
45
+ "full",
46
+ "shared",
47
+ "shared",
48
+ "shared",
49
+ "full",
50
+ "shared",
51
+ "shared",
52
+ "shared",
53
+ "full",
54
+ "shared",
55
+ "shared",
56
+ "shared",
57
+ "full",
58
+ "shared",
59
+ "shared",
60
+ "shared",
61
+ "full",
62
+ "shared",
63
+ "shared",
64
+ "shared",
65
+ "full",
66
+ "shared",
67
+ "shared",
68
+ "shared",
69
+ "full",
70
+ "shared",
71
+ "shared",
72
+ "shared",
73
+ "full",
74
+ "shared",
75
+ "shared",
76
+ "shared",
77
+ "full",
78
+ "shared",
79
+ "shared",
80
+ "shared",
81
+ "full",
82
+ "shared",
83
+ "shared",
84
+ "shared",
85
+ "full",
86
+ "shared",
87
+ "shared",
88
+ "shared",
89
+ "full",
90
+ "shared",
91
+ "shared",
92
+ "shared",
93
+ "full",
94
+ "shared",
95
+ "shared",
96
+ "shared",
97
+ "full",
98
+ "shared",
99
+ "shared",
100
+ "shared",
101
+ "full",
102
+ "shared",
103
+ "shared",
104
+ "shared"
105
+ ],
106
+ "initializer_range": 0.02,
107
+ "intermediate_size": 12288,
108
+ "kv_lora_rank": 512,
109
+ "max_position_embeddings": 1048576,
110
+ "mlp_layer_types": [
111
+ "dense",
112
+ "dense",
113
+ "dense",
114
+ "sparse",
115
+ "sparse",
116
+ "sparse",
117
+ "sparse",
118
+ "sparse",
119
+ "sparse",
120
+ "sparse",
121
+ "sparse",
122
+ "sparse",
123
+ "sparse",
124
+ "sparse",
125
+ "sparse",
126
+ "sparse",
127
+ "sparse",
128
+ "sparse",
129
+ "sparse",
130
+ "sparse",
131
+ "sparse",
132
+ "sparse",
133
+ "sparse",
134
+ "sparse",
135
+ "sparse",
136
+ "sparse",
137
+ "sparse",
138
+ "sparse",
139
+ "sparse",
140
+ "sparse",
141
+ "sparse",
142
+ "sparse",
143
+ "sparse",
144
+ "sparse",
145
+ "sparse",
146
+ "sparse",
147
+ "sparse",
148
+ "sparse",
149
+ "sparse",
150
+ "sparse",
151
+ "sparse",
152
+ "sparse",
153
+ "sparse",
154
+ "sparse",
155
+ "sparse",
156
+ "sparse",
157
+ "sparse",
158
+ "sparse",
159
+ "sparse",
160
+ "sparse",
161
+ "sparse",
162
+ "sparse",
163
+ "sparse",
164
+ "sparse",
165
+ "sparse",
166
+ "sparse",
167
+ "sparse",
168
+ "sparse",
169
+ "sparse",
170
+ "sparse",
171
+ "sparse",
172
+ "sparse",
173
+ "sparse",
174
+ "sparse",
175
+ "sparse",
176
+ "sparse",
177
+ "sparse",
178
+ "sparse",
179
+ "sparse",
180
+ "sparse",
181
+ "sparse",
182
+ "sparse",
183
+ "sparse",
184
+ "sparse",
185
+ "sparse",
186
+ "sparse",
187
+ "sparse",
188
+ "sparse"
189
+ ],
190
+ "model_type": "glm_moe_dsa",
191
+ "moe_intermediate_size": 2048,
192
+ "moe_layer_freq": 1,
193
+ "n_group": 1,
194
+ "n_routed_experts": 256,
195
+ "n_shared_experts": 1,
196
+ "norm_topk_prob": true,
197
+ "num_attention_heads": 64,
198
+ "num_experts_per_tok": 8,
199
+ "num_hidden_layers": 78,
200
+ "num_key_value_heads": 64,
201
+ "num_nextn_predict_layers": 1,
202
+ "pad_token_id": 154820,
203
+ "pretraining_tp": 1,
204
+ "q_lora_rank": 2048,
205
+ "qk_head_dim": 256,
206
+ "qk_nope_head_dim": 192,
207
+ "qk_rope_head_dim": 64,
208
+ "quantization": {
209
+ "group_size": 64,
210
+ "bits": 6,
211
+ "mode": "affine"
212
+ },
213
+ "quantization_config": {
214
+ "group_size": 64,
215
+ "bits": 6,
216
+ "mode": "affine"
217
+ },
218
+ "rms_norm_eps": 1e-05,
219
+ "rope_interleave": true,
220
+ "rope_parameters": {
221
+ "rope_theta": 8000000,
222
+ "rope_type": "default"
223
+ },
224
+ "routed_scaling_factor": 2.5,
225
+ "scoring_func": "sigmoid",
226
+ "tie_word_embeddings": false,
227
+ "topk_group": 1,
228
+ "topk_method": "noaux_tc",
229
+ "transformers_version": "5.12.0",
230
+ "use_cache": true,
231
+ "v_head_dim": 256,
232
+ "vocab_size": 154880,
233
+ "model_file": "glm_moe_dsa.py"
234
  }
glm_moe_dsa.py ADDED
@@ -0,0 +1,703 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright © 2025 Apple Inc.
2
+
3
+ import math
4
+ from dataclasses import dataclass
5
+ from typing import Any, Dict, Optional
6
+
7
+ import mlx.core as mx
8
+ import mlx.nn as nn
9
+ from mlx.nn.layers.distributed import shard_inplace, shard_linear, sum_gradients
10
+
11
+ from mlx_lm.models.activations import swiglu
12
+ from mlx_lm.models.base import BaseModelArgs, create_attention_mask, scaled_dot_product_attention
13
+ from mlx_lm.models.cache import CacheList, KVCache
14
+ from mlx_lm.models.mla import MultiLinear
15
+ from mlx_lm.models.rope_utils import initialize_rope
16
+ from mlx_lm.models.switch_layers import SwitchGLU
17
+
18
+
19
+ @dataclass
20
+ class ModelArgs(BaseModelArgs):
21
+ model_type: str
22
+ vocab_size: int
23
+ hidden_size: int
24
+ index_head_dim: int
25
+ index_n_heads: int
26
+ index_topk: int
27
+ intermediate_size: int
28
+ moe_intermediate_size: int
29
+ num_hidden_layers: int
30
+ num_attention_heads: int
31
+ num_key_value_heads: int
32
+ n_shared_experts: Optional[int]
33
+ n_routed_experts: Optional[int]
34
+ routed_scaling_factor: float
35
+ kv_lora_rank: int
36
+ q_lora_rank: int
37
+ qk_rope_head_dim: int
38
+ v_head_dim: int
39
+ qk_nope_head_dim: int
40
+ topk_method: str
41
+ scoring_func: str
42
+ norm_topk_prob: bool
43
+ n_group: int
44
+ topk_group: int
45
+ num_experts_per_tok: int
46
+ moe_layer_freq: int
47
+ first_k_dense_replace: int
48
+ max_position_embeddings: int
49
+ rms_norm_eps: float
50
+ rope_parameters: Dict
51
+ attention_bias: bool
52
+ rope_scaling: Dict = None
53
+ rope_theta: Optional[float] = None
54
+ indexer_rope_interleave: bool = True
55
+ rope_interleave: bool = True
56
+ indexer_types: Optional[list] = None
57
+ index_topk_freq: int = 1
58
+ index_skip_topk_offset: int = 0
59
+ index_topk_pattern: Optional[Any] = None
60
+ num_nextn_predict_layers: int = 0
61
+ mlp_layer_types: Optional[list] = None
62
+
63
+ def __post_init__(self):
64
+ self.rope_scaling = self.rope_parameters
65
+ self.rope_theta = self.rope_parameters["rope_theta"]
66
+ if self.indexer_types is None:
67
+ # Reference schedule: a layer runs its own indexer ("full") iff
68
+ # max(i - offset + 1, 0) % freq == 0; the others ("shared") reuse the most recent
69
+ # full layer's top-k selection and carry no indexer weights.
70
+ f, o = self.index_topk_freq, self.index_skip_topk_offset
71
+ self.indexer_types = [
72
+ "full" if max(i - o + 1, 0) % f == 0 else "shared"
73
+ for i in range(self.num_hidden_layers)
74
+ ]
75
+
76
+
77
+
78
+ class Indexer(nn.Module):
79
+ def __init__(self, args: ModelArgs):
80
+ super().__init__()
81
+ self.dim = args.hidden_size
82
+ self.n_heads = args.index_n_heads
83
+ self.head_dim = args.index_head_dim
84
+ self.rope_head_dim = args.qk_rope_head_dim
85
+ self.index_topk = args.index_topk
86
+ self.q_lora_rank = args.q_lora_rank
87
+ self.wq_b = nn.Linear(
88
+ self.q_lora_rank, self.n_heads * self.head_dim, bias=False
89
+ )
90
+ self.wk = nn.Linear(self.dim, self.head_dim, bias=False)
91
+ self.k_norm = nn.LayerNorm(self.head_dim, eps=1e-6)
92
+ self.weights_proj = nn.Linear(self.dim, self.n_heads, bias=False)
93
+ self.softmax_scale = self.head_dim**-0.5
94
+ self.rope = initialize_rope(
95
+ dims=args.qk_rope_head_dim,
96
+ base=args.rope_theta,
97
+ traditional=args.indexer_rope_interleave,
98
+ max_position_embeddings=args.max_position_embeddings,
99
+ scaling_config=args.rope_scaling,
100
+ )
101
+
102
+ def __call__(
103
+ self,
104
+ x: mx.array,
105
+ qr: mx.array,
106
+ mask: Optional[mx.array],
107
+ cache: Optional[Any] = None,
108
+ ):
109
+ # Computes top_k indices for attention
110
+ b, s, _ = x.shape
111
+ q = self.wq_b(qr)
112
+ q = q.reshape(b, s, self.n_heads, self.head_dim).swapaxes(1, 2)
113
+ k = self.wk(x)
114
+ k = self.k_norm(k)
115
+ k = mx.reshape(k, (b, 1, s, self.head_dim))
116
+
117
+ offset = cache.offset if cache is not None else 0
118
+
119
+ q = self.rope(q, offset=offset)
120
+ k = self.rope(k, offset=offset)
121
+
122
+ if cache is not None:
123
+ k, _ = cache.update_and_fetch(k, mx.zeros([b, 1, s, 0]))
124
+ if k.shape[2] <= self.index_topk:
125
+ return None
126
+ # Scores in float32, as the reference (which also keeps weights_proj in fp32): in bf16
127
+ # keys at the top-k boundary flip.
128
+ scores = q.astype(mx.float32) @ k.astype(mx.float32).swapaxes(-1, -2)
129
+ scores = mx.maximum(scores, 0)
130
+ weights = self.weights_proj(x).astype(mx.float32) * (self.n_heads**-0.5 * self.softmax_scale)
131
+ weights = weights.swapaxes(-1, -2)[..., None]
132
+ scores = scores * weights
133
+ scores = scores.sum(axis=1, keepdims=True)
134
+ if mask is not None:
135
+ scores = mx.where(mask, scores, -float("inf"))
136
+ return mx.argpartition(scores, kth=-self.index_topk, axis=-1)[
137
+ ..., -self.index_topk :
138
+ ]
139
+
140
+
141
+ class DeepseekV32Attention(nn.Module):
142
+ def __init__(self, config: ModelArgs, layer_idx: int = 0):
143
+ super().__init__()
144
+ self.config = config
145
+ self.hidden_size = config.hidden_size
146
+ self.num_heads = config.num_attention_heads
147
+ self.max_position_embeddings = config.max_position_embeddings
148
+ self.rope_theta = config.rope_theta
149
+ self.q_lora_rank = config.q_lora_rank
150
+ self.qk_rope_head_dim = config.qk_rope_head_dim
151
+ self.kv_lora_rank = config.kv_lora_rank
152
+ self.v_head_dim = config.v_head_dim
153
+ self.qk_nope_head_dim = config.qk_nope_head_dim
154
+ self.q_head_dim = config.qk_nope_head_dim + config.qk_rope_head_dim
155
+
156
+ self.scale = self.q_head_dim**-0.5
157
+
158
+ self.q_a_proj = nn.Linear(
159
+ self.hidden_size, self.q_lora_rank, bias=config.attention_bias
160
+ )
161
+ self.q_a_layernorm = nn.RMSNorm(self.q_lora_rank, eps=1e-6)
162
+ self.q_b_proj = nn.Linear(
163
+ self.q_lora_rank, self.num_heads * self.q_head_dim, bias=False
164
+ )
165
+
166
+ self.kv_a_proj_with_mqa = nn.Linear(
167
+ self.hidden_size,
168
+ self.kv_lora_rank + self.qk_rope_head_dim,
169
+ bias=config.attention_bias,
170
+ )
171
+ self.kv_a_layernorm = nn.RMSNorm(self.kv_lora_rank, eps=1e-6)
172
+ self.embed_q = MultiLinear(
173
+ self.qk_nope_head_dim, self.kv_lora_rank, self.num_heads
174
+ )
175
+ self.unembed_out = MultiLinear(
176
+ self.kv_lora_rank, self.v_head_dim, self.num_heads
177
+ )
178
+
179
+ self.o_proj = nn.Linear(
180
+ self.num_heads * self.v_head_dim,
181
+ self.hidden_size,
182
+ bias=config.attention_bias,
183
+ )
184
+
185
+ if self.config.rope_scaling is not None:
186
+ mscale_all_dim = self.config.rope_scaling.get("mscale_all_dim", 0)
187
+ if mscale_all_dim:
188
+ scaling_factor = self.config.rope_scaling["factor"]
189
+ if scaling_factor > 1:
190
+ s = 0.1 * mscale_all_dim * math.log(scaling_factor) + 1.0
191
+ self.scale = self.scale * s * s
192
+
193
+ # "shared" layers carry no indexer weights: they reuse the previous full layer's top-k.
194
+ self.indexer = Indexer(config) if config.indexer_types[layer_idx] == "full" else None
195
+ self.rope = initialize_rope(
196
+ dims=self.qk_rope_head_dim,
197
+ base=self.rope_theta,
198
+ traditional=True,
199
+ max_position_embeddings=self.max_position_embeddings,
200
+ scaling_config=self.config.rope_scaling,
201
+ )
202
+
203
+ def __call__(
204
+ self,
205
+ x: mx.array,
206
+ mask: Optional[mx.array] = None,
207
+ cache: Optional[Any] = None,
208
+ prev_topk_indices: Optional[mx.array] = None,
209
+ ):
210
+ B, L, D = x.shape
211
+
212
+ qr = self.q_a_layernorm(self.q_a_proj(x))
213
+ q = self.q_b_proj(qr)
214
+
215
+ q = q.reshape(B, L, self.num_heads, self.q_head_dim).transpose(0, 2, 1, 3)
216
+ q_nope, q_pe = mx.split(q, [self.qk_nope_head_dim], axis=-1)
217
+ compressed_kv = self.kv_a_proj_with_mqa(x)
218
+ compressed_kv, k_pe = mx.split(compressed_kv, [self.kv_lora_rank], axis=-1)
219
+ k_pe = k_pe.reshape(B, L, 1, self.qk_rope_head_dim).transpose(0, 2, 1, 3)
220
+ kv_latent = self.kv_a_layernorm(compressed_kv)
221
+
222
+ offset = cache[0].offset if cache is not None else 0
223
+ q_pe = self.rope(q_pe, offset)
224
+ k_pe = self.rope(k_pe, offset)
225
+
226
+ kv_latent = mx.expand_dims(kv_latent, axis=1)
227
+
228
+ if cache is not None:
229
+ kv_latent, k_pe = cache[0].update_and_fetch(kv_latent, k_pe)
230
+ else:
231
+ cache = [None] * 2
232
+
233
+ if self.indexer is not None:
234
+ topk_indices = self.indexer(x, qr, mask, cache=cache[1])
235
+ else:
236
+ topk_indices = prev_topk_indices
237
+ if topk_indices is not None:
238
+ if L == 1:
239
+ idx = topk_indices[:, :, 0, :, None]
240
+ kv_latent = mx.take_along_axis(
241
+ kv_latent,
242
+ mx.broadcast_to(idx, idx.shape[:-1] + (kv_latent.shape[-1],)),
243
+ axis=2,
244
+ )
245
+ k_pe = mx.take_along_axis(
246
+ k_pe,
247
+ mx.broadcast_to(idx, idx.shape[:-1] + (k_pe.shape[-1],)),
248
+ axis=2,
249
+ )
250
+ if mask is not None:
251
+ mask = mx.take_along_axis(mask, topk_indices, axis=-1)
252
+ else:
253
+ shape = list(topk_indices.shape)
254
+ shape[-1] = kv_latent.shape[2]
255
+ sparse_mask = mx.zeros(shape, dtype=mx.bool_)
256
+ sparse_mask = mx.put_along_axis(
257
+ sparse_mask, topk_indices, mx.array(True), axis=-1
258
+ )
259
+ if mask is not None:
260
+ sparse_mask = sparse_mask & mask
261
+ mask = sparse_mask
262
+ # Ensure the indexer cache is evaluated even if the topk_indices are unused
263
+ # to keep the graph from getting too large
264
+ if self.indexer is not None and cache is not None and cache[0] is not None:
265
+ cache[0].keys = mx.depends(cache[0].keys, (cache[1].keys, cache[1].values))
266
+
267
+ pe_scores = (q_pe * self.scale) @ k_pe.swapaxes(-1, -2)
268
+ if mask is not None:
269
+ pe_scores = mx.where(
270
+ mask,
271
+ pe_scores,
272
+ mx.array(mx.finfo(pe_scores.dtype).min, pe_scores.dtype),
273
+ )
274
+
275
+ if L == 1:
276
+ q_nope = self.embed_q(q_nope)
277
+ k = v = kv_latent
278
+ else:
279
+ k = self.embed_q(kv_latent, transpose=False)
280
+ v = self.unembed_out(kv_latent)
281
+
282
+ output = scaled_dot_product_attention(
283
+ q_nope, k, v, cache=cache, scale=self.scale, mask=pe_scores
284
+ )
285
+ if L == 1:
286
+ output = self.unembed_out(output)
287
+
288
+ output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)
289
+ return self.o_proj(output), topk_indices
290
+
291
+
292
+ class DeepseekV32MLP(nn.Module):
293
+ def __init__(
294
+ self, config: ModelArgs, hidden_size: int = None, intermediate_size: int = None
295
+ ):
296
+ super().__init__()
297
+ self.config = config
298
+ self.hidden_size = config.hidden_size if hidden_size is None else hidden_size
299
+ self.intermediate_size = (
300
+ config.intermediate_size if intermediate_size is None else intermediate_size
301
+ )
302
+
303
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
304
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
305
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
306
+
307
+ def __call__(self, x):
308
+ down_proj = self.down_proj(swiglu(self.gate_proj(x), self.up_proj(x)))
309
+ return down_proj
310
+
311
+
312
+ @mx.compile
313
+ def group_expert_select(
314
+ gates,
315
+ e_score_correction_bias,
316
+ top_k,
317
+ n_group,
318
+ topk_group,
319
+ routed_scaling_factor,
320
+ norm_topk_prob,
321
+ ):
322
+
323
+ scores = mx.sigmoid(gates.astype(mx.float32))
324
+ orig_scores = scores
325
+ scores = scores + e_score_correction_bias
326
+ if n_group > 1:
327
+ scores = mx.unflatten(scores, axis=-1, shape=(n_group, -1))
328
+ group_scores = mx.topk(scores, 2, axis=-1).sum(axis=-1, keepdims=True)
329
+ k = n_group - topk_group
330
+ group_idx = mx.argpartition(group_scores, kth=k - 1, axis=-2)[..., :k, :]
331
+ scores = mx.put_along_axis(
332
+ scores, mx.stop_gradient(group_idx), mx.array(0.0), axis=-2
333
+ )
334
+ scores = mx.flatten(scores, -2, -1)
335
+
336
+ k = top_k
337
+ inds = mx.argpartition(-scores, kth=k - 1, axis=-1)[..., :k]
338
+ scores = mx.take_along_axis(orig_scores, inds, axis=-1)
339
+ if top_k > 1 and norm_topk_prob:
340
+ denominator = scores.sum(axis=-1, keepdims=True)
341
+ scores = scores / denominator
342
+ scores = scores * routed_scaling_factor
343
+
344
+ return inds, scores
345
+
346
+
347
+ class MoEGate(nn.Module):
348
+ def __init__(self, config: ModelArgs):
349
+ super().__init__()
350
+ self.config = config
351
+ self.top_k = config.num_experts_per_tok
352
+ self.norm_topk_prob = config.norm_topk_prob
353
+ self.n_routed_experts = config.n_routed_experts
354
+ self.routed_scaling_factor = config.routed_scaling_factor
355
+ self.n_group = config.n_group
356
+ self.topk_group = config.topk_group
357
+ self.weight = mx.zeros((self.n_routed_experts, config.hidden_size))
358
+ self.e_score_correction_bias = mx.zeros((self.n_routed_experts,))
359
+ assert config.topk_method == "noaux_tc", "Unsupported topk method."
360
+
361
+ def __call__(self, x):
362
+ return group_expert_select(
363
+ x.astype(mx.float32) @ self.weight.astype(mx.float32).T,
364
+ self.e_score_correction_bias,
365
+ self.top_k,
366
+ self.n_group,
367
+ self.topk_group,
368
+ self.routed_scaling_factor,
369
+ self.norm_topk_prob,
370
+ )
371
+
372
+
373
+ class DeepseekV32MoE(nn.Module):
374
+ def __init__(self, config: ModelArgs):
375
+ super().__init__()
376
+ self.config = config
377
+ self.num_experts_per_tok = config.num_experts_per_tok
378
+ self.switch_mlp = SwitchGLU(
379
+ config.hidden_size,
380
+ config.moe_intermediate_size,
381
+ config.n_routed_experts,
382
+ )
383
+
384
+ self.gate = MoEGate(config)
385
+ if config.n_shared_experts is not None:
386
+ intermediate_size = config.moe_intermediate_size * config.n_shared_experts
387
+ self.shared_experts = DeepseekV32MLP(
388
+ config=config, intermediate_size=intermediate_size
389
+ )
390
+
391
+ self.sharding_group = None
392
+
393
+ def __call__(self, x):
394
+ if self.sharding_group is not None:
395
+ x = sum_gradients(self.sharding_group)(x)
396
+
397
+ inds, scores = self.gate(x)
398
+ y = self.switch_mlp(x, inds)
399
+ y = (y * scores[..., None]).sum(axis=-2).astype(y.dtype)
400
+ if self.config.n_shared_experts is not None:
401
+ y = y + self.shared_experts(x)
402
+
403
+ if self.sharding_group is not None:
404
+ y = mx.distributed.all_sum(y, group=self.sharding_group)
405
+
406
+ return y
407
+
408
+
409
+ class DeepseekV32DecoderLayer(nn.Module):
410
+ def __init__(self, config: ModelArgs, layer_idx: int):
411
+ super().__init__()
412
+ self.self_attn = DeepseekV32Attention(config, layer_idx)
413
+ self.mlp = (
414
+ DeepseekV32MoE(config)
415
+ if (
416
+ config.n_routed_experts is not None
417
+ and layer_idx >= config.first_k_dense_replace
418
+ and layer_idx % config.moe_layer_freq == 0
419
+ )
420
+ else DeepseekV32MLP(config)
421
+ )
422
+ self.input_layernorm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
423
+ self.post_attention_layernorm = nn.RMSNorm(
424
+ config.hidden_size, eps=config.rms_norm_eps
425
+ )
426
+
427
+ def __call__(
428
+ self,
429
+ x: mx.array,
430
+ mask: Optional[mx.array] = None,
431
+ cache: Optional[Any] = None,
432
+ prev_topk_indices: Optional[mx.array] = None,
433
+ ):
434
+ r, topk_indices = self.self_attn(self.input_layernorm(x), mask, cache, prev_topk_indices)
435
+ h = x + r
436
+ r = self.mlp(self.post_attention_layernorm(h))
437
+ return h + r, topk_indices
438
+
439
+
440
+ class DeepseekV32Model(nn.Module):
441
+ def __init__(self, config: ModelArgs):
442
+ super().__init__()
443
+ self.vocab_size = config.vocab_size
444
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
445
+ self.layers = [
446
+ DeepseekV32DecoderLayer(config, idx)
447
+ for idx in range(config.num_hidden_layers)
448
+ ]
449
+ self.start_idx = 0
450
+ self.end_idx = len(self.layers)
451
+ self.num_layers = self.end_idx
452
+
453
+ self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
454
+ self.pipeline_rank = 0
455
+ self.pipeline_size = 1
456
+
457
+ def pipeline(self, group):
458
+ # Split layers in reverse so rank=0 gets the last layers and
459
+ # rank=pipeline_size-1 gets the first
460
+ self.pipeline_rank = group.rank()
461
+ self.pipeline_size = group.size()
462
+ layers_per_rank = len(self.layers) // self.pipeline_size
463
+ extra = len(self.layers) - layers_per_rank * self.pipeline_size
464
+ if self.pipeline_rank < extra:
465
+ layers_per_rank += 1
466
+ self.start_idx = (self.pipeline_size - self.pipeline_rank - 1) * layers_per_rank
467
+ self.end_idx = self.start_idx + layers_per_rank
468
+ self.layers = self.layers[: self.end_idx]
469
+ self.layers[: self.start_idx] = [None] * self.start_idx
470
+ self.num_layers = len(self.layers) - self.start_idx
471
+
472
+ def __call__(
473
+ self,
474
+ x: mx.array,
475
+ cache: Optional[Any] = None,
476
+ ) -> mx.array:
477
+ h = self.embed_tokens(x)
478
+
479
+ pipeline_rank = self.pipeline_rank
480
+ pipeline_size = self.pipeline_size
481
+
482
+ if cache is None:
483
+ cache = [None] * self.num_layers
484
+ mask = create_attention_mask(
485
+ h, cache[0][0] if cache[0] else None, return_array=True
486
+ )
487
+
488
+ # Receive from the previous process in the pipeline
489
+
490
+ if pipeline_rank < pipeline_size - 1:
491
+ h = mx.distributed.recv_like(h, (pipeline_rank + 1))
492
+
493
+ # A "shared" layer reuses the top-k selection of the most recent "full" layer.
494
+ topk = None
495
+ for i in range(self.num_layers):
496
+ h, topk = self.layers[self.start_idx + i](h, mask, cache[i], topk)
497
+
498
+ # Send to the next process in the pipeline
499
+ if pipeline_rank != 0:
500
+ h = mx.distributed.send(h, (pipeline_rank - 1) % pipeline_size)
501
+ if cache[-1] is not None:
502
+ cache[-1][0].keys = mx.depends(cache[-1][0].keys, h)
503
+
504
+ # Broadcast h while keeping it in the graph
505
+ if pipeline_size > 1:
506
+ h = mx.distributed.all_gather(h)[: h.shape[0]]
507
+
508
+ return self.norm(h)
509
+
510
+
511
+ class Model(nn.Module):
512
+ def __init__(self, config: ModelArgs):
513
+ super().__init__()
514
+ self.args = config
515
+ self.model_type = config.model_type
516
+ self.model = DeepseekV32Model(config)
517
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
518
+
519
+ def __call__(
520
+ self,
521
+ inputs: mx.array,
522
+ cache: Optional[Any] = None,
523
+ ):
524
+ out = self.model(inputs, cache)
525
+ return self.lm_head(out)
526
+
527
+ def sanitize(self, weights):
528
+ # Remove multi-token prediction layers
529
+ mpt_layer = self.args.num_hidden_layers
530
+ new_weights = {}
531
+ for k, v in weights.items():
532
+ parts = k.split(".")
533
+ if len(parts) >= 3 and parts[1] == "layers" and int(parts[2]) >= mpt_layer:
534
+ continue
535
+ new_weights[k] = v
536
+ weights = new_weights
537
+
538
+ def dequant(weight, scale_inv):
539
+ dtype = mx.bfloat16
540
+ weight = mx.from_fp8(weight, dtype=mx.bfloat16)
541
+ bs = 128 # block size
542
+ m, n = weight.shape
543
+ pad_bottom = (-m) % bs
544
+ pad_side = (-n) % bs
545
+ weight = mx.pad(weight, ((0, pad_bottom), (0, pad_side)))
546
+ weight = weight.reshape(
547
+ ((m + pad_bottom) // bs, bs, (n + pad_side) // bs, bs)
548
+ )
549
+ weight = (weight * scale_inv[:, None, :, None]).reshape(
550
+ m + pad_bottom, n + pad_side
551
+ )
552
+ return weight[:m, :n].astype(dtype)
553
+
554
+ # Dequantize
555
+ new_weights = {}
556
+ for k, v in weights.items():
557
+ if "weight_scale_inv" in k:
558
+ scale_inv = v
559
+ wk = k.replace("_scale_inv", "")
560
+ weight = weights[wk]
561
+ weight = dequant(weight, scale_inv)
562
+ new_weights[wk] = weight
563
+ elif k not in new_weights:
564
+ new_weights[k] = v
565
+ weights = new_weights
566
+
567
+ # Stack experts
568
+ for l in range(self.args.num_hidden_layers):
569
+ prefix = f"model.layers.{l}"
570
+ for n, m in [("w1", "gate_proj"), ("w2", "down_proj"), ("w3", "up_proj")]:
571
+ for k in ["weight", "scales", "biases"]:
572
+ if f"{prefix}.mlp.experts.0.{m}.{k}" in weights:
573
+ to_join = [
574
+ weights.pop(f"{prefix}.mlp.experts.{e}.{m}.{k}")
575
+ for e in range(self.args.n_routed_experts)
576
+ ]
577
+ weights[f"{prefix}.mlp.switch_mlp.{m}.{k}"] = mx.stack(to_join)
578
+ prefix = f"model.layers.{l}.self_attn"
579
+ if f"{prefix}.kv_b_proj.weight" in weights:
580
+ layer = self.model.layers[l].self_attn.embed_q
581
+ quantized = f"{prefix}.kv_b_proj.scales" in weights
582
+ v = weights.pop(f"{prefix}.kv_b_proj.weight")
583
+ head_dim = self.args.qk_nope_head_dim + self.args.v_head_dim
584
+
585
+ if quantized:
586
+ dims = self.args.kv_lora_rank
587
+ scales = weights.pop(f"{prefix}.kv_b_proj.scales")
588
+ biases = weights.pop(f"{prefix}.kv_b_proj.biases")
589
+ # Try to infer bits and group size
590
+ bits = (v.shape[-1] * 32) // dims
591
+ group_size = dims // scales.shape[-1]
592
+ v = mx.dequantize(
593
+ v, scales, biases, bits=bits, group_size=group_size
594
+ )
595
+ num_heads = self.args.num_attention_heads
596
+ v = v.reshape(num_heads, head_dim, -1)
597
+ wk = mx.contiguous(
598
+ v[:, : self.args.qk_nope_head_dim, :].swapaxes(-1, -2)
599
+ )
600
+ wv = mx.contiguous(v[:, self.args.qk_nope_head_dim :, :])
601
+ if quantized:
602
+ wk, wk_scales, wk_biases = mx.quantize(
603
+ wk, bits=bits, group_size=group_size
604
+ )
605
+ wv, wv_scales, wv_biases = mx.quantize(
606
+ wv, bits=bits, group_size=group_size
607
+ )
608
+ weights[f"{prefix}.embed_q.scales"] = wk_scales
609
+ weights[f"{prefix}.unembed_out.scales"] = wv_scales
610
+ weights[f"{prefix}.embed_q.biases"] = wk_biases
611
+ weights[f"{prefix}.unembed_out.biases"] = wv_biases
612
+ weights[f"{prefix}.embed_q.weight"] = wk
613
+ weights[f"{prefix}.unembed_out.weight"] = wv
614
+
615
+ return weights
616
+
617
+ def shard(self, group: Optional[mx.distributed.Group] = None):
618
+ group = group or mx.distributed.init()
619
+ N = group.size()
620
+ rank = group.rank()
621
+ for layer in self.model.layers:
622
+ layer.self_attn.q_b_proj = shard_linear(
623
+ layer.self_attn.q_b_proj, "all-to-sharded", group=group
624
+ )
625
+
626
+ layer.self_attn.o_proj = shard_linear(
627
+ layer.self_attn.o_proj, "sharded-to-all", group=group
628
+ )
629
+ layer.self_attn.num_heads //= N
630
+ num_heads = layer.self_attn.num_heads
631
+ sh = rank * num_heads
632
+ eh = sh + num_heads
633
+
634
+ def shard_heads(w):
635
+ return w[sh:eh]
636
+
637
+ layer.self_attn.embed_q.apply(shard_heads)
638
+ layer.self_attn.unembed_out.apply(shard_heads)
639
+
640
+ # Shard the MLP
641
+ if isinstance(layer.mlp, DeepseekV32MLP):
642
+ layer.mlp.gate_proj = shard_linear(
643
+ layer.mlp.gate_proj, "all-to-sharded", group=group
644
+ )
645
+ layer.mlp.down_proj = shard_linear(
646
+ layer.mlp.down_proj, "sharded-to-all", group=group
647
+ )
648
+ layer.mlp.up_proj = shard_linear(
649
+ layer.mlp.up_proj, "all-to-sharded", group=group
650
+ )
651
+
652
+ # Shard the MoE. Shard in place since the MoE should be responsible
653
+ # for aggregating the results.
654
+ else:
655
+ layer.mlp.sharding_group = group = group
656
+ shard_inplace(
657
+ layer.mlp.shared_experts.gate_proj, "all-to-sharded", group=group
658
+ )
659
+ shard_inplace(
660
+ layer.mlp.shared_experts.down_proj, "sharded-to-all", group=group
661
+ )
662
+ shard_inplace(
663
+ layer.mlp.shared_experts.up_proj, "all-to-sharded", group=group
664
+ )
665
+ shard_inplace(
666
+ layer.mlp.switch_mlp.gate_proj, "all-to-sharded", group=group
667
+ )
668
+ shard_inplace(
669
+ layer.mlp.switch_mlp.down_proj, "sharded-to-all", group=group
670
+ )
671
+ shard_inplace(
672
+ layer.mlp.switch_mlp.up_proj, "all-to-sharded", group=group
673
+ )
674
+
675
+ @property
676
+ def layers(self):
677
+ return self.model.layers[self.model.start_idx : self.model.end_idx]
678
+
679
+ @property
680
+ def cast_predicate(self):
681
+ def predicate(k):
682
+ return "e_score_correction_bias" not in k
683
+
684
+ return predicate
685
+
686
+ @property
687
+ def quant_predicate(self):
688
+ def predicate(path, module):
689
+ # The lightning indexer (0.03% of parameters, reference keeps weights_proj in fp32) and the
690
+ # MoE router decide *what* is computed; keep them as stored.
691
+ if ".indexer." in path or path.endswith("mlp.gate"):
692
+ return False
693
+ return True
694
+
695
+ return predicate
696
+
697
+ def make_cache(self):
698
+ # Shared layers have no indexer, so no indexer key cache: an empty second KVCache would
699
+ # trip mlx-lm's `cache.state` evaluation (keys is None) during generation.
700
+ return [
701
+ CacheList(KVCache(), KVCache()) if layer.self_attn.indexer is not None else CacheList(KVCache())
702
+ for layer in self.layers
703
+ ]