Hugging Face's logo

cs2764
/
Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6

Text Generation
Transformers
Safetensors
MLX
qwen3_moe
conversational
4-bit precision
Model card Files
xet
Community
cs2764 commited on
Commit
5097ccc
·
verified ·
1 Parent(s): 9799579

Upload MLX converted model with quantization settings

Browse files
Files changed (4) hide show
  1. README.md +8 -1
  2. chat_template.jinja +31 -45
  3. config.json +2 -746
  4. qwen3coder_tool_parser.py +320 -306
README.md CHANGED
@@ -10,7 +10,14 @@ tags:
10
 
11
  # cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6
12
 
13
- The Model [cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6](https://huggingface.co/cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6) was converted to MLX format from [Qwen/Qwen3-Coder-480B-A35B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct) using mlx-lm version **0.26.2**.
 
 
 
 
 
 
 
14
 
15
  ## Use with mlx
16
 
 
10
 
11
  # cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6
12
 
13
+ The Model [cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6](https://huggingface.co/cs2764/Qwen3-Coder-480B-A35B-Instruct-mlx-mixed_4_6) was converted to MLX format from [Qwen/Qwen3-Coder-480B-A35B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct) using mlx-lm version **0.28.0**.
14
+
15
+ ## Quantization Details
16
+
17
+ This model was converted with the following quantization settings:
18
+
19
+ - **Quantization Strategy**: mixed_4_6 (Mixed precision)
20
+ - **Average bits per weight**: 4.819
21
 
22
  ## Use with mlx
23
 
chat_template.jinja CHANGED
@@ -1,17 +1,12 @@
1
- {% macro render_item_list(item_list, tag_name='required') %}
2
- {%- if item_list is defined and item_list is iterable and item_list | length > 0 %}
3
- {%- if tag_name %}{{- '\n<' ~ tag_name ~ '>' -}}{% endif %}
4
- {{- '[' }}
5
- {%- for item in item_list -%}
6
- {%- if loop.index > 1 %}{{- ", "}}{% endif -%}
7
- {%- if item is string -%}
8
- {{ "`" ~ item ~ "`" }}
9
- {%- else -%}
10
- {{ item }}
11
- {%- endif -%}
12
- {%- endfor -%}
13
- {{- ']' }}
14
- {%- if tag_name %}{{- '</' ~ tag_name ~ '>' -}}{% endif %}
15
  {%- endif %}
16
  {% endmacro %}
17
 
@@ -34,46 +29,37 @@
34
  {%- endif %}
35
  {%- endif %}
36
  {%- if tools is iterable and tools | length > 0 %}
37
- {{- "\n\nYou have access to the following functions:\n\n" }}
38
  {{- "<tools>" }}
39
  {%- for tool in tools %}
40
  {%- if tool.function is defined %}
41
  {%- set tool = tool.function %}
42
  {%- endif %}
43
  {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
44
- {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
 
 
45
  {{- '\n<parameters>' }}
46
- {%- for param_name, param_fields in tool.parameters.properties|items %}
47
- {{- '\n<parameter>' }}
48
- {{- '\n<name>' ~ param_name ~ '</name>' }}
49
- {%- if param_fields.type is defined %}
50
- {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
51
- {%- endif %}
52
- {%- if param_fields.description is defined %}
53
- {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
54
- {%- endif %}
55
- {{- render_item_list(param_fields.enum, 'enum') }}
56
- {%- set handled_keys = ['type', 'description', 'enum', 'required'] %}
57
- {%- for json_key in param_fields.keys() | reject("in", handled_keys) %}
58
- {%- set normed_json_key = json_key | replace("-", "_") | replace(" ", "_") | replace("$", "") %}
59
- {%- if param_fields[json_key] is mapping %}
60
- {{- '\n<' ~ normed_json_key ~ '>' ~ (param_fields[json_key] | tojson | safe) ~ '</' ~ normed_json_key ~ '>' }}
61
- {%- else %}
62
- {{-'\n<' ~ normed_json_key ~ '>' ~ (param_fields[json_key] | string) ~ '</' ~ normed_json_key ~ '>' }}
63
  {%- endif %}
 
 
 
 
 
 
64
  {%- endfor %}
65
- {{- render_item_list(param_fields.required, 'required') }}
66
- {{- '\n</parameter>' }}
67
- {%- endfor %}
68
- {{- render_item_list(tool.parameters.required, 'required') }}
69
- {{- '\n</parameters>' }}
70
- {%- if tool.return is defined %}
71
- {%- if tool.return is mapping %}
72
- {{- '\n<return>' ~ (tool.return | tojson | safe) ~ '</return>' }}
73
- {%- else %}
74
- {{- '\n<return>' ~ (tool.return | string) ~ '</return>' }}
75
- {%- endif %}
76
  {%- endif %}
 
 
 
 
 
77
  {{- '\n</function>' }}
78
  {%- endfor %}
79
  {{- "\n</tools>" }}
@@ -100,7 +86,7 @@
100
  {%- if tool_call.arguments is defined %}
101
  {%- for args_name, args_value in tool_call.arguments|items %}
102
  {{- '<parameter=' + args_name + '>\n' }}
103
- {%- set args_value = args_value if args_value is string else args_value | string %}
104
  {{- args_value }}
105
  {{- '\n</parameter>\n' }}
106
  {%- endfor %}
 
1
+ {% macro render_extra_keys(json_dict, handled_keys) %}
2
+ {%- if json_dict is mapping %}
3
+ {%- for json_key in json_dict if json_key not in handled_keys %}
4
+ {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
5
+ {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
6
+ {%- else %}
7
+ {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
8
+ {%- endif %}
9
+ {%- endfor %}
 
 
 
 
 
10
  {%- endif %}
11
  {% endmacro %}
12
 
 
29
  {%- endif %}
30
  {%- endif %}
31
  {%- if tools is iterable and tools | length > 0 %}
32
+ {{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
33
  {{- "<tools>" }}
34
  {%- for tool in tools %}
35
  {%- if tool.function is defined %}
36
  {%- set tool = tool.function %}
37
  {%- endif %}
38
  {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
39
+ {%- if tool.description is defined %}
40
+ {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
41
+ {%- endif %}
42
  {{- '\n<parameters>' }}
43
+ {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
44
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
45
+ {{- '\n<parameter>' }}
46
+ {{- '\n<name>' ~ param_name ~ '</name>' }}
47
+ {%- if param_fields.type is defined %}
48
+ {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
 
 
 
 
 
 
 
 
 
 
 
49
  {%- endif %}
50
+ {%- if param_fields.description is defined %}
51
+ {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
52
+ {%- endif %}
53
+ {%- set handled_keys = ['name', 'type', 'description'] %}
54
+ {{- render_extra_keys(param_fields, handled_keys) }}
55
+ {{- '\n</parameter>' }}
56
  {%- endfor %}
 
 
 
 
 
 
 
 
 
 
 
57
  {%- endif %}
58
+ {% set handled_keys = ['type', 'properties'] %}
59
+ {{- render_extra_keys(tool.parameters, handled_keys) }}
60
+ {{- '\n</parameters>' }}
61
+ {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
62
+ {{- render_extra_keys(tool, handled_keys) }}
63
  {{- '\n</function>' }}
64
  {%- endfor %}
65
  {{- "\n</tools>" }}
 
86
  {%- if tool_call.arguments is defined %}
87
  {%- for args_name, args_value in tool_call.arguments|items %}
88
  {{- '<parameter=' + args_name + '>\n' }}
89
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
90
  {{- args_value }}
91
  {{- '\n</parameter>\n' }}
92
  {%- endfor %}
config.json CHANGED
@@ -26,6 +26,7 @@
26
  "quantization": {
27
  "group_size": 64,
28
  "bits": 4,
 
29
  "model.embed_tokens": {
30
  "group_size": 64,
31
  "bits": 4
@@ -46,11 +47,6 @@
46
  "group_size": 64,
47
  "bits": 4
48
  },
49
- "model.layers.0.self_attn.q_norm": false,
50
- "model.layers.0.self_attn.k_norm": false,
51
- "model.layers.0.self_attn.rope": false,
52
- "model.layers.0.input_layernorm": false,
53
- "model.layers.0.post_attention_layernorm": false,
54
  "model.layers.0.mlp.gate": {
55
  "group_size": 64,
56
  "bits": 4
@@ -67,7 +63,6 @@
67
  "group_size": 64,
68
  "bits": 6
69
  },
70
- "model.layers.0.mlp.switch_mlp.activation": false,
71
  "model.layers.1.self_attn.q_proj": {
72
  "group_size": 64,
73
  "bits": 4
@@ -84,11 +79,6 @@
84
  "group_size": 64,
85
  "bits": 4
86
  },
87
- "model.layers.1.self_attn.q_norm": false,
88
- "model.layers.1.self_attn.k_norm": false,
89
- "model.layers.1.self_attn.rope": false,
90
- "model.layers.1.input_layernorm": false,
91
- "model.layers.1.post_attention_layernorm": false,
92
  "model.layers.1.mlp.gate": {
93
  "group_size": 64,
94
  "bits": 4
@@ -105,7 +95,6 @@
105
  "group_size": 64,
106
  "bits": 6
107
  },
108
- "model.layers.1.mlp.switch_mlp.activation": false,
109
  "model.layers.2.self_attn.q_proj": {
110
  "group_size": 64,
111
  "bits": 4
@@ -122,11 +111,6 @@
122
  "group_size": 64,
123
  "bits": 4
124
  },
125
- "model.layers.2.self_attn.q_norm": false,
126
- "model.layers.2.self_attn.k_norm": false,
127
- "model.layers.2.self_attn.rope": false,
128
- "model.layers.2.input_layernorm": false,
129
- "model.layers.2.post_attention_layernorm": false,
130
  "model.layers.2.mlp.gate": {
131
  "group_size": 64,
132
  "bits": 4
@@ -143,7 +127,6 @@
143
  "group_size": 64,
144
  "bits": 6
145
  },
146
- "model.layers.2.mlp.switch_mlp.activation": false,
147
  "model.layers.3.self_attn.q_proj": {
148
  "group_size": 64,
149
  "bits": 4
@@ -160,11 +143,6 @@
160
  "group_size": 64,
161
  "bits": 4
162
  },
163
- "model.layers.3.self_attn.q_norm": false,
164
- "model.layers.3.self_attn.k_norm": false,
165
- "model.layers.3.self_attn.rope": false,
166
- "model.layers.3.input_layernorm": false,
167
- "model.layers.3.post_attention_layernorm": false,
168
  "model.layers.3.mlp.gate": {
169
  "group_size": 64,
170
  "bits": 4
@@ -181,7 +159,6 @@
181
  "group_size": 64,
182
  "bits": 6
183
  },
184
- "model.layers.3.mlp.switch_mlp.activation": false,
185
  "model.layers.4.self_attn.q_proj": {
186
  "group_size": 64,
187
  "bits": 4
@@ -198,11 +175,6 @@
198
  "group_size": 64,
199
  "bits": 4
200
  },
201
- "model.layers.4.self_attn.q_norm": false,
202
- "model.layers.4.self_attn.k_norm": false,
203
- "model.layers.4.self_attn.rope": false,
204
- "model.layers.4.input_layernorm": false,
205
- "model.layers.4.post_attention_layernorm": false,
206
  "model.layers.4.mlp.gate": {
207
  "group_size": 64,
208
  "bits": 4
@@ -219,7 +191,6 @@
219
  "group_size": 64,
220
  "bits": 6
221
  },
222
- "model.layers.4.mlp.switch_mlp.activation": false,
223
  "model.layers.5.self_attn.q_proj": {
224
  "group_size": 64,
225
  "bits": 4
@@ -236,11 +207,6 @@
236
  "group_size": 64,
237
  "bits": 4
238
  },
239
- "model.layers.5.self_attn.q_norm": false,
240
- "model.layers.5.self_attn.k_norm": false,
241
- "model.layers.5.self_attn.rope": false,
242
- "model.layers.5.input_layernorm": false,
243
- "model.layers.5.post_attention_layernorm": false,
244
  "model.layers.5.mlp.gate": {
245
  "group_size": 64,
246
  "bits": 4
@@ -257,7 +223,6 @@
257
  "group_size": 64,
258
  "bits": 6
259
  },
260
- "model.layers.5.mlp.switch_mlp.activation": false,
261
  "model.layers.6.self_attn.q_proj": {
262
  "group_size": 64,
263
  "bits": 4
@@ -274,11 +239,6 @@
274
  "group_size": 64,
275
  "bits": 4
276
  },
277
- "model.layers.6.self_attn.q_norm": false,
278
- "model.layers.6.self_attn.k_norm": false,
279
- "model.layers.6.self_attn.rope": false,
280
- "model.layers.6.input_layernorm": false,
281
- "model.layers.6.post_attention_layernorm": false,
282
  "model.layers.6.mlp.gate": {
283
  "group_size": 64,
284
  "bits": 4
@@ -295,7 +255,6 @@
295
  "group_size": 64,
296
  "bits": 6
297
  },
298
- "model.layers.6.mlp.switch_mlp.activation": false,
299
  "model.layers.7.self_attn.q_proj": {
300
  "group_size": 64,
301
  "bits": 4
@@ -312,11 +271,6 @@
312
  "group_size": 64,
313
  "bits": 4
314
  },
315
- "model.layers.7.self_attn.q_norm": false,
316
- "model.layers.7.self_attn.k_norm": false,
317
- "model.layers.7.self_attn.rope": false,
318
- "model.layers.7.input_layernorm": false,
319
- "model.layers.7.post_attention_layernorm": false,
320
  "model.layers.7.mlp.gate": {
321
  "group_size": 64,
322
  "bits": 4
@@ -333,7 +287,6 @@
333
  "group_size": 64,
334
  "bits": 4
335
  },
336
- "model.layers.7.mlp.switch_mlp.activation": false,
337
  "model.layers.8.self_attn.q_proj": {
338
  "group_size": 64,
339
  "bits": 4
@@ -350,11 +303,6 @@
350
  "group_size": 64,
351
  "bits": 4
352
  },
353
- "model.layers.8.self_attn.q_norm": false,
354
- "model.layers.8.self_attn.k_norm": false,
355
- "model.layers.8.self_attn.rope": false,
356
- "model.layers.8.input_layernorm": false,
357
- "model.layers.8.post_attention_layernorm": false,
358
  "model.layers.8.mlp.gate": {
359
  "group_size": 64,
360
  "bits": 4
@@ -371,7 +319,6 @@
371
  "group_size": 64,
372
  "bits": 4
373
  },
374
- "model.layers.8.mlp.switch_mlp.activation": false,
375
  "model.layers.9.self_attn.q_proj": {
376
  "group_size": 64,
377
  "bits": 4
@@ -388,11 +335,6 @@
388
  "group_size": 64,
389
  "bits": 4
390
  },
391
- "model.layers.9.self_attn.q_norm": false,
392
- "model.layers.9.self_attn.k_norm": false,
393
- "model.layers.9.self_attn.rope": false,
394
- "model.layers.9.input_layernorm": false,
395
- "model.layers.9.post_attention_layernorm": false,
396
  "model.layers.9.mlp.gate": {
397
  "group_size": 64,
398
  "bits": 4
@@ -409,7 +351,6 @@
409
  "group_size": 64,
410
  "bits": 6
411
  },
412
- "model.layers.9.mlp.switch_mlp.activation": false,
413
  "model.layers.10.self_attn.q_proj": {
414
  "group_size": 64,
415
  "bits": 4
@@ -426,11 +367,6 @@
426
  "group_size": 64,
427
  "bits": 4
428
  },
429
- "model.layers.10.self_attn.q_norm": false,
430
- "model.layers.10.self_attn.k_norm": false,
431
- "model.layers.10.self_attn.rope": false,
432
- "model.layers.10.input_layernorm": false,
433
- "model.layers.10.post_attention_layernorm": false,
434
  "model.layers.10.mlp.gate": {
435
  "group_size": 64,
436
  "bits": 4
@@ -447,7 +383,6 @@
447
  "group_size": 64,
448
  "bits": 4
449
  },
450
- "model.layers.10.mlp.switch_mlp.activation": false,
451
  "model.layers.11.self_attn.q_proj": {
452
  "group_size": 64,
453
  "bits": 4
@@ -464,11 +399,6 @@
464
  "group_size": 64,
465
  "bits": 4
466
  },
467
- "model.layers.11.self_attn.q_norm": false,
468
- "model.layers.11.self_attn.k_norm": false,
469
- "model.layers.11.self_attn.rope": false,
470
- "model.layers.11.input_layernorm": false,
471
- "model.layers.11.post_attention_layernorm": false,
472
  "model.layers.11.mlp.gate": {
473
  "group_size": 64,
474
  "bits": 4
@@ -485,7 +415,6 @@
485
  "group_size": 64,
486
  "bits": 4
487
  },
488
- "model.layers.11.mlp.switch_mlp.activation": false,
489
  "model.layers.12.self_attn.q_proj": {
490
  "group_size": 64,
491
  "bits": 4
@@ -502,11 +431,6 @@
502
  "group_size": 64,
503
  "bits": 4
504
  },
505
- "model.layers.12.self_attn.q_norm": false,
506
- "model.layers.12.self_attn.k_norm": false,
507
- "model.layers.12.self_attn.rope": false,
508
- "model.layers.12.input_layernorm": false,
509
- "model.layers.12.post_attention_layernorm": false,
510
  "model.layers.12.mlp.gate": {
511
  "group_size": 64,
512
  "bits": 4
@@ -523,7 +447,6 @@
523
  "group_size": 64,
524
  "bits": 6
525
  },
526
- "model.layers.12.mlp.switch_mlp.activation": false,
527
  "model.layers.13.self_attn.q_proj": {
528
  "group_size": 64,
529
  "bits": 4
@@ -540,11 +463,6 @@
540
  "group_size": 64,
541
  "bits": 4
542
  },
543
- "model.layers.13.self_attn.q_norm": false,
544
- "model.layers.13.self_attn.k_norm": false,
545
- "model.layers.13.self_attn.rope": false,
546
- "model.layers.13.input_layernorm": false,
547
- "model.layers.13.post_attention_layernorm": false,
548
  "model.layers.13.mlp.gate": {
549
  "group_size": 64,
550
  "bits": 4
@@ -561,7 +479,6 @@
561
  "group_size": 64,
562
  "bits": 4
563
  },
564
- "model.layers.13.mlp.switch_mlp.activation": false,
565
  "model.layers.14.self_attn.q_proj": {
566
  "group_size": 64,
567
  "bits": 4
@@ -578,11 +495,6 @@
578
  "group_size": 64,
579
  "bits": 4
580
  },
581
- "model.layers.14.self_attn.q_norm": false,
582
- "model.layers.14.self_attn.k_norm": false,
583
- "model.layers.14.self_attn.rope": false,
584
- "model.layers.14.input_layernorm": false,
585
- "model.layers.14.post_attention_layernorm": false,
586
  "model.layers.14.mlp.gate": {
587
  "group_size": 64,
588
  "bits": 4
@@ -599,7 +511,6 @@
599
  "group_size": 64,
600
  "bits": 4
601
  },
602
- "model.layers.14.mlp.switch_mlp.activation": false,
603
  "model.layers.15.self_attn.q_proj": {
604
  "group_size": 64,
605
  "bits": 4
@@ -616,11 +527,6 @@
616
  "group_size": 64,
617
  "bits": 4
618
  },
619
- "model.layers.15.self_attn.q_norm": false,
620
- "model.layers.15.self_attn.k_norm": false,
621
- "model.layers.15.self_attn.rope": false,
622
- "model.layers.15.input_layernorm": false,
623
- "model.layers.15.post_attention_layernorm": false,
624
  "model.layers.15.mlp.gate": {
625
  "group_size": 64,
626
  "bits": 4
@@ -637,7 +543,6 @@
637
  "group_size": 64,
638
  "bits": 6
639
  },
640
- "model.layers.15.mlp.switch_mlp.activation": false,
641
  "model.layers.16.self_attn.q_proj": {
642
  "group_size": 64,
643
  "bits": 4
@@ -654,11 +559,6 @@
654
  "group_size": 64,
655
  "bits": 4
656
  },
657
- "model.layers.16.self_attn.q_norm": false,
658
- "model.layers.16.self_attn.k_norm": false,
659
- "model.layers.16.self_attn.rope": false,
660
- "model.layers.16.input_layernorm": false,
661
- "model.layers.16.post_attention_layernorm": false,
662
  "model.layers.16.mlp.gate": {
663
  "group_size": 64,
664
  "bits": 4
@@ -675,7 +575,6 @@
675
  "group_size": 64,
676
  "bits": 4
677
  },
678
- "model.layers.16.mlp.switch_mlp.activation": false,
679
  "model.layers.17.self_attn.q_proj": {
680
  "group_size": 64,
681
  "bits": 4
@@ -692,11 +591,6 @@
692
  "group_size": 64,
693
  "bits": 4
694
  },
695
- "model.layers.17.self_attn.q_norm": false,
696
- "model.layers.17.self_attn.k_norm": false,
697
- "model.layers.17.self_attn.rope": false,
698
- "model.layers.17.input_layernorm": false,
699
- "model.layers.17.post_attention_layernorm": false,
700
  "model.layers.17.mlp.gate": {
701
  "group_size": 64,
702
  "bits": 4
@@ -713,7 +607,6 @@
713
  "group_size": 64,
714
  "bits": 4
715
  },
716
- "model.layers.17.mlp.switch_mlp.activation": false,
717
  "model.layers.18.self_attn.q_proj": {
718
  "group_size": 64,
719
  "bits": 4
@@ -730,11 +623,6 @@
730
  "group_size": 64,
731
  "bits": 4
732
  },
733
- "model.layers.18.self_attn.q_norm": false,
734
- "model.layers.18.self_attn.k_norm": false,
735
- "model.layers.18.self_attn.rope": false,
736
- "model.layers.18.input_layernorm": false,
737
- "model.layers.18.post_attention_layernorm": false,
738
  "model.layers.18.mlp.gate": {
739
  "group_size": 64,
740
  "bits": 4
@@ -751,7 +639,6 @@
751
  "group_size": 64,
752
  "bits": 6
753
  },
754
- "model.layers.18.mlp.switch_mlp.activation": false,
755
  "model.layers.19.self_attn.q_proj": {
756
  "group_size": 64,
757
  "bits": 4
@@ -768,11 +655,6 @@
768
  "group_size": 64,
769
  "bits": 4
770
  },
771
- "model.layers.19.self_attn.q_norm": false,
772
- "model.layers.19.self_attn.k_norm": false,
773
- "model.layers.19.self_attn.rope": false,
774
- "model.layers.19.input_layernorm": false,
775
- "model.layers.19.post_attention_layernorm": false,
776
  "model.layers.19.mlp.gate": {
777
  "group_size": 64,
778
  "bits": 4
@@ -789,7 +671,6 @@
789
  "group_size": 64,
790
  "bits": 4
791
  },
792
- "model.layers.19.mlp.switch_mlp.activation": false,
793
  "model.layers.20.self_attn.q_proj": {
794
  "group_size": 64,
795
  "bits": 4
@@ -806,11 +687,6 @@
806
  "group_size": 64,
807
  "bits": 4
808
  },
809
- "model.layers.20.self_attn.q_norm": false,
810
- "model.layers.20.self_attn.k_norm": false,
811
- "model.layers.20.self_attn.rope": false,
812
- "model.layers.20.input_layernorm": false,
813
- "model.layers.20.post_attention_layernorm": false,
814
  "model.layers.20.mlp.gate": {
815
  "group_size": 64,
816
  "bits": 4
@@ -827,7 +703,6 @@
827
  "group_size": 64,
828
  "bits": 4
829
  },
830
- "model.layers.20.mlp.switch_mlp.activation": false,
831
  "model.layers.21.self_attn.q_proj": {
832
  "group_size": 64,
833
  "bits": 4
@@ -844,11 +719,6 @@
844
  "group_size": 64,
845
  "bits": 4
846
  },
847
- "model.layers.21.self_attn.q_norm": false,
848
- "model.layers.21.self_attn.k_norm": false,
849
- "model.layers.21.self_attn.rope": false,
850
- "model.layers.21.input_layernorm": false,
851
- "model.layers.21.post_attention_layernorm": false,
852
  "model.layers.21.mlp.gate": {
853
  "group_size": 64,
854
  "bits": 4
@@ -865,7 +735,6 @@
865
  "group_size": 64,
866
  "bits": 6
867
  },
868
- "model.layers.21.mlp.switch_mlp.activation": false,
869
  "model.layers.22.self_attn.q_proj": {
870
  "group_size": 64,
871
  "bits": 4
@@ -882,11 +751,6 @@
882
  "group_size": 64,
883
  "bits": 4
884
  },
885
- "model.layers.22.self_attn.q_norm": false,
886
- "model.layers.22.self_attn.k_norm": false,
887
- "model.layers.22.self_attn.rope": false,
888
- "model.layers.22.input_layernorm": false,
889
- "model.layers.22.post_attention_layernorm": false,
890
  "model.layers.22.mlp.gate": {
891
  "group_size": 64,
892
  "bits": 4
@@ -903,7 +767,6 @@
903
  "group_size": 64,
904
  "bits": 4
905
  },
906
- "model.layers.22.mlp.switch_mlp.activation": false,
907
  "model.layers.23.self_attn.q_proj": {
908
  "group_size": 64,
909
  "bits": 4
@@ -920,11 +783,6 @@
920
  "group_size": 64,
921
  "bits": 4
922
  },
923
- "model.layers.23.self_attn.q_norm": false,
924
- "model.layers.23.self_attn.k_norm": false,
925
- "model.layers.23.self_attn.rope": false,
926
- "model.layers.23.input_layernorm": false,
927
- "model.layers.23.post_attention_layernorm": false,
928
  "model.layers.23.mlp.gate": {
929
  "group_size": 64,
930
  "bits": 4
@@ -941,7 +799,6 @@
941
  "group_size": 64,
942
  "bits": 4
943
  },
944
- "model.layers.23.mlp.switch_mlp.activation": false,
945
  "model.layers.24.self_attn.q_proj": {
946
  "group_size": 64,
947
  "bits": 4
@@ -958,11 +815,6 @@
958
  "group_size": 64,
959
  "bits": 4
960
  },
961
- "model.layers.24.self_attn.q_norm": false,
962
- "model.layers.24.self_attn.k_norm": false,
963
- "model.layers.24.self_attn.rope": false,
964
- "model.layers.24.input_layernorm": false,
965
- "model.layers.24.post_attention_layernorm": false,
966
  "model.layers.24.mlp.gate": {
967
  "group_size": 64,
968
  "bits": 4
@@ -979,7 +831,6 @@
979
  "group_size": 64,
980
  "bits": 6
981
  },
982
- "model.layers.24.mlp.switch_mlp.activation": false,
983
  "model.layers.25.self_attn.q_proj": {
984
  "group_size": 64,
985
  "bits": 4
@@ -996,11 +847,6 @@
996
  "group_size": 64,
997
  "bits": 4
998
  },
999
- "model.layers.25.self_attn.q_norm": false,
1000
- "model.layers.25.self_attn.k_norm": false,
1001
- "model.layers.25.self_attn.rope": false,
1002
- "model.layers.25.input_layernorm": false,
1003
- "model.layers.25.post_attention_layernorm": false,
1004
  "model.layers.25.mlp.gate": {
1005
  "group_size": 64,
1006
  "bits": 4
@@ -1017,7 +863,6 @@
1017
  "group_size": 64,
1018
  "bits": 4
1019
  },
1020
- "model.layers.25.mlp.switch_mlp.activation": false,
1021
  "model.layers.26.self_attn.q_proj": {
1022
  "group_size": 64,
1023
  "bits": 4
@@ -1034,11 +879,6 @@
1034
  "group_size": 64,
1035
  "bits": 4
1036
  },
1037
- "model.layers.26.self_attn.q_norm": false,
1038
- "model.layers.26.self_attn.k_norm": false,
1039
- "model.layers.26.self_attn.rope": false,
1040
- "model.layers.26.input_layernorm": false,
1041
- "model.layers.26.post_attention_layernorm": false,
1042
  "model.layers.26.mlp.gate": {
1043
  "group_size": 64,
1044
  "bits": 4
@@ -1055,7 +895,6 @@
1055
  "group_size": 64,
1056
  "bits": 4
1057
  },
1058
- "model.layers.26.mlp.switch_mlp.activation": false,
1059
  "model.layers.27.self_attn.q_proj": {
1060
  "group_size": 64,
1061
  "bits": 4
@@ -1072,11 +911,6 @@
1072
  "group_size": 64,
1073
  "bits": 4
1074
  },
1075
- "model.layers.27.self_attn.q_norm": false,
1076
- "model.layers.27.self_attn.k_norm": false,
1077
- "model.layers.27.self_attn.rope": false,
1078
- "model.layers.27.input_layernorm": false,
1079
- "model.layers.27.post_attention_layernorm": false,
1080
  "model.layers.27.mlp.gate": {
1081
  "group_size": 64,
1082
  "bits": 4
@@ -1093,7 +927,6 @@
1093
  "group_size": 64,
1094
  "bits": 6
1095
  },
1096
- "model.layers.27.mlp.switch_mlp.activation": false,
1097
  "model.layers.28.self_attn.q_proj": {
1098
  "group_size": 64,
1099
  "bits": 4
@@ -1110,11 +943,6 @@
1110
  "group_size": 64,
1111
  "bits": 4
1112
  },
1113
- "model.layers.28.self_attn.q_norm": false,
1114
- "model.layers.28.self_attn.k_norm": false,
1115
- "model.layers.28.self_attn.rope": false,
1116
- "model.layers.28.input_layernorm": false,
1117
- "model.layers.28.post_attention_layernorm": false,
1118
  "model.layers.28.mlp.gate": {
1119
  "group_size": 64,
1120
  "bits": 4
@@ -1131,7 +959,6 @@
1131
  "group_size": 64,
1132
  "bits": 4
1133
  },
1134
- "model.layers.28.mlp.switch_mlp.activation": false,
1135
  "model.layers.29.self_attn.q_proj": {
1136
  "group_size": 64,
1137
  "bits": 4
@@ -1148,11 +975,6 @@
1148
  "group_size": 64,
1149
  "bits": 4
1150
  },
1151
- "model.layers.29.self_attn.q_norm": false,
1152
- "model.layers.29.self_attn.k_norm": false,
1153
- "model.layers.29.self_attn.rope": false,
1154
- "model.layers.29.input_layernorm": false,
1155
- "model.layers.29.post_attention_layernorm": false,
1156
  "model.layers.29.mlp.gate": {
1157
  "group_size": 64,
1158
  "bits": 4
@@ -1169,7 +991,6 @@
1169
  "group_size": 64,
1170
  "bits": 4
1171
  },
1172
- "model.layers.29.mlp.switch_mlp.activation": false,
1173
  "model.layers.30.self_attn.q_proj": {
1174
  "group_size": 64,
1175
  "bits": 4
@@ -1186,11 +1007,6 @@
1186
  "group_size": 64,
1187
  "bits": 4
1188
  },
1189
- "model.layers.30.self_attn.q_norm": false,
1190
- "model.layers.30.self_attn.k_norm": false,
1191
- "model.layers.30.self_attn.rope": false,
1192
- "model.layers.30.input_layernorm": false,
1193
- "model.layers.30.post_attention_layernorm": false,
1194
  "model.layers.30.mlp.gate": {
1195
  "group_size": 64,
1196
  "bits": 4
@@ -1207,7 +1023,6 @@
1207
  "group_size": 64,
1208
  "bits": 6
1209
  },
1210
- "model.layers.30.mlp.switch_mlp.activation": false,
1211
  "model.layers.31.self_attn.q_proj": {
1212
  "group_size": 64,
1213
  "bits": 4
@@ -1224,11 +1039,6 @@
1224
  "group_size": 64,
1225
  "bits": 4
1226
  },
1227
- "model.layers.31.self_attn.q_norm": false,
1228
- "model.layers.31.self_attn.k_norm": false,
1229
- "model.layers.31.self_attn.rope": false,
1230
- "model.layers.31.input_layernorm": false,
1231
- "model.layers.31.post_attention_layernorm": false,
1232
  "model.layers.31.mlp.gate": {
1233
  "group_size": 64,
1234
  "bits": 4
@@ -1245,7 +1055,6 @@
1245
  "group_size": 64,
1246
  "bits": 4
1247
  },
1248
- "model.layers.31.mlp.switch_mlp.activation": false,
1249
  "model.layers.32.self_attn.q_proj": {
1250
  "group_size": 64,
1251
  "bits": 4
@@ -1262,11 +1071,6 @@
1262
  "group_size": 64,
1263
  "bits": 4
1264
  },
1265
- "model.layers.32.self_attn.q_norm": false,
1266
- "model.layers.32.self_attn.k_norm": false,
1267
- "model.layers.32.self_attn.rope": false,
1268
- "model.layers.32.input_layernorm": false,
1269
- "model.layers.32.post_attention_layernorm": false,
1270
  "model.layers.32.mlp.gate": {
1271
  "group_size": 64,
1272
  "bits": 4
@@ -1283,7 +1087,6 @@
1283
  "group_size": 64,
1284
  "bits": 4
1285
  },
1286
- "model.layers.32.mlp.switch_mlp.activation": false,
1287
  "model.layers.33.self_attn.q_proj": {
1288
  "group_size": 64,
1289
  "bits": 4
@@ -1300,11 +1103,6 @@
1300
  "group_size": 64,
1301
  "bits": 4
1302
  },
1303
- "model.layers.33.self_attn.q_norm": false,
1304
- "model.layers.33.self_attn.k_norm": false,
1305
- "model.layers.33.self_attn.rope": false,
1306
- "model.layers.33.input_layernorm": false,
1307
- "model.layers.33.post_attention_layernorm": false,
1308
  "model.layers.33.mlp.gate": {
1309
  "group_size": 64,
1310
  "bits": 4
@@ -1321,7 +1119,6 @@
1321
  "group_size": 64,
1322
  "bits": 6
1323
  },
1324
- "model.layers.33.mlp.switch_mlp.activation": false,
1325
  "model.layers.34.self_attn.q_proj": {
1326
  "group_size": 64,
1327
  "bits": 4
@@ -1338,11 +1135,6 @@
1338
  "group_size": 64,
1339
  "bits": 4
1340
  },
1341
- "model.layers.34.self_attn.q_norm": false,
1342
- "model.layers.34.self_attn.k_norm": false,
1343
- "model.layers.34.self_attn.rope": false,
1344
- "model.layers.34.input_layernorm": false,
1345
- "model.layers.34.post_attention_layernorm": false,
1346
  "model.layers.34.mlp.gate": {
1347
  "group_size": 64,
1348
  "bits": 4
@@ -1359,7 +1151,6 @@
1359
  "group_size": 64,
1360
  "bits": 4
1361
  },
1362
- "model.layers.34.mlp.switch_mlp.activation": false,
1363
  "model.layers.35.self_attn.q_proj": {
1364
  "group_size": 64,
1365
  "bits": 4
@@ -1376,11 +1167,6 @@
1376
  "group_size": 64,
1377
  "bits": 4
1378
  },
1379
- "model.layers.35.self_attn.q_norm": false,
1380
- "model.layers.35.self_attn.k_norm": false,
1381
- "model.layers.35.self_attn.rope": false,
1382
- "model.layers.35.input_layernorm": false,
1383
- "model.layers.35.post_attention_layernorm": false,
1384
  "model.layers.35.mlp.gate": {
1385
  "group_size": 64,
1386
  "bits": 4
@@ -1397,7 +1183,6 @@
1397
  "group_size": 64,
1398
  "bits": 4
1399
  },
1400
- "model.layers.35.mlp.switch_mlp.activation": false,
1401
  "model.layers.36.self_attn.q_proj": {
1402
  "group_size": 64,
1403
  "bits": 4
@@ -1414,11 +1199,6 @@
1414
  "group_size": 64,
1415
  "bits": 4
1416
  },
1417
- "model.layers.36.self_attn.q_norm": false,
1418
- "model.layers.36.self_attn.k_norm": false,
1419
- "model.layers.36.self_attn.rope": false,
1420
- "model.layers.36.input_layernorm": false,
1421
- "model.layers.36.post_attention_layernorm": false,
1422
  "model.layers.36.mlp.gate": {
1423
  "group_size": 64,
1424
  "bits": 4
@@ -1435,7 +1215,6 @@
1435
  "group_size": 64,
1436
  "bits": 6
1437
  },
1438
- "model.layers.36.mlp.switch_mlp.activation": false,
1439
  "model.layers.37.self_attn.q_proj": {
1440
  "group_size": 64,
1441
  "bits": 4
@@ -1452,11 +1231,6 @@
1452
  "group_size": 64,
1453
  "bits": 4
1454
  },
1455
- "model.layers.37.self_attn.q_norm": false,
1456
- "model.layers.37.self_attn.k_norm": false,
1457
- "model.layers.37.self_attn.rope": false,
1458
- "model.layers.37.input_layernorm": false,
1459
- "model.layers.37.post_attention_layernorm": false,
1460
  "model.layers.37.mlp.gate": {
1461
  "group_size": 64,
1462
  "bits": 4
@@ -1473,7 +1247,6 @@
1473
  "group_size": 64,
1474
  "bits": 4
1475
  },
1476
- "model.layers.37.mlp.switch_mlp.activation": false,
1477
  "model.layers.38.self_attn.q_proj": {
1478
  "group_size": 64,
1479
  "bits": 4
@@ -1490,11 +1263,6 @@
1490
  "group_size": 64,
1491
  "bits": 4
1492
  },
1493
- "model.layers.38.self_attn.q_norm": false,
1494
- "model.layers.38.self_attn.k_norm": false,
1495
- "model.layers.38.self_attn.rope": false,
1496
- "model.layers.38.input_layernorm": false,
1497
- "model.layers.38.post_attention_layernorm": false,
1498
  "model.layers.38.mlp.gate": {
1499
  "group_size": 64,
1500
  "bits": 4
@@ -1511,7 +1279,6 @@
1511
  "group_size": 64,
1512
  "bits": 4
1513
  },
1514
- "model.layers.38.mlp.switch_mlp.activation": false,
1515
  "model.layers.39.self_attn.q_proj": {
1516
  "group_size": 64,
1517
  "bits": 4
@@ -1528,11 +1295,6 @@
1528
  "group_size": 64,
1529
  "bits": 4
1530
  },
1531
- "model.layers.39.self_attn.q_norm": false,
1532
- "model.layers.39.self_attn.k_norm": false,
1533
- "model.layers.39.self_attn.rope": false,
1534
- "model.layers.39.input_layernorm": false,
1535
- "model.layers.39.post_attention_layernorm": false,
1536
  "model.layers.39.mlp.gate": {
1537
  "group_size": 64,
1538
  "bits": 4
@@ -1549,7 +1311,6 @@
1549
  "group_size": 64,
1550
  "bits": 6
1551
  },
1552
- "model.layers.39.mlp.switch_mlp.activation": false,
1553
  "model.layers.40.self_attn.q_proj": {
1554
  "group_size": 64,
1555
  "bits": 4
@@ -1566,11 +1327,6 @@
1566
  "group_size": 64,
1567
  "bits": 4
1568
  },
1569
- "model.layers.40.self_attn.q_norm": false,
1570
- "model.layers.40.self_attn.k_norm": false,
1571
- "model.layers.40.self_attn.rope": false,
1572
- "model.layers.40.input_layernorm": false,
1573
- "model.layers.40.post_attention_layernorm": false,
1574
  "model.layers.40.mlp.gate": {
1575
  "group_size": 64,
1576
  "bits": 4
@@ -1587,7 +1343,6 @@
1587
  "group_size": 64,
1588
  "bits": 4
1589
  },
1590
- "model.layers.40.mlp.switch_mlp.activation": false,
1591
  "model.layers.41.self_attn.q_proj": {
1592
  "group_size": 64,
1593
  "bits": 4
@@ -1604,11 +1359,6 @@
1604
  "group_size": 64,
1605
  "bits": 4
1606
  },
1607
- "model.layers.41.self_attn.q_norm": false,
1608
- "model.layers.41.self_attn.k_norm": false,
1609
- "model.layers.41.self_attn.rope": false,
1610
- "model.layers.41.input_layernorm": false,
1611
- "model.layers.41.post_attention_layernorm": false,
1612
  "model.layers.41.mlp.gate": {
1613
  "group_size": 64,
1614
  "bits": 4
@@ -1625,7 +1375,6 @@
1625
  "group_size": 64,
1626
  "bits": 4
1627
  },
1628
- "model.layers.41.mlp.switch_mlp.activation": false,
1629
  "model.layers.42.self_attn.q_proj": {
1630
  "group_size": 64,
1631
  "bits": 4
@@ -1642,11 +1391,6 @@
1642
  "group_size": 64,
1643
  "bits": 4
1644
  },
1645
- "model.layers.42.self_attn.q_norm": false,
1646
- "model.layers.42.self_attn.k_norm": false,
1647
- "model.layers.42.self_attn.rope": false,
1648
- "model.layers.42.input_layernorm": false,
1649
- "model.layers.42.post_attention_layernorm": false,
1650
  "model.layers.42.mlp.gate": {
1651
  "group_size": 64,
1652
  "bits": 4
@@ -1663,7 +1407,6 @@
1663
  "group_size": 64,
1664
  "bits": 6
1665
  },
1666
- "model.layers.42.mlp.switch_mlp.activation": false,
1667
  "model.layers.43.self_attn.q_proj": {
1668
  "group_size": 64,
1669
  "bits": 4
@@ -1680,11 +1423,6 @@
1680
  "group_size": 64,
1681
  "bits": 4
1682
  },
1683
- "model.layers.43.self_attn.q_norm": false,
1684
- "model.layers.43.self_attn.k_norm": false,
1685
- "model.layers.43.self_attn.rope": false,
1686
- "model.layers.43.input_layernorm": false,
1687
- "model.layers.43.post_attention_layernorm": false,
1688
  "model.layers.43.mlp.gate": {
1689
  "group_size": 64,
1690
  "bits": 4
@@ -1701,7 +1439,6 @@
1701
  "group_size": 64,
1702
  "bits": 4
1703
  },
1704
- "model.layers.43.mlp.switch_mlp.activation": false,
1705
  "model.layers.44.self_attn.q_proj": {
1706
  "group_size": 64,
1707
  "bits": 4
@@ -1718,11 +1455,6 @@
1718
  "group_size": 64,
1719
  "bits": 4
1720
  },
1721
- "model.layers.44.self_attn.q_norm": false,
1722
- "model.layers.44.self_attn.k_norm": false,
1723
- "model.layers.44.self_attn.rope": false,
1724
- "model.layers.44.input_layernorm": false,
1725
- "model.layers.44.post_attention_layernorm": false,
1726
  "model.layers.44.mlp.gate": {
1727
  "group_size": 64,
1728
  "bits": 4
@@ -1739,7 +1471,6 @@
1739
  "group_size": 64,
1740
  "bits": 4
1741
  },
1742
- "model.layers.44.mlp.switch_mlp.activation": false,
1743
  "model.layers.45.self_attn.q_proj": {
1744
  "group_size": 64,
1745
  "bits": 4
@@ -1756,11 +1487,6 @@
1756
  "group_size": 64,
1757
  "bits": 4
1758
  },
1759
- "model.layers.45.self_attn.q_norm": false,
1760
- "model.layers.45.self_attn.k_norm": false,
1761
- "model.layers.45.self_attn.rope": false,
1762
- "model.layers.45.input_layernorm": false,
1763
- "model.layers.45.post_attention_layernorm": false,
1764
  "model.layers.45.mlp.gate": {
1765
  "group_size": 64,
1766
  "bits": 4
@@ -1777,7 +1503,6 @@
1777
  "group_size": 64,
1778
  "bits": 6
1779
  },
1780
- "model.layers.45.mlp.switch_mlp.activation": false,
1781
  "model.layers.46.self_attn.q_proj": {
1782
  "group_size": 64,
1783
  "bits": 4
@@ -1794,11 +1519,6 @@
1794
  "group_size": 64,
1795
  "bits": 4
1796
  },
1797
- "model.layers.46.self_attn.q_norm": false,
1798
- "model.layers.46.self_attn.k_norm": false,
1799
- "model.layers.46.self_attn.rope": false,
1800
- "model.layers.46.input_layernorm": false,
1801
- "model.layers.46.post_attention_layernorm": false,
1802
  "model.layers.46.mlp.gate": {
1803
  "group_size": 64,
1804
  "bits": 4
@@ -1815,7 +1535,6 @@
1815
  "group_size": 64,
1816
  "bits": 4
1817
  },
1818
- "model.layers.46.mlp.switch_mlp.activation": false,
1819
  "model.layers.47.self_attn.q_proj": {
1820
  "group_size": 64,
1821
  "bits": 4
@@ -1832,11 +1551,6 @@
1832
  "group_size": 64,
1833
  "bits": 4
1834
  },
1835
- "model.layers.47.self_attn.q_norm": false,
1836
- "model.layers.47.self_attn.k_norm": false,
1837
- "model.layers.47.self_attn.rope": false,
1838
- "model.layers.47.input_layernorm": false,
1839
- "model.layers.47.post_attention_layernorm": false,
1840
  "model.layers.47.mlp.gate": {
1841
  "group_size": 64,
1842
  "bits": 4
@@ -1853,7 +1567,6 @@
1853
  "group_size": 64,
1854
  "bits": 4
1855
  },
1856
- "model.layers.47.mlp.switch_mlp.activation": false,
1857
  "model.layers.48.self_attn.q_proj": {
1858
  "group_size": 64,
1859
  "bits": 4
@@ -1870,11 +1583,6 @@
1870
  "group_size": 64,
1871
  "bits": 4
1872
  },
1873
- "model.layers.48.self_attn.q_norm": false,
1874
- "model.layers.48.self_attn.k_norm": false,
1875
- "model.layers.48.self_attn.rope": false,
1876
- "model.layers.48.input_layernorm": false,
1877
- "model.layers.48.post_attention_layernorm": false,
1878
  "model.layers.48.mlp.gate": {
1879
  "group_size": 64,
1880
  "bits": 4
@@ -1891,7 +1599,6 @@
1891
  "group_size": 64,
1892
  "bits": 6
1893
  },
1894
- "model.layers.48.mlp.switch_mlp.activation": false,
1895
  "model.layers.49.self_attn.q_proj": {
1896
  "group_size": 64,
1897
  "bits": 4
@@ -1908,11 +1615,6 @@
1908
  "group_size": 64,
1909
  "bits": 4
1910
  },
1911
- "model.layers.49.self_attn.q_norm": false,
1912
- "model.layers.49.self_attn.k_norm": false,
1913
- "model.layers.49.self_attn.rope": false,
1914
- "model.layers.49.input_layernorm": false,
1915
- "model.layers.49.post_attention_layernorm": false,
1916
  "model.layers.49.mlp.gate": {
1917
  "group_size": 64,
1918
  "bits": 4
@@ -1929,7 +1631,6 @@
1929
  "group_size": 64,
1930
  "bits": 4
1931
  },
1932
- "model.layers.49.mlp.switch_mlp.activation": false,
1933
  "model.layers.50.self_attn.q_proj": {
1934
  "group_size": 64,
1935
  "bits": 4
@@ -1946,11 +1647,6 @@
1946
  "group_size": 64,
1947
  "bits": 4
1948
  },
1949
- "model.layers.50.self_attn.q_norm": false,
1950
- "model.layers.50.self_attn.k_norm": false,
1951
- "model.layers.50.self_attn.rope": false,
1952
- "model.layers.50.input_layernorm": false,
1953
- "model.layers.50.post_attention_layernorm": false,
1954
  "model.layers.50.mlp.gate": {
1955
  "group_size": 64,
1956
  "bits": 4
@@ -1967,7 +1663,6 @@
1967
  "group_size": 64,
1968
  "bits": 4
1969
  },
1970
- "model.layers.50.mlp.switch_mlp.activation": false,
1971
  "model.layers.51.self_attn.q_proj": {
1972
  "group_size": 64,
1973
  "bits": 4
@@ -1984,11 +1679,6 @@
1984
  "group_size": 64,
1985
  "bits": 4
1986
  },
1987
- "model.layers.51.self_attn.q_norm": false,
1988
- "model.layers.51.self_attn.k_norm": false,
1989
- "model.layers.51.self_attn.rope": false,
1990
- "model.layers.51.input_layernorm": false,
1991
- "model.layers.51.post_attention_layernorm": false,
1992
  "model.layers.51.mlp.gate": {
1993
  "group_size": 64,
1994
  "bits": 4
@@ -2005,7 +1695,6 @@
2005
  "group_size": 64,
2006
  "bits": 6
2007
  },
2008
- "model.layers.51.mlp.switch_mlp.activation": false,
2009
  "model.layers.52.self_attn.q_proj": {
2010
  "group_size": 64,
2011
  "bits": 4
@@ -2022,11 +1711,6 @@
2022
  "group_size": 64,
2023
  "bits": 4
2024
  },
2025
- "model.layers.52.self_attn.q_norm": false,
2026
- "model.layers.52.self_attn.k_norm": false,
2027
- "model.layers.52.self_attn.rope": false,
2028
- "model.layers.52.input_layernorm": false,
2029
- "model.layers.52.post_attention_layernorm": false,
2030
  "model.layers.52.mlp.gate": {
2031
  "group_size": 64,
2032
  "bits": 4
@@ -2043,7 +1727,6 @@
2043
  "group_size": 64,
2044
  "bits": 4
2045
  },
2046
- "model.layers.52.mlp.switch_mlp.activation": false,
2047
  "model.layers.53.self_attn.q_proj": {
2048
  "group_size": 64,
2049
  "bits": 4
@@ -2060,11 +1743,6 @@
2060
  "group_size": 64,
2061
  "bits": 4
2062
  },
2063
- "model.layers.53.self_attn.q_norm": false,
2064
- "model.layers.53.self_attn.k_norm": false,
2065
- "model.layers.53.self_attn.rope": false,
2066
- "model.layers.53.input_layernorm": false,
2067
- "model.layers.53.post_attention_layernorm": false,
2068
  "model.layers.53.mlp.gate": {
2069
  "group_size": 64,
2070
  "bits": 4
@@ -2081,7 +1759,6 @@
2081
  "group_size": 64,
2082
  "bits": 4
2083
  },
2084
- "model.layers.53.mlp.switch_mlp.activation": false,
2085
  "model.layers.54.self_attn.q_proj": {
2086
  "group_size": 64,
2087
  "bits": 4
@@ -2098,11 +1775,6 @@
2098
  "group_size": 64,
2099
  "bits": 4
2100
  },
2101
- "model.layers.54.self_attn.q_norm": false,
2102
- "model.layers.54.self_attn.k_norm": false,
2103
- "model.layers.54.self_attn.rope": false,
2104
- "model.layers.54.input_layernorm": false,
2105
- "model.layers.54.post_attention_layernorm": false,
2106
  "model.layers.54.mlp.gate": {
2107
  "group_size": 64,
2108
  "bits": 4
@@ -2119,7 +1791,6 @@
2119
  "group_size": 64,
2120
  "bits": 6
2121
  },
2122
- "model.layers.54.mlp.switch_mlp.activation": false,
2123
  "model.layers.55.self_attn.q_proj": {
2124
  "group_size": 64,
2125
  "bits": 4
@@ -2136,11 +1807,6 @@
2136
  "group_size": 64,
2137
  "bits": 4
2138
  },
2139
- "model.layers.55.self_attn.q_norm": false,
2140
- "model.layers.55.self_attn.k_norm": false,
2141
- "model.layers.55.self_attn.rope": false,
2142
- "model.layers.55.input_layernorm": false,
2143
- "model.layers.55.post_attention_layernorm": false,
2144
  "model.layers.55.mlp.gate": {
2145
  "group_size": 64,
2146
  "bits": 4
@@ -2157,7 +1823,6 @@
2157
  "group_size": 64,
2158
  "bits": 6
2159
  },
2160
- "model.layers.55.mlp.switch_mlp.activation": false,
2161
  "model.layers.56.self_attn.q_proj": {
2162
  "group_size": 64,
2163
  "bits": 4
@@ -2174,11 +1839,6 @@
2174
  "group_size": 64,
2175
  "bits": 4
2176
  },
2177
- "model.layers.56.self_attn.q_norm": false,
2178
- "model.layers.56.self_attn.k_norm": false,
2179
- "model.layers.56.self_attn.rope": false,
2180
- "model.layers.56.input_layernorm": false,
2181
- "model.layers.56.post_attention_layernorm": false,
2182
  "model.layers.56.mlp.gate": {
2183
  "group_size": 64,
2184
  "bits": 4
@@ -2195,7 +1855,6 @@
2195
  "group_size": 64,
2196
  "bits": 6
2197
  },
2198
- "model.layers.56.mlp.switch_mlp.activation": false,
2199
  "model.layers.57.self_attn.q_proj": {
2200
  "group_size": 64,
2201
  "bits": 4
@@ -2212,11 +1871,6 @@
2212
  "group_size": 64,
2213
  "bits": 4
2214
  },
2215
- "model.layers.57.self_attn.q_norm": false,
2216
- "model.layers.57.self_attn.k_norm": false,
2217
- "model.layers.57.self_attn.rope": false,
2218
- "model.layers.57.input_layernorm": false,
2219
- "model.layers.57.post_attention_layernorm": false,
2220
  "model.layers.57.mlp.gate": {
2221
  "group_size": 64,
2222
  "bits": 4
@@ -2233,7 +1887,6 @@
2233
  "group_size": 64,
2234
  "bits": 6
2235
  },
2236
- "model.layers.57.mlp.switch_mlp.activation": false,
2237
  "model.layers.58.self_attn.q_proj": {
2238
  "group_size": 64,
2239
  "bits": 4
@@ -2250,11 +1903,6 @@
2250
  "group_size": 64,
2251
  "bits": 4
2252
  },
2253
- "model.layers.58.self_attn.q_norm": false,
2254
- "model.layers.58.self_attn.k_norm": false,
2255
- "model.layers.58.self_attn.rope": false,
2256
- "model.layers.58.input_layernorm": false,
2257
- "model.layers.58.post_attention_layernorm": false,
2258
  "model.layers.58.mlp.gate": {
2259
  "group_size": 64,
2260
  "bits": 4
@@ -2271,7 +1919,6 @@
2271
  "group_size": 64,
2272
  "bits": 6
2273
  },
2274
- "model.layers.58.mlp.switch_mlp.activation": false,
2275
  "model.layers.59.self_attn.q_proj": {
2276
  "group_size": 64,
2277
  "bits": 4
@@ -2288,11 +1935,6 @@
2288
  "group_size": 64,
2289
  "bits": 4
2290
  },
2291
- "model.layers.59.self_attn.q_norm": false,
2292
- "model.layers.59.self_attn.k_norm": false,
2293
- "model.layers.59.self_attn.rope": false,
2294
- "model.layers.59.input_layernorm": false,
2295
- "model.layers.59.post_attention_layernorm": false,
2296
  "model.layers.59.mlp.gate": {
2297
  "group_size": 64,
2298
  "bits": 4
@@ -2309,7 +1951,6 @@
2309
  "group_size": 64,
2310
  "bits": 6
2311
  },
2312
- "model.layers.59.mlp.switch_mlp.activation": false,
2313
  "model.layers.60.self_attn.q_proj": {
2314
  "group_size": 64,
2315
  "bits": 4
@@ -2326,11 +1967,6 @@
2326
  "group_size": 64,
2327
  "bits": 4
2328
  },
2329
- "model.layers.60.self_attn.q_norm": false,
2330
- "model.layers.60.self_attn.k_norm": false,
2331
- "model.layers.60.self_attn.rope": false,
2332
- "model.layers.60.input_layernorm": false,
2333
- "model.layers.60.post_attention_layernorm": false,
2334
  "model.layers.60.mlp.gate": {
2335
  "group_size": 64,
2336
  "bits": 4
@@ -2347,7 +1983,6 @@
2347
  "group_size": 64,
2348
  "bits": 6
2349
  },
2350
- "model.layers.60.mlp.switch_mlp.activation": false,
2351
  "model.layers.61.self_attn.q_proj": {
2352
  "group_size": 64,
2353
  "bits": 4
@@ -2364,11 +1999,6 @@
2364
  "group_size": 64,
2365
  "bits": 4
2366
  },
2367
- "model.layers.61.self_attn.q_norm": false,
2368
- "model.layers.61.self_attn.k_norm": false,
2369
- "model.layers.61.self_attn.rope": false,
2370
- "model.layers.61.input_layernorm": false,
2371
- "model.layers.61.post_attention_layernorm": false,
2372
  "model.layers.61.mlp.gate": {
2373
  "group_size": 64,
2374
  "bits": 4
@@ -2385,8 +2015,6 @@
2385
  "group_size": 64,
2386
  "bits": 6
2387
  },
2388
- "model.layers.61.mlp.switch_mlp.activation": false,
2389
- "model.norm": false,
2390
  "lm_head": {
2391
  "group_size": 64,
2392
  "bits": 6
@@ -2395,6 +2023,7 @@
2395
  "quantization_config": {
2396
  "group_size": 64,
2397
  "bits": 4,
 
2398
  "model.embed_tokens": {
2399
  "group_size": 64,
2400
  "bits": 4
@@ -2415,11 +2044,6 @@
2415
  "group_size": 64,
2416
  "bits": 4
2417
  },
2418
- "model.layers.0.self_attn.q_norm": false,
2419
- "model.layers.0.self_attn.k_norm": false,
2420
- "model.layers.0.self_attn.rope": false,
2421
- "model.layers.0.input_layernorm": false,
2422
- "model.layers.0.post_attention_layernorm": false,
2423
  "model.layers.0.mlp.gate": {
2424
  "group_size": 64,
2425
  "bits": 4
@@ -2436,7 +2060,6 @@
2436
  "group_size": 64,
2437
  "bits": 6
2438
  },
2439
- "model.layers.0.mlp.switch_mlp.activation": false,
2440
  "model.layers.1.self_attn.q_proj": {
2441
  "group_size": 64,
2442
  "bits": 4
@@ -2453,11 +2076,6 @@
2453
  "group_size": 64,
2454
  "bits": 4
2455
  },
2456
- "model.layers.1.self_attn.q_norm": false,
2457
- "model.layers.1.self_attn.k_norm": false,
2458
- "model.layers.1.self_attn.rope": false,
2459
- "model.layers.1.input_layernorm": false,
2460
- "model.layers.1.post_attention_layernorm": false,
2461
  "model.layers.1.mlp.gate": {
2462
  "group_size": 64,
2463
  "bits": 4
@@ -2474,7 +2092,6 @@
2474
  "group_size": 64,
2475
  "bits": 6
2476
  },
2477
- "model.layers.1.mlp.switch_mlp.activation": false,
2478
  "model.layers.2.self_attn.q_proj": {
2479
  "group_size": 64,
2480
  "bits": 4
@@ -2491,11 +2108,6 @@
2491
  "group_size": 64,
2492
  "bits": 4
2493
  },
2494
- "model.layers.2.self_attn.q_norm": false,
2495
- "model.layers.2.self_attn.k_norm": false,
2496
- "model.layers.2.self_attn.rope": false,
2497
- "model.layers.2.input_layernorm": false,
2498
- "model.layers.2.post_attention_layernorm": false,
2499
  "model.layers.2.mlp.gate": {
2500
  "group_size": 64,
2501
  "bits": 4
@@ -2512,7 +2124,6 @@
2512
  "group_size": 64,
2513
  "bits": 6
2514
  },
2515
- "model.layers.2.mlp.switch_mlp.activation": false,
2516
  "model.layers.3.self_attn.q_proj": {
2517
  "group_size": 64,
2518
  "bits": 4
@@ -2529,11 +2140,6 @@
2529
  "group_size": 64,
2530
  "bits": 4
2531
  },
2532
- "model.layers.3.self_attn.q_norm": false,
2533
- "model.layers.3.self_attn.k_norm": false,
2534
- "model.layers.3.self_attn.rope": false,
2535
- "model.layers.3.input_layernorm": false,
2536
- "model.layers.3.post_attention_layernorm": false,
2537
  "model.layers.3.mlp.gate": {
2538
  "group_size": 64,
2539
  "bits": 4
@@ -2550,7 +2156,6 @@
2550
  "group_size": 64,
2551
  "bits": 6
2552
  },
2553
- "model.layers.3.mlp.switch_mlp.activation": false,
2554
  "model.layers.4.self_attn.q_proj": {
2555
  "group_size": 64,
2556
  "bits": 4
@@ -2567,11 +2172,6 @@
2567
  "group_size": 64,
2568
  "bits": 4
2569
  },
2570
- "model.layers.4.self_attn.q_norm": false,
2571
- "model.layers.4.self_attn.k_norm": false,
2572
- "model.layers.4.self_attn.rope": false,
2573
- "model.layers.4.input_layernorm": false,
2574
- "model.layers.4.post_attention_layernorm": false,
2575
  "model.layers.4.mlp.gate": {
2576
  "group_size": 64,
2577
  "bits": 4
@@ -2588,7 +2188,6 @@
2588
  "group_size": 64,
2589
  "bits": 6
2590
  },
2591
- "model.layers.4.mlp.switch_mlp.activation": false,
2592
  "model.layers.5.self_attn.q_proj": {
2593
  "group_size": 64,
2594
  "bits": 4
@@ -2605,11 +2204,6 @@
2605
  "group_size": 64,
2606
  "bits": 4
2607
  },
2608
- "model.layers.5.self_attn.q_norm": false,
2609
- "model.layers.5.self_attn.k_norm": false,
2610
- "model.layers.5.self_attn.rope": false,
2611
- "model.layers.5.input_layernorm": false,
2612
- "model.layers.5.post_attention_layernorm": false,
2613
  "model.layers.5.mlp.gate": {
2614
  "group_size": 64,
2615
  "bits": 4
@@ -2626,7 +2220,6 @@
2626
  "group_size": 64,
2627
  "bits": 6
2628
  },
2629
- "model.layers.5.mlp.switch_mlp.activation": false,
2630
  "model.layers.6.self_attn.q_proj": {
2631
  "group_size": 64,
2632
  "bits": 4
@@ -2643,11 +2236,6 @@
2643
  "group_size": 64,
2644
  "bits": 4
2645
  },
2646
- "model.layers.6.self_attn.q_norm": false,
2647
- "model.layers.6.self_attn.k_norm": false,
2648
- "model.layers.6.self_attn.rope": false,
2649
- "model.layers.6.input_layernorm": false,
2650
- "model.layers.6.post_attention_layernorm": false,
2651
  "model.layers.6.mlp.gate": {
2652
  "group_size": 64,
2653
  "bits": 4
@@ -2664,7 +2252,6 @@
2664
  "group_size": 64,
2665
  "bits": 6
2666
  },
2667
- "model.layers.6.mlp.switch_mlp.activation": false,
2668
  "model.layers.7.self_attn.q_proj": {
2669
  "group_size": 64,
2670
  "bits": 4
@@ -2681,11 +2268,6 @@
2681
  "group_size": 64,
2682
  "bits": 4
2683
  },
2684
- "model.layers.7.self_attn.q_norm": false,
2685
- "model.layers.7.self_attn.k_norm": false,
2686
- "model.layers.7.self_attn.rope": false,
2687
- "model.layers.7.input_layernorm": false,
2688
- "model.layers.7.post_attention_layernorm": false,
2689
  "model.layers.7.mlp.gate": {
2690
  "group_size": 64,
2691
  "bits": 4
@@ -2702,7 +2284,6 @@
2702
  "group_size": 64,
2703
  "bits": 4
2704
  },
2705
- "model.layers.7.mlp.switch_mlp.activation": false,
2706
  "model.layers.8.self_attn.q_proj": {
2707
  "group_size": 64,
2708
  "bits": 4
@@ -2719,11 +2300,6 @@
2719
  "group_size": 64,
2720
  "bits": 4
2721
  },
2722
- "model.layers.8.self_attn.q_norm": false,
2723
- "model.layers.8.self_attn.k_norm": false,
2724
- "model.layers.8.self_attn.rope": false,
2725
- "model.layers.8.input_layernorm": false,
2726
- "model.layers.8.post_attention_layernorm": false,
2727
  "model.layers.8.mlp.gate": {
2728
  "group_size": 64,
2729
  "bits": 4
@@ -2740,7 +2316,6 @@
2740
  "group_size": 64,
2741
  "bits": 4
2742
  },
2743
- "model.layers.8.mlp.switch_mlp.activation": false,
2744
  "model.layers.9.self_attn.q_proj": {
2745
  "group_size": 64,
2746
  "bits": 4
@@ -2757,11 +2332,6 @@
2757
  "group_size": 64,
2758
  "bits": 4
2759
  },
2760
- "model.layers.9.self_attn.q_norm": false,
2761
- "model.layers.9.self_attn.k_norm": false,
2762
- "model.layers.9.self_attn.rope": false,
2763
- "model.layers.9.input_layernorm": false,
2764
- "model.layers.9.post_attention_layernorm": false,
2765
  "model.layers.9.mlp.gate": {
2766
  "group_size": 64,
2767
  "bits": 4
@@ -2778,7 +2348,6 @@
2778
  "group_size": 64,
2779
  "bits": 6
2780
  },
2781
- "model.layers.9.mlp.switch_mlp.activation": false,
2782
  "model.layers.10.self_attn.q_proj": {
2783
  "group_size": 64,
2784
  "bits": 4
@@ -2795,11 +2364,6 @@
2795
  "group_size": 64,
2796
  "bits": 4
2797
  },
2798
- "model.layers.10.self_attn.q_norm": false,
2799
- "model.layers.10.self_attn.k_norm": false,
2800
- "model.layers.10.self_attn.rope": false,
2801
- "model.layers.10.input_layernorm": false,
2802
- "model.layers.10.post_attention_layernorm": false,
2803
  "model.layers.10.mlp.gate": {
2804
  "group_size": 64,
2805
  "bits": 4
@@ -2816,7 +2380,6 @@
2816
  "group_size": 64,
2817
  "bits": 4
2818
  },
2819
- "model.layers.10.mlp.switch_mlp.activation": false,
2820
  "model.layers.11.self_attn.q_proj": {
2821
  "group_size": 64,
2822
  "bits": 4
@@ -2833,11 +2396,6 @@
2833
  "group_size": 64,
2834
  "bits": 4
2835
  },
2836
- "model.layers.11.self_attn.q_norm": false,
2837
- "model.layers.11.self_attn.k_norm": false,
2838
- "model.layers.11.self_attn.rope": false,
2839
- "model.layers.11.input_layernorm": false,
2840
- "model.layers.11.post_attention_layernorm": false,
2841
  "model.layers.11.mlp.gate": {
2842
  "group_size": 64,
2843
  "bits": 4
@@ -2854,7 +2412,6 @@
2854
  "group_size": 64,
2855
  "bits": 4
2856
  },
2857
- "model.layers.11.mlp.switch_mlp.activation": false,
2858
  "model.layers.12.self_attn.q_proj": {
2859
  "group_size": 64,
2860
  "bits": 4
@@ -2871,11 +2428,6 @@
2871
  "group_size": 64,
2872
  "bits": 4
2873
  },
2874
- "model.layers.12.self_attn.q_norm": false,
2875
- "model.layers.12.self_attn.k_norm": false,
2876
- "model.layers.12.self_attn.rope": false,
2877
- "model.layers.12.input_layernorm": false,
2878
- "model.layers.12.post_attention_layernorm": false,
2879
  "model.layers.12.mlp.gate": {
2880
  "group_size": 64,
2881
  "bits": 4
@@ -2892,7 +2444,6 @@
2892
  "group_size": 64,
2893
  "bits": 6
2894
  },
2895
- "model.layers.12.mlp.switch_mlp.activation": false,
2896
  "model.layers.13.self_attn.q_proj": {
2897
  "group_size": 64,
2898
  "bits": 4
@@ -2909,11 +2460,6 @@
2909
  "group_size": 64,
2910
  "bits": 4
2911
  },
2912
- "model.layers.13.self_attn.q_norm": false,
2913
- "model.layers.13.self_attn.k_norm": false,
2914
- "model.layers.13.self_attn.rope": false,
2915
- "model.layers.13.input_layernorm": false,
2916
- "model.layers.13.post_attention_layernorm": false,
2917
  "model.layers.13.mlp.gate": {
2918
  "group_size": 64,
2919
  "bits": 4
@@ -2930,7 +2476,6 @@
2930
  "group_size": 64,
2931
  "bits": 4
2932
  },
2933
- "model.layers.13.mlp.switch_mlp.activation": false,
2934
  "model.layers.14.self_attn.q_proj": {
2935
  "group_size": 64,
2936
  "bits": 4
@@ -2947,11 +2492,6 @@
2947
  "group_size": 64,
2948
  "bits": 4
2949
  },
2950
- "model.layers.14.self_attn.q_norm": false,
2951
- "model.layers.14.self_attn.k_norm": false,
2952
- "model.layers.14.self_attn.rope": false,
2953
- "model.layers.14.input_layernorm": false,
2954
- "model.layers.14.post_attention_layernorm": false,
2955
  "model.layers.14.mlp.gate": {
2956
  "group_size": 64,
2957
  "bits": 4
@@ -2968,7 +2508,6 @@
2968
  "group_size": 64,
2969
  "bits": 4
2970
  },
2971
- "model.layers.14.mlp.switch_mlp.activation": false,
2972
  "model.layers.15.self_attn.q_proj": {
2973
  "group_size": 64,
2974
  "bits": 4
@@ -2985,11 +2524,6 @@
2985
  "group_size": 64,
2986
  "bits": 4
2987
  },
2988
- "model.layers.15.self_attn.q_norm": false,
2989
- "model.layers.15.self_attn.k_norm": false,
2990
- "model.layers.15.self_attn.rope": false,
2991
- "model.layers.15.input_layernorm": false,
2992
- "model.layers.15.post_attention_layernorm": false,
2993
  "model.layers.15.mlp.gate": {
2994
  "group_size": 64,
2995
  "bits": 4
@@ -3006,7 +2540,6 @@
3006
  "group_size": 64,
3007
  "bits": 6
3008
  },
3009
- "model.layers.15.mlp.switch_mlp.activation": false,
3010
  "model.layers.16.self_attn.q_proj": {
3011
  "group_size": 64,
3012
  "bits": 4
@@ -3023,11 +2556,6 @@
3023
  "group_size": 64,
3024
  "bits": 4
3025
  },
3026
- "model.layers.16.self_attn.q_norm": false,
3027
- "model.layers.16.self_attn.k_norm": false,
3028
- "model.layers.16.self_attn.rope": false,
3029
- "model.layers.16.input_layernorm": false,
3030
- "model.layers.16.post_attention_layernorm": false,
3031
  "model.layers.16.mlp.gate": {
3032
  "group_size": 64,
3033
  "bits": 4
@@ -3044,7 +2572,6 @@
3044
  "group_size": 64,
3045
  "bits": 4
3046
  },
3047
- "model.layers.16.mlp.switch_mlp.activation": false,
3048
  "model.layers.17.self_attn.q_proj": {
3049
  "group_size": 64,
3050
  "bits": 4
@@ -3061,11 +2588,6 @@
3061
  "group_size": 64,
3062
  "bits": 4
3063
  },
3064
- "model.layers.17.self_attn.q_norm": false,
3065
- "model.layers.17.self_attn.k_norm": false,
3066
- "model.layers.17.self_attn.rope": false,
3067
- "model.layers.17.input_layernorm": false,
3068
- "model.layers.17.post_attention_layernorm": false,
3069
  "model.layers.17.mlp.gate": {
3070
  "group_size": 64,
3071
  "bits": 4
@@ -3082,7 +2604,6 @@
3082
  "group_size": 64,
3083
  "bits": 4
3084
  },
3085
- "model.layers.17.mlp.switch_mlp.activation": false,
3086
  "model.layers.18.self_attn.q_proj": {
3087
  "group_size": 64,
3088
  "bits": 4
@@ -3099,11 +2620,6 @@
3099
  "group_size": 64,
3100
  "bits": 4
3101
  },
3102
- "model.layers.18.self_attn.q_norm": false,
3103
- "model.layers.18.self_attn.k_norm": false,
3104
- "model.layers.18.self_attn.rope": false,
3105
- "model.layers.18.input_layernorm": false,
3106
- "model.layers.18.post_attention_layernorm": false,
3107
  "model.layers.18.mlp.gate": {
3108
  "group_size": 64,
3109
  "bits": 4
@@ -3120,7 +2636,6 @@
3120
  "group_size": 64,
3121
  "bits": 6
3122
  },
3123
- "model.layers.18.mlp.switch_mlp.activation": false,
3124
  "model.layers.19.self_attn.q_proj": {
3125
  "group_size": 64,
3126
  "bits": 4
@@ -3137,11 +2652,6 @@
3137
  "group_size": 64,
3138
  "bits": 4
3139
  },
3140
- "model.layers.19.self_attn.q_norm": false,
3141
- "model.layers.19.self_attn.k_norm": false,
3142
- "model.layers.19.self_attn.rope": false,
3143
- "model.layers.19.input_layernorm": false,
3144
- "model.layers.19.post_attention_layernorm": false,
3145
  "model.layers.19.mlp.gate": {
3146
  "group_size": 64,
3147
  "bits": 4
@@ -3158,7 +2668,6 @@
3158
  "group_size": 64,
3159
  "bits": 4
3160
  },
3161
- "model.layers.19.mlp.switch_mlp.activation": false,
3162
  "model.layers.20.self_attn.q_proj": {
3163
  "group_size": 64,
3164
  "bits": 4
@@ -3175,11 +2684,6 @@
3175
  "group_size": 64,
3176
  "bits": 4
3177
  },
3178
- "model.layers.20.self_attn.q_norm": false,
3179
- "model.layers.20.self_attn.k_norm": false,
3180
- "model.layers.20.self_attn.rope": false,
3181
- "model.layers.20.input_layernorm": false,
3182
- "model.layers.20.post_attention_layernorm": false,
3183
  "model.layers.20.mlp.gate": {
3184
  "group_size": 64,
3185
  "bits": 4
@@ -3196,7 +2700,6 @@
3196
  "group_size": 64,
3197
  "bits": 4
3198
  },
3199
- "model.layers.20.mlp.switch_mlp.activation": false,
3200
  "model.layers.21.self_attn.q_proj": {
3201
  "group_size": 64,
3202
  "bits": 4
@@ -3213,11 +2716,6 @@
3213
  "group_size": 64,
3214
  "bits": 4
3215
  },
3216
- "model.layers.21.self_attn.q_norm": false,
3217
- "model.layers.21.self_attn.k_norm": false,
3218
- "model.layers.21.self_attn.rope": false,
3219
- "model.layers.21.input_layernorm": false,
3220
- "model.layers.21.post_attention_layernorm": false,
3221
  "model.layers.21.mlp.gate": {
3222
  "group_size": 64,
3223
  "bits": 4
@@ -3234,7 +2732,6 @@
3234
  "group_size": 64,
3235
  "bits": 6
3236
  },
3237
- "model.layers.21.mlp.switch_mlp.activation": false,
3238
  "model.layers.22.self_attn.q_proj": {
3239
  "group_size": 64,
3240
  "bits": 4
@@ -3251,11 +2748,6 @@
3251
  "group_size": 64,
3252
  "bits": 4
3253
  },
3254
- "model.layers.22.self_attn.q_norm": false,
3255
- "model.layers.22.self_attn.k_norm": false,
3256
- "model.layers.22.self_attn.rope": false,
3257
- "model.layers.22.input_layernorm": false,
3258
- "model.layers.22.post_attention_layernorm": false,
3259
  "model.layers.22.mlp.gate": {
3260
  "group_size": 64,
3261
  "bits": 4
@@ -3272,7 +2764,6 @@
3272
  "group_size": 64,
3273
  "bits": 4
3274
  },
3275
- "model.layers.22.mlp.switch_mlp.activation": false,
3276
  "model.layers.23.self_attn.q_proj": {
3277
  "group_size": 64,
3278
  "bits": 4
@@ -3289,11 +2780,6 @@
3289
  "group_size": 64,
3290
  "bits": 4
3291
  },
3292
- "model.layers.23.self_attn.q_norm": false,
3293
- "model.layers.23.self_attn.k_norm": false,
3294
- "model.layers.23.self_attn.rope": false,
3295
- "model.layers.23.input_layernorm": false,
3296
- "model.layers.23.post_attention_layernorm": false,
3297
  "model.layers.23.mlp.gate": {
3298
  "group_size": 64,
3299
  "bits": 4
@@ -3310,7 +2796,6 @@
3310
  "group_size": 64,
3311
  "bits": 4
3312
  },
3313
- "model.layers.23.mlp.switch_mlp.activation": false,
3314
  "model.layers.24.self_attn.q_proj": {
3315
  "group_size": 64,
3316
  "bits": 4
@@ -3327,11 +2812,6 @@
3327
  "group_size": 64,
3328
  "bits": 4
3329
  },
3330
- "model.layers.24.self_attn.q_norm": false,
3331
- "model.layers.24.self_attn.k_norm": false,
3332
- "model.layers.24.self_attn.rope": false,
3333
- "model.layers.24.input_layernorm": false,
3334
- "model.layers.24.post_attention_layernorm": false,
3335
  "model.layers.24.mlp.gate": {
3336
  "group_size": 64,
3337
  "bits": 4
@@ -3348,7 +2828,6 @@
3348
  "group_size": 64,
3349
  "bits": 6
3350
  },
3351
- "model.layers.24.mlp.switch_mlp.activation": false,
3352
  "model.layers.25.self_attn.q_proj": {
3353
  "group_size": 64,
3354
  "bits": 4
@@ -3365,11 +2844,6 @@
3365
  "group_size": 64,
3366
  "bits": 4
3367
  },
3368
- "model.layers.25.self_attn.q_norm": false,
3369
- "model.layers.25.self_attn.k_norm": false,
3370
- "model.layers.25.self_attn.rope": false,
3371
- "model.layers.25.input_layernorm": false,
3372
- "model.layers.25.post_attention_layernorm": false,
3373
  "model.layers.25.mlp.gate": {
3374
  "group_size": 64,
3375
  "bits": 4
@@ -3386,7 +2860,6 @@
3386
  "group_size": 64,
3387
  "bits": 4
3388
  },
3389
- "model.layers.25.mlp.switch_mlp.activation": false,
3390
  "model.layers.26.self_attn.q_proj": {
3391
  "group_size": 64,
3392
  "bits": 4
@@ -3403,11 +2876,6 @@
3403
  "group_size": 64,
3404
  "bits": 4
3405
  },
3406
- "model.layers.26.self_attn.q_norm": false,
3407
- "model.layers.26.self_attn.k_norm": false,
3408
- "model.layers.26.self_attn.rope": false,
3409
- "model.layers.26.input_layernorm": false,
3410
- "model.layers.26.post_attention_layernorm": false,
3411
  "model.layers.26.mlp.gate": {
3412
  "group_size": 64,
3413
  "bits": 4
@@ -3424,7 +2892,6 @@
3424
  "group_size": 64,
3425
  "bits": 4
3426
  },
3427
- "model.layers.26.mlp.switch_mlp.activation": false,
3428
  "model.layers.27.self_attn.q_proj": {
3429
  "group_size": 64,
3430
  "bits": 4
@@ -3441,11 +2908,6 @@
3441
  "group_size": 64,
3442
  "bits": 4
3443
  },
3444
- "model.layers.27.self_attn.q_norm": false,
3445
- "model.layers.27.self_attn.k_norm": false,
3446
- "model.layers.27.self_attn.rope": false,
3447
- "model.layers.27.input_layernorm": false,
3448
- "model.layers.27.post_attention_layernorm": false,
3449
  "model.layers.27.mlp.gate": {
3450
  "group_size": 64,
3451
  "bits": 4
@@ -3462,7 +2924,6 @@
3462
  "group_size": 64,
3463
  "bits": 6
3464
  },
3465
- "model.layers.27.mlp.switch_mlp.activation": false,
3466
  "model.layers.28.self_attn.q_proj": {
3467
  "group_size": 64,
3468
  "bits": 4
@@ -3479,11 +2940,6 @@
3479
  "group_size": 64,
3480
  "bits": 4
3481
  },
3482
- "model.layers.28.self_attn.q_norm": false,
3483
- "model.layers.28.self_attn.k_norm": false,
3484
- "model.layers.28.self_attn.rope": false,
3485
- "model.layers.28.input_layernorm": false,
3486
- "model.layers.28.post_attention_layernorm": false,
3487
  "model.layers.28.mlp.gate": {
3488
  "group_size": 64,
3489
  "bits": 4
@@ -3500,7 +2956,6 @@
3500
  "group_size": 64,
3501
  "bits": 4
3502
  },
3503
- "model.layers.28.mlp.switch_mlp.activation": false,
3504
  "model.layers.29.self_attn.q_proj": {
3505
  "group_size": 64,
3506
  "bits": 4
@@ -3517,11 +2972,6 @@
3517
  "group_size": 64,
3518
  "bits": 4
3519
  },
3520
- "model.layers.29.self_attn.q_norm": false,
3521
- "model.layers.29.self_attn.k_norm": false,
3522
- "model.layers.29.self_attn.rope": false,
3523
- "model.layers.29.input_layernorm": false,
3524
- "model.layers.29.post_attention_layernorm": false,
3525
  "model.layers.29.mlp.gate": {
3526
  "group_size": 64,
3527
  "bits": 4
@@ -3538,7 +2988,6 @@
3538
  "group_size": 64,
3539
  "bits": 4
3540
  },
3541
- "model.layers.29.mlp.switch_mlp.activation": false,
3542
  "model.layers.30.self_attn.q_proj": {
3543
  "group_size": 64,
3544
  "bits": 4
@@ -3555,11 +3004,6 @@
3555
  "group_size": 64,
3556
  "bits": 4
3557
  },
3558
- "model.layers.30.self_attn.q_norm": false,
3559
- "model.layers.30.self_attn.k_norm": false,
3560
- "model.layers.30.self_attn.rope": false,
3561
- "model.layers.30.input_layernorm": false,
3562
- "model.layers.30.post_attention_layernorm": false,
3563
  "model.layers.30.mlp.gate": {
3564
  "group_size": 64,
3565
  "bits": 4
@@ -3576,7 +3020,6 @@
3576
  "group_size": 64,
3577
  "bits": 6
3578
  },
3579
- "model.layers.30.mlp.switch_mlp.activation": false,
3580
  "model.layers.31.self_attn.q_proj": {
3581
  "group_size": 64,
3582
  "bits": 4
@@ -3593,11 +3036,6 @@
3593
  "group_size": 64,
3594
  "bits": 4
3595
  },
3596
- "model.layers.31.self_attn.q_norm": false,
3597
- "model.layers.31.self_attn.k_norm": false,
3598
- "model.layers.31.self_attn.rope": false,
3599
- "model.layers.31.input_layernorm": false,
3600
- "model.layers.31.post_attention_layernorm": false,
3601
  "model.layers.31.mlp.gate": {
3602
  "group_size": 64,
3603
  "bits": 4
@@ -3614,7 +3052,6 @@
3614
  "group_size": 64,
3615
  "bits": 4
3616
  },
3617
- "model.layers.31.mlp.switch_mlp.activation": false,
3618
  "model.layers.32.self_attn.q_proj": {
3619
  "group_size": 64,
3620
  "bits": 4
@@ -3631,11 +3068,6 @@
3631
  "group_size": 64,
3632
  "bits": 4
3633
  },
3634
- "model.layers.32.self_attn.q_norm": false,
3635
- "model.layers.32.self_attn.k_norm": false,
3636
- "model.layers.32.self_attn.rope": false,
3637
- "model.layers.32.input_layernorm": false,
3638
- "model.layers.32.post_attention_layernorm": false,
3639
  "model.layers.32.mlp.gate": {
3640
  "group_size": 64,
3641
  "bits": 4
@@ -3652,7 +3084,6 @@
3652
  "group_size": 64,
3653
  "bits": 4
3654
  },
3655
- "model.layers.32.mlp.switch_mlp.activation": false,
3656
  "model.layers.33.self_attn.q_proj": {
3657
  "group_size": 64,
3658
  "bits": 4
@@ -3669,11 +3100,6 @@
3669
  "group_size": 64,
3670
  "bits": 4
3671
  },
3672
- "model.layers.33.self_attn.q_norm": false,
3673
- "model.layers.33.self_attn.k_norm": false,
3674
- "model.layers.33.self_attn.rope": false,
3675
- "model.layers.33.input_layernorm": false,
3676
- "model.layers.33.post_attention_layernorm": false,
3677
  "model.layers.33.mlp.gate": {
3678
  "group_size": 64,
3679
  "bits": 4
@@ -3690,7 +3116,6 @@
3690
  "group_size": 64,
3691
  "bits": 6
3692
  },
3693
- "model.layers.33.mlp.switch_mlp.activation": false,
3694
  "model.layers.34.self_attn.q_proj": {
3695
  "group_size": 64,
3696
  "bits": 4
@@ -3707,11 +3132,6 @@
3707
  "group_size": 64,
3708
  "bits": 4
3709
  },
3710
- "model.layers.34.self_attn.q_norm": false,
3711
- "model.layers.34.self_attn.k_norm": false,
3712
- "model.layers.34.self_attn.rope": false,
3713
- "model.layers.34.input_layernorm": false,
3714
- "model.layers.34.post_attention_layernorm": false,
3715
  "model.layers.34.mlp.gate": {
3716
  "group_size": 64,
3717
  "bits": 4
@@ -3728,7 +3148,6 @@
3728
  "group_size": 64,
3729
  "bits": 4
3730
  },
3731
- "model.layers.34.mlp.switch_mlp.activation": false,
3732
  "model.layers.35.self_attn.q_proj": {
3733
  "group_size": 64,
3734
  "bits": 4
@@ -3745,11 +3164,6 @@
3745
  "group_size": 64,
3746
  "bits": 4
3747
  },
3748
- "model.layers.35.self_attn.q_norm": false,
3749
- "model.layers.35.self_attn.k_norm": false,
3750
- "model.layers.35.self_attn.rope": false,
3751
- "model.layers.35.input_layernorm": false,
3752
- "model.layers.35.post_attention_layernorm": false,
3753
  "model.layers.35.mlp.gate": {
3754
  "group_size": 64,
3755
  "bits": 4
@@ -3766,7 +3180,6 @@
3766
  "group_size": 64,
3767
  "bits": 4
3768
  },
3769
- "model.layers.35.mlp.switch_mlp.activation": false,
3770
  "model.layers.36.self_attn.q_proj": {
3771
  "group_size": 64,
3772
  "bits": 4
@@ -3783,11 +3196,6 @@
3783
  "group_size": 64,
3784
  "bits": 4
3785
  },
3786
- "model.layers.36.self_attn.q_norm": false,
3787
- "model.layers.36.self_attn.k_norm": false,
3788
- "model.layers.36.self_attn.rope": false,
3789
- "model.layers.36.input_layernorm": false,
3790
- "model.layers.36.post_attention_layernorm": false,
3791
  "model.layers.36.mlp.gate": {
3792
  "group_size": 64,
3793
  "bits": 4
@@ -3804,7 +3212,6 @@
3804
  "group_size": 64,
3805
  "bits": 6
3806
  },
3807
- "model.layers.36.mlp.switch_mlp.activation": false,
3808
  "model.layers.37.self_attn.q_proj": {
3809
  "group_size": 64,
3810
  "bits": 4
@@ -3821,11 +3228,6 @@
3821
  "group_size": 64,
3822
  "bits": 4
3823
  },
3824
- "model.layers.37.self_attn.q_norm": false,
3825
- "model.layers.37.self_attn.k_norm": false,
3826
- "model.layers.37.self_attn.rope": false,
3827
- "model.layers.37.input_layernorm": false,
3828
- "model.layers.37.post_attention_layernorm": false,
3829
  "model.layers.37.mlp.gate": {
3830
  "group_size": 64,
3831
  "bits": 4
@@ -3842,7 +3244,6 @@
3842
  "group_size": 64,
3843
  "bits": 4
3844
  },
3845
- "model.layers.37.mlp.switch_mlp.activation": false,
3846
  "model.layers.38.self_attn.q_proj": {
3847
  "group_size": 64,
3848
  "bits": 4
@@ -3859,11 +3260,6 @@
3859
  "group_size": 64,
3860
  "bits": 4
3861
  },
3862
- "model.layers.38.self_attn.q_norm": false,
3863
- "model.layers.38.self_attn.k_norm": false,
3864
- "model.layers.38.self_attn.rope": false,
3865
- "model.layers.38.input_layernorm": false,
3866
- "model.layers.38.post_attention_layernorm": false,
3867
  "model.layers.38.mlp.gate": {
3868
  "group_size": 64,
3869
  "bits": 4
@@ -3880,7 +3276,6 @@
3880
  "group_size": 64,
3881
  "bits": 4
3882
  },
3883
- "model.layers.38.mlp.switch_mlp.activation": false,
3884
  "model.layers.39.self_attn.q_proj": {
3885
  "group_size": 64,
3886
  "bits": 4
@@ -3897,11 +3292,6 @@
3897
  "group_size": 64,
3898
  "bits": 4
3899
  },
3900
- "model.layers.39.self_attn.q_norm": false,
3901
- "model.layers.39.self_attn.k_norm": false,
3902
- "model.layers.39.self_attn.rope": false,
3903
- "model.layers.39.input_layernorm": false,
3904
- "model.layers.39.post_attention_layernorm": false,
3905
  "model.layers.39.mlp.gate": {
3906
  "group_size": 64,
3907
  "bits": 4
@@ -3918,7 +3308,6 @@
3918
  "group_size": 64,
3919
  "bits": 6
3920
  },
3921
- "model.layers.39.mlp.switch_mlp.activation": false,
3922
  "model.layers.40.self_attn.q_proj": {
3923
  "group_size": 64,
3924
  "bits": 4
@@ -3935,11 +3324,6 @@
3935
  "group_size": 64,
3936
  "bits": 4
3937
  },
3938
- "model.layers.40.self_attn.q_norm": false,
3939
- "model.layers.40.self_attn.k_norm": false,
3940
- "model.layers.40.self_attn.rope": false,
3941
- "model.layers.40.input_layernorm": false,
3942
- "model.layers.40.post_attention_layernorm": false,
3943
  "model.layers.40.mlp.gate": {
3944
  "group_size": 64,
3945
  "bits": 4
@@ -3956,7 +3340,6 @@
3956
  "group_size": 64,
3957
  "bits": 4
3958
  },
3959
- "model.layers.40.mlp.switch_mlp.activation": false,
3960
  "model.layers.41.self_attn.q_proj": {
3961
  "group_size": 64,
3962
  "bits": 4
@@ -3973,11 +3356,6 @@
3973
  "group_size": 64,
3974
  "bits": 4
3975
  },
3976
- "model.layers.41.self_attn.q_norm": false,
3977
- "model.layers.41.self_attn.k_norm": false,
3978
- "model.layers.41.self_attn.rope": false,
3979
- "model.layers.41.input_layernorm": false,
3980
- "model.layers.41.post_attention_layernorm": false,
3981
  "model.layers.41.mlp.gate": {
3982
  "group_size": 64,
3983
  "bits": 4
@@ -3994,7 +3372,6 @@
3994
  "group_size": 64,
3995
  "bits": 4
3996
  },
3997
- "model.layers.41.mlp.switch_mlp.activation": false,
3998
  "model.layers.42.self_attn.q_proj": {
3999
  "group_size": 64,
4000
  "bits": 4
@@ -4011,11 +3388,6 @@
4011
  "group_size": 64,
4012
  "bits": 4
4013
  },
4014
- "model.layers.42.self_attn.q_norm": false,
4015
- "model.layers.42.self_attn.k_norm": false,
4016
- "model.layers.42.self_attn.rope": false,
4017
- "model.layers.42.input_layernorm": false,
4018
- "model.layers.42.post_attention_layernorm": false,
4019
  "model.layers.42.mlp.gate": {
4020
  "group_size": 64,
4021
  "bits": 4
@@ -4032,7 +3404,6 @@
4032
  "group_size": 64,
4033
  "bits": 6
4034
  },
4035
- "model.layers.42.mlp.switch_mlp.activation": false,
4036
  "model.layers.43.self_attn.q_proj": {
4037
  "group_size": 64,
4038
  "bits": 4
@@ -4049,11 +3420,6 @@
4049
  "group_size": 64,
4050
  "bits": 4
4051
  },
4052
- "model.layers.43.self_attn.q_norm": false,
4053
- "model.layers.43.self_attn.k_norm": false,
4054
- "model.layers.43.self_attn.rope": false,
4055
- "model.layers.43.input_layernorm": false,
4056
- "model.layers.43.post_attention_layernorm": false,
4057
  "model.layers.43.mlp.gate": {
4058
  "group_size": 64,
4059
  "bits": 4
@@ -4070,7 +3436,6 @@
4070
  "group_size": 64,
4071
  "bits": 4
4072
  },
4073
- "model.layers.43.mlp.switch_mlp.activation": false,
4074
  "model.layers.44.self_attn.q_proj": {
4075
  "group_size": 64,
4076
  "bits": 4
@@ -4087,11 +3452,6 @@
4087
  "group_size": 64,
4088
  "bits": 4
4089
  },
4090
- "model.layers.44.self_attn.q_norm": false,
4091
- "model.layers.44.self_attn.k_norm": false,
4092
- "model.layers.44.self_attn.rope": false,
4093
- "model.layers.44.input_layernorm": false,
4094
- "model.layers.44.post_attention_layernorm": false,
4095
  "model.layers.44.mlp.gate": {
4096
  "group_size": 64,
4097
  "bits": 4
@@ -4108,7 +3468,6 @@
4108
  "group_size": 64,
4109
  "bits": 4
4110
  },
4111
- "model.layers.44.mlp.switch_mlp.activation": false,
4112
  "model.layers.45.self_attn.q_proj": {
4113
  "group_size": 64,
4114
  "bits": 4
@@ -4125,11 +3484,6 @@
4125
  "group_size": 64,
4126
  "bits": 4
4127
  },
4128
- "model.layers.45.self_attn.q_norm": false,
4129
- "model.layers.45.self_attn.k_norm": false,
4130
- "model.layers.45.self_attn.rope": false,
4131
- "model.layers.45.input_layernorm": false,
4132
- "model.layers.45.post_attention_layernorm": false,
4133
  "model.layers.45.mlp.gate": {
4134
  "group_size": 64,
4135
  "bits": 4
@@ -4146,7 +3500,6 @@
4146
  "group_size": 64,
4147
  "bits": 6
4148
  },
4149
- "model.layers.45.mlp.switch_mlp.activation": false,
4150
  "model.layers.46.self_attn.q_proj": {
4151
  "group_size": 64,
4152
  "bits": 4
@@ -4163,11 +3516,6 @@
4163
  "group_size": 64,
4164
  "bits": 4
4165
  },
4166
- "model.layers.46.self_attn.q_norm": false,
4167
- "model.layers.46.self_attn.k_norm": false,
4168
- "model.layers.46.self_attn.rope": false,
4169
- "model.layers.46.input_layernorm": false,
4170
- "model.layers.46.post_attention_layernorm": false,
4171
  "model.layers.46.mlp.gate": {
4172
  "group_size": 64,
4173
  "bits": 4
@@ -4184,7 +3532,6 @@
4184
  "group_size": 64,
4185
  "bits": 4
4186
  },
4187
- "model.layers.46.mlp.switch_mlp.activation": false,
4188
  "model.layers.47.self_attn.q_proj": {
4189
  "group_size": 64,
4190
  "bits": 4
@@ -4201,11 +3548,6 @@
4201
  "group_size": 64,
4202
  "bits": 4
4203
  },
4204
- "model.layers.47.self_attn.q_norm": false,
4205
- "model.layers.47.self_attn.k_norm": false,
4206
- "model.layers.47.self_attn.rope": false,
4207
- "model.layers.47.input_layernorm": false,
4208
- "model.layers.47.post_attention_layernorm": false,
4209
  "model.layers.47.mlp.gate": {
4210
  "group_size": 64,
4211
  "bits": 4
@@ -4222,7 +3564,6 @@
4222
  "group_size": 64,
4223
  "bits": 4
4224
  },
4225
- "model.layers.47.mlp.switch_mlp.activation": false,
4226
  "model.layers.48.self_attn.q_proj": {
4227
  "group_size": 64,
4228
  "bits": 4
@@ -4239,11 +3580,6 @@
4239
  "group_size": 64,
4240
  "bits": 4
4241
  },
4242
- "model.layers.48.self_attn.q_norm": false,
4243
- "model.layers.48.self_attn.k_norm": false,
4244
- "model.layers.48.self_attn.rope": false,
4245
- "model.layers.48.input_layernorm": false,
4246
- "model.layers.48.post_attention_layernorm": false,
4247
  "model.layers.48.mlp.gate": {
4248
  "group_size": 64,
4249
  "bits": 4
@@ -4260,7 +3596,6 @@
4260
  "group_size": 64,
4261
  "bits": 6
4262
  },
4263
- "model.layers.48.mlp.switch_mlp.activation": false,
4264
  "model.layers.49.self_attn.q_proj": {
4265
  "group_size": 64,
4266
  "bits": 4
@@ -4277,11 +3612,6 @@
4277
  "group_size": 64,
4278
  "bits": 4
4279
  },
4280
- "model.layers.49.self_attn.q_norm": false,
4281
- "model.layers.49.self_attn.k_norm": false,
4282
- "model.layers.49.self_attn.rope": false,
4283
- "model.layers.49.input_layernorm": false,
4284
- "model.layers.49.post_attention_layernorm": false,
4285
  "model.layers.49.mlp.gate": {
4286
  "group_size": 64,
4287
  "bits": 4
@@ -4298,7 +3628,6 @@
4298
  "group_size": 64,
4299
  "bits": 4
4300
  },
4301
- "model.layers.49.mlp.switch_mlp.activation": false,
4302
  "model.layers.50.self_attn.q_proj": {
4303
  "group_size": 64,
4304
  "bits": 4
@@ -4315,11 +3644,6 @@
4315
  "group_size": 64,
4316
  "bits": 4
4317
  },
4318
- "model.layers.50.self_attn.q_norm": false,
4319
- "model.layers.50.self_attn.k_norm": false,
4320
- "model.layers.50.self_attn.rope": false,
4321
- "model.layers.50.input_layernorm": false,
4322
- "model.layers.50.post_attention_layernorm": false,
4323
  "model.layers.50.mlp.gate": {
4324
  "group_size": 64,
4325
  "bits": 4
@@ -4336,7 +3660,6 @@
4336
  "group_size": 64,
4337
  "bits": 4
4338
  },
4339
- "model.layers.50.mlp.switch_mlp.activation": false,
4340
  "model.layers.51.self_attn.q_proj": {
4341
  "group_size": 64,
4342
  "bits": 4
@@ -4353,11 +3676,6 @@
4353
  "group_size": 64,
4354
  "bits": 4
4355
  },
4356
- "model.layers.51.self_attn.q_norm": false,
4357
- "model.layers.51.self_attn.k_norm": false,
4358
- "model.layers.51.self_attn.rope": false,
4359
- "model.layers.51.input_layernorm": false,
4360
- "model.layers.51.post_attention_layernorm": false,
4361
  "model.layers.51.mlp.gate": {
4362
  "group_size": 64,
4363
  "bits": 4
@@ -4374,7 +3692,6 @@
4374
  "group_size": 64,
4375
  "bits": 6
4376
  },
4377
- "model.layers.51.mlp.switch_mlp.activation": false,
4378
  "model.layers.52.self_attn.q_proj": {
4379
  "group_size": 64,
4380
  "bits": 4
@@ -4391,11 +3708,6 @@
4391
  "group_size": 64,
4392
  "bits": 4
4393
  },
4394
- "model.layers.52.self_attn.q_norm": false,
4395
- "model.layers.52.self_attn.k_norm": false,
4396
- "model.layers.52.self_attn.rope": false,
4397
- "model.layers.52.input_layernorm": false,
4398
- "model.layers.52.post_attention_layernorm": false,
4399
  "model.layers.52.mlp.gate": {
4400
  "group_size": 64,
4401
  "bits": 4
@@ -4412,7 +3724,6 @@
4412
  "group_size": 64,
4413
  "bits": 4
4414
  },
4415
- "model.layers.52.mlp.switch_mlp.activation": false,
4416
  "model.layers.53.self_attn.q_proj": {
4417
  "group_size": 64,
4418
  "bits": 4
@@ -4429,11 +3740,6 @@
4429
  "group_size": 64,
4430
  "bits": 4
4431
  },
4432
- "model.layers.53.self_attn.q_norm": false,
4433
- "model.layers.53.self_attn.k_norm": false,
4434
- "model.layers.53.self_attn.rope": false,
4435
- "model.layers.53.input_layernorm": false,
4436
- "model.layers.53.post_attention_layernorm": false,
4437
  "model.layers.53.mlp.gate": {
4438
  "group_size": 64,
4439
  "bits": 4
@@ -4450,7 +3756,6 @@
4450
  "group_size": 64,
4451
  "bits": 4
4452
  },
4453
- "model.layers.53.mlp.switch_mlp.activation": false,
4454
  "model.layers.54.self_attn.q_proj": {
4455
  "group_size": 64,
4456
  "bits": 4
@@ -4467,11 +3772,6 @@
4467
  "group_size": 64,
4468
  "bits": 4
4469
  },
4470
- "model.layers.54.self_attn.q_norm": false,
4471
- "model.layers.54.self_attn.k_norm": false,
4472
- "model.layers.54.self_attn.rope": false,
4473
- "model.layers.54.input_layernorm": false,
4474
- "model.layers.54.post_attention_layernorm": false,
4475
  "model.layers.54.mlp.gate": {
4476
  "group_size": 64,
4477
  "bits": 4
@@ -4488,7 +3788,6 @@
4488
  "group_size": 64,
4489
  "bits": 6
4490
  },
4491
- "model.layers.54.mlp.switch_mlp.activation": false,
4492
  "model.layers.55.self_attn.q_proj": {
4493
  "group_size": 64,
4494
  "bits": 4
@@ -4505,11 +3804,6 @@
4505
  "group_size": 64,
4506
  "bits": 4
4507
  },
4508
- "model.layers.55.self_attn.q_norm": false,
4509
- "model.layers.55.self_attn.k_norm": false,
4510
- "model.layers.55.self_attn.rope": false,
4511
- "model.layers.55.input_layernorm": false,
4512
- "model.layers.55.post_attention_layernorm": false,
4513
  "model.layers.55.mlp.gate": {
4514
  "group_size": 64,
4515
  "bits": 4
@@ -4526,7 +3820,6 @@
4526
  "group_size": 64,
4527
  "bits": 6
4528
  },
4529
- "model.layers.55.mlp.switch_mlp.activation": false,
4530
  "model.layers.56.self_attn.q_proj": {
4531
  "group_size": 64,
4532
  "bits": 4
@@ -4543,11 +3836,6 @@
4543
  "group_size": 64,
4544
  "bits": 4
4545
  },
4546
- "model.layers.56.self_attn.q_norm": false,
4547
- "model.layers.56.self_attn.k_norm": false,
4548
- "model.layers.56.self_attn.rope": false,
4549
- "model.layers.56.input_layernorm": false,
4550
- "model.layers.56.post_attention_layernorm": false,
4551
  "model.layers.56.mlp.gate": {
4552
  "group_size": 64,
4553
  "bits": 4
@@ -4564,7 +3852,6 @@
4564
  "group_size": 64,
4565
  "bits": 6
4566
  },
4567
- "model.layers.56.mlp.switch_mlp.activation": false,
4568
  "model.layers.57.self_attn.q_proj": {
4569
  "group_size": 64,
4570
  "bits": 4
@@ -4581,11 +3868,6 @@
4581
  "group_size": 64,
4582
  "bits": 4
4583
  },
4584
- "model.layers.57.self_attn.q_norm": false,
4585
- "model.layers.57.self_attn.k_norm": false,
4586
- "model.layers.57.self_attn.rope": false,
4587
- "model.layers.57.input_layernorm": false,
4588
- "model.layers.57.post_attention_layernorm": false,
4589
  "model.layers.57.mlp.gate": {
4590
  "group_size": 64,
4591
  "bits": 4
@@ -4602,7 +3884,6 @@
4602
  "group_size": 64,
4603
  "bits": 6
4604
  },
4605
- "model.layers.57.mlp.switch_mlp.activation": false,
4606
  "model.layers.58.self_attn.q_proj": {
4607
  "group_size": 64,
4608
  "bits": 4
@@ -4619,11 +3900,6 @@
4619
  "group_size": 64,
4620
  "bits": 4
4621
  },
4622
- "model.layers.58.self_attn.q_norm": false,
4623
- "model.layers.58.self_attn.k_norm": false,
4624
- "model.layers.58.self_attn.rope": false,
4625
- "model.layers.58.input_layernorm": false,
4626
- "model.layers.58.post_attention_layernorm": false,
4627
  "model.layers.58.mlp.gate": {
4628
  "group_size": 64,
4629
  "bits": 4
@@ -4640,7 +3916,6 @@
4640
  "group_size": 64,
4641
  "bits": 6
4642
  },
4643
- "model.layers.58.mlp.switch_mlp.activation": false,
4644
  "model.layers.59.self_attn.q_proj": {
4645
  "group_size": 64,
4646
  "bits": 4
@@ -4657,11 +3932,6 @@
4657
  "group_size": 64,
4658
  "bits": 4
4659
  },
4660
- "model.layers.59.self_attn.q_norm": false,
4661
- "model.layers.59.self_attn.k_norm": false,
4662
- "model.layers.59.self_attn.rope": false,
4663
- "model.layers.59.input_layernorm": false,
4664
- "model.layers.59.post_attention_layernorm": false,
4665
  "model.layers.59.mlp.gate": {
4666
  "group_size": 64,
4667
  "bits": 4
@@ -4678,7 +3948,6 @@
4678
  "group_size": 64,
4679
  "bits": 6
4680
  },
4681
- "model.layers.59.mlp.switch_mlp.activation": false,
4682
  "model.layers.60.self_attn.q_proj": {
4683
  "group_size": 64,
4684
  "bits": 4
@@ -4695,11 +3964,6 @@
4695
  "group_size": 64,
4696
  "bits": 4
4697
  },
4698
- "model.layers.60.self_attn.q_norm": false,
4699
- "model.layers.60.self_attn.k_norm": false,
4700
- "model.layers.60.self_attn.rope": false,
4701
- "model.layers.60.input_layernorm": false,
4702
- "model.layers.60.post_attention_layernorm": false,
4703
  "model.layers.60.mlp.gate": {
4704
  "group_size": 64,
4705
  "bits": 4
@@ -4716,7 +3980,6 @@
4716
  "group_size": 64,
4717
  "bits": 6
4718
  },
4719
- "model.layers.60.mlp.switch_mlp.activation": false,
4720
  "model.layers.61.self_attn.q_proj": {
4721
  "group_size": 64,
4722
  "bits": 4
@@ -4733,11 +3996,6 @@
4733
  "group_size": 64,
4734
  "bits": 4
4735
  },
4736
- "model.layers.61.self_attn.q_norm": false,
4737
- "model.layers.61.self_attn.k_norm": false,
4738
- "model.layers.61.self_attn.rope": false,
4739
- "model.layers.61.input_layernorm": false,
4740
- "model.layers.61.post_attention_layernorm": false,
4741
  "model.layers.61.mlp.gate": {
4742
  "group_size": 64,
4743
  "bits": 4
@@ -4754,8 +4012,6 @@
4754
  "group_size": 64,
4755
  "bits": 6
4756
  },
4757
- "model.layers.61.mlp.switch_mlp.activation": false,
4758
- "model.norm": false,
4759
  "lm_head": {
4760
  "group_size": 64,
4761
  "bits": 6
 
26
  "quantization": {
27
  "group_size": 64,
28
  "bits": 4,
29
+ "mode": "affine",
30
  "model.embed_tokens": {
31
  "group_size": 64,
32
  "bits": 4
 
47
  "group_size": 64,
48
  "bits": 4
49
  },
 
 
 
 
 
50
  "model.layers.0.mlp.gate": {
51
  "group_size": 64,
52
  "bits": 4
 
63
  "group_size": 64,
64
  "bits": 6
65
  },
 
66
  "model.layers.1.self_attn.q_proj": {
67
  "group_size": 64,
68
  "bits": 4
 
79
  "group_size": 64,
80
  "bits": 4
81
  },
 
 
 
 
 
82
  "model.layers.1.mlp.gate": {
83
  "group_size": 64,
84
  "bits": 4
 
95
  "group_size": 64,
96
  "bits": 6
97
  },
 
98
  "model.layers.2.self_attn.q_proj": {
99
  "group_size": 64,
100
  "bits": 4
 
111
  "group_size": 64,
112
  "bits": 4
113
  },
 
 
 
 
 
114
  "model.layers.2.mlp.gate": {
115
  "group_size": 64,
116
  "bits": 4
 
127
  "group_size": 64,
128
  "bits": 6
129
  },
 
130
  "model.layers.3.self_attn.q_proj": {
131
  "group_size": 64,
132
  "bits": 4
 
143
  "group_size": 64,
144
  "bits": 4
145
  },
 
 
 
 
 
146
  "model.layers.3.mlp.gate": {
147
  "group_size": 64,
148
  "bits": 4
 
159
  "group_size": 64,
160
  "bits": 6
161
  },
 
162
  "model.layers.4.self_attn.q_proj": {
163
  "group_size": 64,
164
  "bits": 4
 
175
  "group_size": 64,
176
  "bits": 4
177
  },
 
 
 
 
 
178
  "model.layers.4.mlp.gate": {
179
  "group_size": 64,
180
  "bits": 4
 
191
  "group_size": 64,
192
  "bits": 6
193
  },
 
194
  "model.layers.5.self_attn.q_proj": {
195
  "group_size": 64,
196
  "bits": 4
 
207
  "group_size": 64,
208
  "bits": 4
209
  },
 
 
 
 
 
210
  "model.layers.5.mlp.gate": {
211
  "group_size": 64,
212
  "bits": 4
 
223
  "group_size": 64,
224
  "bits": 6
225
  },
 
226
  "model.layers.6.self_attn.q_proj": {
227
  "group_size": 64,
228
  "bits": 4
 
239
  "group_size": 64,
240
  "bits": 4
241
  },
 
 
 
 
 
242
  "model.layers.6.mlp.gate": {
243
  "group_size": 64,
244
  "bits": 4
 
255
  "group_size": 64,
256
  "bits": 6
257
  },
 
258
  "model.layers.7.self_attn.q_proj": {
259
  "group_size": 64,
260
  "bits": 4
 
271
  "group_size": 64,
272
  "bits": 4
273
  },
 
 
 
 
 
274
  "model.layers.7.mlp.gate": {
275
  "group_size": 64,
276
  "bits": 4
 
287
  "group_size": 64,
288
  "bits": 4
289
  },
 
290
  "model.layers.8.self_attn.q_proj": {
291
  "group_size": 64,
292
  "bits": 4
 
303
  "group_size": 64,
304
  "bits": 4
305
  },
 
 
 
 
 
306
  "model.layers.8.mlp.gate": {
307
  "group_size": 64,
308
  "bits": 4
 
319
  "group_size": 64,
320
  "bits": 4
321
  },
 
322
  "model.layers.9.self_attn.q_proj": {
323
  "group_size": 64,
324
  "bits": 4
 
335
  "group_size": 64,
336
  "bits": 4
337
  },
 
 
 
 
 
338
  "model.layers.9.mlp.gate": {
339
  "group_size": 64,
340
  "bits": 4
 
351
  "group_size": 64,
352
  "bits": 6
353
  },
 
354
  "model.layers.10.self_attn.q_proj": {
355
  "group_size": 64,
356
  "bits": 4
 
367
  "group_size": 64,
368
  "bits": 4
369
  },
 
 
 
 
 
370
  "model.layers.10.mlp.gate": {
371
  "group_size": 64,
372
  "bits": 4
 
383
  "group_size": 64,
384
  "bits": 4
385
  },
 
386
  "model.layers.11.self_attn.q_proj": {
387
  "group_size": 64,
388
  "bits": 4
 
399
  "group_size": 64,
400
  "bits": 4
401
  },
 
 
 
 
 
402
  "model.layers.11.mlp.gate": {
403
  "group_size": 64,
404
  "bits": 4
 
415
  "group_size": 64,
416
  "bits": 4
417
  },
 
418
  "model.layers.12.self_attn.q_proj": {
419
  "group_size": 64,
420
  "bits": 4
 
431
  "group_size": 64,
432
  "bits": 4
433
  },
 
 
 
 
 
434
  "model.layers.12.mlp.gate": {
435
  "group_size": 64,
436
  "bits": 4
 
447
  "group_size": 64,
448
  "bits": 6
449
  },
 
450
  "model.layers.13.self_attn.q_proj": {
451
  "group_size": 64,
452
  "bits": 4
 
463
  "group_size": 64,
464
  "bits": 4
465
  },
 
 
 
 
 
466
  "model.layers.13.mlp.gate": {
467
  "group_size": 64,
468
  "bits": 4
 
479
  "group_size": 64,
480
  "bits": 4
481
  },
 
482
  "model.layers.14.self_attn.q_proj": {
483
  "group_size": 64,
484
  "bits": 4
 
495
  "group_size": 64,
496
  "bits": 4
497
  },
 
 
 
 
 
498
  "model.layers.14.mlp.gate": {
499
  "group_size": 64,
500
  "bits": 4
 
511
  "group_size": 64,
512
  "bits": 4
513
  },
 
514
  "model.layers.15.self_attn.q_proj": {
515
  "group_size": 64,
516
  "bits": 4
 
527
  "group_size": 64,
528
  "bits": 4
529
  },
 
 
 
 
 
530
  "model.layers.15.mlp.gate": {
531
  "group_size": 64,
532
  "bits": 4
 
543
  "group_size": 64,
544
  "bits": 6
545
  },
 
546
  "model.layers.16.self_attn.q_proj": {
547
  "group_size": 64,
548
  "bits": 4
 
559
  "group_size": 64,
560
  "bits": 4
561
  },
 
 
 
 
 
562
  "model.layers.16.mlp.gate": {
563
  "group_size": 64,
564
  "bits": 4
 
575
  "group_size": 64,
576
  "bits": 4
577
  },
 
578
  "model.layers.17.self_attn.q_proj": {
579
  "group_size": 64,
580
  "bits": 4
 
591
  "group_size": 64,
592
  "bits": 4
593
  },
 
 
 
 
 
594
  "model.layers.17.mlp.gate": {
595
  "group_size": 64,
596
  "bits": 4
 
607
  "group_size": 64,
608
  "bits": 4
609
  },
 
610
  "model.layers.18.self_attn.q_proj": {
611
  "group_size": 64,
612
  "bits": 4
 
623
  "group_size": 64,
624
  "bits": 4
625
  },
 
 
 
 
 
626
  "model.layers.18.mlp.gate": {
627
  "group_size": 64,
628
  "bits": 4
 
639
  "group_size": 64,
640
  "bits": 6
641
  },
 
642
  "model.layers.19.self_attn.q_proj": {
643
  "group_size": 64,
644
  "bits": 4
 
655
  "group_size": 64,
656
  "bits": 4
657
  },
 
 
 
 
 
658
  "model.layers.19.mlp.gate": {
659
  "group_size": 64,
660
  "bits": 4
 
671
  "group_size": 64,
672
  "bits": 4
673
  },
 
674
  "model.layers.20.self_attn.q_proj": {
675
  "group_size": 64,
676
  "bits": 4
 
687
  "group_size": 64,
688
  "bits": 4
689
  },
 
 
 
 
 
690
  "model.layers.20.mlp.gate": {
691
  "group_size": 64,
692
  "bits": 4
 
703
  "group_size": 64,
704
  "bits": 4
705
  },
 
706
  "model.layers.21.self_attn.q_proj": {
707
  "group_size": 64,
708
  "bits": 4
 
719
  "group_size": 64,
720
  "bits": 4
721
  },
 
 
 
 
 
722
  "model.layers.21.mlp.gate": {
723
  "group_size": 64,
724
  "bits": 4
 
735
  "group_size": 64,
736
  "bits": 6
737
  },
 
738
  "model.layers.22.self_attn.q_proj": {
739
  "group_size": 64,
740
  "bits": 4
 
751
  "group_size": 64,
752
  "bits": 4
753
  },
 
 
 
 
 
754
  "model.layers.22.mlp.gate": {
755
  "group_size": 64,
756
  "bits": 4
 
767
  "group_size": 64,
768
  "bits": 4
769
  },
 
770
  "model.layers.23.self_attn.q_proj": {
771
  "group_size": 64,
772
  "bits": 4
 
783
  "group_size": 64,
784
  "bits": 4
785
  },
 
 
 
 
 
786
  "model.layers.23.mlp.gate": {
787
  "group_size": 64,
788
  "bits": 4
 
799
  "group_size": 64,
800
  "bits": 4
801
  },
 
802
  "model.layers.24.self_attn.q_proj": {
803
  "group_size": 64,
804
  "bits": 4
 
815
  "group_size": 64,
816
  "bits": 4
817
  },
 
 
 
 
 
818
  "model.layers.24.mlp.gate": {
819
  "group_size": 64,
820
  "bits": 4
 
831
  "group_size": 64,
832
  "bits": 6
833
  },
 
834
  "model.layers.25.self_attn.q_proj": {
835
  "group_size": 64,
836
  "bits": 4
 
847
  "group_size": 64,
848
  "bits": 4
849
  },
 
 
 
 
 
850
  "model.layers.25.mlp.gate": {
851
  "group_size": 64,
852
  "bits": 4
 
863
  "group_size": 64,
864
  "bits": 4
865
  },
 
866
  "model.layers.26.self_attn.q_proj": {
867
  "group_size": 64,
868
  "bits": 4
 
879
  "group_size": 64,
880
  "bits": 4
881
  },
 
 
 
 
 
882
  "model.layers.26.mlp.gate": {
883
  "group_size": 64,
884
  "bits": 4
 
895
  "group_size": 64,
896
  "bits": 4
897
  },
 
898
  "model.layers.27.self_attn.q_proj": {
899
  "group_size": 64,
900
  "bits": 4
 
911
  "group_size": 64,
912
  "bits": 4
913
  },
 
 
 
 
 
914
  "model.layers.27.mlp.gate": {
915
  "group_size": 64,
916
  "bits": 4
 
927
  "group_size": 64,
928
  "bits": 6
929
  },
 
930
  "model.layers.28.self_attn.q_proj": {
931
  "group_size": 64,
932
  "bits": 4
 
943
  "group_size": 64,
944
  "bits": 4
945
  },
 
 
 
 
 
946
  "model.layers.28.mlp.gate": {
947
  "group_size": 64,
948
  "bits": 4
 
959
  "group_size": 64,
960
  "bits": 4
961
  },
 
962
  "model.layers.29.self_attn.q_proj": {
963
  "group_size": 64,
964
  "bits": 4
 
975
  "group_size": 64,
976
  "bits": 4
977
  },
 
 
 
 
 
978
  "model.layers.29.mlp.gate": {
979
  "group_size": 64,
980
  "bits": 4
 
991
  "group_size": 64,
992
  "bits": 4
993
  },
 
994
  "model.layers.30.self_attn.q_proj": {
995
  "group_size": 64,
996
  "bits": 4
 
1007
  "group_size": 64,
1008
  "bits": 4
1009
  },
 
 
 
 
 
1010
  "model.layers.30.mlp.gate": {
1011
  "group_size": 64,
1012
  "bits": 4
 
1023
  "group_size": 64,
1024
  "bits": 6
1025
  },
 
1026
  "model.layers.31.self_attn.q_proj": {
1027
  "group_size": 64,
1028
  "bits": 4
 
1039
  "group_size": 64,
1040
  "bits": 4
1041
  },
 
 
 
 
 
1042
  "model.layers.31.mlp.gate": {
1043
  "group_size": 64,
1044
  "bits": 4
 
1055
  "group_size": 64,
1056
  "bits": 4
1057
  },
 
1058
  "model.layers.32.self_attn.q_proj": {
1059
  "group_size": 64,
1060
  "bits": 4
 
1071
  "group_size": 64,
1072
  "bits": 4
1073
  },
 
 
 
 
 
1074
  "model.layers.32.mlp.gate": {
1075
  "group_size": 64,
1076
  "bits": 4
 
1087
  "group_size": 64,
1088
  "bits": 4
1089
  },
 
1090
  "model.layers.33.self_attn.q_proj": {
1091
  "group_size": 64,
1092
  "bits": 4
 
1103
  "group_size": 64,
1104
  "bits": 4
1105
  },
 
 
 
 
 
1106
  "model.layers.33.mlp.gate": {
1107
  "group_size": 64,
1108
  "bits": 4
 
1119
  "group_size": 64,
1120
  "bits": 6
1121
  },
 
1122
  "model.layers.34.self_attn.q_proj": {
1123
  "group_size": 64,
1124
  "bits": 4
 
1135
  "group_size": 64,
1136
  "bits": 4
1137
  },
 
 
 
 
 
1138
  "model.layers.34.mlp.gate": {
1139
  "group_size": 64,
1140
  "bits": 4
 
1151
  "group_size": 64,
1152
  "bits": 4
1153
  },
 
1154
  "model.layers.35.self_attn.q_proj": {
1155
  "group_size": 64,
1156
  "bits": 4
 
1167
  "group_size": 64,
1168
  "bits": 4
1169
  },
 
 
 
 
 
1170
  "model.layers.35.mlp.gate": {
1171
  "group_size": 64,
1172
  "bits": 4
 
1183
  "group_size": 64,
1184
  "bits": 4
1185
  },
 
1186
  "model.layers.36.self_attn.q_proj": {
1187
  "group_size": 64,
1188
  "bits": 4
 
1199
  "group_size": 64,
1200
  "bits": 4
1201
  },
 
 
 
 
 
1202
  "model.layers.36.mlp.gate": {
1203
  "group_size": 64,
1204
  "bits": 4
 
1215
  "group_size": 64,
1216
  "bits": 6
1217
  },
 
1218
  "model.layers.37.self_attn.q_proj": {
1219
  "group_size": 64,
1220
  "bits": 4
 
1231
  "group_size": 64,
1232
  "bits": 4
1233
  },
 
 
 
 
 
1234
  "model.layers.37.mlp.gate": {
1235
  "group_size": 64,
1236
  "bits": 4
 
1247
  "group_size": 64,
1248
  "bits": 4
1249
  },
 
1250
  "model.layers.38.self_attn.q_proj": {
1251
  "group_size": 64,
1252
  "bits": 4
 
1263
  "group_size": 64,
1264
  "bits": 4
1265
  },
 
 
 
 
 
1266
  "model.layers.38.mlp.gate": {
1267
  "group_size": 64,
1268
  "bits": 4
 
1279
  "group_size": 64,
1280
  "bits": 4
1281
  },
 
1282
  "model.layers.39.self_attn.q_proj": {
1283
  "group_size": 64,
1284
  "bits": 4
 
1295
  "group_size": 64,
1296
  "bits": 4
1297
  },
 
 
 
 
 
1298
  "model.layers.39.mlp.gate": {
1299
  "group_size": 64,
1300
  "bits": 4
 
1311
  "group_size": 64,
1312
  "bits": 6
1313
  },
 
1314
  "model.layers.40.self_attn.q_proj": {
1315
  "group_size": 64,
1316
  "bits": 4
 
1327
  "group_size": 64,
1328
  "bits": 4
1329
  },
 
 
 
 
 
1330
  "model.layers.40.mlp.gate": {
1331
  "group_size": 64,
1332
  "bits": 4
 
1343
  "group_size": 64,
1344
  "bits": 4
1345
  },
 
1346
  "model.layers.41.self_attn.q_proj": {
1347
  "group_size": 64,
1348
  "bits": 4
 
1359
  "group_size": 64,
1360
  "bits": 4
1361
  },
 
 
 
 
 
1362
  "model.layers.41.mlp.gate": {
1363
  "group_size": 64,
1364
  "bits": 4
 
1375
  "group_size": 64,
1376
  "bits": 4
1377
  },
 
1378
  "model.layers.42.self_attn.q_proj": {
1379
  "group_size": 64,
1380
  "bits": 4
 
1391
  "group_size": 64,
1392
  "bits": 4
1393
  },
 
 
 
 
 
1394
  "model.layers.42.mlp.gate": {
1395
  "group_size": 64,
1396
  "bits": 4
 
1407
  "group_size": 64,
1408
  "bits": 6
1409
  },
 
1410
  "model.layers.43.self_attn.q_proj": {
1411
  "group_size": 64,
1412
  "bits": 4
 
1423
  "group_size": 64,
1424
  "bits": 4
1425
  },
 
 
 
 
 
1426
  "model.layers.43.mlp.gate": {
1427
  "group_size": 64,
1428
  "bits": 4
 
1439
  "group_size": 64,
1440
  "bits": 4
1441
  },
 
1442
  "model.layers.44.self_attn.q_proj": {
1443
  "group_size": 64,
1444
  "bits": 4
 
1455
  "group_size": 64,
1456
  "bits": 4
1457
  },
 
 
 
 
 
1458
  "model.layers.44.mlp.gate": {
1459
  "group_size": 64,
1460
  "bits": 4
 
1471
  "group_size": 64,
1472
  "bits": 4
1473
  },
 
1474
  "model.layers.45.self_attn.q_proj": {
1475
  "group_size": 64,
1476
  "bits": 4
 
1487
  "group_size": 64,
1488
  "bits": 4
1489
  },
 
 
 
 
 
1490
  "model.layers.45.mlp.gate": {
1491
  "group_size": 64,
1492
  "bits": 4
 
1503
  "group_size": 64,
1504
  "bits": 6
1505
  },
 
1506
  "model.layers.46.self_attn.q_proj": {
1507
  "group_size": 64,
1508
  "bits": 4
 
1519
  "group_size": 64,
1520
  "bits": 4
1521
  },
 
 
 
 
 
1522
  "model.layers.46.mlp.gate": {
1523
  "group_size": 64,
1524
  "bits": 4
 
1535
  "group_size": 64,
1536
  "bits": 4
1537
  },
 
1538
  "model.layers.47.self_attn.q_proj": {
1539
  "group_size": 64,
1540
  "bits": 4
 
1551
  "group_size": 64,
1552
  "bits": 4
1553
  },
 
 
 
 
 
1554
  "model.layers.47.mlp.gate": {
1555
  "group_size": 64,
1556
  "bits": 4
 
1567
  "group_size": 64,
1568
  "bits": 4
1569
  },
 
1570
  "model.layers.48.self_attn.q_proj": {
1571
  "group_size": 64,
1572
  "bits": 4
 
1583
  "group_size": 64,
1584
  "bits": 4
1585
  },
 
 
 
 
 
1586
  "model.layers.48.mlp.gate": {
1587
  "group_size": 64,
1588
  "bits": 4
 
1599
  "group_size": 64,
1600
  "bits": 6
1601
  },
 
1602
  "model.layers.49.self_attn.q_proj": {
1603
  "group_size": 64,
1604
  "bits": 4
 
1615
  "group_size": 64,
1616
  "bits": 4
1617
  },
 
 
 
 
 
1618
  "model.layers.49.mlp.gate": {
1619
  "group_size": 64,
1620
  "bits": 4
 
1631
  "group_size": 64,
1632
  "bits": 4
1633
  },
 
1634
  "model.layers.50.self_attn.q_proj": {
1635
  "group_size": 64,
1636
  "bits": 4
 
1647
  "group_size": 64,
1648
  "bits": 4
1649
  },
 
 
 
 
 
1650
  "model.layers.50.mlp.gate": {
1651
  "group_size": 64,
1652
  "bits": 4
 
1663
  "group_size": 64,
1664
  "bits": 4
1665
  },
 
1666
  "model.layers.51.self_attn.q_proj": {
1667
  "group_size": 64,
1668
  "bits": 4
 
1679
  "group_size": 64,
1680
  "bits": 4
1681
  },
 
 
 
 
 
1682
  "model.layers.51.mlp.gate": {
1683
  "group_size": 64,
1684
  "bits": 4
 
1695
  "group_size": 64,
1696
  "bits": 6
1697
  },
 
1698
  "model.layers.52.self_attn.q_proj": {
1699
  "group_size": 64,
1700
  "bits": 4
 
1711
  "group_size": 64,
1712
  "bits": 4
1713
  },
 
 
 
 
 
1714
  "model.layers.52.mlp.gate": {
1715
  "group_size": 64,
1716
  "bits": 4
 
1727
  "group_size": 64,
1728
  "bits": 4
1729
  },
 
1730
  "model.layers.53.self_attn.q_proj": {
1731
  "group_size": 64,
1732
  "bits": 4
 
1743
  "group_size": 64,
1744
  "bits": 4
1745
  },
 
 
 
 
 
1746
  "model.layers.53.mlp.gate": {
1747
  "group_size": 64,
1748
  "bits": 4
 
1759
  "group_size": 64,
1760
  "bits": 4
1761
  },
 
1762
  "model.layers.54.self_attn.q_proj": {
1763
  "group_size": 64,
1764
  "bits": 4
 
1775
  "group_size": 64,
1776
  "bits": 4
1777
  },
 
 
 
 
 
1778
  "model.layers.54.mlp.gate": {
1779
  "group_size": 64,
1780
  "bits": 4
 
1791
  "group_size": 64,
1792
  "bits": 6
1793
  },
 
1794
  "model.layers.55.self_attn.q_proj": {
1795
  "group_size": 64,
1796
  "bits": 4
 
1807
  "group_size": 64,
1808
  "bits": 4
1809
  },
 
 
 
 
 
1810
  "model.layers.55.mlp.gate": {
1811
  "group_size": 64,
1812
  "bits": 4
 
1823
  "group_size": 64,
1824
  "bits": 6
1825
  },
 
1826
  "model.layers.56.self_attn.q_proj": {
1827
  "group_size": 64,
1828
  "bits": 4
 
1839
  "group_size": 64,
1840
  "bits": 4
1841
  },
 
 
 
 
 
1842
  "model.layers.56.mlp.gate": {
1843
  "group_size": 64,
1844
  "bits": 4
 
1855
  "group_size": 64,
1856
  "bits": 6
1857
  },
 
1858
  "model.layers.57.self_attn.q_proj": {
1859
  "group_size": 64,
1860
  "bits": 4
 
1871
  "group_size": 64,
1872
  "bits": 4
1873
  },
 
 
 
 
 
1874
  "model.layers.57.mlp.gate": {
1875
  "group_size": 64,
1876
  "bits": 4
 
1887
  "group_size": 64,
1888
  "bits": 6
1889
  },
 
1890
  "model.layers.58.self_attn.q_proj": {
1891
  "group_size": 64,
1892
  "bits": 4
 
1903
  "group_size": 64,
1904
  "bits": 4
1905
  },
 
 
 
 
 
1906
  "model.layers.58.mlp.gate": {
1907
  "group_size": 64,
1908
  "bits": 4
 
1919
  "group_size": 64,
1920
  "bits": 6
1921
  },
 
1922
  "model.layers.59.self_attn.q_proj": {
1923
  "group_size": 64,
1924
  "bits": 4
 
1935
  "group_size": 64,
1936
  "bits": 4
1937
  },
 
 
 
 
 
1938
  "model.layers.59.mlp.gate": {
1939
  "group_size": 64,
1940
  "bits": 4
 
1951
  "group_size": 64,
1952
  "bits": 6
1953
  },
 
1954
  "model.layers.60.self_attn.q_proj": {
1955
  "group_size": 64,
1956
  "bits": 4
 
1967
  "group_size": 64,
1968
  "bits": 4
1969
  },
 
 
 
 
 
1970
  "model.layers.60.mlp.gate": {
1971
  "group_size": 64,
1972
  "bits": 4
 
1983
  "group_size": 64,
1984
  "bits": 6
1985
  },
 
1986
  "model.layers.61.self_attn.q_proj": {
1987
  "group_size": 64,
1988
  "bits": 4
 
1999
  "group_size": 64,
2000
  "bits": 4
2001
  },
 
 
 
 
 
2002
  "model.layers.61.mlp.gate": {
2003
  "group_size": 64,
2004
  "bits": 4
 
2015
  "group_size": 64,
2016
  "bits": 6
2017
  },
 
 
2018
  "lm_head": {
2019
  "group_size": 64,
2020
  "bits": 6
 
2023
  "quantization_config": {
2024
  "group_size": 64,
2025
  "bits": 4,
2026
+ "mode": "affine",
2027
  "model.embed_tokens": {
2028
  "group_size": 64,
2029
  "bits": 4
 
2044
  "group_size": 64,
2045
  "bits": 4
2046
  },
 
 
 
 
 
2047
  "model.layers.0.mlp.gate": {
2048
  "group_size": 64,
2049
  "bits": 4
 
2060
  "group_size": 64,
2061
  "bits": 6
2062
  },
 
2063
  "model.layers.1.self_attn.q_proj": {
2064
  "group_size": 64,
2065
  "bits": 4
 
2076
  "group_size": 64,
2077
  "bits": 4
2078
  },
 
 
 
 
 
2079
  "model.layers.1.mlp.gate": {
2080
  "group_size": 64,
2081
  "bits": 4
 
2092
  "group_size": 64,
2093
  "bits": 6
2094
  },
 
2095
  "model.layers.2.self_attn.q_proj": {
2096
  "group_size": 64,
2097
  "bits": 4
 
2108
  "group_size": 64,
2109
  "bits": 4
2110
  },
 
 
 
 
 
2111
  "model.layers.2.mlp.gate": {
2112
  "group_size": 64,
2113
  "bits": 4
 
2124
  "group_size": 64,
2125
  "bits": 6
2126
  },
 
2127
  "model.layers.3.self_attn.q_proj": {
2128
  "group_size": 64,
2129
  "bits": 4
 
2140
  "group_size": 64,
2141
  "bits": 4
2142
  },
 
 
 
 
 
2143
  "model.layers.3.mlp.gate": {
2144
  "group_size": 64,
2145
  "bits": 4
 
2156
  "group_size": 64,
2157
  "bits": 6
2158
  },
 
2159
  "model.layers.4.self_attn.q_proj": {
2160
  "group_size": 64,
2161
  "bits": 4
 
2172
  "group_size": 64,
2173
  "bits": 4
2174
  },
 
 
 
 
 
2175
  "model.layers.4.mlp.gate": {
2176
  "group_size": 64,
2177
  "bits": 4
 
2188
  "group_size": 64,
2189
  "bits": 6
2190
  },
 
2191
  "model.layers.5.self_attn.q_proj": {
2192
  "group_size": 64,
2193
  "bits": 4
 
2204
  "group_size": 64,
2205
  "bits": 4
2206
  },
 
 
 
 
 
2207
  "model.layers.5.mlp.gate": {
2208
  "group_size": 64,
2209
  "bits": 4
 
2220
  "group_size": 64,
2221
  "bits": 6
2222
  },
 
2223
  "model.layers.6.self_attn.q_proj": {
2224
  "group_size": 64,
2225
  "bits": 4
 
2236
  "group_size": 64,
2237
  "bits": 4
2238
  },
 
 
 
 
 
2239
  "model.layers.6.mlp.gate": {
2240
  "group_size": 64,
2241
  "bits": 4
 
2252
  "group_size": 64,
2253
  "bits": 6
2254
  },
 
2255
  "model.layers.7.self_attn.q_proj": {
2256
  "group_size": 64,
2257
  "bits": 4
 
2268
  "group_size": 64,
2269
  "bits": 4
2270
  },
 
 
 
 
 
2271
  "model.layers.7.mlp.gate": {
2272
  "group_size": 64,
2273
  "bits": 4
 
2284
  "group_size": 64,
2285
  "bits": 4
2286
  },
 
2287
  "model.layers.8.self_attn.q_proj": {
2288
  "group_size": 64,
2289
  "bits": 4
 
2300
  "group_size": 64,
2301
  "bits": 4
2302
  },
 
 
 
 
 
2303
  "model.layers.8.mlp.gate": {
2304
  "group_size": 64,
2305
  "bits": 4
 
2316
  "group_size": 64,
2317
  "bits": 4
2318
  },
 
2319
  "model.layers.9.self_attn.q_proj": {
2320
  "group_size": 64,
2321
  "bits": 4
 
2332
  "group_size": 64,
2333
  "bits": 4
2334
  },
 
 
 
 
 
2335
  "model.layers.9.mlp.gate": {
2336
  "group_size": 64,
2337
  "bits": 4
 
2348
  "group_size": 64,
2349
  "bits": 6
2350
  },
 
2351
  "model.layers.10.self_attn.q_proj": {
2352
  "group_size": 64,
2353
  "bits": 4
 
2364
  "group_size": 64,
2365
  "bits": 4
2366
  },
 
 
 
 
 
2367
  "model.layers.10.mlp.gate": {
2368
  "group_size": 64,
2369
  "bits": 4
 
2380
  "group_size": 64,
2381
  "bits": 4
2382
  },
 
2383
  "model.layers.11.self_attn.q_proj": {
2384
  "group_size": 64,
2385
  "bits": 4
 
2396
  "group_size": 64,
2397
  "bits": 4
2398
  },
 
 
 
 
 
2399
  "model.layers.11.mlp.gate": {
2400
  "group_size": 64,
2401
  "bits": 4
 
2412
  "group_size": 64,
2413
  "bits": 4
2414
  },
 
2415
  "model.layers.12.self_attn.q_proj": {
2416
  "group_size": 64,
2417
  "bits": 4
 
2428
  "group_size": 64,
2429
  "bits": 4
2430
  },
 
 
 
 
 
2431
  "model.layers.12.mlp.gate": {
2432
  "group_size": 64,
2433
  "bits": 4
 
2444
  "group_size": 64,
2445
  "bits": 6
2446
  },
 
2447
  "model.layers.13.self_attn.q_proj": {
2448
  "group_size": 64,
2449
  "bits": 4
 
2460
  "group_size": 64,
2461
  "bits": 4
2462
  },
 
 
 
 
 
2463
  "model.layers.13.mlp.gate": {
2464
  "group_size": 64,
2465
  "bits": 4
 
2476
  "group_size": 64,
2477
  "bits": 4
2478
  },
 
2479
  "model.layers.14.self_attn.q_proj": {
2480
  "group_size": 64,
2481
  "bits": 4
 
2492
  "group_size": 64,
2493
  "bits": 4
2494
  },
 
 
 
 
 
2495
  "model.layers.14.mlp.gate": {
2496
  "group_size": 64,
2497
  "bits": 4
 
2508
  "group_size": 64,
2509
  "bits": 4
2510
  },
 
2511
  "model.layers.15.self_attn.q_proj": {
2512
  "group_size": 64,
2513
  "bits": 4
 
2524
  "group_size": 64,
2525
  "bits": 4
2526
  },
 
 
 
 
 
2527
  "model.layers.15.mlp.gate": {
2528
  "group_size": 64,
2529
  "bits": 4
 
2540
  "group_size": 64,
2541
  "bits": 6
2542
  },
 
2543
  "model.layers.16.self_attn.q_proj": {
2544
  "group_size": 64,
2545
  "bits": 4
 
2556
  "group_size": 64,
2557
  "bits": 4
2558
  },
 
 
 
 
 
2559
  "model.layers.16.mlp.gate": {
2560
  "group_size": 64,
2561
  "bits": 4
 
2572
  "group_size": 64,
2573
  "bits": 4
2574
  },
 
2575
  "model.layers.17.self_attn.q_proj": {
2576
  "group_size": 64,
2577
  "bits": 4
 
2588
  "group_size": 64,
2589
  "bits": 4
2590
  },
 
 
 
 
 
2591
  "model.layers.17.mlp.gate": {
2592
  "group_size": 64,
2593
  "bits": 4
 
2604
  "group_size": 64,
2605
  "bits": 4
2606
  },
 
2607
  "model.layers.18.self_attn.q_proj": {
2608
  "group_size": 64,
2609
  "bits": 4
 
2620
  "group_size": 64,
2621
  "bits": 4
2622
  },
 
 
 
 
 
2623
  "model.layers.18.mlp.gate": {
2624
  "group_size": 64,
2625
  "bits": 4
 
2636
  "group_size": 64,
2637
  "bits": 6
2638
  },
 
2639
  "model.layers.19.self_attn.q_proj": {
2640
  "group_size": 64,
2641
  "bits": 4
 
2652
  "group_size": 64,
2653
  "bits": 4
2654
  },
 
 
 
 
 
2655
  "model.layers.19.mlp.gate": {
2656
  "group_size": 64,
2657
  "bits": 4
 
2668
  "group_size": 64,
2669
  "bits": 4
2670
  },
 
2671
  "model.layers.20.self_attn.q_proj": {
2672
  "group_size": 64,
2673
  "bits": 4
 
2684
  "group_size": 64,
2685
  "bits": 4
2686
  },
 
 
 
 
 
2687
  "model.layers.20.mlp.gate": {
2688
  "group_size": 64,
2689
  "bits": 4
 
2700
  "group_size": 64,
2701
  "bits": 4
2702
  },
 
2703
  "model.layers.21.self_attn.q_proj": {
2704
  "group_size": 64,
2705
  "bits": 4
 
2716
  "group_size": 64,
2717
  "bits": 4
2718
  },
 
 
 
 
 
2719
  "model.layers.21.mlp.gate": {
2720
  "group_size": 64,
2721
  "bits": 4
 
2732
  "group_size": 64,
2733
  "bits": 6
2734
  },
 
2735
  "model.layers.22.self_attn.q_proj": {
2736
  "group_size": 64,
2737
  "bits": 4
 
2748
  "group_size": 64,
2749
  "bits": 4
2750
  },
 
 
 
 
 
2751
  "model.layers.22.mlp.gate": {
2752
  "group_size": 64,
2753
  "bits": 4
 
2764
  "group_size": 64,
2765
  "bits": 4
2766
  },
 
2767
  "model.layers.23.self_attn.q_proj": {
2768
  "group_size": 64,
2769
  "bits": 4
 
2780
  "group_size": 64,
2781
  "bits": 4
2782
  },
 
 
 
 
 
2783
  "model.layers.23.mlp.gate": {
2784
  "group_size": 64,
2785
  "bits": 4
 
2796
  "group_size": 64,
2797
  "bits": 4
2798
  },
 
2799
  "model.layers.24.self_attn.q_proj": {
2800
  "group_size": 64,
2801
  "bits": 4
 
2812
  "group_size": 64,
2813
  "bits": 4
2814
  },
 
 
 
 
 
2815
  "model.layers.24.mlp.gate": {
2816
  "group_size": 64,
2817
  "bits": 4
 
2828
  "group_size": 64,
2829
  "bits": 6
2830
  },
 
2831
  "model.layers.25.self_attn.q_proj": {
2832
  "group_size": 64,
2833
  "bits": 4
 
2844
  "group_size": 64,
2845
  "bits": 4
2846
  },
 
 
 
 
 
2847
  "model.layers.25.mlp.gate": {
2848
  "group_size": 64,
2849
  "bits": 4
 
2860
  "group_size": 64,
2861
  "bits": 4
2862
  },
 
2863
  "model.layers.26.self_attn.q_proj": {
2864
  "group_size": 64,
2865
  "bits": 4
 
2876
  "group_size": 64,
2877
  "bits": 4
2878
  },
 
 
 
 
 
2879
  "model.layers.26.mlp.gate": {
2880
  "group_size": 64,
2881
  "bits": 4
 
2892
  "group_size": 64,
2893
  "bits": 4
2894
  },
 
2895
  "model.layers.27.self_attn.q_proj": {
2896
  "group_size": 64,
2897
  "bits": 4
 
2908
  "group_size": 64,
2909
  "bits": 4
2910
  },
 
 
 
 
 
2911
  "model.layers.27.mlp.gate": {
2912
  "group_size": 64,
2913
  "bits": 4
 
2924
  "group_size": 64,
2925
  "bits": 6
2926
  },
 
2927
  "model.layers.28.self_attn.q_proj": {
2928
  "group_size": 64,
2929
  "bits": 4
 
2940
  "group_size": 64,
2941
  "bits": 4
2942
  },
 
 
 
 
 
2943
  "model.layers.28.mlp.gate": {
2944
  "group_size": 64,
2945
  "bits": 4
 
2956
  "group_size": 64,
2957
  "bits": 4
2958
  },
 
2959
  "model.layers.29.self_attn.q_proj": {
2960
  "group_size": 64,
2961
  "bits": 4
 
2972
  "group_size": 64,
2973
  "bits": 4
2974
  },
 
 
 
 
 
2975
  "model.layers.29.mlp.gate": {
2976
  "group_size": 64,
2977
  "bits": 4
 
2988
  "group_size": 64,
2989
  "bits": 4
2990
  },
 
2991
  "model.layers.30.self_attn.q_proj": {
2992
  "group_size": 64,
2993
  "bits": 4
 
3004
  "group_size": 64,
3005
  "bits": 4
3006
  },
 
 
 
 
 
3007
  "model.layers.30.mlp.gate": {
3008
  "group_size": 64,
3009
  "bits": 4
 
3020
  "group_size": 64,
3021
  "bits": 6
3022
  },
 
3023
  "model.layers.31.self_attn.q_proj": {
3024
  "group_size": 64,
3025
  "bits": 4
 
3036
  "group_size": 64,
3037
  "bits": 4
3038
  },
 
 
 
 
 
3039
  "model.layers.31.mlp.gate": {
3040
  "group_size": 64,
3041
  "bits": 4
 
3052
  "group_size": 64,
3053
  "bits": 4
3054
  },
 
3055
  "model.layers.32.self_attn.q_proj": {
3056
  "group_size": 64,
3057
  "bits": 4
 
3068
  "group_size": 64,
3069
  "bits": 4
3070
  },
 
 
 
 
 
3071
  "model.layers.32.mlp.gate": {
3072
  "group_size": 64,
3073
  "bits": 4
 
3084
  "group_size": 64,
3085
  "bits": 4
3086
  },
 
3087
  "model.layers.33.self_attn.q_proj": {
3088
  "group_size": 64,
3089
  "bits": 4
 
3100
  "group_size": 64,
3101
  "bits": 4
3102
  },
 
 
 
 
 
3103
  "model.layers.33.mlp.gate": {
3104
  "group_size": 64,
3105
  "bits": 4
 
3116
  "group_size": 64,
3117
  "bits": 6
3118
  },
 
3119
  "model.layers.34.self_attn.q_proj": {
3120
  "group_size": 64,
3121
  "bits": 4
 
3132
  "group_size": 64,
3133
  "bits": 4
3134
  },
 
 
 
 
 
3135
  "model.layers.34.mlp.gate": {
3136
  "group_size": 64,
3137
  "bits": 4
 
3148
  "group_size": 64,
3149
  "bits": 4
3150
  },
 
3151
  "model.layers.35.self_attn.q_proj": {
3152
  "group_size": 64,
3153
  "bits": 4
 
3164
  "group_size": 64,
3165
  "bits": 4
3166
  },
 
 
 
 
 
3167
  "model.layers.35.mlp.gate": {
3168
  "group_size": 64,
3169
  "bits": 4
 
3180
  "group_size": 64,
3181
  "bits": 4
3182
  },
 
3183
  "model.layers.36.self_attn.q_proj": {
3184
  "group_size": 64,
3185
  "bits": 4
 
3196
  "group_size": 64,
3197
  "bits": 4
3198
  },
 
 
 
 
 
3199
  "model.layers.36.mlp.gate": {
3200
  "group_size": 64,
3201
  "bits": 4
 
3212
  "group_size": 64,
3213
  "bits": 6
3214
  },
 
3215
  "model.layers.37.self_attn.q_proj": {
3216
  "group_size": 64,
3217
  "bits": 4
 
3228
  "group_size": 64,
3229
  "bits": 4
3230
  },
 
 
 
 
 
3231
  "model.layers.37.mlp.gate": {
3232
  "group_size": 64,
3233
  "bits": 4
 
3244
  "group_size": 64,
3245
  "bits": 4
3246
  },
 
3247
  "model.layers.38.self_attn.q_proj": {
3248
  "group_size": 64,
3249
  "bits": 4
 
3260
  "group_size": 64,
3261
  "bits": 4
3262
  },
 
 
 
 
 
3263
  "model.layers.38.mlp.gate": {
3264
  "group_size": 64,
3265
  "bits": 4
 
3276
  "group_size": 64,
3277
  "bits": 4
3278
  },
 
3279
  "model.layers.39.self_attn.q_proj": {
3280
  "group_size": 64,
3281
  "bits": 4
 
3292
  "group_size": 64,
3293
  "bits": 4
3294
  },
 
 
 
 
 
3295
  "model.layers.39.mlp.gate": {
3296
  "group_size": 64,
3297
  "bits": 4
 
3308
  "group_size": 64,
3309
  "bits": 6
3310
  },
 
3311
  "model.layers.40.self_attn.q_proj": {
3312
  "group_size": 64,
3313
  "bits": 4
 
3324
  "group_size": 64,
3325
  "bits": 4
3326
  },
 
 
 
 
 
3327
  "model.layers.40.mlp.gate": {
3328
  "group_size": 64,
3329
  "bits": 4
 
3340
  "group_size": 64,
3341
  "bits": 4
3342
  },
 
3343
  "model.layers.41.self_attn.q_proj": {
3344
  "group_size": 64,
3345
  "bits": 4
 
3356
  "group_size": 64,
3357
  "bits": 4
3358
  },
 
 
 
 
 
3359
  "model.layers.41.mlp.gate": {
3360
  "group_size": 64,
3361
  "bits": 4
 
3372
  "group_size": 64,
3373
  "bits": 4
3374
  },
 
3375
  "model.layers.42.self_attn.q_proj": {
3376
  "group_size": 64,
3377
  "bits": 4
 
3388
  "group_size": 64,
3389
  "bits": 4
3390
  },
 
 
 
 
 
3391
  "model.layers.42.mlp.gate": {
3392
  "group_size": 64,
3393
  "bits": 4
 
3404
  "group_size": 64,
3405
  "bits": 6
3406
  },
 
3407
  "model.layers.43.self_attn.q_proj": {
3408
  "group_size": 64,
3409
  "bits": 4
 
3420
  "group_size": 64,
3421
  "bits": 4
3422
  },
 
 
 
 
 
3423
  "model.layers.43.mlp.gate": {
3424
  "group_size": 64,
3425
  "bits": 4
 
3436
  "group_size": 64,
3437
  "bits": 4
3438
  },
 
3439
  "model.layers.44.self_attn.q_proj": {
3440
  "group_size": 64,
3441
  "bits": 4
 
3452
  "group_size": 64,
3453
  "bits": 4
3454
  },
 
 
 
 
 
3455
  "model.layers.44.mlp.gate": {
3456
  "group_size": 64,
3457
  "bits": 4
 
3468
  "group_size": 64,
3469
  "bits": 4
3470
  },
 
3471
  "model.layers.45.self_attn.q_proj": {
3472
  "group_size": 64,
3473
  "bits": 4
 
3484
  "group_size": 64,
3485
  "bits": 4
3486
  },
 
 
 
 
 
3487
  "model.layers.45.mlp.gate": {
3488
  "group_size": 64,
3489
  "bits": 4
 
3500
  "group_size": 64,
3501
  "bits": 6
3502
  },
 
3503
  "model.layers.46.self_attn.q_proj": {
3504
  "group_size": 64,
3505
  "bits": 4
 
3516
  "group_size": 64,
3517
  "bits": 4
3518
  },
 
 
 
 
 
3519
  "model.layers.46.mlp.gate": {
3520
  "group_size": 64,
3521
  "bits": 4
 
3532
  "group_size": 64,
3533
  "bits": 4
3534
  },
 
3535
  "model.layers.47.self_attn.q_proj": {
3536
  "group_size": 64,
3537
  "bits": 4
 
3548
  "group_size": 64,
3549
  "bits": 4
3550
  },
 
 
 
 
 
3551
  "model.layers.47.mlp.gate": {
3552
  "group_size": 64,
3553
  "bits": 4
 
3564
  "group_size": 64,
3565
  "bits": 4
3566
  },
 
3567
  "model.layers.48.self_attn.q_proj": {
3568
  "group_size": 64,
3569
  "bits": 4
 
3580
  "group_size": 64,
3581
  "bits": 4
3582
  },
 
 
 
 
 
3583
  "model.layers.48.mlp.gate": {
3584
  "group_size": 64,
3585
  "bits": 4
 
3596
  "group_size": 64,
3597
  "bits": 6
3598
  },
 
3599
  "model.layers.49.self_attn.q_proj": {
3600
  "group_size": 64,
3601
  "bits": 4
 
3612
  "group_size": 64,
3613
  "bits": 4
3614
  },
 
 
 
 
 
3615
  "model.layers.49.mlp.gate": {
3616
  "group_size": 64,
3617
  "bits": 4
 
3628
  "group_size": 64,
3629
  "bits": 4
3630
  },
 
3631
  "model.layers.50.self_attn.q_proj": {
3632
  "group_size": 64,
3633
  "bits": 4
 
3644
  "group_size": 64,
3645
  "bits": 4
3646
  },
 
 
 
 
 
3647
  "model.layers.50.mlp.gate": {
3648
  "group_size": 64,
3649
  "bits": 4
 
3660
  "group_size": 64,
3661
  "bits": 4
3662
  },
 
3663
  "model.layers.51.self_attn.q_proj": {
3664
  "group_size": 64,
3665
  "bits": 4
 
3676
  "group_size": 64,
3677
  "bits": 4
3678
  },
 
 
 
 
 
3679
  "model.layers.51.mlp.gate": {
3680
  "group_size": 64,
3681
  "bits": 4
 
3692
  "group_size": 64,
3693
  "bits": 6
3694
  },
 
3695
  "model.layers.52.self_attn.q_proj": {
3696
  "group_size": 64,
3697
  "bits": 4
 
3708
  "group_size": 64,
3709
  "bits": 4
3710
  },
 
 
 
 
 
3711
  "model.layers.52.mlp.gate": {
3712
  "group_size": 64,
3713
  "bits": 4
 
3724
  "group_size": 64,
3725
  "bits": 4
3726
  },
 
3727
  "model.layers.53.self_attn.q_proj": {
3728
  "group_size": 64,
3729
  "bits": 4
 
3740
  "group_size": 64,
3741
  "bits": 4
3742
  },
 
 
 
 
 
3743
  "model.layers.53.mlp.gate": {
3744
  "group_size": 64,
3745
  "bits": 4
 
3756
  "group_size": 64,
3757
  "bits": 4
3758
  },
 
3759
  "model.layers.54.self_attn.q_proj": {
3760
  "group_size": 64,
3761
  "bits": 4
 
3772
  "group_size": 64,
3773
  "bits": 4
3774
  },
 
 
 
 
 
3775
  "model.layers.54.mlp.gate": {
3776
  "group_size": 64,
3777
  "bits": 4
 
3788
  "group_size": 64,
3789
  "bits": 6
3790
  },
 
3791
  "model.layers.55.self_attn.q_proj": {
3792
  "group_size": 64,
3793
  "bits": 4
 
3804
  "group_size": 64,
3805
  "bits": 4
3806
  },
 
 
 
 
 
3807
  "model.layers.55.mlp.gate": {
3808
  "group_size": 64,
3809
  "bits": 4
 
3820
  "group_size": 64,
3821
  "bits": 6
3822
  },
 
3823
  "model.layers.56.self_attn.q_proj": {
3824
  "group_size": 64,
3825
  "bits": 4
 
3836
  "group_size": 64,
3837
  "bits": 4
3838
  },
 
 
 
 
 
3839
  "model.layers.56.mlp.gate": {
3840
  "group_size": 64,
3841
  "bits": 4
 
3852
  "group_size": 64,
3853
  "bits": 6
3854
  },
 
3855
  "model.layers.57.self_attn.q_proj": {
3856
  "group_size": 64,
3857
  "bits": 4
 
3868
  "group_size": 64,
3869
  "bits": 4
3870
  },
 
 
 
 
 
3871
  "model.layers.57.mlp.gate": {
3872
  "group_size": 64,
3873
  "bits": 4
 
3884
  "group_size": 64,
3885
  "bits": 6
3886
  },
 
3887
  "model.layers.58.self_attn.q_proj": {
3888
  "group_size": 64,
3889
  "bits": 4
 
3900
  "group_size": 64,
3901
  "bits": 4
3902
  },
 
 
 
 
 
3903
  "model.layers.58.mlp.gate": {
3904
  "group_size": 64,
3905
  "bits": 4
 
3916
  "group_size": 64,
3917
  "bits": 6
3918
  },
 
3919
  "model.layers.59.self_attn.q_proj": {
3920
  "group_size": 64,
3921
  "bits": 4
 
3932
  "group_size": 64,
3933
  "bits": 4
3934
  },
 
 
 
 
 
3935
  "model.layers.59.mlp.gate": {
3936
  "group_size": 64,
3937
  "bits": 4
 
3948
  "group_size": 64,
3949
  "bits": 6
3950
  },
 
3951
  "model.layers.60.self_attn.q_proj": {
3952
  "group_size": 64,
3953
  "bits": 4
 
3964
  "group_size": 64,
3965
  "bits": 4
3966
  },
 
 
 
 
 
3967
  "model.layers.60.mlp.gate": {
3968
  "group_size": 64,
3969
  "bits": 4
 
3980
  "group_size": 64,
3981
  "bits": 6
3982
  },
 
3983
  "model.layers.61.self_attn.q_proj": {
3984
  "group_size": 64,
3985
  "bits": 4
 
3996
  "group_size": 64,
3997
  "bits": 4
3998
  },
 
 
 
 
 
3999
  "model.layers.61.mlp.gate": {
4000
  "group_size": 64,
4001
  "bits": 4
 
4012
  "group_size": 64,
4013
  "bits": 6
4014
  },
 
 
4015
  "lm_head": {
4016
  "group_size": 64,
4017
  "bits": 6
qwen3coder_tool_parser.py CHANGED
@@ -1,34 +1,30 @@
1
  # SPDX-License-Identifier: Apache-2.0
2
-
 
3
  import json
4
- import re
5
  import uuid
6
  from collections.abc import Sequence
7
- from typing import Union, Optional, Any, List, Dict
8
- from enum import Enum
9
-
10
- from vllm.entrypoints.openai.protocol import (
11
- ChatCompletionRequest,
12
- ChatCompletionToolsParam,
13
- DeltaMessage,
14
- DeltaToolCall,
15
- DeltaFunctionCall,
16
- ExtractedToolCallInformation,
17
- FunctionCall,
18
- ToolCall,
19
- )
20
  from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
21
- ToolParser,
22
- ToolParserManager,
23
- )
24
  from vllm.logger import init_logger
25
  from vllm.transformers_utils.tokenizer import AnyTokenizer
26
 
27
  logger = init_logger(__name__)
28
 
29
 
30
- @ToolParserManager.register_module("qwen3_xml")
31
- class Qwen3XMLToolParser(ToolParser):
 
32
  def __init__(self, tokenizer: AnyTokenizer):
33
  super().__init__(tokenizer)
34
 
@@ -52,34 +48,32 @@ class Qwen3XMLToolParser(ToolParser):
52
 
53
  # Regex patterns
54
  self.tool_call_complete_regex = re.compile(
55
- r"<tool_call>(.*?)</tool_call>", re.DOTALL
56
- )
57
  self.tool_call_regex = re.compile(
58
- r"<tool_call>(.*?)</tool_call>|<tool_call>(.*?)$", re.DOTALL
59
- )
60
  self.tool_call_function_regex = re.compile(
61
- r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL
62
- )
63
  self.tool_call_parameter_regex = re.compile(
64
- r"<parameter=(.*?)</parameter>|<parameter=(.*?)$", re.DOTALL
65
- )
66
 
67
  if not self.model_tokenizer:
68
  raise ValueError(
69
  "The model tokenizer must be passed to the ToolParser "
70
- "constructor during construction."
71
- )
72
 
73
- self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
 
74
  self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
75
 
76
  if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None:
77
  raise RuntimeError(
78
  "Qwen3 XML Tool parser could not locate tool call start/end "
79
- "tokens in the tokenizer!"
80
- )
81
 
82
- logger.info(f"vLLM Successfully import tool parser {self.__class__.__name__} !")
 
 
83
 
84
  def _generate_tool_call_id(self) -> str:
85
  """Generate a unique tool call ID."""
@@ -100,130 +94,130 @@ class Qwen3XMLToolParser(ToolParser):
100
  self.accumulated_text = ""
101
  self.json_started = False
102
  self.json_closed = False
103
-
104
- def _parse_xml_function_call(
105
- self, function_call_str: str, tools: Optional[list[ChatCompletionToolsParam]]
106
- ) -> Optional[ToolCall]:
107
- def get_arguments_config(func_name: str) -> dict:
108
- if tools is None:
109
- return {}
110
- for config in tools:
111
- if not hasattr(config, "type") or not (
112
- hasattr(config, "function") and hasattr(config.function, "name")
113
- ):
114
- continue
115
- if config.type == "function" and config.function.name == func_name:
116
- if not hasattr(config.function, "parameters"):
117
- return {}
118
- params = config.function.parameters
119
- if isinstance(params, dict) and "properties" in params:
120
- return params["properties"]
121
- elif isinstance(params, dict):
122
- return params
123
- else:
124
- return {}
125
- logger.warning(f"Tool '{func_name}' is not defined in the tools list.")
126
  return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
 
128
- def convert_param_value(
129
- param_value: str, param_name: str, param_config: dict, func_name: str
130
- ) -> Any:
131
- # Handle null value for any type
132
- if param_value.lower() == "null":
133
- return None
134
-
135
- if param_name not in param_config:
136
- if param_config != {}:
137
- logger.warning(
138
- f"Parsed parameter '{param_name}' is not defined in the tool "
139
- f"parameters for tool '{func_name}', directly returning the string value."
140
- )
141
- return param_value
142
 
143
- if (
144
- isinstance(param_config[param_name], dict)
145
- and "type" in param_config[param_name]
146
- ):
147
- param_type = str(param_config[param_name]["type"]).strip().lower()
148
- else:
149
- param_type = "string"
150
- if param_type in ["string", "str", "text", "varchar", "char", "enum"]:
151
- return param_value
152
- elif (
153
- param_type.startswith("int")
154
- or param_type.startswith("uint")
155
- or param_type.startswith("long")
156
- or param_type.startswith("short")
157
- or param_type.startswith("unsigned")
158
- ):
159
- try:
160
- param_value = int(param_value)
161
- except:
162
- logger.warning(
163
- f"Parsed value '{param_value}' of parameter '{param_name}' is not an integer in tool "
164
- f"'{func_name}', degenerating to string."
165
- )
166
- return param_value
167
- elif param_type.startswith("num") or param_type.startswith("float"):
168
- try:
169
- float_param_value = float(param_value)
170
- param_value = float_param_value if float_param_value - int(float_param_value) != 0 else int(float_param_value)
171
- except:
172
- logger.warning(
173
- f"Parsed value '{param_value}' of parameter '{param_name}' is not a float in tool "
174
- f"'{func_name}', degenerating to string."
175
- )
176
- return param_value
177
- elif param_type in ["boolean", "bool", "binary"]:
178
- param_value = param_value.lower()
179
- if param_value not in ["true", "false"]:
180
- logger.warning(
181
- f"Parsed value '{param_value}' of parameter '{param_name}' is not a boolean (`true` of `false`) in tool '{func_name}', degenerating to false."
182
- )
183
- return param_value == "true"
184
- else:
185
- if param_type == "object" or param_type.startswith("dict"):
186
- try:
187
- param_value = json.loads(param_value)
188
- return param_value
189
- except:
190
- logger.warning(
191
- f"Parsed value '{param_value}' of parameter '{param_name}' is not a valid JSON object in tool "
192
- f"'{func_name}', will try other methods to parse it."
193
- )
194
  try:
195
- param_value = eval(param_value)
 
196
  except:
197
  logger.warning(
198
- f"Parsed value '{param_value}' of parameter '{param_name}' cannot be converted via Python `eval()` in tool '{func_name}', degenerating to string."
199
- )
200
- return param_value
 
 
 
 
 
 
 
 
 
 
 
201
 
202
  # Extract function name
203
  end_index = function_call_str.index(">")
204
  function_name = function_call_str[:end_index]
205
- param_config = get_arguments_config(function_name)
206
- parameters = function_call_str[end_index + 1 :]
207
  param_dict = {}
208
- for match in self.tool_call_parameter_regex.findall(parameters):
209
- match_text = match[0] if match[0] else match[1]
210
  idx = match_text.index(">")
211
  param_name = match_text[:idx]
212
- param_value = str(match_text[idx + 1 :])
213
  # Remove prefix and trailing \n
214
  if param_value.startswith("\n"):
215
  param_value = param_value[1:]
216
  if param_value.endswith("\n"):
217
  param_value = param_value[:-1]
218
 
219
- param_dict[param_name] = convert_param_value(
220
- param_value, param_name, param_config, function_name
221
- )
222
  return ToolCall(
223
  type="function",
224
- function=FunctionCall(
225
- name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False)
226
- ),
227
  )
228
 
229
  def _get_function_calls(self, model_output: str) -> List[str]:
@@ -239,7 +233,8 @@ class Qwen3XMLToolParser(ToolParser):
239
 
240
  raw_function_calls = []
241
  for tool_call in raw_tool_calls:
242
- raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call))
 
243
 
244
  function_calls = [
245
  match[0] if match[0] else match[1] for match in raw_function_calls
@@ -253,16 +248,16 @@ class Qwen3XMLToolParser(ToolParser):
253
  ) -> ExtractedToolCallInformation:
254
  # Quick check to avoid unnecessary processing
255
  if self.tool_call_prefix not in model_output:
256
- return ExtractedToolCallInformation(
257
- tools_called=False, tool_calls=[], content=model_output
258
- )
259
 
260
  try:
261
  function_calls = self._get_function_calls(model_output)
262
  if len(function_calls) == 0:
263
- return ExtractedToolCallInformation(
264
- tools_called=False, tool_calls=[], content=model_output
265
- )
266
 
267
  tool_calls = [
268
  self._parse_xml_function_call(function_call_str, request.tools)
@@ -273,20 +268,17 @@ class Qwen3XMLToolParser(ToolParser):
273
  self.prev_tool_call_arr.clear() # Clear previous calls
274
  for tool_call in tool_calls:
275
  if tool_call:
276
- self.prev_tool_call_arr.append(
277
- {
278
- "name": tool_call.function.name,
279
- "arguments": tool_call.function.arguments,
280
- }
281
- )
282
 
283
  # Extract content before tool calls
284
  content_index = model_output.find(self.tool_call_start_token)
285
- content_index = (
286
- content_index
287
- if content_index >= 0
288
- else model_output.find(self.tool_call_prefix)
289
- )
290
  content = model_output[:content_index] # .rstrip()
291
 
292
  return ExtractedToolCallInformation(
@@ -297,9 +289,9 @@ class Qwen3XMLToolParser(ToolParser):
297
 
298
  except Exception:
299
  logger.exception("Error in extracting tool call from response.")
300
- return ExtractedToolCallInformation(
301
- tools_called=False, tool_calls=[], content=model_output
302
- )
303
 
304
  def extract_tool_calls_streaming(
305
  self,
@@ -311,6 +303,11 @@ class Qwen3XMLToolParser(ToolParser):
311
  delta_token_ids: Sequence[int],
312
  request: ChatCompletionRequest,
313
  ) -> Union[DeltaMessage, None]:
 
 
 
 
 
314
  # If no delta text, return None unless it's an EOS token after tool calls
315
  if not delta_text:
316
  # Check if this is an EOS token after all tool calls are complete
@@ -319,15 +316,14 @@ class Qwen3XMLToolParser(ToolParser):
319
  if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids:
320
  # Count complete tool calls
321
  complete_calls = len(
322
- self.tool_call_complete_regex.findall(current_text)
323
- )
324
 
325
  # If we have completed tool calls and populated prev_tool_call_arr
326
  if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
327
  # Check if all tool calls are closed
328
  open_calls = current_text.count(
329
- self.tool_call_start_token
330
- ) - current_text.count(self.tool_call_end_token)
331
  if open_calls == 0:
332
  # Return empty delta message to allow finish_reason processing
333
  return DeltaMessage(content="")
@@ -336,10 +332,6 @@ class Qwen3XMLToolParser(ToolParser):
336
  return DeltaMessage(content="")
337
  return None
338
 
339
- # Check if this is the first call (reset state if needed)
340
- if not previous_text:
341
- self._reset_streaming_state()
342
-
343
  # Update accumulated text
344
  self.accumulated_text = current_text
345
 
@@ -354,6 +346,7 @@ class Qwen3XMLToolParser(ToolParser):
354
  self.param_count = 0
355
  self.json_started = False
356
  self.json_closed = False
 
357
 
358
  # Check if there are more tool calls
359
  tool_starts = current_text.count(self.tool_call_start_token)
@@ -366,16 +359,12 @@ class Qwen3XMLToolParser(ToolParser):
366
  # Handle normal content before tool calls
367
  if not self.is_tool_call_started:
368
  # Check if tool call is starting
369
- if (
370
- self.tool_call_start_token_id in delta_token_ids
371
- or self.tool_call_start_token in delta_text
372
- ):
373
  self.is_tool_call_started = True
374
  # Return any content before the tool call
375
  if self.tool_call_start_token in delta_text:
376
- content_before = delta_text[
377
- : delta_text.index(self.tool_call_start_token)
378
- ]
379
  if content_before:
380
  return DeltaMessage(content=content_before)
381
  return None
@@ -412,20 +401,19 @@ class Qwen3XMLToolParser(ToolParser):
412
 
413
  tool_start_idx = tool_starts[self.current_tool_index]
414
  # Find where this tool call ends (or current position if not ended yet)
415
- tool_end_idx = current_text.find(self.tool_call_end_token, tool_start_idx)
 
416
  if tool_end_idx == -1:
417
  tool_text = current_text[tool_start_idx:]
418
  else:
419
- tool_text = current_text[
420
- tool_start_idx : tool_end_idx + len(self.tool_call_end_token)
421
- ]
422
 
423
  # Looking for function header
424
  if not self.header_sent:
425
  if self.tool_call_prefix in tool_text:
426
  func_start = tool_text.find(self.tool_call_prefix) + len(
427
- self.tool_call_prefix
428
- )
429
  func_end = tool_text.find(">", func_start)
430
 
431
  if func_end != -1:
@@ -439,44 +427,37 @@ class Qwen3XMLToolParser(ToolParser):
439
  # This ensures finish_reason="tool_calls" even if parsing isn't complete
440
  already_added = any(
441
  tool.get("name") == self.current_function_name
442
- for tool in self.prev_tool_call_arr
443
- )
444
  if not already_added:
445
- self.prev_tool_call_arr.append(
446
- {
447
- "name": self.current_function_name,
448
- "arguments": "{}", # Placeholder, will be updated later
449
- }
450
- )
451
 
452
  # Send header with function info
453
- return DeltaMessage(
454
- tool_calls=[
455
- DeltaToolCall(
456
- index=self.current_tool_index,
457
- id=self.current_tool_id,
458
- function=DeltaFunctionCall(
459
- name=self.current_function_name, arguments=""
460
- ),
461
- type="function",
462
- )
463
- ]
464
- )
465
  return None
466
 
467
  # We've sent header, now handle function body
468
  if self.in_function:
469
  # Send opening brace if not sent yet
470
- if not self.json_started and not self.parameter_prefix in delta_text:
471
  self.json_started = True
472
- return DeltaMessage(
473
- tool_calls=[
474
- DeltaToolCall(
475
- index=self.current_tool_index,
476
- function=DeltaFunctionCall(arguments="{"),
477
- )
478
- ]
479
- )
480
 
481
  # Make sure json_started is set if we're processing parameters
482
  if not self.json_started:
@@ -490,58 +471,54 @@ class Qwen3XMLToolParser(ToolParser):
490
  # Extract the complete tool call to update prev_tool_call_arr with final arguments
491
  # Find the function content
492
  func_start = tool_text.find(self.tool_call_prefix) + len(
493
- self.tool_call_prefix
494
- )
495
- func_content_end = tool_text.find(self.function_end_token, func_start)
496
  if func_content_end != -1:
497
  func_content = tool_text[func_start:func_content_end]
498
  # Parse to get the complete arguments
499
  try:
500
  parsed_tool = self._parse_xml_function_call(
501
- func_content, request.tools if request else None
502
- )
503
  if parsed_tool:
504
  # Update existing entry in prev_tool_call_arr with complete arguments
505
  for i, tool in enumerate(self.prev_tool_call_arr):
506
- if tool.get("name") == parsed_tool.function.name:
507
- self.prev_tool_call_arr[i]["arguments"] = (
508
- parsed_tool.function.arguments
509
- )
510
  break
511
  except Exception:
512
  pass # Ignore parsing errors during streaming
513
 
514
- result = DeltaMessage(
515
- tool_calls=[
516
- DeltaToolCall(
517
- index=self.current_tool_index,
518
- function=DeltaFunctionCall(arguments="}"),
519
- )
520
- ]
521
- )
522
 
523
  # Reset state for next tool
524
  self.in_function = False
525
  self.json_closed = True
 
526
 
527
  return result
528
 
529
  # Look for parameters
530
- # Count how many complete parameters we have processed
531
- complete_params = tool_text.count(self.parameter_end_token)
 
 
 
 
 
 
 
532
 
533
  # Check if we should start a new parameter
534
- if not self.in_param and self.param_count < complete_params:
535
- # Find the unprocessed parameter
536
- # Count parameter starts
537
- param_starts = []
538
- idx = 0
539
- while True:
540
- idx = tool_text.find(self.parameter_prefix, idx)
541
- if idx == -1:
542
- break
543
- param_starts.append(idx)
544
- idx += len(self.parameter_prefix)
545
 
546
  if len(param_starts) > self.param_count:
547
  # Process the next parameter
@@ -561,45 +538,74 @@ class Qwen3XMLToolParser(ToolParser):
561
  value_text = value_text[1:]
562
 
563
  # Find where this parameter ends
564
- param_end_idx = value_text.find(self.parameter_end_token)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
565
  if param_end_idx != -1:
566
  # Complete parameter found
567
  param_value = value_text[:param_end_idx]
568
  if param_value.endswith("\n"):
569
  param_value = param_value[:-1]
570
 
571
- # Build complete JSON fragment for this parameter
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
572
  if self.param_count == 0:
573
- json_fragment = (
574
- '"'
575
- + self.current_param_name
576
- + '": "'
577
- + json.dumps(param_value)[1:-1]
578
- + '"'
579
- )
580
  else:
581
- json_fragment = (
582
- ', "'
583
- + self.current_param_name
584
- + '": "'
585
- + json.dumps(param_value)[1:-1]
586
- + '"'
587
- )
588
 
589
  self.param_count += 1
590
 
591
- return DeltaMessage(
592
- tool_calls=[
593
- DeltaToolCall(
594
- index=self.current_tool_index,
595
- function=DeltaFunctionCall(
596
- arguments=json_fragment
597
- ),
598
- )
599
- ]
600
- )
601
-
602
- # Continue parameter value
603
  if self.in_param:
604
  if self.parameter_end_token in delta_text:
605
  # End of parameter
@@ -609,34 +615,45 @@ class Qwen3XMLToolParser(ToolParser):
609
  # Skip past > if at start
610
  if not self.current_param_value and ">" in value_chunk:
611
  gt_idx = value_chunk.find(">")
612
- value_chunk = value_chunk[gt_idx + 1 :]
613
 
614
- if not self.current_param_value and value_chunk.startswith("\n"):
 
615
  value_chunk = value_chunk[1:]
616
 
617
- # Calculate incremental JSON
618
  full_value = self.current_param_value + value_chunk
619
- prev_escaped = (
620
- json.dumps(self.current_param_value)[1:-1]
621
- if self.current_param_value
622
- else ""
623
- )
624
- full_escaped = json.dumps(full_value)[1:-1]
625
- delta_escaped = full_escaped[len(prev_escaped) :]
626
-
 
 
 
 
 
 
 
 
 
 
 
 
627
  self.in_param = False
628
  self.current_param_value = ""
629
 
630
- return DeltaMessage(
631
- tool_calls=[
632
- DeltaToolCall(
633
- index=self.current_tool_index,
634
- function=DeltaFunctionCall(
635
- arguments=delta_escaped + '"'
636
- ),
637
- )
638
- ]
639
- )
640
  else:
641
  # Continue accumulating value
642
  value_chunk = delta_text
@@ -644,32 +661,29 @@ class Qwen3XMLToolParser(ToolParser):
644
  # Handle first chunk after param name
645
  if not self.current_param_value and ">" in value_chunk:
646
  gt_idx = value_chunk.find(">")
647
- value_chunk = value_chunk[gt_idx + 1 :]
648
 
649
- if not self.current_param_value and value_chunk.startswith("\n"):
 
650
  value_chunk = value_chunk[1:]
651
 
652
  if value_chunk:
653
  # Stream the escaped delta
654
- prev_escaped = (
655
- json.dumps(self.current_param_value)[1:-1]
656
- if self.current_param_value
657
- else ""
658
- )
659
  self.current_param_value += value_chunk
660
- full_escaped = json.dumps(self.current_param_value)[1:-1]
661
- delta_escaped = full_escaped[len(prev_escaped) :]
 
662
 
663
  if delta_escaped:
664
- return DeltaMessage(
665
- tool_calls=[
666
- DeltaToolCall(
667
- index=self.current_tool_index,
668
- function=DeltaFunctionCall(
669
- arguments=delta_escaped
670
- ),
671
- )
672
- ]
673
- )
674
 
675
  return None
 
1
  # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ import ast
4
  import json
 
5
  import uuid
6
  from collections.abc import Sequence
7
+ from typing import Any, List, Optional, Union
8
+
9
+ import regex as re
10
+
11
+ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
+ ChatCompletionToolsParam,
13
+ DeltaFunctionCall, DeltaMessage,
14
+ DeltaToolCall,
15
+ ExtractedToolCallInformation,
16
+ FunctionCall, ToolCall)
 
 
 
17
  from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
18
+ ToolParser, ToolParserManager)
 
 
19
  from vllm.logger import init_logger
20
  from vllm.transformers_utils.tokenizer import AnyTokenizer
21
 
22
  logger = init_logger(__name__)
23
 
24
 
25
+ @ToolParserManager.register_module("qwen3_coder")
26
+ class Qwen3CoderToolParser(ToolParser):
27
+
28
  def __init__(self, tokenizer: AnyTokenizer):
29
  super().__init__(tokenizer)
30
 
 
48
 
49
  # Regex patterns
50
  self.tool_call_complete_regex = re.compile(
51
+ r"<tool_call>(.*?)</tool_call>", re.DOTALL)
 
52
  self.tool_call_regex = re.compile(
53
+ r"<tool_call>(.*?)</tool_call>|<tool_call>(.*?)$", re.DOTALL)
 
54
  self.tool_call_function_regex = re.compile(
55
+ r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL)
 
56
  self.tool_call_parameter_regex = re.compile(
57
+ r"<parameter=(.*?)(?:</parameter>|(?=<parameter=)|(?=</function>)|$)",
58
+ re.DOTALL)
59
 
60
  if not self.model_tokenizer:
61
  raise ValueError(
62
  "The model tokenizer must be passed to the ToolParser "
63
+ "constructor during construction.")
 
64
 
65
+ self.tool_call_start_token_id = self.vocab.get(
66
+ self.tool_call_start_token)
67
  self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
68
 
69
  if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None:
70
  raise RuntimeError(
71
  "Qwen3 XML Tool parser could not locate tool call start/end "
72
+ "tokens in the tokenizer!")
 
73
 
74
+ logger.info(
75
+ f"vLLM Successfully import tool parser {self.__class__.__name__} !"
76
+ )
77
 
78
  def _generate_tool_call_id(self) -> str:
79
  """Generate a unique tool call ID."""
 
94
  self.accumulated_text = ""
95
  self.json_started = False
96
  self.json_closed = False
97
+ # Store accumulated parameters for type conversion
98
+ self.accumulated_params = {}
99
+ self.streaming_request = None
100
+
101
+ def _get_arguments_config(
102
+ self, func_name: str,
103
+ tools: Optional[list[ChatCompletionToolsParam]]) -> dict:
104
+ """Extract argument configuration for a function."""
105
+ if tools is None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  return {}
107
+ for config in tools:
108
+ if not hasattr(config, "type") or not (hasattr(
109
+ config, "function") and hasattr(config.function, "name")):
110
+ continue
111
+ if config.type == "function" and config.function.name == func_name:
112
+ if not hasattr(config.function, "parameters"):
113
+ return {}
114
+ params = config.function.parameters
115
+ if isinstance(params, dict) and "properties" in params:
116
+ return params["properties"]
117
+ elif isinstance(params, dict):
118
+ return params
119
+ else:
120
+ return {}
121
+ logger.warning(f"Tool '{func_name}' is not defined in the tools list.")
122
+ return {}
123
+
124
+ def _convert_param_value(self, param_value: str, param_name: str,
125
+ param_config: dict, func_name: str) -> Any:
126
+ """Convert parameter value based on its type in the schema."""
127
+ # Handle null value for any type
128
+ if param_value.lower() == "null":
129
+ return None
130
 
131
+ if param_name not in param_config:
132
+ if param_config != {}:
133
+ logger.warning(
134
+ f"Parsed parameter '{param_name}' is not defined in the tool "
135
+ f"parameters for tool '{func_name}', directly returning the string value."
136
+ )
137
+ return param_value
 
 
 
 
 
 
 
138
 
139
+ if isinstance(param_config[param_name],
140
+ dict) and "type" in param_config[param_name]:
141
+ param_type = str(param_config[param_name]["type"]).strip().lower()
142
+ else:
143
+ param_type = "string"
144
+ if param_type in ["string", "str", "text", "varchar", "char", "enum"]:
145
+ return param_value
146
+ elif param_type.startswith("int") or param_type.startswith(
147
+ "uint") or param_type.startswith(
148
+ "long") or param_type.startswith(
149
+ "short") or param_type.startswith("unsigned"):
150
+ try:
151
+ param_value = int(param_value)
152
+ except:
153
+ logger.warning(
154
+ f"Parsed value '{param_value}' of parameter '{param_name}' is not an integer in tool "
155
+ f"'{func_name}', degenerating to string.")
156
+ return param_value
157
+ elif param_type.startswith("num") or param_type.startswith("float"):
158
+ try:
159
+ float_param_value = float(param_value)
160
+ param_value = float_param_value if float_param_value - int(
161
+ float_param_value) != 0 else int(float_param_value)
162
+ except:
163
+ logger.warning(
164
+ f"Parsed value '{param_value}' of parameter '{param_name}' is not a float in tool "
165
+ f"'{func_name}', degenerating to string.")
166
+ return param_value
167
+ elif param_type in ["boolean", "bool", "binary"]:
168
+ param_value = param_value.lower()
169
+ if param_value not in ["true", "false"]:
170
+ logger.warning(
171
+ f"Parsed value '{param_value}' of parameter '{param_name}' is not a boolean (`true` of `false`) in tool '{func_name}', degenerating to false."
172
+ )
173
+ return param_value == "true"
174
+ else:
175
+ if param_type in ["object", "array", "arr"
176
+ ] or param_type.startswith(
177
+ "dict") or param_type.startswith("list"):
 
 
 
 
 
 
 
 
 
 
 
 
178
  try:
179
+ param_value = json.loads(param_value)
180
+ return param_value
181
  except:
182
  logger.warning(
183
+ f"Parsed value '{param_value}' of parameter '{param_name}' cannot be parsed with json.loads in tool "
184
+ f"'{func_name}', will try other methods to parse it.")
185
+ try:
186
+ param_value = ast.literal_eval(param_value) # safer
187
+ except:
188
+ logger.warning(
189
+ f"Parsed value '{param_value}' of parameter '{param_name}' cannot be converted via Python `ast.literal_eval()` in tool '{func_name}', degenerating to string."
190
+ )
191
+ return param_value
192
+
193
+ def _parse_xml_function_call(
194
+ self, function_call_str: str,
195
+ tools: Optional[list[ChatCompletionToolsParam]]
196
+ ) -> Optional[ToolCall]:
197
 
198
  # Extract function name
199
  end_index = function_call_str.index(">")
200
  function_name = function_call_str[:end_index]
201
+ param_config = self._get_arguments_config(function_name, tools)
202
+ parameters = function_call_str[end_index + 1:]
203
  param_dict = {}
204
+ for match_text in self.tool_call_parameter_regex.findall(parameters):
 
205
  idx = match_text.index(">")
206
  param_name = match_text[:idx]
207
+ param_value = str(match_text[idx + 1:])
208
  # Remove prefix and trailing \n
209
  if param_value.startswith("\n"):
210
  param_value = param_value[1:]
211
  if param_value.endswith("\n"):
212
  param_value = param_value[:-1]
213
 
214
+ param_dict[param_name] = self._convert_param_value(
215
+ param_value, param_name, param_config, function_name)
 
216
  return ToolCall(
217
  type="function",
218
+ function=FunctionCall(name=function_name,
219
+ arguments=json.dumps(param_dict,
220
+ ensure_ascii=False)),
221
  )
222
 
223
  def _get_function_calls(self, model_output: str) -> List[str]:
 
233
 
234
  raw_function_calls = []
235
  for tool_call in raw_tool_calls:
236
+ raw_function_calls.extend(
237
+ self.tool_call_function_regex.findall(tool_call))
238
 
239
  function_calls = [
240
  match[0] if match[0] else match[1] for match in raw_function_calls
 
248
  ) -> ExtractedToolCallInformation:
249
  # Quick check to avoid unnecessary processing
250
  if self.tool_call_prefix not in model_output:
251
+ return ExtractedToolCallInformation(tools_called=False,
252
+ tool_calls=[],
253
+ content=model_output)
254
 
255
  try:
256
  function_calls = self._get_function_calls(model_output)
257
  if len(function_calls) == 0:
258
+ return ExtractedToolCallInformation(tools_called=False,
259
+ tool_calls=[],
260
+ content=model_output)
261
 
262
  tool_calls = [
263
  self._parse_xml_function_call(function_call_str, request.tools)
 
268
  self.prev_tool_call_arr.clear() # Clear previous calls
269
  for tool_call in tool_calls:
270
  if tool_call:
271
+ self.prev_tool_call_arr.append({
272
+ "name":
273
+ tool_call.function.name,
274
+ "arguments":
275
+ tool_call.function.arguments,
276
+ })
277
 
278
  # Extract content before tool calls
279
  content_index = model_output.find(self.tool_call_start_token)
280
+ content_index = content_index if content_index >= 0 else model_output.find(
281
+ self.tool_call_prefix)
 
 
 
282
  content = model_output[:content_index] # .rstrip()
283
 
284
  return ExtractedToolCallInformation(
 
289
 
290
  except Exception:
291
  logger.exception("Error in extracting tool call from response.")
292
+ return ExtractedToolCallInformation(tools_called=False,
293
+ tool_calls=[],
294
+ content=model_output)
295
 
296
  def extract_tool_calls_streaming(
297
  self,
 
303
  delta_token_ids: Sequence[int],
304
  request: ChatCompletionRequest,
305
  ) -> Union[DeltaMessage, None]:
306
+ # Store request for type conversion
307
+ if not previous_text:
308
+ self._reset_streaming_state()
309
+ self.streaming_request = request
310
+
311
  # If no delta text, return None unless it's an EOS token after tool calls
312
  if not delta_text:
313
  # Check if this is an EOS token after all tool calls are complete
 
316
  if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids:
317
  # Count complete tool calls
318
  complete_calls = len(
319
+ self.tool_call_complete_regex.findall(current_text))
 
320
 
321
  # If we have completed tool calls and populated prev_tool_call_arr
322
  if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
323
  # Check if all tool calls are closed
324
  open_calls = current_text.count(
325
+ self.tool_call_start_token) - current_text.count(
326
+ self.tool_call_end_token)
327
  if open_calls == 0:
328
  # Return empty delta message to allow finish_reason processing
329
  return DeltaMessage(content="")
 
332
  return DeltaMessage(content="")
333
  return None
334
 
 
 
 
 
335
  # Update accumulated text
336
  self.accumulated_text = current_text
337
 
 
346
  self.param_count = 0
347
  self.json_started = False
348
  self.json_closed = False
349
+ self.accumulated_params = {}
350
 
351
  # Check if there are more tool calls
352
  tool_starts = current_text.count(self.tool_call_start_token)
 
359
  # Handle normal content before tool calls
360
  if not self.is_tool_call_started:
361
  # Check if tool call is starting
362
+ if self.tool_call_start_token_id in delta_token_ids or self.tool_call_start_token in delta_text:
 
 
 
363
  self.is_tool_call_started = True
364
  # Return any content before the tool call
365
  if self.tool_call_start_token in delta_text:
366
+ content_before = delta_text[:delta_text.index(
367
+ self.tool_call_start_token)]
 
368
  if content_before:
369
  return DeltaMessage(content=content_before)
370
  return None
 
401
 
402
  tool_start_idx = tool_starts[self.current_tool_index]
403
  # Find where this tool call ends (or current position if not ended yet)
404
+ tool_end_idx = current_text.find(self.tool_call_end_token,
405
+ tool_start_idx)
406
  if tool_end_idx == -1:
407
  tool_text = current_text[tool_start_idx:]
408
  else:
409
+ tool_text = current_text[tool_start_idx:tool_end_idx +
410
+ len(self.tool_call_end_token)]
 
411
 
412
  # Looking for function header
413
  if not self.header_sent:
414
  if self.tool_call_prefix in tool_text:
415
  func_start = tool_text.find(self.tool_call_prefix) + len(
416
+ self.tool_call_prefix)
 
417
  func_end = tool_text.find(">", func_start)
418
 
419
  if func_end != -1:
 
427
  # This ensures finish_reason="tool_calls" even if parsing isn't complete
428
  already_added = any(
429
  tool.get("name") == self.current_function_name
430
+ for tool in self.prev_tool_call_arr)
 
431
  if not already_added:
432
+ self.prev_tool_call_arr.append({
433
+ "name": self.current_function_name,
434
+ "arguments":
435
+ "{}", # Placeholder, will be updated later
436
+ })
 
437
 
438
  # Send header with function info
439
+ return DeltaMessage(tool_calls=[
440
+ DeltaToolCall(
441
+ index=self.current_tool_index,
442
+ id=self.current_tool_id,
443
+ function=DeltaFunctionCall(
444
+ name=self.current_function_name, arguments=""),
445
+ type="function",
446
+ )
447
+ ])
 
 
 
448
  return None
449
 
450
  # We've sent header, now handle function body
451
  if self.in_function:
452
  # Send opening brace if not sent yet
453
+ if not self.json_started and self.parameter_prefix not in delta_text:
454
  self.json_started = True
455
+ return DeltaMessage(tool_calls=[
456
+ DeltaToolCall(
457
+ index=self.current_tool_index,
458
+ function=DeltaFunctionCall(arguments="{"),
459
+ )
460
+ ])
 
 
461
 
462
  # Make sure json_started is set if we're processing parameters
463
  if not self.json_started:
 
471
  # Extract the complete tool call to update prev_tool_call_arr with final arguments
472
  # Find the function content
473
  func_start = tool_text.find(self.tool_call_prefix) + len(
474
+ self.tool_call_prefix)
475
+ func_content_end = tool_text.find(self.function_end_token,
476
+ func_start)
477
  if func_content_end != -1:
478
  func_content = tool_text[func_start:func_content_end]
479
  # Parse to get the complete arguments
480
  try:
481
  parsed_tool = self._parse_xml_function_call(
482
+ func_content, self.streaming_request.tools
483
+ if self.streaming_request else None)
484
  if parsed_tool:
485
  # Update existing entry in prev_tool_call_arr with complete arguments
486
  for i, tool in enumerate(self.prev_tool_call_arr):
487
+ if tool.get(
488
+ "name") == parsed_tool.function.name:
489
+ self.prev_tool_call_arr[i][
490
+ "arguments"] = parsed_tool.function.arguments
491
  break
492
  except Exception:
493
  pass # Ignore parsing errors during streaming
494
 
495
+ result = DeltaMessage(tool_calls=[
496
+ DeltaToolCall(
497
+ index=self.current_tool_index,
498
+ function=DeltaFunctionCall(arguments="}"),
499
+ )
500
+ ])
 
 
501
 
502
  # Reset state for next tool
503
  self.in_function = False
504
  self.json_closed = True
505
+ self.accumulated_params = {}
506
 
507
  return result
508
 
509
  # Look for parameters
510
+ # Find all parameter starts
511
+ param_starts = []
512
+ idx = 0
513
+ while True:
514
+ idx = tool_text.find(self.parameter_prefix, idx)
515
+ if idx == -1:
516
+ break
517
+ param_starts.append(idx)
518
+ idx += len(self.parameter_prefix)
519
 
520
  # Check if we should start a new parameter
521
+ if not self.in_param and self.param_count < len(param_starts):
 
 
 
 
 
 
 
 
 
 
522
 
523
  if len(param_starts) > self.param_count:
524
  # Process the next parameter
 
538
  value_text = value_text[1:]
539
 
540
  # Find where this parameter ends
541
+ param_end_idx = value_text.find(
542
+ self.parameter_end_token)
543
+ if param_end_idx == -1:
544
+ # No closing tag, look for next parameter or function end
545
+ next_param_idx = value_text.find(
546
+ self.parameter_prefix)
547
+ func_end_idx = value_text.find(
548
+ self.function_end_token)
549
+
550
+ if next_param_idx != -1 and (func_end_idx == -1
551
+ or next_param_idx
552
+ < func_end_idx):
553
+ param_end_idx = next_param_idx
554
+ elif func_end_idx != -1:
555
+ param_end_idx = func_end_idx
556
+ else:
557
+ # Neither found, check if tool call is complete
558
+ if self.tool_call_end_token in tool_text:
559
+ # Tool call is complete, so parameter must be complete too
560
+ # Use all remaining text before function end as value
561
+ param_end_idx = len(value_text)
562
+ else:
563
+ # Still streaming, wait for more content
564
+ return None
565
+
566
  if param_end_idx != -1:
567
  # Complete parameter found
568
  param_value = value_text[:param_end_idx]
569
  if param_value.endswith("\n"):
570
  param_value = param_value[:-1]
571
 
572
+ # Store raw value for later processing
573
+ self.accumulated_params[
574
+ self.current_param_name] = param_value
575
+
576
+ # Get parameter configuration for type conversion
577
+ param_config = self._get_arguments_config(
578
+ self.current_function_name,
579
+ self.streaming_request.tools
580
+ if self.streaming_request else None)
581
+
582
+ # Convert the parameter value to the appropriate type
583
+ converted_value = self._convert_param_value(
584
+ param_value, self.current_param_name,
585
+ param_config, self.current_function_name)
586
+
587
+ # Build JSON fragment based on the converted type
588
+ # Use json.dumps to properly serialize the value
589
+ serialized_value = json.dumps(converted_value,
590
+ ensure_ascii=False)
591
+
592
  if self.param_count == 0:
593
+ json_fragment = f'"{self.current_param_name}": {serialized_value}'
 
 
 
 
 
 
594
  else:
595
+ json_fragment = f', "{self.current_param_name}": {serialized_value}'
 
 
 
 
 
 
596
 
597
  self.param_count += 1
598
 
599
+ return DeltaMessage(tool_calls=[
600
+ DeltaToolCall(
601
+ index=self.current_tool_index,
602
+ function=DeltaFunctionCall(
603
+ arguments=json_fragment),
604
+ )
605
+ ])
606
+
607
+ # Continue parameter value - Not used in the current implementation
608
+ # since we process complete parameters above
 
 
609
  if self.in_param:
610
  if self.parameter_end_token in delta_text:
611
  # End of parameter
 
615
  # Skip past > if at start
616
  if not self.current_param_value and ">" in value_chunk:
617
  gt_idx = value_chunk.find(">")
618
+ value_chunk = value_chunk[gt_idx + 1:]
619
 
620
+ if not self.current_param_value and value_chunk.startswith(
621
+ "\n"):
622
  value_chunk = value_chunk[1:]
623
 
624
+ # Store complete value
625
  full_value = self.current_param_value + value_chunk
626
+ self.accumulated_params[
627
+ self.current_param_name] = full_value
628
+
629
+ # Get parameter configuration for type conversion
630
+ param_config = self._get_arguments_config(
631
+ self.current_function_name,
632
+ self.streaming_request.tools
633
+ if self.streaming_request else None)
634
+
635
+ # Convert the parameter value to the appropriate type
636
+ converted_value = self._convert_param_value(
637
+ full_value, self.current_param_name, param_config,
638
+ self.current_function_name)
639
+
640
+ # Serialize the converted value
641
+ serialized_value = json.dumps(converted_value,
642
+ ensure_ascii=False)
643
+
644
+ # Since we've been streaming the quoted version, we need to close it properly
645
+ # This is complex - for now just complete the value
646
  self.in_param = False
647
  self.current_param_value = ""
648
 
649
+ # Just close the current parameter string
650
+ return DeltaMessage(tool_calls=[
651
+ DeltaToolCall(
652
+ index=self.current_tool_index,
653
+ function=DeltaFunctionCall(
654
+ arguments='"'), # Close the string quote
655
+ )
656
+ ])
 
 
657
  else:
658
  # Continue accumulating value
659
  value_chunk = delta_text
 
661
  # Handle first chunk after param name
662
  if not self.current_param_value and ">" in value_chunk:
663
  gt_idx = value_chunk.find(">")
664
+ value_chunk = value_chunk[gt_idx + 1:]
665
 
666
+ if not self.current_param_value and value_chunk.startswith(
667
+ "\n"):
668
  value_chunk = value_chunk[1:]
669
 
670
  if value_chunk:
671
  # Stream the escaped delta
672
+ prev_escaped = json.dumps(
673
+ self.current_param_value, ensure_ascii=False
674
+ )[1:-1] if self.current_param_value else ""
 
 
675
  self.current_param_value += value_chunk
676
+ full_escaped = json.dumps(self.current_param_value,
677
+ ensure_ascii=False)[1:-1]
678
+ delta_escaped = full_escaped[len(prev_escaped):]
679
 
680
  if delta_escaped:
681
+ return DeltaMessage(tool_calls=[
682
+ DeltaToolCall(
683
+ index=self.current_tool_index,
684
+ function=DeltaFunctionCall(
685
+ arguments=delta_escaped),
686
+ )
687
+ ])
 
 
 
688
 
689
  return None