revert back to old config
Browse files- config.json +4 -12
config.json
CHANGED
@@ -4,6 +4,7 @@
|
|
4 |
],
|
5 |
"attention_bias": true,
|
6 |
"attention_dropout": 0.0,
|
|
|
7 |
"eos_token_id": 200002,
|
8 |
"experts_per_token": 4,
|
9 |
"head_dim": 64,
|
@@ -46,16 +47,7 @@
|
|
46 |
"num_key_value_heads": 8,
|
47 |
"num_local_experts": 32,
|
48 |
"output_router_logits": false,
|
49 |
-
"pad_token_id":
|
50 |
-
"quantization_config": {
|
51 |
-
"modules_to_not_convert": [
|
52 |
-
"model.layers.*.self_attn",
|
53 |
-
"model.layers.*.mlp.router",
|
54 |
-
"model.embed_tokens",
|
55 |
-
"lm_head"
|
56 |
-
],
|
57 |
-
"quant_method": "mxfp4"
|
58 |
-
},
|
59 |
"rms_norm_eps": 1e-05,
|
60 |
"rope_scaling": {
|
61 |
"beta_fast": 32.0,
|
@@ -70,8 +62,8 @@
|
|
70 |
"sliding_window": 128,
|
71 |
"swiglu_limit": 7.0,
|
72 |
"tie_word_embeddings": false,
|
73 |
-
"transformers_version": "4.
|
74 |
-
"
|
75 |
"use_cache": true,
|
76 |
"vocab_size": 201088
|
77 |
}
|
|
|
4 |
],
|
5 |
"attention_bias": true,
|
6 |
"attention_dropout": 0.0,
|
7 |
+
"dtype": "bfloat16",
|
8 |
"eos_token_id": 200002,
|
9 |
"experts_per_token": 4,
|
10 |
"head_dim": 64,
|
|
|
47 |
"num_key_value_heads": 8,
|
48 |
"num_local_experts": 32,
|
49 |
"output_router_logits": false,
|
50 |
+
"pad_token_id": 199999,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
"rms_norm_eps": 1e-05,
|
52 |
"rope_scaling": {
|
53 |
"beta_fast": 32.0,
|
|
|
62 |
"sliding_window": 128,
|
63 |
"swiglu_limit": 7.0,
|
64 |
"tie_word_embeddings": false,
|
65 |
+
"transformers_version": "4.56.0",
|
66 |
+
"unsloth_version": "2025.9.4",
|
67 |
"use_cache": true,
|
68 |
"vocab_size": 201088
|
69 |
}
|