BramVanroy commited on
Commit
93676c8
·
1 Parent(s): acf9f2b

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +1 -0
config.json CHANGED
@@ -17,6 +17,7 @@
17
  "layer_norm_epsilon": 1e-05,
18
  "num_attention_heads": 71,
19
  "num_hidden_layers": 32,
 
20
  "parallel_attn": true,
21
  "torch_dtype": "bfloat16",
22
  "transformers_version": "4.30.2",
 
17
  "layer_norm_epsilon": 1e-05,
18
  "num_attention_heads": 71,
19
  "num_hidden_layers": 32,
20
+ "num_kv_heads": 3,
21
  "parallel_attn": true,
22
  "torch_dtype": "bfloat16",
23
  "transformers_version": "4.30.2",