Hugging Face's logo

junrushao
/
pythia-1.4b-q4f16_1-MLC

Model card Files Community
pythia-1.4b-q4f16_1-MLC / ndarray-cache.json
junrushao's picture
junrushao
Initial commit
af5c900
raw
history blame contribute delete
153 kB
 {
"metadata": {
"ParamSize": 390,
"ParamBytes": 797155328.0,
"BitsPerParam": 4.508007284877509
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "f5d37acca328f86a3f93dfe0c9e8c01a"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 25362432,
"records": [
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6438912,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6438912
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6443008
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6447104
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 6451200
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 6455296
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 12746752
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 13533184
},
{
"name": "gpt_neox.layers.0.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 13545472
},
{
"name": "gpt_neox.layers.0.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 15642624
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 15904768
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 15908864
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 24297472
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 25346048
}
],
"md5sum": "fa3fb11f5f3661e92a97b89d7107ca60"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.1.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.1.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "71eb20679356efe834e0c6e77755ddc8"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.2.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.2.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "fa4edd7b8fd843acc87ee9515b8043fd"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.3.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.3.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "71177eed571066070046e827041186ac"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.4.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.4.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "380060a9d9486cda7c2cb9755b0d9a52"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.5.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.5.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "75790c9c4a4ddaccb7d827837d88c122"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.6.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.6.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "2e4ebfc0b291caa82e796ca5067b4fa0"
},
{
"dataPath": "params_shard_8.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.7.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.7.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "9cb930bc8c46cfcd61e9a9ca5450ac54"
},
{
"dataPath": "params_shard_9.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.8.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.8.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "9dbf704abf1a51f698f1be931117a263"
},
{
"dataPath": "params_shard_10.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.9.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.9.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "7c982e3bb6d18b0d8130de250f5fe02e"
},
{
"dataPath": "params_shard_11.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.10.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.10.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "91ab53f14b2bb99c7b7782d476775c7a"
},
{
"dataPath": "params_shard_12.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.11.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.11.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "f149d1fd266d1801e5cdb407078fa0b7"
},
{
"dataPath": "params_shard_13.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.12.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.12.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.12.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.12.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.12.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "995b2bde580f0c9350b3b8485b758c92"
},
{
"dataPath": "params_shard_14.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.13.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.13.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.13.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.13.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.13.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "9f0470526e94e0044512aa766e54c1b7"
},
{
"dataPath": "params_shard_15.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.14.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.14.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.14.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.14.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.14.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "5b3a6256ae792a91fe186024fcec8610"
},
{
"dataPath": "params_shard_16.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.15.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.15.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.15.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.15.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.15.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "67cf0fc6773e7035fd15ceb9d46f9763"
},
{
"dataPath": "params_shard_17.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.16.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.16.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.16.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.16.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.16.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "661358a9a4e87d9922bc2f10e4be20ce"
},
{
"dataPath": "params_shard_18.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.17.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.17.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.17.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.17.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.17.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "e613587844b14403d294cc773679dd8c"
},
{
"dataPath": "params_shard_19.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.18.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.18.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.18.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.18.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.18.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "d262176e0042d57d78262ef73fbfd9c6"
},
{
"dataPath": "params_shard_20.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.19.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.19.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.19.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.19.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.19.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "d7906624163bc8a2db190a4ad309134e"
},
{
"dataPath": "params_shard_21.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.20.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.20.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.20.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.20.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.20.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "1b5802c8b104e0e1de6a405b81c01540"
},
{
"dataPath": "params_shard_22.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.21.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.21.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.21.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.21.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.21.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "802d593e30cb4d64f5e81ebf9d08eba9"
},
{
"dataPath": "params_shard_23.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.22.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.22.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.22.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.22.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.22.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "26943d5dde6ef42f086e920277e3135a"
},
{
"dataPath": "params_shard_24.bin",
"format": "raw-shard",
"nbytes": 28364800,
"records": [
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.layers.23.input_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.layers.23.input_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9453568
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_weight",
"shape": [
6144,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 6291456,
"byteOffset": 9457664
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_scale",
"shape": [
6144,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 786432,
"byteOffset": 15749120
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.bias",
"shape": [
6144
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 12288,
"byteOffset": 16535552
},
{
"name": "gpt_neox.layers.23.attention.dense.q_weight",
"shape": [
2048,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16547840
},
{
"name": "gpt_neox.layers.23.attention.dense.q_scale",
"shape": [
2048,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18644992
},
{
"name": "gpt_neox.layers.23.attention.dense.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 18907136
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight",
"shape": [
8192,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 18911232
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale",
"shape": [
8192,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 27299840
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias",
"shape": [
8192
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 16384,
"byteOffset": 28348416
}
],
"md5sum": "a71f0572037259be8e3e9c0e33a09031"
},
{
"dataPath": "params_shard_25.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "embed_out.q_weight",
"shape": [
50304,
256
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "baaf1146d31c6f73e0eb0caa9d9d72d7"
},
{
"dataPath": "params_shard_26.bin",
"format": "raw-shard",
"nbytes": 15888384,
"records": [
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight",
"shape": [
2048,
1024
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 8388608,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale",
"shape": [
2048,
256
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576,
"byteOffset": 8388608
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9437184
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9441280
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 9445376
},
{
"name": "embed_out.q_scale",
"shape": [
50304,
64
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6438912,
"byteOffset": 9449472
}
],
"md5sum": "ab7dcc074643dccfd287236a4d0e2060"
}
]
}