| { | |
| "metadata": { | |
| "ParamSize": 137, | |
| "ParamBytes": 394450304.0, | |
| "BitsPerParam": 4.501226222315055 | |
| }, | |
| "records": [ | |
| { | |
| "dataPath": "params_shard_0.bin", | |
| "format": "raw-shard", | |
| "nbytes": 18435776, | |
| "records": [ | |
| { | |
| "name": "model.embed_tokens.q_weight", | |
| "shape": [ | |
| 32003, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16385536, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.embed_tokens.q_scale", | |
| "shape": [ | |
| 32003, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048192, | |
| "byteOffset": 16385536 | |
| }, | |
| { | |
| "name": "model.norm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18433728 | |
| } | |
| ], | |
| "md5sum": "39262692eadce1dc2cc71f107dde7dba" | |
| }, | |
| { | |
| "dataPath": "params_shard_1.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.0.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "4b4790e347b682a2d1625d4ca8cf2fdc" | |
| }, | |
| { | |
| "dataPath": "params_shard_2.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.1.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "96cd48c89abfb63f5e47ac29c2dea29f" | |
| }, | |
| { | |
| "dataPath": "params_shard_3.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.2.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "2ebe7fb32b921287504c204504a3a3bf" | |
| }, | |
| { | |
| "dataPath": "params_shard_4.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.3.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "040d60d3b1c140992c37ae6f47157639" | |
| }, | |
| { | |
| "dataPath": "params_shard_5.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.4.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "78d5dc519482da4b6fa5a6fdbf3075fc" | |
| }, | |
| { | |
| "dataPath": "params_shard_6.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.5.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "ddf0d5b05b33b228c73a67b12bdf5181" | |
| }, | |
| { | |
| "dataPath": "params_shard_7.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.6.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "6393f0828fe8a7be924c6a58b122b655" | |
| }, | |
| { | |
| "dataPath": "params_shard_8.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.7.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "747277f3a82adb4957c54b7fbcc2265e" | |
| }, | |
| { | |
| "dataPath": "params_shard_9.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33138368, | |
| "records": [ | |
| { | |
| "name": "lm_head.q_weight", | |
| "shape": [ | |
| 32003, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16385536, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "lm_head.q_scale", | |
| "shape": [ | |
| 32003, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048192, | |
| "byteOffset": 16385536 | |
| }, | |
| { | |
| "name": "model.layers.0.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18433728 | |
| }, | |
| { | |
| "name": "model.layers.1.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18435776 | |
| }, | |
| { | |
| "name": "model.layers.2.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18437824 | |
| }, | |
| { | |
| "name": "model.layers.3.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18439872 | |
| }, | |
| { | |
| "name": "model.layers.4.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18441920 | |
| }, | |
| { | |
| "name": "model.layers.5.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18443968 | |
| }, | |
| { | |
| "name": "model.layers.6.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18446016 | |
| }, | |
| { | |
| "name": "model.layers.7.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18448064 | |
| }, | |
| { | |
| "name": "model.layers.8.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18450112 | |
| }, | |
| { | |
| "name": "model.layers.9.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18452160 | |
| }, | |
| { | |
| "name": "model.layers.10.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18454208 | |
| }, | |
| { | |
| "name": "model.layers.11.input_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 18456256 | |
| }, | |
| { | |
| "name": "model.layers.0.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18458304 | |
| }, | |
| { | |
| "name": "model.layers.1.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 20555456 | |
| }, | |
| { | |
| "name": "model.layers.2.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 22652608 | |
| }, | |
| { | |
| "name": "model.layers.3.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 24749760 | |
| }, | |
| { | |
| "name": "model.layers.4.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 26846912 | |
| }, | |
| { | |
| "name": "model.layers.5.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 28944064 | |
| }, | |
| { | |
| "name": "model.layers.6.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 31041216 | |
| } | |
| ], | |
| "md5sum": "15fc560eff0beef7363843096cd3f83f" | |
| }, | |
| { | |
| "dataPath": "params_shard_10.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.9.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "225eec8ab6c3a421d5f563f229fcf42f" | |
| }, | |
| { | |
| "dataPath": "params_shard_11.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.10.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "0b7d9bfd90e4239969f587c972110172" | |
| }, | |
| { | |
| "dataPath": "params_shard_12.bin", | |
| "format": "raw-shard", | |
| "nbytes": 16777216, | |
| "records": [ | |
| { | |
| "name": "model.layers.11.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 0 | |
| } | |
| ], | |
| "md5sum": "49d56d62a2c5cbeb10306793e6aa6f70" | |
| }, | |
| { | |
| "dataPath": "params_shard_13.bin", | |
| "format": "raw-shard", | |
| "nbytes": 27262976, | |
| "records": [ | |
| { | |
| "name": "model.layers.7.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.8.moe.e1_e3.q_weight", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 16777216, | |
| "byteOffset": 2097152 | |
| }, | |
| { | |
| "name": "model.layers.8.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 18874368 | |
| }, | |
| { | |
| "name": "model.layers.9.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 20971520 | |
| }, | |
| { | |
| "name": "model.layers.10.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 23068672 | |
| }, | |
| { | |
| "name": "model.layers.11.moe.e1_e3.q_scale", | |
| "shape": [ | |
| 4, | |
| 8192, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2097152, | |
| "byteOffset": 25165824 | |
| } | |
| ], | |
| "md5sum": "655bd70482a3f3d9e74580733bbc15d1" | |
| }, | |
| { | |
| "dataPath": "params_shard_14.bin", | |
| "format": "raw-shard", | |
| "nbytes": 28311552, | |
| "records": [ | |
| { | |
| "name": "model.layers.0.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.0.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.1.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.1.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 17825792 | |
| }, | |
| { | |
| "name": "model.layers.2.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 18874368 | |
| }, | |
| { | |
| "name": "model.layers.2.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 27262976 | |
| } | |
| ], | |
| "md5sum": "b8058afbdee107c4dd664bf5677bb64b" | |
| }, | |
| { | |
| "dataPath": "params_shard_15.bin", | |
| "format": "raw-shard", | |
| "nbytes": 28311552, | |
| "records": [ | |
| { | |
| "name": "model.layers.3.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.3.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.4.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.4.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 17825792 | |
| }, | |
| { | |
| "name": "model.layers.5.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 18874368 | |
| }, | |
| { | |
| "name": "model.layers.5.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 27262976 | |
| } | |
| ], | |
| "md5sum": "ea81f78fb218262780ff13b5c95662dc" | |
| }, | |
| { | |
| "dataPath": "params_shard_16.bin", | |
| "format": "raw-shard", | |
| "nbytes": 28311552, | |
| "records": [ | |
| { | |
| "name": "model.layers.6.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.6.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.7.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.7.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 17825792 | |
| }, | |
| { | |
| "name": "model.layers.8.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 18874368 | |
| }, | |
| { | |
| "name": "model.layers.8.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 27262976 | |
| } | |
| ], | |
| "md5sum": "2ba93f293ac038d9c735a1bcdea65dc3" | |
| }, | |
| { | |
| "dataPath": "params_shard_17.bin", | |
| "format": "raw-shard", | |
| "nbytes": 33546240, | |
| "records": [ | |
| { | |
| "name": "model.layers.9.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.9.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 8388608 | |
| }, | |
| { | |
| "name": "model.layers.10.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 9437184 | |
| }, | |
| { | |
| "name": "model.layers.10.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 17825792 | |
| }, | |
| { | |
| "name": "model.layers.11.moe.e2.q_weight", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 512 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8388608, | |
| "byteOffset": 18874368 | |
| }, | |
| { | |
| "name": "model.layers.11.moe.e2.q_scale", | |
| "shape": [ | |
| 4, | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 1048576, | |
| "byteOffset": 27262976 | |
| }, | |
| { | |
| "name": "model.layers.0.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28311552 | |
| }, | |
| { | |
| "name": "model.layers.1.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28313600 | |
| }, | |
| { | |
| "name": "model.layers.2.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28315648 | |
| }, | |
| { | |
| "name": "model.layers.3.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28317696 | |
| }, | |
| { | |
| "name": "model.layers.4.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28319744 | |
| }, | |
| { | |
| "name": "model.layers.5.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28321792 | |
| }, | |
| { | |
| "name": "model.layers.6.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28323840 | |
| }, | |
| { | |
| "name": "model.layers.7.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28325888 | |
| }, | |
| { | |
| "name": "model.layers.8.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28327936 | |
| }, | |
| { | |
| "name": "model.layers.9.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28329984 | |
| }, | |
| { | |
| "name": "model.layers.10.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28332032 | |
| }, | |
| { | |
| "name": "model.layers.11.post_attention_layernorm.weight", | |
| "shape": [ | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 2048, | |
| "byteOffset": 28334080 | |
| }, | |
| { | |
| "name": "model.layers.0.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 28336128 | |
| }, | |
| { | |
| "name": "model.layers.0.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 29122560 | |
| }, | |
| { | |
| "name": "model.layers.1.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 29220864 | |
| }, | |
| { | |
| "name": "model.layers.1.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 30007296 | |
| }, | |
| { | |
| "name": "model.layers.2.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 30105600 | |
| }, | |
| { | |
| "name": "model.layers.2.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 30892032 | |
| }, | |
| { | |
| "name": "model.layers.3.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 30990336 | |
| }, | |
| { | |
| "name": "model.layers.3.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 31776768 | |
| }, | |
| { | |
| "name": "model.layers.4.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 31875072 | |
| }, | |
| { | |
| "name": "model.layers.4.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 32661504 | |
| }, | |
| { | |
| "name": "model.layers.5.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 32759808 | |
| } | |
| ], | |
| "md5sum": "6cea3c1ce36c0eeebc1e0cd0c211e553" | |
| }, | |
| { | |
| "dataPath": "params_shard_18.bin", | |
| "format": "raw-shard", | |
| "nbytes": 12582912, | |
| "records": [ | |
| { | |
| "name": "model.layers.5.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 0 | |
| }, | |
| { | |
| "name": "model.layers.6.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 98304 | |
| }, | |
| { | |
| "name": "model.layers.6.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 884736 | |
| }, | |
| { | |
| "name": "model.layers.7.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 983040 | |
| }, | |
| { | |
| "name": "model.layers.7.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 1769472 | |
| }, | |
| { | |
| "name": "model.layers.8.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 1867776 | |
| }, | |
| { | |
| "name": "model.layers.8.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 2654208 | |
| }, | |
| { | |
| "name": "model.layers.9.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 2752512 | |
| }, | |
| { | |
| "name": "model.layers.9.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 3538944 | |
| }, | |
| { | |
| "name": "model.layers.10.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 3637248 | |
| }, | |
| { | |
| "name": "model.layers.10.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 4423680 | |
| }, | |
| { | |
| "name": "model.layers.11.self_attn.qkv_proj.q_weight", | |
| "shape": [ | |
| 1536, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 786432, | |
| "byteOffset": 4521984 | |
| }, | |
| { | |
| "name": "model.layers.11.self_attn.qkv_proj.q_scale", | |
| "shape": [ | |
| 1536, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 98304, | |
| "byteOffset": 5308416 | |
| }, | |
| { | |
| "name": "model.layers.0.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 5406720 | |
| }, | |
| { | |
| "name": "model.layers.0.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 5931008 | |
| }, | |
| { | |
| "name": "model.layers.1.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 5996544 | |
| }, | |
| { | |
| "name": "model.layers.1.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 6520832 | |
| }, | |
| { | |
| "name": "model.layers.2.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 6586368 | |
| }, | |
| { | |
| "name": "model.layers.2.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 7110656 | |
| }, | |
| { | |
| "name": "model.layers.3.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 7176192 | |
| }, | |
| { | |
| "name": "model.layers.3.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 7700480 | |
| }, | |
| { | |
| "name": "model.layers.4.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 7766016 | |
| }, | |
| { | |
| "name": "model.layers.4.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 8290304 | |
| }, | |
| { | |
| "name": "model.layers.5.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 8355840 | |
| }, | |
| { | |
| "name": "model.layers.5.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 8880128 | |
| }, | |
| { | |
| "name": "model.layers.6.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 8945664 | |
| }, | |
| { | |
| "name": "model.layers.6.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 9469952 | |
| }, | |
| { | |
| "name": "model.layers.7.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 9535488 | |
| }, | |
| { | |
| "name": "model.layers.7.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 10059776 | |
| }, | |
| { | |
| "name": "model.layers.8.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 10125312 | |
| }, | |
| { | |
| "name": "model.layers.8.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 10649600 | |
| }, | |
| { | |
| "name": "model.layers.9.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 10715136 | |
| }, | |
| { | |
| "name": "model.layers.9.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 11239424 | |
| }, | |
| { | |
| "name": "model.layers.10.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 11304960 | |
| }, | |
| { | |
| "name": "model.layers.10.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 11829248 | |
| }, | |
| { | |
| "name": "model.layers.11.self_attn.o_proj.q_weight", | |
| "shape": [ | |
| 1024, | |
| 128 | |
| ], | |
| "dtype": "uint32", | |
| "format": "f32-to-bf16", | |
| "nbytes": 524288, | |
| "byteOffset": 11894784 | |
| }, | |
| { | |
| "name": "model.layers.11.self_attn.o_proj.q_scale", | |
| "shape": [ | |
| 1024, | |
| 32 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 65536, | |
| "byteOffset": 12419072 | |
| }, | |
| { | |
| "name": "model.layers.0.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12484608 | |
| }, | |
| { | |
| "name": "model.layers.1.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12492800 | |
| }, | |
| { | |
| "name": "model.layers.2.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12500992 | |
| }, | |
| { | |
| "name": "model.layers.3.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12509184 | |
| }, | |
| { | |
| "name": "model.layers.4.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12517376 | |
| }, | |
| { | |
| "name": "model.layers.5.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12525568 | |
| }, | |
| { | |
| "name": "model.layers.6.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12533760 | |
| }, | |
| { | |
| "name": "model.layers.7.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12541952 | |
| }, | |
| { | |
| "name": "model.layers.8.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12550144 | |
| }, | |
| { | |
| "name": "model.layers.9.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12558336 | |
| }, | |
| { | |
| "name": "model.layers.10.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12566528 | |
| }, | |
| { | |
| "name": "model.layers.11.moe.gate.weight", | |
| "shape": [ | |
| 4, | |
| 1024 | |
| ], | |
| "dtype": "float16", | |
| "format": "f32-to-bf16", | |
| "nbytes": 8192, | |
| "byteOffset": 12574720 | |
| } | |
| ], | |
| "md5sum": "c81b7f0ed2e3e42b10a9de6ffc778b2e" | |
| } | |
| ] | |
| } |