diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,4175 @@ +{ + "metadata": { + "ParamSize": 325, + "ParamBytes": 2149644288.0, + "BitsPerParam": 4.500600961055312 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 49250304, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 384, + 32064 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 49250304, + "byteOffset": 0 + } + ], + "md5sum": "15573574494a1c2a922401367f6e7f3b" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 49250304, + "records": [ + { + "name": "transformer.embd.q_weight", + "shape": [ + 32064, + 384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 49250304, + "byteOffset": 0 + } + ], + "md5sum": "e0811e2336da65b98ffc27b4f7a39c39" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.0.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5bb70357e6b750882927e6da7c5a36ef" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 29626368, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 96, + 32064 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6156288, + "byteOffset": 0 + }, + { + "name": "transformer.embd.q_scale", + "shape": [ + 32064, + 96 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6156288, + "byteOffset": 6156288 + }, + { + "name": "transformer.h.0.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 12312576 + }, + { + "name": "transformer.h.0.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 12318720 + }, + { + "name": "transformer.h.0.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 24901632 + }, + { + "name": "transformer.h.0.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 26474496 + }, + { + "name": "transformer.h.0.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 29620224 + } + ], + "md5sum": "daeaba161fee0d2587777bc0a0439353" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.0.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.0.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.0.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.0.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.1.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "ec4cb7aa5757e874fc5d4697b6af1cf4" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.1.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "0875cfe99fbf3462e115a60626bad3a5" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.1.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.1.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.1.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.1.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.1.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "0d27b00632b1fd873029bb57a32b524e" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.1.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.1.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.10.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 20643840 + }, + { + "name": "transformer.h.2.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "15ebd758e534ab423e8009f4e310d709" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.2.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c8c75ab57aa43d8ad8965169136f000f" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.2.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.2.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.2.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.2.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.2.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "39c27177c60769b8d8932e9b63f1a99c" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.3.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "970536f0c4a4f648455085dd416095e5" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.2.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.2.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.3.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.3.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.3.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.3.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.3.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "ea7c9f8d68686047f25e1065b0b773b1" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.3.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.3.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.3.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.3.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.4.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "2230b5716309774b6faf558626cf2be4" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.4.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "304bf71ac1bfd23c37aff008c376c5ea" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.4.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.4.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.4.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.4.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.4.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "b1f6ac560113bc4f8710d033535ab163" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.5.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c87170941f3c7d2e78659e0e1cb7150c" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.4.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.4.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.5.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.5.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.5.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.5.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.5.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "7c7ffddce1370de2362b78e21a6591d0" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.5.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.5.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.5.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.5.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.6.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "4f53a78d3d44704fc76cf9dabbb00818" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.6.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "03f34c97555e4068e4357453ff503354" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.6.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.6.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.6.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.6.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.6.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.6.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "adf9b9bbf3beb390b6b4e99276fb5a5f" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.7.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1f8ebeb1757f0bc120c3ea3ec8a2d460" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.6.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.6.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.7.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.7.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.7.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.7.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.7.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "daba95cdd00ada8acbcc62d90e980524" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.7.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.7.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.7.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.7.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.8.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "be2252e5d16a20091919c7706ed5bb58" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.8.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "365fbbd1d1df787874647f25d4976b20" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.8.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.8.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.8.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.8.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.8.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.8.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "eb430e51053e1179645f896d2c53d68b" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.9.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d9495eeb87346f6b6ab0cd302e034607" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.8.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.8.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.9.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.9.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.9.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.9.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.9.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "dd9bc7a4591dd45c901cf5d3d31bde16" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.9.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.9.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.9.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.9.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.10.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "8746eefff81605500a690b8ef2877bea" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.10.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1d8e3cf9581ff1784cfe1152510e44bf" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.10.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.10.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.10.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.10.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.10.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.10.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 31463424 + }, + { + "name": "transformer.h.11.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "9319ba01839c3fcedc29740659216c32" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.11.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c2550beb4502957aad93df81f22dacdd" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.11.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.11.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.11.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.11.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "761c825320673a67712af5c57abc883f" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.12.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5d95f3d52fd77e93926f6eb6b3772621" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.11.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.11.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.12.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.12.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.12.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.12.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.12.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "f7c2de90187c932d792d32342f5193fa" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.12.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.12.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.12.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.12.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.13.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "630fa88d211032d0993608c98298219d" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.13.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e67cb0286bb64f55ba50b09d54df50d1" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.13.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.13.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.13.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.13.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.13.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "ba58d88ba5ec66a170289ac3f182ad9b" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.14.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "289f88d75aef6cc18657b8c402a18234" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.13.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.13.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.14.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.14.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.14.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.14.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.14.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "4c43924843a0e2247711d1bb651d304c" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.14.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.14.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.14.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.14.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.15.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "a58fd333bc8ef158f9b1290b7c857eba" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.15.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "418f614d2efb8ef2f3679befae13836c" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.15.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.15.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.15.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.15.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.15.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "e09006870e844e8aa9d80003c5cc4956" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.16.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5d6398415ad3ec220f3362ec28becdef" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.15.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.15.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.16.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.16.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.16.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.16.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.16.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "f6690cf5b5bcdf56c3b0c8a4c8493931" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.16.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.16.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.16.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.16.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.17.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "d1464dd3d1ec5d243d67d16df08a8758" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.17.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "41fd3b9996a7b8e62ed889b4337a6b30" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.17.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.17.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.17.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.17.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.17.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "b1809c6cbefbe78a378454fe90292c8f" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.18.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e788d1611072f50a21eb930083dea03b" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.17.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.17.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.18.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.18.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.18.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.18.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.18.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "3449ca5272b2b01bedf4a467bd47da96" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.18.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.18.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.18.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.18.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.19.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "fd6ccccd3d9dafc004e8401cd048e812" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.19.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "7840919ad8415bfa5974c0160d256a49" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.19.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.19.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.19.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.19.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.19.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "7390c4aeb09fc03807a7635882017919" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.20.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "deb7aee1e739628ba7c9d0e89e87c5c1" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.19.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.19.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.20.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.20.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.20.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.20.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.20.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "48147ad5096b40fd52d906e77d1fb724" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 26548224, + "records": [ + { + "name": "transformer.h.20.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.20.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.20.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.20.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 21233664 + }, + { + "name": "transformer.h.21.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 25952256 + }, + { + "name": "transformer.h.21.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 26542080 + } + ], + "md5sum": "8efb76077f593cb38fdbb99a5d791300" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.21.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c2fabae7f63fb4d0f14899fa4ff8b4d3" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.21.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.21.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.21.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.21.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.21.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.21.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 31463424 + }, + { + "name": "transformer.h.22.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "55c31627f27d656313eca4eb874a0b07" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.22.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "b5a949deef4bfae09a6cbacda55f4a75" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.22.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.22.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.22.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.22.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.22.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "26b55242cfab8e1434b1130d546113d7" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.23.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "5417e1f3806f3868059f8e2cacf732c8" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.22.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.22.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.23.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.23.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.23.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.23.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.23.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "066e3b3303ae7d6b096af85d9262f987" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.23.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.23.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.23.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.23.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.24.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "5fa60e4b10e1a93ad4d7eb5dc4550648" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.24.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "eab49416f0a8ceda4075c045754b7c4d" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.24.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.24.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.24.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.24.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.24.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.24.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "962f764d49142f6b448b294323664f66" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.25.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8709e6e8ca6a8cd1a782e066dd79440c" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.24.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.24.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.25.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.25.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.25.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.25.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.25.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "91fc548eed1f4d8f3806ebdf5d2b1d88" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.25.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.25.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.25.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.25.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.26.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "fd68e353c2d0679990e3392f8d06018e" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.26.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "ce05bcf99093f3c51edfc42ad5140749" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.26.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.26.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.26.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.26.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.26.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.26.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "196c81d6ee6cabbf2d505b3e3cfce54b" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.27.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "b13dadb3b336e75f9edbc658b0251b8a" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.26.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.26.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.27.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.27.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.27.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.27.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.27.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "5efd36043298d77fbb67aa42228c1612" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.27.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.27.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.27.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.27.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.28.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "f3d05381835875dad7fe7c83b7aa4fb4" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.28.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "4bf4d2cdc342fe9bfbfd86dd604ead64" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.28.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.28.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.28.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.28.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.28.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.28.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "5bf3ee91205a0d4a5bf8d905c3c5b7ff" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.29.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d5aa4aaa907026b054b11f60acb7f257" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.28.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.28.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.29.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.29.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.29.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.29.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.29.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "8f057188fe213432cfd8667c5a9857c8" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.29.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.29.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.29.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.29.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.h.30.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "b48817fbd545288d46d68eba2bb293c6" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.30.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9d72872b683a5f18a21ba945acd3b303" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 22616064, + "records": [ + { + "name": "transformer.h.30.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 0 + }, + { + "name": "transformer.h.30.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 12582912 + }, + { + "name": "transformer.h.30.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.30.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 17301504 + }, + { + "name": "transformer.h.30.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 17307648 + }, + { + "name": "transformer.h.30.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 22026240 + } + ], + "md5sum": "c342abf30d9b7ee00b1bf225aed25d8d" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "transformer.h.31.mlp.gate_up_proj.q_weight", + "shape": [ + 384, + 16384 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d66b4c8a4d8f017f96957e1d545932e7" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 33239040, + "records": [ + { + "name": "transformer.h.30.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 0 + }, + { + "name": "transformer.h.30.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 14155776 + }, + { + "name": "transformer.h.31.ln.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 15925248 + }, + { + "name": "transformer.h.31.mlp.down_proj.q_weight", + "shape": [ + 1024, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 12582912, + "byteOffset": 15931392 + }, + { + "name": "transformer.h.31.mlp.down_proj.q_scale", + "shape": [ + 256, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 28514304 + }, + { + "name": "transformer.h.31.mlp.gate_up_proj.q_scale", + "shape": [ + 96, + 16384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 30087168 + }, + { + "name": "transformer.h.31.post_attention_layernorm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 33232896 + } + ], + "md5sum": "9392f9269cc80331df56625c788da0b1" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 21239808, + "records": [ + { + "name": "transformer.h.31.mixer.out_proj.q_weight", + "shape": [ + 384, + 3072 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 4718592, + "byteOffset": 0 + }, + { + "name": "transformer.h.31.mixer.out_proj.q_scale", + "shape": [ + 96, + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 589824, + "byteOffset": 4718592 + }, + { + "name": "transformer.h.31.mixer.qkv_proj.q_weight", + "shape": [ + 384, + 9216 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 14155776, + "byteOffset": 5308416 + }, + { + "name": "transformer.h.31.mixer.qkv_proj.q_scale", + "shape": [ + 96, + 9216 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1769472, + "byteOffset": 19464192 + }, + { + "name": "transformer.norm.weight", + "shape": [ + 3072 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6144, + "byteOffset": 21233664 + } + ], + "md5sum": "dc8b9cc5fcccb2885b98b22586fffcf5" + } + ] +} \ No newline at end of file