gemma-1.1-2b-it-q0f16 / ndarray-cache.json
Amai
Add weights
681b1c9
Raw History Blame Contribute Delete
52.7 kB
{ "metadata": { "ParamSize": 110, "ParamBytes": 5012344832.0, "BitsPerParam": 16.0 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 1048576000, "records": [ { "name": "model.embed_tokens.weight", "shape": [ 256000, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1048576000, "byteOffset": 0 } ], "md5sum": "610b9d73bf7dc83ac6ed9468f13a6b54" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.0.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "49b0af5d56713ffd3fa1decf35ea8868" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.0.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "25ff4970334be601c65f54ea82bed4a2" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.1.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "ad6f7709f155ba84dc6756d286db6778" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.1.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "330eba418c9caede3735f8b7e726b5d5" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 29376512, "records": [ { "name": "model.layers.0.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 0 }, { "name": "model.layers.0.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 4096 }, { "name": "model.layers.0.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8192 }, { "name": "model.layers.0.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10493952 }, { "name": "model.layers.1.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18882560 }, { "name": "model.layers.1.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18886656 }, { "name": "model.layers.1.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18890752 } ], "md5sum": "f7c960d3f758931df295359b2f2eeb7c" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.10.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "fd519c1c6d835720ad141a878ec98b57" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.10.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "b93154c3cdde1daaf41a0feb251502ef" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.11.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "4e22835a816aea13a1b35fb85f888100" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.11.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "46fc10a74eab368ecf1ea5723e8d410b" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 27279360, "records": [ { "name": "model.layers.1.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.10.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.10.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.10.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.10.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 }, { "name": "model.layers.11.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27271168 }, { "name": "model.layers.11.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27275264 } ], "md5sum": "c27177e8ddd302e0d5ebfd35871fdf36" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.12.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "ee86eb5ec0ef925c305a8f098ff0ce84" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.12.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "7870c542c62a866fdd92208fe13b4dbe" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 29368320, "records": [ { "name": "model.layers.11.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 0 }, { "name": "model.layers.11.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10485760 }, { "name": "model.layers.12.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18874368 }, { "name": "model.layers.12.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18878464 }, { "name": "model.layers.12.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18882560 } ], "md5sum": "b34d3fa2ba6c76031e463ceb6b577a47" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.13.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "97c3f86ffd0a1e1bbb9a2b5d99b19495" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.13.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "b67c819617d88ce9de770defa733a4ac" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.14.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "a2ad12ff30d075189f9f08c13c8f8255" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.14.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "e12bd309683f3c104191c1acffdb5404" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 27279360, "records": [ { "name": "model.layers.12.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.13.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.13.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.13.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.13.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 }, { "name": "model.layers.14.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27271168 }, { "name": "model.layers.14.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27275264 } ], "md5sum": "3675c48036899f0b5bce4c05a98fbd00" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.15.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "a4b7e3d87817a62bae737a294057ec92" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.15.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "9d6cf5a4339ea12d0ff1d8cb7289bc64" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 29368320, "records": [ { "name": "model.layers.14.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 0 }, { "name": "model.layers.14.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10485760 }, { "name": "model.layers.15.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18874368 }, { "name": "model.layers.15.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18878464 }, { "name": "model.layers.15.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18882560 } ], "md5sum": "42e04050cae3d259ecd1770155430953" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.16.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "cbf2874309283fed084b0107649c9dc6" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.16.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "7f082fd649a0b82ea6d2708b66a8c326" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.17.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "356e1888c3025747a605f751d3726165" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 27271168, "records": [ { "name": "model.layers.15.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.16.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.16.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.16.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.16.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 } ], "md5sum": "3df6efc79725d093c67f8818c9e29c27" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.2.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "484e0e95b2f89b527f78b56f387ba4ec" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.2.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "a8db107a7090655d18642980f6beece9" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 29368320, "records": [ { "name": "model.layers.17.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 0 }, { "name": "model.layers.17.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10485760 }, { "name": "model.layers.2.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18874368 }, { "name": "model.layers.2.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18878464 }, { "name": "model.layers.2.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18882560 } ], "md5sum": "12dd22fca1d8c7b6b56b05456e8e33ff" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.3.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "9fd0b63802be90e7facabff3a0871693" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.3.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "83c152660a4532914895699803a378d0" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.4.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "9bd5db71ee225ea8527318781e1d5ad4" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.4.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "30a989ff91d613ece956a27b30270af2" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 27279360, "records": [ { "name": "model.layers.2.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.3.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.3.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.3.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.3.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 }, { "name": "model.layers.4.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27271168 }, { "name": "model.layers.4.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27275264 } ], "md5sum": "8138ddac70785362d64528ccacb70eaa" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.5.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "0bead22571193395ae89d6338fc5a9a7" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.5.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "ce7814334477096ba6fdec3c232ebafc" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 29368320, "records": [ { "name": "model.layers.4.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 0 }, { "name": "model.layers.4.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10485760 }, { "name": "model.layers.5.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18874368 }, { "name": "model.layers.5.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18878464 }, { "name": "model.layers.5.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18882560 } ], "md5sum": "f1120bc6939205accb9e9a267e2562de" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.6.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "00320de9a89c0b08340539a56a30c686" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.6.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "90b28e61f8d68f690b9b5ee33b8b1217" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.7.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "f7f3d33105ec8a04e0b13c38878cda81" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.7.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "e72469b6d6a48449e967d0a7af6cf643" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 27279360, "records": [ { "name": "model.layers.5.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.6.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.6.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.6.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.6.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 }, { "name": "model.layers.7.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27271168 }, { "name": "model.layers.7.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27275264 } ], "md5sum": "fceac59c76806a3bf0a2e851919e7ed4" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.8.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "156d84e98a53cabb3eac82292e764410" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.8.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "1e82df8cd1177ced424fc79d7cb70b6e" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 29368320, "records": [ { "name": "model.layers.7.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 0 }, { "name": "model.layers.7.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 10485760 }, { "name": "model.layers.8.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18874368 }, { "name": "model.layers.8.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 18878464 }, { "name": "model.layers.8.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 18882560 } ], "md5sum": "b329da7b8298f019c4c0ec66a9dae5bf" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.9.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "7eb0c015f2ac170b0c2e7856b3d5e8d9" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 134217728, "records": [ { "name": "model.layers.9.mlp.gate_up_proj.weight", "shape": [ 32768, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 134217728, "byteOffset": 0 } ], "md5sum": "585c41da1e4be48c4ee1572d66345659" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 67108864, "records": [ { "name": "model.layers.17.mlp.down_proj.weight", "shape": [ 2048, 16384 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 67108864, "byteOffset": 0 } ], "md5sum": "17fe66496d212296b0202d8a47e95e1f" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 27283456, "records": [ { "name": "model.layers.8.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 0 }, { "name": "model.layers.9.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8388608 }, { "name": "model.layers.9.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 8392704 }, { "name": "model.layers.9.self_attn.qkv_proj.weight", "shape": [ 2560, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 10485760, "byteOffset": 8396800 }, { "name": "model.layers.9.self_attn.o_proj.weight", "shape": [ 2048, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 8388608, "byteOffset": 18882560 }, { "name": "model.layers.17.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27271168 }, { "name": "model.layers.17.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27275264 }, { "name": "model.norm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27279360 } ], "md5sum": "1089bbae03b4c29a366767053843c0b6" } ] }
{
"metadata": {
"ParamSize": 110,
"ParamBytes": 5012344832.0,
"BitsPerParam": 16.0
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 1048576000,
"records": [
{
"name": "model.embed_tokens.weight",
"shape": [
256000,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1048576000,
"byteOffset": 0
}
],
"md5sum": "610b9d73bf7dc83ac6ed9468f13a6b54"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 67108864,
"records": [
{
"name": "model.layers.0.mlp.down_proj.weight",
"shape": [
2048,
16384
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 67108864,
"byteOffset": 0
}
],
"md5sum": "49b0af5d56713ffd3fa1decf35ea8868"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 134217728,
"records": [
{
"name": "model.layers.0.mlp.gate_up_proj.weight",
"shape": [
32768,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 134217728,
"byteOffset": 0
}
],
"md5sum": "25ff4970334be601c65f54ea82bed4a2"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 67108864,
"records": [
{
"name": "model.layers.1.mlp.down_proj.weight",
"shape": [
2048,
16384
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 67108864,
"byteOffset": 0
}
],
"md5sum": "ad6f7709f155ba84dc6756d286db6778"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 134217728,
"records": [
{
"name": "model.layers.1.mlp.gate_up_proj.weight",
"shape": [
32768,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 134217728,
"byteOffset": 0
}
],