| { |
| "format": "zonos3-web-v1", |
| "model": "encoder", |
| "meta": { |
| "general.architecture": "mossdual_encoder", |
| "general.name": "fixed600k frozen dual Opus reference encoder", |
| "sample_rate": 44100, |
| "internal_sample_rate": 44160, |
| "frame_rate": 15, |
| "hop": 2940, |
| "internal_hop": 2944, |
| "n_codebooks": 10, |
| "codebook_size": 4096, |
| "latent_dim": 1024, |
| "semantic_dim": 768, |
| "semantic_depth": 4, |
| "lane_frames": 300, |
| "context_frames": 150, |
| "window_ratio": 2.0, |
| "ln_eps": 9.999999747378752e-06, |
| "convnext_eps": 9.999999974752427e-07, |
| "stage.in_dim": [ |
| 5888, |
| 768, |
| 768, |
| 768, |
| 1280 |
| ], |
| "stage.out_dim": [ |
| 384, |
| 384, |
| 384, |
| 640, |
| 1024 |
| ], |
| "stage.d_model": [ |
| 768, |
| 768, |
| 768, |
| 768, |
| 1280 |
| ], |
| "stage.n_head": [ |
| 12, |
| 12, |
| 12, |
| 12, |
| 20 |
| ], |
| "stage.n_layer": [ |
| 12, |
| 12, |
| 12, |
| 12, |
| 32 |
| ], |
| "stage.n_ff": [ |
| 3072, |
| 3072, |
| 3072, |
| 3072, |
| 5120 |
| ], |
| "stage.context": [ |
| 480, |
| 480, |
| 480, |
| 300, |
| 150 |
| ], |
| "stage.patch": [ |
| 23, |
| 2, |
| 2, |
| 2, |
| 2 |
| ], |
| "resample.orig": 735, |
| "resample.new": 736, |
| "resample.width": 17 |
| }, |
| "shards": [ |
| "shard-000.bin", |
| "shard-001.bin", |
| "shard-002.bin", |
| "shard-003.bin", |
| "shard-004.bin", |
| "shard-005.bin", |
| "shard-006.bin", |
| "shard-007.bin", |
| "shard-008.bin", |
| "shard-009.bin", |
| "shard-010.bin", |
| "shard-011.bin", |
| "shard-012.bin", |
| "shard-013.bin", |
| "shard-014.bin", |
| "shard-015.bin", |
| "shard-016.bin", |
| "shard-017.bin", |
| "shard-018.bin", |
| "shard-019.bin", |
| "shard-020.bin", |
| "shard-021.bin", |
| "shard-022.bin", |
| "shard-023.bin", |
| "shard-024.bin", |
| "shard-025.bin", |
| "shard-026.bin", |
| "shard-027.bin", |
| "shard-028.bin", |
| "shard-029.bin", |
| "shard-030.bin" |
| ], |
| "tensors": { |
| "resample.kernel": { |
| "dtype": "f32", |
| "shape": [ |
| 769, |
| 736 |
| ], |
| "shard": 0, |
| "offset": 0, |
| "nbytes": 2263936, |
| "host": true |
| }, |
| "enc.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 16, |
| 1024, |
| 2 |
| ], |
| "shard": 0, |
| "offset": 2264064, |
| "nbytes": 65536 |
| }, |
| "enc.in.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2329600, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2329856, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2330112, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 2330368, |
| "nbytes": 57344 |
| }, |
| "enc.b0.res0.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2387712, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2387968, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2388224, |
| "nbytes": 256 |
| }, |
| "enc.b0.res0.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 2388480, |
| "nbytes": 8192 |
| }, |
| "enc.b0.res0.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2396672, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2396928, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2397184, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 2397440, |
| "nbytes": 57344 |
| }, |
| "enc.b0.res1.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2454784, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2455040, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2455296, |
| "nbytes": 256 |
| }, |
| "enc.b0.res1.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 2455552, |
| "nbytes": 8192 |
| }, |
| "enc.b0.res1.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2463744, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2464000, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2464256, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 2464512, |
| "nbytes": 57344 |
| }, |
| "enc.b0.res2.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2521856, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2522112, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2522368, |
| "nbytes": 256 |
| }, |
| "enc.b0.res2.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 64, |
| 64, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 2522624, |
| "nbytes": 8192 |
| }, |
| "enc.b0.res2.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2530816, |
| "nbytes": 256 |
| }, |
| "enc.b0.snake.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2531072, |
| "nbytes": 256 |
| }, |
| "enc.b0.snake.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 64 |
| ], |
| "shard": 0, |
| "offset": 2531328, |
| "nbytes": 256 |
| }, |
| "enc.b0.conv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 2 |
| ], |
| "shard": 0, |
| "offset": 2531584, |
| "nbytes": 65536 |
| }, |
| "enc.b0.conv.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2597120, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2597632, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2598144, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 2598656, |
| "nbytes": 229376 |
| }, |
| "enc.b1.res0.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2828032, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2828544, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2829056, |
| "nbytes": 512 |
| }, |
| "enc.b1.res0.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 2829568, |
| "nbytes": 32768 |
| }, |
| "enc.b1.res0.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2862336, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2862848, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 2863360, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 2863872, |
| "nbytes": 229376 |
| }, |
| "enc.b1.res1.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3093248, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3093760, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3094272, |
| "nbytes": 512 |
| }, |
| "enc.b1.res1.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 3094784, |
| "nbytes": 32768 |
| }, |
| "enc.b1.res1.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3127552, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.snake1.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3128064, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.snake1.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3128576, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.conv1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 7 |
| ], |
| "shard": 0, |
| "offset": 3129088, |
| "nbytes": 229376 |
| }, |
| "enc.b1.res2.conv1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3358464, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.snake2.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3358976, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.snake2.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3359488, |
| "nbytes": 512 |
| }, |
| "enc.b1.res2.conv2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 128, |
| 128, |
| 1 |
| ], |
| "shard": 0, |
| "offset": 3360000, |
| "nbytes": 32768 |
| }, |
| "enc.b1.res2.conv2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3392768, |
| "nbytes": 512 |
| }, |
| "enc.b1.snake.alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3393280, |
| "nbytes": 512 |
| }, |
| "enc.b1.snake.inv_alpha": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 128 |
| ], |
| "shard": 0, |
| "offset": 3393792, |
| "nbytes": 512 |
| }, |
| "enc.b1.conv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 512, |
| 256, |
| 2 |
| ], |
| "shard": 0, |
| "offset": 3394304, |
| "nbytes": 524288 |
| }, |
| "enc.b1.conv.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1, |
| 256 |
| ], |
| "shard": 0, |
| "offset": 3918592, |
| "nbytes": 1024 |
| }, |
| "stage.0.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5888, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 3919616, |
| "nbytes": 9043968 |
| }, |
| "stage.0.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 384 |
| ], |
| "shard": 0, |
| "offset": 12963584, |
| "nbytes": 589824 |
| }, |
| "stage.0.l0.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13553408, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13556480, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13559552, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13562624, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13565696, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 13568768, |
| "nbytes": 3072 |
| }, |
| "stage.0.l0.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 0, |
| "offset": 13571840, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l0.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 17110784, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l0.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 0, |
| "offset": 18290432, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l0.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 23009024, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l1.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27727616, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27730688, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27733760, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27736832, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27739904, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 27742976, |
| "nbytes": 3072 |
| }, |
| "stage.0.l1.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 0, |
| "offset": 27746048, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l1.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 31284992, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l1.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 0, |
| "offset": 32464640, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l1.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 37183232, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l2.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41901824, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41904896, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41907968, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41911040, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41914112, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 41917184, |
| "nbytes": 3072 |
| }, |
| "stage.0.l2.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 0, |
| "offset": 41920256, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l2.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 45459200, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l2.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 0, |
| "offset": 46638848, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l2.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 51357440, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l3.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56076032, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56079104, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56082176, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56085248, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56088320, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 0, |
| "offset": 56091392, |
| "nbytes": 3072 |
| }, |
| "stage.0.l3.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 0, |
| "offset": 56094464, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l3.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 0, |
| "offset": 59633408, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l3.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 0, |
| "offset": 60813056, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l3.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l4.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4718592, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4721664, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4724736, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4727808, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4730880, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 4733952, |
| "nbytes": 3072 |
| }, |
| "stage.0.l4.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 1, |
| "offset": 4737024, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l4.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 8275968, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l4.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 1, |
| "offset": 9455616, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l4.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l5.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18892800, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18895872, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18898944, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18902016, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18905088, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 18908160, |
| "nbytes": 3072 |
| }, |
| "stage.0.l5.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 1, |
| "offset": 18911232, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l5.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 22450176, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l5.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 1, |
| "offset": 23629824, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l5.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l6.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33067008, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33070080, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33073152, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33076224, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33079296, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 33082368, |
| "nbytes": 3072 |
| }, |
| "stage.0.l6.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 1, |
| "offset": 33085440, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l6.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 36624384, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l6.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 1, |
| "offset": 37804032, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l6.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l7.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47241216, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47244288, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47247360, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47250432, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47253504, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 47256576, |
| "nbytes": 3072 |
| }, |
| "stage.0.l7.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 1, |
| "offset": 47259648, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l7.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 50798592, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l7.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 1, |
| "offset": 51978240, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l7.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 56696832, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l8.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61415424, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61418496, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61421568, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61424640, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61427712, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 1, |
| "offset": 61430784, |
| "nbytes": 3072 |
| }, |
| "stage.0.l8.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 1, |
| "offset": 61433856, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l8.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 1, |
| "offset": 64972800, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l8.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 2, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l8.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l9.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9437184, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9440256, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9443328, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9446400, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9449472, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 9452544, |
| "nbytes": 3072 |
| }, |
| "stage.0.l9.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 2, |
| "offset": 9455616, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l9.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 12994560, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l9.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 2, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l9.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l10.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23611392, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23614464, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23617536, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23620608, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23623680, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 23626752, |
| "nbytes": 3072 |
| }, |
| "stage.0.l10.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 2, |
| "offset": 23629824, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l10.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 27168768, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l10.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 2, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l10.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 33067008, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l11.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37785600, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37788672, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37791744, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37794816, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37797888, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 37800960, |
| "nbytes": 3072 |
| }, |
| "stage.0.l11.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 2, |
| "offset": 37804032, |
| "nbytes": 3538944 |
| }, |
| "stage.0.l11.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 41342976, |
| "nbytes": 1179648 |
| }, |
| "stage.0.l11.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 2, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.0.l11.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 47241216, |
| "nbytes": 4718592 |
| }, |
| "stage.1.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 51959808, |
| "nbytes": 1179648 |
| }, |
| "stage.1.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 384 |
| ], |
| "shard": 2, |
| "offset": 53139456, |
| "nbytes": 589824 |
| }, |
| "stage.1.l0.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53729280, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53732352, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53735424, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53738496, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53741568, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 2, |
| "offset": 53744640, |
| "nbytes": 3072 |
| }, |
| "stage.1.l0.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 2, |
| "offset": 53747712, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l0.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 2, |
| "offset": 57286656, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l0.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 2, |
| "offset": 58466304, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l0.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l1.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4718592, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4721664, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4724736, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4727808, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4730880, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 4733952, |
| "nbytes": 3072 |
| }, |
| "stage.1.l1.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 3, |
| "offset": 4737024, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l1.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 8275968, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l1.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 3, |
| "offset": 9455616, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l1.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l2.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18892800, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18895872, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18898944, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18902016, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18905088, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 18908160, |
| "nbytes": 3072 |
| }, |
| "stage.1.l2.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 3, |
| "offset": 18911232, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l2.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 22450176, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l2.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 3, |
| "offset": 23629824, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l2.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l3.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33067008, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33070080, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33073152, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33076224, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33079296, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 33082368, |
| "nbytes": 3072 |
| }, |
| "stage.1.l3.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 3, |
| "offset": 33085440, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l3.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 36624384, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l3.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 3, |
| "offset": 37804032, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l3.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l4.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47241216, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47244288, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47247360, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47250432, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47253504, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 47256576, |
| "nbytes": 3072 |
| }, |
| "stage.1.l4.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 3, |
| "offset": 47259648, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l4.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 50798592, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l4.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 3, |
| "offset": 51978240, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l4.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 56696832, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l5.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61415424, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61418496, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61421568, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61424640, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61427712, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 3, |
| "offset": 61430784, |
| "nbytes": 3072 |
| }, |
| "stage.1.l5.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 3, |
| "offset": 61433856, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l5.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 3, |
| "offset": 64972800, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l5.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 4, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l5.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l6.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9437184, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9440256, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9443328, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9446400, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9449472, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 9452544, |
| "nbytes": 3072 |
| }, |
| "stage.1.l6.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 4, |
| "offset": 9455616, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l6.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 12994560, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l6.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 4, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l6.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l7.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23611392, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23614464, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23617536, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23620608, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23623680, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 23626752, |
| "nbytes": 3072 |
| }, |
| "stage.1.l7.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 4, |
| "offset": 23629824, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l7.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 27168768, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l7.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 4, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l7.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 33067008, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l8.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37785600, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37788672, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37791744, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37794816, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37797888, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 37800960, |
| "nbytes": 3072 |
| }, |
| "stage.1.l8.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 4, |
| "offset": 37804032, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l8.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 41342976, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l8.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 4, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l8.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 47241216, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l9.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51959808, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51962880, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51965952, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51969024, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51972096, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 51975168, |
| "nbytes": 3072 |
| }, |
| "stage.1.l9.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 4, |
| "offset": 51978240, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l9.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 55517184, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l9.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 4, |
| "offset": 56696832, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l9.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 4, |
| "offset": 61415424, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l10.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66134016, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66137088, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66140160, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66143232, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66146304, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 4, |
| "offset": 66149376, |
| "nbytes": 3072 |
| }, |
| "stage.1.l10.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 5, |
| "offset": 0, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l10.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 3538944, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l10.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 5, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l10.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 9437184, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l11.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14155776, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14158848, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14161920, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14164992, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14168064, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 14171136, |
| "nbytes": 3072 |
| }, |
| "stage.1.l11.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 5, |
| "offset": 14174208, |
| "nbytes": 3538944 |
| }, |
| "stage.1.l11.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 17713152, |
| "nbytes": 1179648 |
| }, |
| "stage.1.l11.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 5, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.1.l11.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 23611392, |
| "nbytes": 4718592 |
| }, |
| "stage.2.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 28329984, |
| "nbytes": 1179648 |
| }, |
| "stage.2.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 384 |
| ], |
| "shard": 5, |
| "offset": 29509632, |
| "nbytes": 589824 |
| }, |
| "stage.2.l0.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30099456, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30102528, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30105600, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30108672, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30111744, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 30114816, |
| "nbytes": 3072 |
| }, |
| "stage.2.l0.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 5, |
| "offset": 30117888, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l0.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 33656832, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l0.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 5, |
| "offset": 34836480, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l0.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 39555072, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l1.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44273664, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44276736, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44279808, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44282880, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44285952, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 44289024, |
| "nbytes": 3072 |
| }, |
| "stage.2.l1.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 5, |
| "offset": 44292096, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l1.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 47831040, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l1.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 5, |
| "offset": 49010688, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l1.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 53729280, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l2.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58447872, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58450944, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58454016, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58457088, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58460160, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 5, |
| "offset": 58463232, |
| "nbytes": 3072 |
| }, |
| "stage.2.l2.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 5, |
| "offset": 58466304, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l2.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 5, |
| "offset": 62005248, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l2.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 6, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l2.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l3.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9437184, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9440256, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9443328, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9446400, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9449472, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 9452544, |
| "nbytes": 3072 |
| }, |
| "stage.2.l3.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 6, |
| "offset": 9455616, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l3.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 12994560, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l3.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 6, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l3.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l4.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23611392, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23614464, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23617536, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23620608, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23623680, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 23626752, |
| "nbytes": 3072 |
| }, |
| "stage.2.l4.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 6, |
| "offset": 23629824, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l4.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 27168768, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l4.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 6, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l4.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 33067008, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l5.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37785600, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37788672, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37791744, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37794816, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37797888, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 37800960, |
| "nbytes": 3072 |
| }, |
| "stage.2.l5.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 6, |
| "offset": 37804032, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l5.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 41342976, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l5.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 6, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l5.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 47241216, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l6.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51959808, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51962880, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51965952, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51969024, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51972096, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 51975168, |
| "nbytes": 3072 |
| }, |
| "stage.2.l6.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 6, |
| "offset": 51978240, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l6.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 55517184, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l6.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 6, |
| "offset": 56696832, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l6.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 6, |
| "offset": 61415424, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l7.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66134016, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66137088, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66140160, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66143232, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66146304, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 6, |
| "offset": 66149376, |
| "nbytes": 3072 |
| }, |
| "stage.2.l7.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 7, |
| "offset": 0, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l7.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 3538944, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l7.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 7, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l7.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 9437184, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l8.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14155776, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14158848, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14161920, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14164992, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14168064, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 14171136, |
| "nbytes": 3072 |
| }, |
| "stage.2.l8.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 7, |
| "offset": 14174208, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l8.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 17713152, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l8.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 7, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l8.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 23611392, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l9.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28329984, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28333056, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28336128, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28339200, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28342272, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 28345344, |
| "nbytes": 3072 |
| }, |
| "stage.2.l9.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 7, |
| "offset": 28348416, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l9.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 31887360, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l9.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 7, |
| "offset": 33067008, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l9.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 37785600, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l10.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42504192, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42507264, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42510336, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42513408, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42516480, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 42519552, |
| "nbytes": 3072 |
| }, |
| "stage.2.l10.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 7, |
| "offset": 42522624, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l10.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 46061568, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l10.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 7, |
| "offset": 47241216, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l10.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 51959808, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l11.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56678400, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56681472, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56684544, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56687616, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56690688, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 7, |
| "offset": 56693760, |
| "nbytes": 3072 |
| }, |
| "stage.2.l11.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 7, |
| "offset": 56696832, |
| "nbytes": 3538944 |
| }, |
| "stage.2.l11.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 7, |
| "offset": 60235776, |
| "nbytes": 1179648 |
| }, |
| "stage.2.l11.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 7, |
| "offset": 61415424, |
| "nbytes": 4718592 |
| }, |
| "stage.2.l11.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.3.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 4718592, |
| "nbytes": 1179648 |
| }, |
| "stage.3.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 640 |
| ], |
| "shard": 8, |
| "offset": 5898240, |
| "nbytes": 983040 |
| }, |
| "stage.3.l0.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6881280, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6884352, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6887424, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6890496, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6893568, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 6896640, |
| "nbytes": 3072 |
| }, |
| "stage.3.l0.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 8, |
| "offset": 6899712, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l0.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 10438656, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l0.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 8, |
| "offset": 11618304, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l0.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 16336896, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l1.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21055488, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21058560, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21061632, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21064704, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21067776, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 21070848, |
| "nbytes": 3072 |
| }, |
| "stage.3.l1.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 8, |
| "offset": 21073920, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l1.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 24612864, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l1.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 8, |
| "offset": 25792512, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l1.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 30511104, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l2.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35229696, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35232768, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35235840, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35238912, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35241984, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 35245056, |
| "nbytes": 3072 |
| }, |
| "stage.3.l2.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 8, |
| "offset": 35248128, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l2.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 38787072, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l2.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 8, |
| "offset": 39966720, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l2.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 44685312, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l3.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49403904, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49406976, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49410048, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49413120, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49416192, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 49419264, |
| "nbytes": 3072 |
| }, |
| "stage.3.l3.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 8, |
| "offset": 49422336, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l3.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 52961280, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l3.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 8, |
| "offset": 54140928, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l3.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 8, |
| "offset": 58859520, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l4.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63578112, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63581184, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63584256, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63587328, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63590400, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 8, |
| "offset": 63593472, |
| "nbytes": 3072 |
| }, |
| "stage.3.l4.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 9, |
| "offset": 0, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l4.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 3538944, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l4.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 9, |
| "offset": 4718592, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l4.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 9437184, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l5.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14155776, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14158848, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14161920, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14164992, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14168064, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 14171136, |
| "nbytes": 3072 |
| }, |
| "stage.3.l5.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 9, |
| "offset": 14174208, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l5.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 17713152, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l5.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 9, |
| "offset": 18892800, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l5.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 23611392, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l6.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28329984, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28333056, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28336128, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28339200, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28342272, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 28345344, |
| "nbytes": 3072 |
| }, |
| "stage.3.l6.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 9, |
| "offset": 28348416, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l6.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 31887360, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l6.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 9, |
| "offset": 33067008, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l6.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 37785600, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l7.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42504192, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42507264, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42510336, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42513408, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42516480, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 42519552, |
| "nbytes": 3072 |
| }, |
| "stage.3.l7.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 9, |
| "offset": 42522624, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l7.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 46061568, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l7.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 9, |
| "offset": 47241216, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l7.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 51959808, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l8.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56678400, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56681472, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56684544, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56687616, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56690688, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 9, |
| "offset": 56693760, |
| "nbytes": 3072 |
| }, |
| "stage.3.l8.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 9, |
| "offset": 56696832, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l8.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 9, |
| "offset": 60235776, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l8.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 9, |
| "offset": 61415424, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l8.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 0, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l9.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4718592, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4721664, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4724736, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4727808, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4730880, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 4733952, |
| "nbytes": 3072 |
| }, |
| "stage.3.l9.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 10, |
| "offset": 4737024, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l9.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 8275968, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l9.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 10, |
| "offset": 9455616, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l9.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 14174208, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l10.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18892800, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18895872, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18898944, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18902016, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18905088, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 18908160, |
| "nbytes": 3072 |
| }, |
| "stage.3.l10.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 10, |
| "offset": 18911232, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l10.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 22450176, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l10.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 10, |
| "offset": 23629824, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l10.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 28348416, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l11.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33067008, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33070080, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33073152, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33076224, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33079296, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 10, |
| "offset": 33082368, |
| "nbytes": 3072 |
| }, |
| "stage.3.l11.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2304 |
| ], |
| "shard": 10, |
| "offset": 33085440, |
| "nbytes": 3538944 |
| }, |
| "stage.3.l11.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 36624384, |
| "nbytes": 1179648 |
| }, |
| "stage.3.l11.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 3072 |
| ], |
| "shard": 10, |
| "offset": 37804032, |
| "nbytes": 4718592 |
| }, |
| "stage.3.l11.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 3072, |
| 768 |
| ], |
| "shard": 10, |
| "offset": 42522624, |
| "nbytes": 4718592 |
| }, |
| "stage.4.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 47241216, |
| "nbytes": 3276800 |
| }, |
| "stage.4.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1024 |
| ], |
| "shard": 10, |
| "offset": 50518016, |
| "nbytes": 2621440 |
| }, |
| "stage.4.l0.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53139456, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53144576, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53149696, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53154816, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53159936, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 53165056, |
| "nbytes": 5120 |
| }, |
| "stage.4.l0.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 10, |
| "offset": 53170176, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l0.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 10, |
| "offset": 63000576, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l0.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 11, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l0.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l1.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l1.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 11, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l1.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l1.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 11, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l1.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l2.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 11, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l2.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 12, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l2.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l2.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 12, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l2.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l3.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l3.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 12, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l3.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 12, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l3.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 12, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l3.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l4.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l4.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 13, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l4.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l4.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 13, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l4.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l5.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l5.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 13, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l5.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 13, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l5.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 14, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l5.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l6.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l6.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 14, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l6.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l6.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 14, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l6.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l7.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 14, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l7.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 15, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l7.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l7.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 15, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l7.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l8.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l8.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 15, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l8.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 15, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l8.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 15, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l8.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l9.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l9.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 16, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l9.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l9.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 16, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l9.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l10.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l10.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 16, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l10.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 16, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l10.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 17, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l10.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l11.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l11.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 17, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l11.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l11.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 17, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l11.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l12.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 17, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l12.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 18, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l12.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l12.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 18, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l12.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l13.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l13.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 18, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l13.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 18, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l13.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 18, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l13.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l14.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l14.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 19, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l14.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l14.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 19, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l14.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l15.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l15.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 19, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l15.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 19, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l15.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 20, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l15.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l16.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l16.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 20, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l16.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l16.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 20, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l16.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l17.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 20, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l17.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 21, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l17.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l17.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 21, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l17.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l18.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l18.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 21, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l18.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 21, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l18.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 21, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l18.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l19.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l19.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 22, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l19.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l19.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 22, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l19.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l20.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l20.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 22, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l20.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 22, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l20.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 23, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l20.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l21.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l21.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 23, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l21.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l21.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 23, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l21.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l22.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 23, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l22.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 24, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l22.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l22.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 24, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l22.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l23.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l23.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 24, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l23.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 24, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l23.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 24, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l23.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l24.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l24.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 25, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l24.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l24.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 25, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l24.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l25.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l25.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 25, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l25.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 25, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l25.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 26, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l25.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l26.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l26.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 26, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l26.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l26.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 26, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l26.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l27.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65566720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65571840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65576960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65582080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65587200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 26, |
| "offset": 65592320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l27.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 27, |
| "offset": 0, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l27.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 9830400, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l27.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 27, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l27.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 26214400, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l28.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39321600, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39326720, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39331840, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39336960, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39342080, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 39347200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l28.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 27, |
| "offset": 39352320, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l28.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 27, |
| "offset": 49182720, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l28.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 27, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l28.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l29.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13107200, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13112320, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13117440, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13122560, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13127680, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 13132800, |
| "nbytes": 5120 |
| }, |
| "stage.4.l29.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 28, |
| "offset": 13137920, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l29.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 22968320, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l29.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 28, |
| "offset": 26245120, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l29.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l30.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52459520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52464640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52469760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52474880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52480000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 52485120, |
| "nbytes": 5120 |
| }, |
| "stage.4.l30.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 28, |
| "offset": 52490240, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l30.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 28, |
| "offset": 62320640, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l30.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 29, |
| "offset": 0, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l30.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 13107200, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l31.norm1.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26214400, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.norm1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26219520, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.norm2.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26224640, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.norm2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26229760, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.attn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26234880, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.ffn_scale": { |
| "dtype": "f32", |
| "shape": [ |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 26240000, |
| "nbytes": 5120 |
| }, |
| "stage.4.l31.qkv.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 3840 |
| ], |
| "shard": 29, |
| "offset": 26245120, |
| "nbytes": 9830400 |
| }, |
| "stage.4.l31.wo.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 36075520, |
| "nbytes": 3276800 |
| }, |
| "stage.4.l31.ffn1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1280, |
| 5120 |
| ], |
| "shard": 29, |
| "offset": 39352320, |
| "nbytes": 13107200 |
| }, |
| "stage.4.l31.ffn2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 5120, |
| 1280 |
| ], |
| "shard": 29, |
| "offset": 52459520, |
| "nbytes": 13107200 |
| }, |
| "sem_enc.in.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 1024, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 0, |
| "nbytes": 1572864 |
| }, |
| "sem_enc.in.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 1572864, |
| "nbytes": 3072 |
| }, |
| "sem_enc.0.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 1575936, |
| "nbytes": 21504 |
| }, |
| "sem_enc.0.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 1597440, |
| "nbytes": 3072 |
| }, |
| "sem_enc.0.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 1600512, |
| "nbytes": 3072 |
| }, |
| "sem_enc.0.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 1603584, |
| "nbytes": 3072 |
| }, |
| "sem_enc.0.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 1606656, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.0.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 4752384, |
| "nbytes": 8192 |
| }, |
| "sem_enc.0.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 4760576, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.0.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 7906304, |
| "nbytes": 3072 |
| }, |
| "sem_head.0.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 7909376, |
| "nbytes": 21504 |
| }, |
| "sem_head.0.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 7930880, |
| "nbytes": 3072 |
| }, |
| "sem_head.0.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 7933952, |
| "nbytes": 3072 |
| }, |
| "sem_head.0.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 7937024, |
| "nbytes": 3072 |
| }, |
| "sem_head.0.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 7940096, |
| "nbytes": 3145728 |
| }, |
| "sem_head.0.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 11085824, |
| "nbytes": 8192 |
| }, |
| "sem_head.0.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 11094016, |
| "nbytes": 3145728 |
| }, |
| "sem_head.0.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 14239744, |
| "nbytes": 3072 |
| }, |
| "sem_enc.1.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 14242816, |
| "nbytes": 21504 |
| }, |
| "sem_enc.1.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 14264320, |
| "nbytes": 3072 |
| }, |
| "sem_enc.1.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 14267392, |
| "nbytes": 3072 |
| }, |
| "sem_enc.1.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 14270464, |
| "nbytes": 3072 |
| }, |
| "sem_enc.1.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 14273536, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.1.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 17419264, |
| "nbytes": 8192 |
| }, |
| "sem_enc.1.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 17427456, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.1.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 20573184, |
| "nbytes": 3072 |
| }, |
| "sem_head.1.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 20576256, |
| "nbytes": 21504 |
| }, |
| "sem_head.1.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 20597760, |
| "nbytes": 3072 |
| }, |
| "sem_head.1.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 20600832, |
| "nbytes": 3072 |
| }, |
| "sem_head.1.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 20603904, |
| "nbytes": 3072 |
| }, |
| "sem_head.1.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 20606976, |
| "nbytes": 3145728 |
| }, |
| "sem_head.1.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 23752704, |
| "nbytes": 8192 |
| }, |
| "sem_head.1.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 23760896, |
| "nbytes": 3145728 |
| }, |
| "sem_head.1.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 26906624, |
| "nbytes": 3072 |
| }, |
| "sem_enc.2.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 26909696, |
| "nbytes": 21504 |
| }, |
| "sem_enc.2.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 26931200, |
| "nbytes": 3072 |
| }, |
| "sem_enc.2.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 26934272, |
| "nbytes": 3072 |
| }, |
| "sem_enc.2.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 26937344, |
| "nbytes": 3072 |
| }, |
| "sem_enc.2.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 26940416, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.2.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 30086144, |
| "nbytes": 8192 |
| }, |
| "sem_enc.2.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 30094336, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.2.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 33240064, |
| "nbytes": 3072 |
| }, |
| "sem_head.2.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 33243136, |
| "nbytes": 21504 |
| }, |
| "sem_head.2.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 33264640, |
| "nbytes": 3072 |
| }, |
| "sem_head.2.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 33267712, |
| "nbytes": 3072 |
| }, |
| "sem_head.2.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 33270784, |
| "nbytes": 3072 |
| }, |
| "sem_head.2.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 33273856, |
| "nbytes": 3145728 |
| }, |
| "sem_head.2.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 36419584, |
| "nbytes": 8192 |
| }, |
| "sem_head.2.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 36427776, |
| "nbytes": 3145728 |
| }, |
| "sem_head.2.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 39573504, |
| "nbytes": 3072 |
| }, |
| "sem_enc.3.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 39576576, |
| "nbytes": 21504 |
| }, |
| "sem_enc.3.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 39598080, |
| "nbytes": 3072 |
| }, |
| "sem_enc.3.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 39601152, |
| "nbytes": 3072 |
| }, |
| "sem_enc.3.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 39604224, |
| "nbytes": 3072 |
| }, |
| "sem_enc.3.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 39607296, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.3.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 42753024, |
| "nbytes": 8192 |
| }, |
| "sem_enc.3.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 42761216, |
| "nbytes": 3145728 |
| }, |
| "sem_enc.3.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 45906944, |
| "nbytes": 3072 |
| }, |
| "sem_head.3.dw.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 7 |
| ], |
| "shard": 30, |
| "offset": 45910016, |
| "nbytes": 21504 |
| }, |
| "sem_head.3.dw.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 45931520, |
| "nbytes": 3072 |
| }, |
| "sem_head.3.norm.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 45934592, |
| "nbytes": 3072 |
| }, |
| "sem_head.3.norm.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 45937664, |
| "nbytes": 3072 |
| }, |
| "sem_head.3.pw1.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 45940736, |
| "nbytes": 3145728 |
| }, |
| "sem_head.3.pw1.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 2048 |
| ], |
| "shard": 30, |
| "offset": 49086464, |
| "nbytes": 8192 |
| }, |
| "sem_head.3.pw2.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 2048, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 49094656, |
| "nbytes": 3145728 |
| }, |
| "sem_head.3.pw2.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 52240384, |
| "nbytes": 3072 |
| }, |
| "sem_head.out.weight": { |
| "dtype": "bf16", |
| "shape": [ |
| 768, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 52243456, |
| "nbytes": 1572864 |
| }, |
| "sem_head.out.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 53816320, |
| "nbytes": 4096 |
| }, |
| "rvq.0.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 768, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 53820416, |
| "nbytes": 24576, |
| "host": true |
| }, |
| "rvq.0.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 53844992, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.0.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 768 |
| ], |
| "shard": 30, |
| "offset": 53845248, |
| "nbytes": 24576, |
| "host": true |
| }, |
| "rvq.0.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 768 |
| ], |
| "shard": 30, |
| "offset": 53869824, |
| "nbytes": 3072, |
| "host": true |
| }, |
| "rvq.0.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 53872896, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.1.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54003968, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.1.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54036736, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.1.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54036992, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.1.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54069760, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.1.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 54073856, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.2.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54204928, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.2.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54237696, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.2.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54237952, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.2.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54270720, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.2.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 54274816, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.3.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54405888, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.3.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54438656, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.3.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54438912, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.3.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54471680, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.3.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 54475776, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.4.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54606848, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.4.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54639616, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.4.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54639872, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.4.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54672640, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.4.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 54676736, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.5.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54807808, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.5.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 54840576, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.5.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54840832, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.5.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 54873600, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.5.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 54877696, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.6.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55008768, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.6.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55041536, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.6.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55041792, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.6.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55074560, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.6.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 55078656, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.7.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55209728, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.7.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55242496, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.7.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55242752, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.7.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55275520, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.7.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 55279616, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.8.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55410688, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.8.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55443456, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.8.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55443712, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.8.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55476480, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.8.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 55480576, |
| "nbytes": 131072, |
| "host": true |
| }, |
| "rvq.9.in_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 1024, |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55611648, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.9.in_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 8 |
| ], |
| "shard": 30, |
| "offset": 55644416, |
| "nbytes": 32, |
| "host": true |
| }, |
| "rvq.9.out_proj.weight": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55644672, |
| "nbytes": 32768, |
| "host": true |
| }, |
| "rvq.9.out_proj.bias": { |
| "dtype": "f32", |
| "shape": [ |
| 1024 |
| ], |
| "shard": 30, |
| "offset": 55677440, |
| "nbytes": 4096, |
| "host": true |
| }, |
| "rvq.9.codebook": { |
| "dtype": "f32", |
| "shape": [ |
| 8, |
| 4096 |
| ], |
| "shard": 30, |
| "offset": 55681536, |
| "nbytes": 131072, |
| "host": true |
| } |
| } |
| } |
