{
 "format": "zonos3-web-v1",
 "model": "encoder",
 "meta": {
  "general.architecture": "mossdual_encoder",
  "general.name": "fixed600k frozen dual Opus reference encoder",
  "sample_rate": 44100,
  "internal_sample_rate": 44160,
  "frame_rate": 15,
  "hop": 2940,
  "internal_hop": 2944,
  "n_codebooks": 10,
  "codebook_size": 4096,
  "latent_dim": 1024,
  "semantic_dim": 768,
  "semantic_depth": 4,
  "lane_frames": 300,
  "context_frames": 150,
  "window_ratio": 2.0,
  "ln_eps": 9.999999747378752e-06,
  "convnext_eps": 9.999999974752427e-07,
  "stage.in_dim": [
   5888,
   768,
   768,
   768,
   1280
  ],
  "stage.out_dim": [
   384,
   384,
   384,
   640,
   1024
  ],
  "stage.d_model": [
   768,
   768,
   768,
   768,
   1280
  ],
  "stage.n_head": [
   12,
   12,
   12,
   12,
   20
  ],
  "stage.n_layer": [
   12,
   12,
   12,
   12,
   32
  ],
  "stage.n_ff": [
   3072,
   3072,
   3072,
   3072,
   5120
  ],
  "stage.context": [
   480,
   480,
   480,
   300,
   150
  ],
  "stage.patch": [
   23,
   2,
   2,
   2,
   2
  ],
  "resample.orig": 735,
  "resample.new": 736,
  "resample.width": 17
 },
 "shards": [
  "shard-000.bin",
  "shard-001.bin",
  "shard-002.bin",
  "shard-003.bin",
  "shard-004.bin",
  "shard-005.bin",
  "shard-006.bin",
  "shard-007.bin",
  "shard-008.bin",
  "shard-009.bin",
  "shard-010.bin",
  "shard-011.bin",
  "shard-012.bin",
  "shard-013.bin",
  "shard-014.bin",
  "shard-015.bin",
  "shard-016.bin",
  "shard-017.bin",
  "shard-018.bin",
  "shard-019.bin",
  "shard-020.bin",
  "shard-021.bin",
  "shard-022.bin",
  "shard-023.bin",
  "shard-024.bin",
  "shard-025.bin",
  "shard-026.bin",
  "shard-027.bin",
  "shard-028.bin",
  "shard-029.bin",
  "shard-030.bin"
 ],
 "tensors": {
  "resample.kernel": {
   "dtype": "f32",
   "shape": [
    769,
    736
   ],
   "shard": 0,
   "offset": 0,
   "nbytes": 2263936,
   "host": true
  },
  "enc.in.weight": {
   "dtype": "bf16",
   "shape": [
    16,
    1024,
    2
   ],
   "shard": 0,
   "offset": 2264064,
   "nbytes": 65536
  },
  "enc.in.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2329600,
   "nbytes": 256
  },
  "enc.b0.res0.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2329856,
   "nbytes": 256
  },
  "enc.b0.res0.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2330112,
   "nbytes": 256
  },
  "enc.b0.res0.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    7
   ],
   "shard": 0,
   "offset": 2330368,
   "nbytes": 57344
  },
  "enc.b0.res0.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2387712,
   "nbytes": 256
  },
  "enc.b0.res0.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2387968,
   "nbytes": 256
  },
  "enc.b0.res0.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2388224,
   "nbytes": 256
  },
  "enc.b0.res0.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    1
   ],
   "shard": 0,
   "offset": 2388480,
   "nbytes": 8192
  },
  "enc.b0.res0.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2396672,
   "nbytes": 256
  },
  "enc.b0.res1.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2396928,
   "nbytes": 256
  },
  "enc.b0.res1.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2397184,
   "nbytes": 256
  },
  "enc.b0.res1.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    7
   ],
   "shard": 0,
   "offset": 2397440,
   "nbytes": 57344
  },
  "enc.b0.res1.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2454784,
   "nbytes": 256
  },
  "enc.b0.res1.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2455040,
   "nbytes": 256
  },
  "enc.b0.res1.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2455296,
   "nbytes": 256
  },
  "enc.b0.res1.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    1
   ],
   "shard": 0,
   "offset": 2455552,
   "nbytes": 8192
  },
  "enc.b0.res1.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2463744,
   "nbytes": 256
  },
  "enc.b0.res2.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2464000,
   "nbytes": 256
  },
  "enc.b0.res2.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2464256,
   "nbytes": 256
  },
  "enc.b0.res2.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    7
   ],
   "shard": 0,
   "offset": 2464512,
   "nbytes": 57344
  },
  "enc.b0.res2.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2521856,
   "nbytes": 256
  },
  "enc.b0.res2.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2522112,
   "nbytes": 256
  },
  "enc.b0.res2.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2522368,
   "nbytes": 256
  },
  "enc.b0.res2.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    64,
    64,
    1
   ],
   "shard": 0,
   "offset": 2522624,
   "nbytes": 8192
  },
  "enc.b0.res2.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2530816,
   "nbytes": 256
  },
  "enc.b0.snake.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2531072,
   "nbytes": 256
  },
  "enc.b0.snake.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    64
   ],
   "shard": 0,
   "offset": 2531328,
   "nbytes": 256
  },
  "enc.b0.conv.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    2
   ],
   "shard": 0,
   "offset": 2531584,
   "nbytes": 65536
  },
  "enc.b0.conv.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2597120,
   "nbytes": 512
  },
  "enc.b1.res0.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2597632,
   "nbytes": 512
  },
  "enc.b1.res0.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2598144,
   "nbytes": 512
  },
  "enc.b1.res0.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    7
   ],
   "shard": 0,
   "offset": 2598656,
   "nbytes": 229376
  },
  "enc.b1.res0.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2828032,
   "nbytes": 512
  },
  "enc.b1.res0.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2828544,
   "nbytes": 512
  },
  "enc.b1.res0.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2829056,
   "nbytes": 512
  },
  "enc.b1.res0.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    1
   ],
   "shard": 0,
   "offset": 2829568,
   "nbytes": 32768
  },
  "enc.b1.res0.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2862336,
   "nbytes": 512
  },
  "enc.b1.res1.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2862848,
   "nbytes": 512
  },
  "enc.b1.res1.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 2863360,
   "nbytes": 512
  },
  "enc.b1.res1.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    7
   ],
   "shard": 0,
   "offset": 2863872,
   "nbytes": 229376
  },
  "enc.b1.res1.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3093248,
   "nbytes": 512
  },
  "enc.b1.res1.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3093760,
   "nbytes": 512
  },
  "enc.b1.res1.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3094272,
   "nbytes": 512
  },
  "enc.b1.res1.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    1
   ],
   "shard": 0,
   "offset": 3094784,
   "nbytes": 32768
  },
  "enc.b1.res1.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3127552,
   "nbytes": 512
  },
  "enc.b1.res2.snake1.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3128064,
   "nbytes": 512
  },
  "enc.b1.res2.snake1.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3128576,
   "nbytes": 512
  },
  "enc.b1.res2.conv1.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    7
   ],
   "shard": 0,
   "offset": 3129088,
   "nbytes": 229376
  },
  "enc.b1.res2.conv1.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3358464,
   "nbytes": 512
  },
  "enc.b1.res2.snake2.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3358976,
   "nbytes": 512
  },
  "enc.b1.res2.snake2.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3359488,
   "nbytes": 512
  },
  "enc.b1.res2.conv2.weight": {
   "dtype": "bf16",
   "shape": [
    128,
    128,
    1
   ],
   "shard": 0,
   "offset": 3360000,
   "nbytes": 32768
  },
  "enc.b1.res2.conv2.bias": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3392768,
   "nbytes": 512
  },
  "enc.b1.snake.alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3393280,
   "nbytes": 512
  },
  "enc.b1.snake.inv_alpha": {
   "dtype": "f32",
   "shape": [
    1,
    128
   ],
   "shard": 0,
   "offset": 3393792,
   "nbytes": 512
  },
  "enc.b1.conv.weight": {
   "dtype": "bf16",
   "shape": [
    512,
    256,
    2
   ],
   "shard": 0,
   "offset": 3394304,
   "nbytes": 524288
  },
  "enc.b1.conv.bias": {
   "dtype": "f32",
   "shape": [
    1,
    256
   ],
   "shard": 0,
   "offset": 3918592,
   "nbytes": 1024
  },
  "stage.0.in.weight": {
   "dtype": "bf16",
   "shape": [
    5888,
    768
   ],
   "shard": 0,
   "offset": 3919616,
   "nbytes": 9043968
  },
  "stage.0.out.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    384
   ],
   "shard": 0,
   "offset": 12963584,
   "nbytes": 589824
  },
  "stage.0.l0.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13553408,
   "nbytes": 3072
  },
  "stage.0.l0.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13556480,
   "nbytes": 3072
  },
  "stage.0.l0.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13559552,
   "nbytes": 3072
  },
  "stage.0.l0.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13562624,
   "nbytes": 3072
  },
  "stage.0.l0.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13565696,
   "nbytes": 3072
  },
  "stage.0.l0.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 13568768,
   "nbytes": 3072
  },
  "stage.0.l0.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 0,
   "offset": 13571840,
   "nbytes": 3538944
  },
  "stage.0.l0.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 0,
   "offset": 17110784,
   "nbytes": 1179648
  },
  "stage.0.l0.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 0,
   "offset": 18290432,
   "nbytes": 4718592
  },
  "stage.0.l0.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 0,
   "offset": 23009024,
   "nbytes": 4718592
  },
  "stage.0.l1.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27727616,
   "nbytes": 3072
  },
  "stage.0.l1.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27730688,
   "nbytes": 3072
  },
  "stage.0.l1.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27733760,
   "nbytes": 3072
  },
  "stage.0.l1.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27736832,
   "nbytes": 3072
  },
  "stage.0.l1.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27739904,
   "nbytes": 3072
  },
  "stage.0.l1.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 27742976,
   "nbytes": 3072
  },
  "stage.0.l1.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 0,
   "offset": 27746048,
   "nbytes": 3538944
  },
  "stage.0.l1.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 0,
   "offset": 31284992,
   "nbytes": 1179648
  },
  "stage.0.l1.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 0,
   "offset": 32464640,
   "nbytes": 4718592
  },
  "stage.0.l1.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 0,
   "offset": 37183232,
   "nbytes": 4718592
  },
  "stage.0.l2.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41901824,
   "nbytes": 3072
  },
  "stage.0.l2.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41904896,
   "nbytes": 3072
  },
  "stage.0.l2.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41907968,
   "nbytes": 3072
  },
  "stage.0.l2.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41911040,
   "nbytes": 3072
  },
  "stage.0.l2.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41914112,
   "nbytes": 3072
  },
  "stage.0.l2.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 41917184,
   "nbytes": 3072
  },
  "stage.0.l2.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 0,
   "offset": 41920256,
   "nbytes": 3538944
  },
  "stage.0.l2.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 0,
   "offset": 45459200,
   "nbytes": 1179648
  },
  "stage.0.l2.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 0,
   "offset": 46638848,
   "nbytes": 4718592
  },
  "stage.0.l2.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 0,
   "offset": 51357440,
   "nbytes": 4718592
  },
  "stage.0.l3.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56076032,
   "nbytes": 3072
  },
  "stage.0.l3.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56079104,
   "nbytes": 3072
  },
  "stage.0.l3.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56082176,
   "nbytes": 3072
  },
  "stage.0.l3.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56085248,
   "nbytes": 3072
  },
  "stage.0.l3.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56088320,
   "nbytes": 3072
  },
  "stage.0.l3.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 0,
   "offset": 56091392,
   "nbytes": 3072
  },
  "stage.0.l3.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 0,
   "offset": 56094464,
   "nbytes": 3538944
  },
  "stage.0.l3.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 0,
   "offset": 59633408,
   "nbytes": 1179648
  },
  "stage.0.l3.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 0,
   "offset": 60813056,
   "nbytes": 4718592
  },
  "stage.0.l3.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 1,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.0.l4.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4718592,
   "nbytes": 3072
  },
  "stage.0.l4.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4721664,
   "nbytes": 3072
  },
  "stage.0.l4.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4724736,
   "nbytes": 3072
  },
  "stage.0.l4.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4727808,
   "nbytes": 3072
  },
  "stage.0.l4.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4730880,
   "nbytes": 3072
  },
  "stage.0.l4.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 4733952,
   "nbytes": 3072
  },
  "stage.0.l4.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 1,
   "offset": 4737024,
   "nbytes": 3538944
  },
  "stage.0.l4.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 1,
   "offset": 8275968,
   "nbytes": 1179648
  },
  "stage.0.l4.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 1,
   "offset": 9455616,
   "nbytes": 4718592
  },
  "stage.0.l4.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 1,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.0.l5.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18892800,
   "nbytes": 3072
  },
  "stage.0.l5.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18895872,
   "nbytes": 3072
  },
  "stage.0.l5.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18898944,
   "nbytes": 3072
  },
  "stage.0.l5.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18902016,
   "nbytes": 3072
  },
  "stage.0.l5.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18905088,
   "nbytes": 3072
  },
  "stage.0.l5.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 18908160,
   "nbytes": 3072
  },
  "stage.0.l5.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 1,
   "offset": 18911232,
   "nbytes": 3538944
  },
  "stage.0.l5.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 1,
   "offset": 22450176,
   "nbytes": 1179648
  },
  "stage.0.l5.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 1,
   "offset": 23629824,
   "nbytes": 4718592
  },
  "stage.0.l5.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 1,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.0.l6.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33067008,
   "nbytes": 3072
  },
  "stage.0.l6.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33070080,
   "nbytes": 3072
  },
  "stage.0.l6.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33073152,
   "nbytes": 3072
  },
  "stage.0.l6.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33076224,
   "nbytes": 3072
  },
  "stage.0.l6.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33079296,
   "nbytes": 3072
  },
  "stage.0.l6.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 33082368,
   "nbytes": 3072
  },
  "stage.0.l6.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 1,
   "offset": 33085440,
   "nbytes": 3538944
  },
  "stage.0.l6.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 1,
   "offset": 36624384,
   "nbytes": 1179648
  },
  "stage.0.l6.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 1,
   "offset": 37804032,
   "nbytes": 4718592
  },
  "stage.0.l6.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 1,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.0.l7.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47241216,
   "nbytes": 3072
  },
  "stage.0.l7.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47244288,
   "nbytes": 3072
  },
  "stage.0.l7.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47247360,
   "nbytes": 3072
  },
  "stage.0.l7.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47250432,
   "nbytes": 3072
  },
  "stage.0.l7.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47253504,
   "nbytes": 3072
  },
  "stage.0.l7.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 47256576,
   "nbytes": 3072
  },
  "stage.0.l7.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 1,
   "offset": 47259648,
   "nbytes": 3538944
  },
  "stage.0.l7.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 1,
   "offset": 50798592,
   "nbytes": 1179648
  },
  "stage.0.l7.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 1,
   "offset": 51978240,
   "nbytes": 4718592
  },
  "stage.0.l7.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 1,
   "offset": 56696832,
   "nbytes": 4718592
  },
  "stage.0.l8.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61415424,
   "nbytes": 3072
  },
  "stage.0.l8.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61418496,
   "nbytes": 3072
  },
  "stage.0.l8.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61421568,
   "nbytes": 3072
  },
  "stage.0.l8.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61424640,
   "nbytes": 3072
  },
  "stage.0.l8.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61427712,
   "nbytes": 3072
  },
  "stage.0.l8.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 1,
   "offset": 61430784,
   "nbytes": 3072
  },
  "stage.0.l8.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 1,
   "offset": 61433856,
   "nbytes": 3538944
  },
  "stage.0.l8.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 1,
   "offset": 64972800,
   "nbytes": 1179648
  },
  "stage.0.l8.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 2,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.0.l8.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 2,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.0.l9.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9437184,
   "nbytes": 3072
  },
  "stage.0.l9.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9440256,
   "nbytes": 3072
  },
  "stage.0.l9.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9443328,
   "nbytes": 3072
  },
  "stage.0.l9.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9446400,
   "nbytes": 3072
  },
  "stage.0.l9.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9449472,
   "nbytes": 3072
  },
  "stage.0.l9.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 9452544,
   "nbytes": 3072
  },
  "stage.0.l9.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 2,
   "offset": 9455616,
   "nbytes": 3538944
  },
  "stage.0.l9.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 2,
   "offset": 12994560,
   "nbytes": 1179648
  },
  "stage.0.l9.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 2,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.0.l9.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 2,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.0.l10.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23611392,
   "nbytes": 3072
  },
  "stage.0.l10.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23614464,
   "nbytes": 3072
  },
  "stage.0.l10.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23617536,
   "nbytes": 3072
  },
  "stage.0.l10.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23620608,
   "nbytes": 3072
  },
  "stage.0.l10.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23623680,
   "nbytes": 3072
  },
  "stage.0.l10.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 23626752,
   "nbytes": 3072
  },
  "stage.0.l10.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 2,
   "offset": 23629824,
   "nbytes": 3538944
  },
  "stage.0.l10.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 2,
   "offset": 27168768,
   "nbytes": 1179648
  },
  "stage.0.l10.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 2,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.0.l10.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 2,
   "offset": 33067008,
   "nbytes": 4718592
  },
  "stage.0.l11.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37785600,
   "nbytes": 3072
  },
  "stage.0.l11.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37788672,
   "nbytes": 3072
  },
  "stage.0.l11.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37791744,
   "nbytes": 3072
  },
  "stage.0.l11.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37794816,
   "nbytes": 3072
  },
  "stage.0.l11.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37797888,
   "nbytes": 3072
  },
  "stage.0.l11.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 37800960,
   "nbytes": 3072
  },
  "stage.0.l11.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 2,
   "offset": 37804032,
   "nbytes": 3538944
  },
  "stage.0.l11.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 2,
   "offset": 41342976,
   "nbytes": 1179648
  },
  "stage.0.l11.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 2,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.0.l11.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 2,
   "offset": 47241216,
   "nbytes": 4718592
  },
  "stage.1.in.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 2,
   "offset": 51959808,
   "nbytes": 1179648
  },
  "stage.1.out.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    384
   ],
   "shard": 2,
   "offset": 53139456,
   "nbytes": 589824
  },
  "stage.1.l0.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53729280,
   "nbytes": 3072
  },
  "stage.1.l0.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53732352,
   "nbytes": 3072
  },
  "stage.1.l0.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53735424,
   "nbytes": 3072
  },
  "stage.1.l0.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53738496,
   "nbytes": 3072
  },
  "stage.1.l0.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53741568,
   "nbytes": 3072
  },
  "stage.1.l0.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 2,
   "offset": 53744640,
   "nbytes": 3072
  },
  "stage.1.l0.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 2,
   "offset": 53747712,
   "nbytes": 3538944
  },
  "stage.1.l0.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 2,
   "offset": 57286656,
   "nbytes": 1179648
  },
  "stage.1.l0.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 2,
   "offset": 58466304,
   "nbytes": 4718592
  },
  "stage.1.l0.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 3,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.1.l1.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4718592,
   "nbytes": 3072
  },
  "stage.1.l1.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4721664,
   "nbytes": 3072
  },
  "stage.1.l1.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4724736,
   "nbytes": 3072
  },
  "stage.1.l1.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4727808,
   "nbytes": 3072
  },
  "stage.1.l1.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4730880,
   "nbytes": 3072
  },
  "stage.1.l1.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 4733952,
   "nbytes": 3072
  },
  "stage.1.l1.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 3,
   "offset": 4737024,
   "nbytes": 3538944
  },
  "stage.1.l1.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 3,
   "offset": 8275968,
   "nbytes": 1179648
  },
  "stage.1.l1.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 3,
   "offset": 9455616,
   "nbytes": 4718592
  },
  "stage.1.l1.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 3,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.1.l2.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18892800,
   "nbytes": 3072
  },
  "stage.1.l2.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18895872,
   "nbytes": 3072
  },
  "stage.1.l2.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18898944,
   "nbytes": 3072
  },
  "stage.1.l2.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18902016,
   "nbytes": 3072
  },
  "stage.1.l2.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18905088,
   "nbytes": 3072
  },
  "stage.1.l2.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 18908160,
   "nbytes": 3072
  },
  "stage.1.l2.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 3,
   "offset": 18911232,
   "nbytes": 3538944
  },
  "stage.1.l2.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 3,
   "offset": 22450176,
   "nbytes": 1179648
  },
  "stage.1.l2.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 3,
   "offset": 23629824,
   "nbytes": 4718592
  },
  "stage.1.l2.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 3,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.1.l3.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33067008,
   "nbytes": 3072
  },
  "stage.1.l3.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33070080,
   "nbytes": 3072
  },
  "stage.1.l3.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33073152,
   "nbytes": 3072
  },
  "stage.1.l3.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33076224,
   "nbytes": 3072
  },
  "stage.1.l3.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33079296,
   "nbytes": 3072
  },
  "stage.1.l3.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 33082368,
   "nbytes": 3072
  },
  "stage.1.l3.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 3,
   "offset": 33085440,
   "nbytes": 3538944
  },
  "stage.1.l3.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 3,
   "offset": 36624384,
   "nbytes": 1179648
  },
  "stage.1.l3.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 3,
   "offset": 37804032,
   "nbytes": 4718592
  },
  "stage.1.l3.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 3,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.1.l4.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47241216,
   "nbytes": 3072
  },
  "stage.1.l4.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47244288,
   "nbytes": 3072
  },
  "stage.1.l4.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47247360,
   "nbytes": 3072
  },
  "stage.1.l4.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47250432,
   "nbytes": 3072
  },
  "stage.1.l4.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47253504,
   "nbytes": 3072
  },
  "stage.1.l4.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 47256576,
   "nbytes": 3072
  },
  "stage.1.l4.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 3,
   "offset": 47259648,
   "nbytes": 3538944
  },
  "stage.1.l4.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 3,
   "offset": 50798592,
   "nbytes": 1179648
  },
  "stage.1.l4.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 3,
   "offset": 51978240,
   "nbytes": 4718592
  },
  "stage.1.l4.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 3,
   "offset": 56696832,
   "nbytes": 4718592
  },
  "stage.1.l5.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61415424,
   "nbytes": 3072
  },
  "stage.1.l5.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61418496,
   "nbytes": 3072
  },
  "stage.1.l5.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61421568,
   "nbytes": 3072
  },
  "stage.1.l5.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61424640,
   "nbytes": 3072
  },
  "stage.1.l5.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61427712,
   "nbytes": 3072
  },
  "stage.1.l5.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 3,
   "offset": 61430784,
   "nbytes": 3072
  },
  "stage.1.l5.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 3,
   "offset": 61433856,
   "nbytes": 3538944
  },
  "stage.1.l5.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 3,
   "offset": 64972800,
   "nbytes": 1179648
  },
  "stage.1.l5.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 4,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.1.l5.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 4,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.1.l6.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9437184,
   "nbytes": 3072
  },
  "stage.1.l6.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9440256,
   "nbytes": 3072
  },
  "stage.1.l6.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9443328,
   "nbytes": 3072
  },
  "stage.1.l6.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9446400,
   "nbytes": 3072
  },
  "stage.1.l6.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9449472,
   "nbytes": 3072
  },
  "stage.1.l6.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 9452544,
   "nbytes": 3072
  },
  "stage.1.l6.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 4,
   "offset": 9455616,
   "nbytes": 3538944
  },
  "stage.1.l6.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 4,
   "offset": 12994560,
   "nbytes": 1179648
  },
  "stage.1.l6.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 4,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.1.l6.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 4,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.1.l7.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23611392,
   "nbytes": 3072
  },
  "stage.1.l7.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23614464,
   "nbytes": 3072
  },
  "stage.1.l7.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23617536,
   "nbytes": 3072
  },
  "stage.1.l7.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23620608,
   "nbytes": 3072
  },
  "stage.1.l7.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23623680,
   "nbytes": 3072
  },
  "stage.1.l7.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 23626752,
   "nbytes": 3072
  },
  "stage.1.l7.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 4,
   "offset": 23629824,
   "nbytes": 3538944
  },
  "stage.1.l7.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 4,
   "offset": 27168768,
   "nbytes": 1179648
  },
  "stage.1.l7.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 4,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.1.l7.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 4,
   "offset": 33067008,
   "nbytes": 4718592
  },
  "stage.1.l8.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37785600,
   "nbytes": 3072
  },
  "stage.1.l8.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37788672,
   "nbytes": 3072
  },
  "stage.1.l8.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37791744,
   "nbytes": 3072
  },
  "stage.1.l8.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37794816,
   "nbytes": 3072
  },
  "stage.1.l8.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37797888,
   "nbytes": 3072
  },
  "stage.1.l8.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 37800960,
   "nbytes": 3072
  },
  "stage.1.l8.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 4,
   "offset": 37804032,
   "nbytes": 3538944
  },
  "stage.1.l8.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 4,
   "offset": 41342976,
   "nbytes": 1179648
  },
  "stage.1.l8.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 4,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.1.l8.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 4,
   "offset": 47241216,
   "nbytes": 4718592
  },
  "stage.1.l9.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51959808,
   "nbytes": 3072
  },
  "stage.1.l9.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51962880,
   "nbytes": 3072
  },
  "stage.1.l9.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51965952,
   "nbytes": 3072
  },
  "stage.1.l9.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51969024,
   "nbytes": 3072
  },
  "stage.1.l9.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51972096,
   "nbytes": 3072
  },
  "stage.1.l9.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 51975168,
   "nbytes": 3072
  },
  "stage.1.l9.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 4,
   "offset": 51978240,
   "nbytes": 3538944
  },
  "stage.1.l9.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 4,
   "offset": 55517184,
   "nbytes": 1179648
  },
  "stage.1.l9.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 4,
   "offset": 56696832,
   "nbytes": 4718592
  },
  "stage.1.l9.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 4,
   "offset": 61415424,
   "nbytes": 4718592
  },
  "stage.1.l10.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66134016,
   "nbytes": 3072
  },
  "stage.1.l10.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66137088,
   "nbytes": 3072
  },
  "stage.1.l10.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66140160,
   "nbytes": 3072
  },
  "stage.1.l10.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66143232,
   "nbytes": 3072
  },
  "stage.1.l10.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66146304,
   "nbytes": 3072
  },
  "stage.1.l10.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 4,
   "offset": 66149376,
   "nbytes": 3072
  },
  "stage.1.l10.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 5,
   "offset": 0,
   "nbytes": 3538944
  },
  "stage.1.l10.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 3538944,
   "nbytes": 1179648
  },
  "stage.1.l10.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 5,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.1.l10.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 5,
   "offset": 9437184,
   "nbytes": 4718592
  },
  "stage.1.l11.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14155776,
   "nbytes": 3072
  },
  "stage.1.l11.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14158848,
   "nbytes": 3072
  },
  "stage.1.l11.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14161920,
   "nbytes": 3072
  },
  "stage.1.l11.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14164992,
   "nbytes": 3072
  },
  "stage.1.l11.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14168064,
   "nbytes": 3072
  },
  "stage.1.l11.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 14171136,
   "nbytes": 3072
  },
  "stage.1.l11.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 5,
   "offset": 14174208,
   "nbytes": 3538944
  },
  "stage.1.l11.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 17713152,
   "nbytes": 1179648
  },
  "stage.1.l11.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 5,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.1.l11.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 5,
   "offset": 23611392,
   "nbytes": 4718592
  },
  "stage.2.in.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 28329984,
   "nbytes": 1179648
  },
  "stage.2.out.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    384
   ],
   "shard": 5,
   "offset": 29509632,
   "nbytes": 589824
  },
  "stage.2.l0.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30099456,
   "nbytes": 3072
  },
  "stage.2.l0.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30102528,
   "nbytes": 3072
  },
  "stage.2.l0.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30105600,
   "nbytes": 3072
  },
  "stage.2.l0.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30108672,
   "nbytes": 3072
  },
  "stage.2.l0.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30111744,
   "nbytes": 3072
  },
  "stage.2.l0.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 30114816,
   "nbytes": 3072
  },
  "stage.2.l0.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 5,
   "offset": 30117888,
   "nbytes": 3538944
  },
  "stage.2.l0.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 33656832,
   "nbytes": 1179648
  },
  "stage.2.l0.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 5,
   "offset": 34836480,
   "nbytes": 4718592
  },
  "stage.2.l0.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 5,
   "offset": 39555072,
   "nbytes": 4718592
  },
  "stage.2.l1.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44273664,
   "nbytes": 3072
  },
  "stage.2.l1.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44276736,
   "nbytes": 3072
  },
  "stage.2.l1.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44279808,
   "nbytes": 3072
  },
  "stage.2.l1.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44282880,
   "nbytes": 3072
  },
  "stage.2.l1.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44285952,
   "nbytes": 3072
  },
  "stage.2.l1.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 44289024,
   "nbytes": 3072
  },
  "stage.2.l1.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 5,
   "offset": 44292096,
   "nbytes": 3538944
  },
  "stage.2.l1.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 47831040,
   "nbytes": 1179648
  },
  "stage.2.l1.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 5,
   "offset": 49010688,
   "nbytes": 4718592
  },
  "stage.2.l1.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 5,
   "offset": 53729280,
   "nbytes": 4718592
  },
  "stage.2.l2.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58447872,
   "nbytes": 3072
  },
  "stage.2.l2.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58450944,
   "nbytes": 3072
  },
  "stage.2.l2.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58454016,
   "nbytes": 3072
  },
  "stage.2.l2.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58457088,
   "nbytes": 3072
  },
  "stage.2.l2.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58460160,
   "nbytes": 3072
  },
  "stage.2.l2.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 5,
   "offset": 58463232,
   "nbytes": 3072
  },
  "stage.2.l2.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 5,
   "offset": 58466304,
   "nbytes": 3538944
  },
  "stage.2.l2.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 5,
   "offset": 62005248,
   "nbytes": 1179648
  },
  "stage.2.l2.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 6,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.2.l2.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 6,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.2.l3.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9437184,
   "nbytes": 3072
  },
  "stage.2.l3.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9440256,
   "nbytes": 3072
  },
  "stage.2.l3.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9443328,
   "nbytes": 3072
  },
  "stage.2.l3.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9446400,
   "nbytes": 3072
  },
  "stage.2.l3.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9449472,
   "nbytes": 3072
  },
  "stage.2.l3.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 9452544,
   "nbytes": 3072
  },
  "stage.2.l3.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 6,
   "offset": 9455616,
   "nbytes": 3538944
  },
  "stage.2.l3.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 6,
   "offset": 12994560,
   "nbytes": 1179648
  },
  "stage.2.l3.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 6,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.2.l3.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 6,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.2.l4.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23611392,
   "nbytes": 3072
  },
  "stage.2.l4.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23614464,
   "nbytes": 3072
  },
  "stage.2.l4.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23617536,
   "nbytes": 3072
  },
  "stage.2.l4.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23620608,
   "nbytes": 3072
  },
  "stage.2.l4.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23623680,
   "nbytes": 3072
  },
  "stage.2.l4.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 23626752,
   "nbytes": 3072
  },
  "stage.2.l4.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 6,
   "offset": 23629824,
   "nbytes": 3538944
  },
  "stage.2.l4.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 6,
   "offset": 27168768,
   "nbytes": 1179648
  },
  "stage.2.l4.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 6,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.2.l4.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 6,
   "offset": 33067008,
   "nbytes": 4718592
  },
  "stage.2.l5.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37785600,
   "nbytes": 3072
  },
  "stage.2.l5.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37788672,
   "nbytes": 3072
  },
  "stage.2.l5.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37791744,
   "nbytes": 3072
  },
  "stage.2.l5.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37794816,
   "nbytes": 3072
  },
  "stage.2.l5.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37797888,
   "nbytes": 3072
  },
  "stage.2.l5.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 37800960,
   "nbytes": 3072
  },
  "stage.2.l5.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 6,
   "offset": 37804032,
   "nbytes": 3538944
  },
  "stage.2.l5.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 6,
   "offset": 41342976,
   "nbytes": 1179648
  },
  "stage.2.l5.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 6,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.2.l5.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 6,
   "offset": 47241216,
   "nbytes": 4718592
  },
  "stage.2.l6.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51959808,
   "nbytes": 3072
  },
  "stage.2.l6.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51962880,
   "nbytes": 3072
  },
  "stage.2.l6.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51965952,
   "nbytes": 3072
  },
  "stage.2.l6.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51969024,
   "nbytes": 3072
  },
  "stage.2.l6.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51972096,
   "nbytes": 3072
  },
  "stage.2.l6.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 51975168,
   "nbytes": 3072
  },
  "stage.2.l6.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 6,
   "offset": 51978240,
   "nbytes": 3538944
  },
  "stage.2.l6.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 6,
   "offset": 55517184,
   "nbytes": 1179648
  },
  "stage.2.l6.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 6,
   "offset": 56696832,
   "nbytes": 4718592
  },
  "stage.2.l6.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 6,
   "offset": 61415424,
   "nbytes": 4718592
  },
  "stage.2.l7.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66134016,
   "nbytes": 3072
  },
  "stage.2.l7.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66137088,
   "nbytes": 3072
  },
  "stage.2.l7.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66140160,
   "nbytes": 3072
  },
  "stage.2.l7.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66143232,
   "nbytes": 3072
  },
  "stage.2.l7.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66146304,
   "nbytes": 3072
  },
  "stage.2.l7.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 6,
   "offset": 66149376,
   "nbytes": 3072
  },
  "stage.2.l7.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 7,
   "offset": 0,
   "nbytes": 3538944
  },
  "stage.2.l7.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 7,
   "offset": 3538944,
   "nbytes": 1179648
  },
  "stage.2.l7.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 7,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.2.l7.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 7,
   "offset": 9437184,
   "nbytes": 4718592
  },
  "stage.2.l8.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14155776,
   "nbytes": 3072
  },
  "stage.2.l8.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14158848,
   "nbytes": 3072
  },
  "stage.2.l8.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14161920,
   "nbytes": 3072
  },
  "stage.2.l8.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14164992,
   "nbytes": 3072
  },
  "stage.2.l8.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14168064,
   "nbytes": 3072
  },
  "stage.2.l8.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 14171136,
   "nbytes": 3072
  },
  "stage.2.l8.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 7,
   "offset": 14174208,
   "nbytes": 3538944
  },
  "stage.2.l8.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 7,
   "offset": 17713152,
   "nbytes": 1179648
  },
  "stage.2.l8.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 7,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.2.l8.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 7,
   "offset": 23611392,
   "nbytes": 4718592
  },
  "stage.2.l9.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28329984,
   "nbytes": 3072
  },
  "stage.2.l9.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28333056,
   "nbytes": 3072
  },
  "stage.2.l9.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28336128,
   "nbytes": 3072
  },
  "stage.2.l9.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28339200,
   "nbytes": 3072
  },
  "stage.2.l9.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28342272,
   "nbytes": 3072
  },
  "stage.2.l9.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 28345344,
   "nbytes": 3072
  },
  "stage.2.l9.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 7,
   "offset": 28348416,
   "nbytes": 3538944
  },
  "stage.2.l9.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 7,
   "offset": 31887360,
   "nbytes": 1179648
  },
  "stage.2.l9.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 7,
   "offset": 33067008,
   "nbytes": 4718592
  },
  "stage.2.l9.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 7,
   "offset": 37785600,
   "nbytes": 4718592
  },
  "stage.2.l10.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42504192,
   "nbytes": 3072
  },
  "stage.2.l10.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42507264,
   "nbytes": 3072
  },
  "stage.2.l10.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42510336,
   "nbytes": 3072
  },
  "stage.2.l10.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42513408,
   "nbytes": 3072
  },
  "stage.2.l10.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42516480,
   "nbytes": 3072
  },
  "stage.2.l10.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 42519552,
   "nbytes": 3072
  },
  "stage.2.l10.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 7,
   "offset": 42522624,
   "nbytes": 3538944
  },
  "stage.2.l10.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 7,
   "offset": 46061568,
   "nbytes": 1179648
  },
  "stage.2.l10.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 7,
   "offset": 47241216,
   "nbytes": 4718592
  },
  "stage.2.l10.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 7,
   "offset": 51959808,
   "nbytes": 4718592
  },
  "stage.2.l11.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56678400,
   "nbytes": 3072
  },
  "stage.2.l11.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56681472,
   "nbytes": 3072
  },
  "stage.2.l11.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56684544,
   "nbytes": 3072
  },
  "stage.2.l11.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56687616,
   "nbytes": 3072
  },
  "stage.2.l11.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56690688,
   "nbytes": 3072
  },
  "stage.2.l11.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 7,
   "offset": 56693760,
   "nbytes": 3072
  },
  "stage.2.l11.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 7,
   "offset": 56696832,
   "nbytes": 3538944
  },
  "stage.2.l11.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 7,
   "offset": 60235776,
   "nbytes": 1179648
  },
  "stage.2.l11.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 7,
   "offset": 61415424,
   "nbytes": 4718592
  },
  "stage.2.l11.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 8,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.3.in.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 8,
   "offset": 4718592,
   "nbytes": 1179648
  },
  "stage.3.out.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    640
   ],
   "shard": 8,
   "offset": 5898240,
   "nbytes": 983040
  },
  "stage.3.l0.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6881280,
   "nbytes": 3072
  },
  "stage.3.l0.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6884352,
   "nbytes": 3072
  },
  "stage.3.l0.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6887424,
   "nbytes": 3072
  },
  "stage.3.l0.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6890496,
   "nbytes": 3072
  },
  "stage.3.l0.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6893568,
   "nbytes": 3072
  },
  "stage.3.l0.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 6896640,
   "nbytes": 3072
  },
  "stage.3.l0.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 8,
   "offset": 6899712,
   "nbytes": 3538944
  },
  "stage.3.l0.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 8,
   "offset": 10438656,
   "nbytes": 1179648
  },
  "stage.3.l0.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 8,
   "offset": 11618304,
   "nbytes": 4718592
  },
  "stage.3.l0.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 8,
   "offset": 16336896,
   "nbytes": 4718592
  },
  "stage.3.l1.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21055488,
   "nbytes": 3072
  },
  "stage.3.l1.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21058560,
   "nbytes": 3072
  },
  "stage.3.l1.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21061632,
   "nbytes": 3072
  },
  "stage.3.l1.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21064704,
   "nbytes": 3072
  },
  "stage.3.l1.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21067776,
   "nbytes": 3072
  },
  "stage.3.l1.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 21070848,
   "nbytes": 3072
  },
  "stage.3.l1.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 8,
   "offset": 21073920,
   "nbytes": 3538944
  },
  "stage.3.l1.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 8,
   "offset": 24612864,
   "nbytes": 1179648
  },
  "stage.3.l1.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 8,
   "offset": 25792512,
   "nbytes": 4718592
  },
  "stage.3.l1.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 8,
   "offset": 30511104,
   "nbytes": 4718592
  },
  "stage.3.l2.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35229696,
   "nbytes": 3072
  },
  "stage.3.l2.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35232768,
   "nbytes": 3072
  },
  "stage.3.l2.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35235840,
   "nbytes": 3072
  },
  "stage.3.l2.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35238912,
   "nbytes": 3072
  },
  "stage.3.l2.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35241984,
   "nbytes": 3072
  },
  "stage.3.l2.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 35245056,
   "nbytes": 3072
  },
  "stage.3.l2.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 8,
   "offset": 35248128,
   "nbytes": 3538944
  },
  "stage.3.l2.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 8,
   "offset": 38787072,
   "nbytes": 1179648
  },
  "stage.3.l2.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 8,
   "offset": 39966720,
   "nbytes": 4718592
  },
  "stage.3.l2.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 8,
   "offset": 44685312,
   "nbytes": 4718592
  },
  "stage.3.l3.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49403904,
   "nbytes": 3072
  },
  "stage.3.l3.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49406976,
   "nbytes": 3072
  },
  "stage.3.l3.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49410048,
   "nbytes": 3072
  },
  "stage.3.l3.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49413120,
   "nbytes": 3072
  },
  "stage.3.l3.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49416192,
   "nbytes": 3072
  },
  "stage.3.l3.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 49419264,
   "nbytes": 3072
  },
  "stage.3.l3.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 8,
   "offset": 49422336,
   "nbytes": 3538944
  },
  "stage.3.l3.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 8,
   "offset": 52961280,
   "nbytes": 1179648
  },
  "stage.3.l3.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 8,
   "offset": 54140928,
   "nbytes": 4718592
  },
  "stage.3.l3.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 8,
   "offset": 58859520,
   "nbytes": 4718592
  },
  "stage.3.l4.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63578112,
   "nbytes": 3072
  },
  "stage.3.l4.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63581184,
   "nbytes": 3072
  },
  "stage.3.l4.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63584256,
   "nbytes": 3072
  },
  "stage.3.l4.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63587328,
   "nbytes": 3072
  },
  "stage.3.l4.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63590400,
   "nbytes": 3072
  },
  "stage.3.l4.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 8,
   "offset": 63593472,
   "nbytes": 3072
  },
  "stage.3.l4.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 9,
   "offset": 0,
   "nbytes": 3538944
  },
  "stage.3.l4.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 9,
   "offset": 3538944,
   "nbytes": 1179648
  },
  "stage.3.l4.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 9,
   "offset": 4718592,
   "nbytes": 4718592
  },
  "stage.3.l4.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 9,
   "offset": 9437184,
   "nbytes": 4718592
  },
  "stage.3.l5.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14155776,
   "nbytes": 3072
  },
  "stage.3.l5.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14158848,
   "nbytes": 3072
  },
  "stage.3.l5.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14161920,
   "nbytes": 3072
  },
  "stage.3.l5.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14164992,
   "nbytes": 3072
  },
  "stage.3.l5.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14168064,
   "nbytes": 3072
  },
  "stage.3.l5.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 14171136,
   "nbytes": 3072
  },
  "stage.3.l5.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 9,
   "offset": 14174208,
   "nbytes": 3538944
  },
  "stage.3.l5.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 9,
   "offset": 17713152,
   "nbytes": 1179648
  },
  "stage.3.l5.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 9,
   "offset": 18892800,
   "nbytes": 4718592
  },
  "stage.3.l5.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 9,
   "offset": 23611392,
   "nbytes": 4718592
  },
  "stage.3.l6.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28329984,
   "nbytes": 3072
  },
  "stage.3.l6.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28333056,
   "nbytes": 3072
  },
  "stage.3.l6.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28336128,
   "nbytes": 3072
  },
  "stage.3.l6.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28339200,
   "nbytes": 3072
  },
  "stage.3.l6.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28342272,
   "nbytes": 3072
  },
  "stage.3.l6.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 28345344,
   "nbytes": 3072
  },
  "stage.3.l6.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 9,
   "offset": 28348416,
   "nbytes": 3538944
  },
  "stage.3.l6.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 9,
   "offset": 31887360,
   "nbytes": 1179648
  },
  "stage.3.l6.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 9,
   "offset": 33067008,
   "nbytes": 4718592
  },
  "stage.3.l6.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 9,
   "offset": 37785600,
   "nbytes": 4718592
  },
  "stage.3.l7.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42504192,
   "nbytes": 3072
  },
  "stage.3.l7.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42507264,
   "nbytes": 3072
  },
  "stage.3.l7.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42510336,
   "nbytes": 3072
  },
  "stage.3.l7.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42513408,
   "nbytes": 3072
  },
  "stage.3.l7.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42516480,
   "nbytes": 3072
  },
  "stage.3.l7.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 42519552,
   "nbytes": 3072
  },
  "stage.3.l7.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 9,
   "offset": 42522624,
   "nbytes": 3538944
  },
  "stage.3.l7.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 9,
   "offset": 46061568,
   "nbytes": 1179648
  },
  "stage.3.l7.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 9,
   "offset": 47241216,
   "nbytes": 4718592
  },
  "stage.3.l7.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 9,
   "offset": 51959808,
   "nbytes": 4718592
  },
  "stage.3.l8.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56678400,
   "nbytes": 3072
  },
  "stage.3.l8.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56681472,
   "nbytes": 3072
  },
  "stage.3.l8.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56684544,
   "nbytes": 3072
  },
  "stage.3.l8.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56687616,
   "nbytes": 3072
  },
  "stage.3.l8.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56690688,
   "nbytes": 3072
  },
  "stage.3.l8.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 9,
   "offset": 56693760,
   "nbytes": 3072
  },
  "stage.3.l8.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 9,
   "offset": 56696832,
   "nbytes": 3538944
  },
  "stage.3.l8.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 9,
   "offset": 60235776,
   "nbytes": 1179648
  },
  "stage.3.l8.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 9,
   "offset": 61415424,
   "nbytes": 4718592
  },
  "stage.3.l8.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 10,
   "offset": 0,
   "nbytes": 4718592
  },
  "stage.3.l9.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4718592,
   "nbytes": 3072
  },
  "stage.3.l9.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4721664,
   "nbytes": 3072
  },
  "stage.3.l9.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4724736,
   "nbytes": 3072
  },
  "stage.3.l9.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4727808,
   "nbytes": 3072
  },
  "stage.3.l9.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4730880,
   "nbytes": 3072
  },
  "stage.3.l9.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 4733952,
   "nbytes": 3072
  },
  "stage.3.l9.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 10,
   "offset": 4737024,
   "nbytes": 3538944
  },
  "stage.3.l9.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 10,
   "offset": 8275968,
   "nbytes": 1179648
  },
  "stage.3.l9.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 10,
   "offset": 9455616,
   "nbytes": 4718592
  },
  "stage.3.l9.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 10,
   "offset": 14174208,
   "nbytes": 4718592
  },
  "stage.3.l10.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18892800,
   "nbytes": 3072
  },
  "stage.3.l10.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18895872,
   "nbytes": 3072
  },
  "stage.3.l10.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18898944,
   "nbytes": 3072
  },
  "stage.3.l10.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18902016,
   "nbytes": 3072
  },
  "stage.3.l10.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18905088,
   "nbytes": 3072
  },
  "stage.3.l10.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 18908160,
   "nbytes": 3072
  },
  "stage.3.l10.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 10,
   "offset": 18911232,
   "nbytes": 3538944
  },
  "stage.3.l10.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 10,
   "offset": 22450176,
   "nbytes": 1179648
  },
  "stage.3.l10.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 10,
   "offset": 23629824,
   "nbytes": 4718592
  },
  "stage.3.l10.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 10,
   "offset": 28348416,
   "nbytes": 4718592
  },
  "stage.3.l11.norm1.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33067008,
   "nbytes": 3072
  },
  "stage.3.l11.norm1.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33070080,
   "nbytes": 3072
  },
  "stage.3.l11.norm2.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33073152,
   "nbytes": 3072
  },
  "stage.3.l11.norm2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33076224,
   "nbytes": 3072
  },
  "stage.3.l11.attn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33079296,
   "nbytes": 3072
  },
  "stage.3.l11.ffn_scale": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 10,
   "offset": 33082368,
   "nbytes": 3072
  },
  "stage.3.l11.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2304
   ],
   "shard": 10,
   "offset": 33085440,
   "nbytes": 3538944
  },
  "stage.3.l11.wo.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    768
   ],
   "shard": 10,
   "offset": 36624384,
   "nbytes": 1179648
  },
  "stage.3.l11.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    3072
   ],
   "shard": 10,
   "offset": 37804032,
   "nbytes": 4718592
  },
  "stage.3.l11.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    3072,
    768
   ],
   "shard": 10,
   "offset": 42522624,
   "nbytes": 4718592
  },
  "stage.4.in.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 10,
   "offset": 47241216,
   "nbytes": 3276800
  },
  "stage.4.out.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1024
   ],
   "shard": 10,
   "offset": 50518016,
   "nbytes": 2621440
  },
  "stage.4.l0.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53139456,
   "nbytes": 5120
  },
  "stage.4.l0.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53144576,
   "nbytes": 5120
  },
  "stage.4.l0.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53149696,
   "nbytes": 5120
  },
  "stage.4.l0.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53154816,
   "nbytes": 5120
  },
  "stage.4.l0.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53159936,
   "nbytes": 5120
  },
  "stage.4.l0.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 10,
   "offset": 53165056,
   "nbytes": 5120
  },
  "stage.4.l0.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 10,
   "offset": 53170176,
   "nbytes": 9830400
  },
  "stage.4.l0.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 10,
   "offset": 63000576,
   "nbytes": 3276800
  },
  "stage.4.l0.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 11,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l0.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 11,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l1.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l1.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l1.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l1.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l1.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l1.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l1.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 11,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l1.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 11,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l1.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 11,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l1.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 11,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l2.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l2.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l2.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l2.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l2.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l2.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 11,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l2.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 12,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l2.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 12,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l2.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 12,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l2.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 12,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l3.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l3.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l3.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l3.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l3.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l3.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 12,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l3.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 12,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l3.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 12,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l3.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 12,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l3.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 13,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l4.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l4.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l4.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l4.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l4.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l4.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l4.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 13,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l4.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 13,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l4.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 13,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l4.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 13,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l5.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l5.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l5.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l5.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l5.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l5.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 13,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l5.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 13,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l5.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 13,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l5.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 14,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l5.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 14,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l6.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l6.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l6.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l6.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l6.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l6.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l6.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 14,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l6.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 14,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l6.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 14,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l6.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 14,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l7.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l7.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l7.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l7.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l7.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l7.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 14,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l7.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 15,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l7.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 15,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l7.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 15,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l7.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 15,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l8.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l8.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l8.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l8.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l8.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l8.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 15,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l8.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 15,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l8.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 15,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l8.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 15,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l8.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 16,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l9.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l9.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l9.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l9.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l9.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l9.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l9.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 16,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l9.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 16,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l9.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 16,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l9.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 16,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l10.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l10.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l10.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l10.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l10.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l10.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 16,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l10.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 16,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l10.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 16,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l10.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 17,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l10.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 17,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l11.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l11.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l11.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l11.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l11.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l11.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l11.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 17,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l11.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 17,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l11.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 17,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l11.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 17,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l12.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l12.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l12.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l12.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l12.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l12.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 17,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l12.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 18,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l12.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 18,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l12.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 18,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l12.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 18,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l13.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l13.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l13.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l13.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l13.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l13.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 18,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l13.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 18,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l13.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 18,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l13.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 18,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l13.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 19,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l14.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l14.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l14.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l14.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l14.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l14.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l14.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 19,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l14.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 19,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l14.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 19,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l14.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 19,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l15.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l15.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l15.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l15.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l15.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l15.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 19,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l15.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 19,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l15.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 19,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l15.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 20,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l15.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 20,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l16.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l16.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l16.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l16.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l16.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l16.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l16.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 20,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l16.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 20,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l16.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 20,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l16.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 20,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l17.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l17.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l17.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l17.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l17.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l17.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 20,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l17.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 21,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l17.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 21,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l17.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 21,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l17.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 21,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l18.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l18.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l18.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l18.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l18.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l18.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 21,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l18.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 21,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l18.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 21,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l18.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 21,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l18.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 22,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l19.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l19.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l19.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l19.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l19.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l19.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l19.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 22,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l19.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 22,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l19.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 22,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l19.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 22,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l20.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l20.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l20.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l20.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l20.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l20.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 22,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l20.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 22,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l20.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 22,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l20.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 23,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l20.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 23,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l21.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l21.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l21.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l21.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l21.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l21.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l21.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 23,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l21.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 23,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l21.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 23,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l21.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 23,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l22.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l22.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l22.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l22.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l22.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l22.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 23,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l22.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 24,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l22.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 24,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l22.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 24,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l22.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 24,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l23.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l23.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l23.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l23.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l23.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l23.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 24,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l23.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 24,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l23.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 24,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l23.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 24,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l23.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 25,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l24.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l24.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l24.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l24.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l24.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l24.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l24.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 25,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l24.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 25,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l24.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 25,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l24.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 25,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l25.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l25.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l25.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l25.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l25.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l25.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 25,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l25.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 25,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l25.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 25,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l25.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 26,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l25.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 26,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l26.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l26.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l26.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l26.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l26.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l26.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l26.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 26,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l26.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 26,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l26.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 26,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l26.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 26,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l27.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65566720,
   "nbytes": 5120
  },
  "stage.4.l27.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65571840,
   "nbytes": 5120
  },
  "stage.4.l27.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65576960,
   "nbytes": 5120
  },
  "stage.4.l27.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65582080,
   "nbytes": 5120
  },
  "stage.4.l27.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65587200,
   "nbytes": 5120
  },
  "stage.4.l27.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 26,
   "offset": 65592320,
   "nbytes": 5120
  },
  "stage.4.l27.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 27,
   "offset": 0,
   "nbytes": 9830400
  },
  "stage.4.l27.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 27,
   "offset": 9830400,
   "nbytes": 3276800
  },
  "stage.4.l27.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 27,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l27.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 27,
   "offset": 26214400,
   "nbytes": 13107200
  },
  "stage.4.l28.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39321600,
   "nbytes": 5120
  },
  "stage.4.l28.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39326720,
   "nbytes": 5120
  },
  "stage.4.l28.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39331840,
   "nbytes": 5120
  },
  "stage.4.l28.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39336960,
   "nbytes": 5120
  },
  "stage.4.l28.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39342080,
   "nbytes": 5120
  },
  "stage.4.l28.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 27,
   "offset": 39347200,
   "nbytes": 5120
  },
  "stage.4.l28.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 27,
   "offset": 39352320,
   "nbytes": 9830400
  },
  "stage.4.l28.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 27,
   "offset": 49182720,
   "nbytes": 3276800
  },
  "stage.4.l28.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 27,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "stage.4.l28.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 28,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l29.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13107200,
   "nbytes": 5120
  },
  "stage.4.l29.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13112320,
   "nbytes": 5120
  },
  "stage.4.l29.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13117440,
   "nbytes": 5120
  },
  "stage.4.l29.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13122560,
   "nbytes": 5120
  },
  "stage.4.l29.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13127680,
   "nbytes": 5120
  },
  "stage.4.l29.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 13132800,
   "nbytes": 5120
  },
  "stage.4.l29.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 28,
   "offset": 13137920,
   "nbytes": 9830400
  },
  "stage.4.l29.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 28,
   "offset": 22968320,
   "nbytes": 3276800
  },
  "stage.4.l29.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 28,
   "offset": 26245120,
   "nbytes": 13107200
  },
  "stage.4.l29.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 28,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l30.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52459520,
   "nbytes": 5120
  },
  "stage.4.l30.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52464640,
   "nbytes": 5120
  },
  "stage.4.l30.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52469760,
   "nbytes": 5120
  },
  "stage.4.l30.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52474880,
   "nbytes": 5120
  },
  "stage.4.l30.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52480000,
   "nbytes": 5120
  },
  "stage.4.l30.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 28,
   "offset": 52485120,
   "nbytes": 5120
  },
  "stage.4.l30.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 28,
   "offset": 52490240,
   "nbytes": 9830400
  },
  "stage.4.l30.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 28,
   "offset": 62320640,
   "nbytes": 3276800
  },
  "stage.4.l30.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 29,
   "offset": 0,
   "nbytes": 13107200
  },
  "stage.4.l30.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 29,
   "offset": 13107200,
   "nbytes": 13107200
  },
  "stage.4.l31.norm1.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26214400,
   "nbytes": 5120
  },
  "stage.4.l31.norm1.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26219520,
   "nbytes": 5120
  },
  "stage.4.l31.norm2.weight": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26224640,
   "nbytes": 5120
  },
  "stage.4.l31.norm2.bias": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26229760,
   "nbytes": 5120
  },
  "stage.4.l31.attn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26234880,
   "nbytes": 5120
  },
  "stage.4.l31.ffn_scale": {
   "dtype": "f32",
   "shape": [
    1280
   ],
   "shard": 29,
   "offset": 26240000,
   "nbytes": 5120
  },
  "stage.4.l31.qkv.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    3840
   ],
   "shard": 29,
   "offset": 26245120,
   "nbytes": 9830400
  },
  "stage.4.l31.wo.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    1280
   ],
   "shard": 29,
   "offset": 36075520,
   "nbytes": 3276800
  },
  "stage.4.l31.ffn1.weight": {
   "dtype": "bf16",
   "shape": [
    1280,
    5120
   ],
   "shard": 29,
   "offset": 39352320,
   "nbytes": 13107200
  },
  "stage.4.l31.ffn2.weight": {
   "dtype": "bf16",
   "shape": [
    5120,
    1280
   ],
   "shard": 29,
   "offset": 52459520,
   "nbytes": 13107200
  },
  "sem_enc.in.weight": {
   "dtype": "bf16",
   "shape": [
    1024,
    768
   ],
   "shard": 30,
   "offset": 0,
   "nbytes": 1572864
  },
  "sem_enc.in.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 1572864,
   "nbytes": 3072
  },
  "sem_enc.0.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 1575936,
   "nbytes": 21504
  },
  "sem_enc.0.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 1597440,
   "nbytes": 3072
  },
  "sem_enc.0.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 1600512,
   "nbytes": 3072
  },
  "sem_enc.0.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 1603584,
   "nbytes": 3072
  },
  "sem_enc.0.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 1606656,
   "nbytes": 3145728
  },
  "sem_enc.0.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 4752384,
   "nbytes": 8192
  },
  "sem_enc.0.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 4760576,
   "nbytes": 3145728
  },
  "sem_enc.0.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 7906304,
   "nbytes": 3072
  },
  "sem_head.0.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 7909376,
   "nbytes": 21504
  },
  "sem_head.0.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 7930880,
   "nbytes": 3072
  },
  "sem_head.0.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 7933952,
   "nbytes": 3072
  },
  "sem_head.0.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 7937024,
   "nbytes": 3072
  },
  "sem_head.0.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 7940096,
   "nbytes": 3145728
  },
  "sem_head.0.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 11085824,
   "nbytes": 8192
  },
  "sem_head.0.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 11094016,
   "nbytes": 3145728
  },
  "sem_head.0.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 14239744,
   "nbytes": 3072
  },
  "sem_enc.1.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 14242816,
   "nbytes": 21504
  },
  "sem_enc.1.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 14264320,
   "nbytes": 3072
  },
  "sem_enc.1.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 14267392,
   "nbytes": 3072
  },
  "sem_enc.1.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 14270464,
   "nbytes": 3072
  },
  "sem_enc.1.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 14273536,
   "nbytes": 3145728
  },
  "sem_enc.1.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 17419264,
   "nbytes": 8192
  },
  "sem_enc.1.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 17427456,
   "nbytes": 3145728
  },
  "sem_enc.1.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 20573184,
   "nbytes": 3072
  },
  "sem_head.1.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 20576256,
   "nbytes": 21504
  },
  "sem_head.1.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 20597760,
   "nbytes": 3072
  },
  "sem_head.1.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 20600832,
   "nbytes": 3072
  },
  "sem_head.1.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 20603904,
   "nbytes": 3072
  },
  "sem_head.1.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 20606976,
   "nbytes": 3145728
  },
  "sem_head.1.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 23752704,
   "nbytes": 8192
  },
  "sem_head.1.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 23760896,
   "nbytes": 3145728
  },
  "sem_head.1.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 26906624,
   "nbytes": 3072
  },
  "sem_enc.2.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 26909696,
   "nbytes": 21504
  },
  "sem_enc.2.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 26931200,
   "nbytes": 3072
  },
  "sem_enc.2.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 26934272,
   "nbytes": 3072
  },
  "sem_enc.2.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 26937344,
   "nbytes": 3072
  },
  "sem_enc.2.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 26940416,
   "nbytes": 3145728
  },
  "sem_enc.2.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 30086144,
   "nbytes": 8192
  },
  "sem_enc.2.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 30094336,
   "nbytes": 3145728
  },
  "sem_enc.2.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 33240064,
   "nbytes": 3072
  },
  "sem_head.2.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 33243136,
   "nbytes": 21504
  },
  "sem_head.2.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 33264640,
   "nbytes": 3072
  },
  "sem_head.2.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 33267712,
   "nbytes": 3072
  },
  "sem_head.2.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 33270784,
   "nbytes": 3072
  },
  "sem_head.2.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 33273856,
   "nbytes": 3145728
  },
  "sem_head.2.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 36419584,
   "nbytes": 8192
  },
  "sem_head.2.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 36427776,
   "nbytes": 3145728
  },
  "sem_head.2.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 39573504,
   "nbytes": 3072
  },
  "sem_enc.3.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 39576576,
   "nbytes": 21504
  },
  "sem_enc.3.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 39598080,
   "nbytes": 3072
  },
  "sem_enc.3.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 39601152,
   "nbytes": 3072
  },
  "sem_enc.3.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 39604224,
   "nbytes": 3072
  },
  "sem_enc.3.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 39607296,
   "nbytes": 3145728
  },
  "sem_enc.3.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 42753024,
   "nbytes": 8192
  },
  "sem_enc.3.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 42761216,
   "nbytes": 3145728
  },
  "sem_enc.3.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 45906944,
   "nbytes": 3072
  },
  "sem_head.3.dw.weight": {
   "dtype": "f32",
   "shape": [
    768,
    7
   ],
   "shard": 30,
   "offset": 45910016,
   "nbytes": 21504
  },
  "sem_head.3.dw.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 45931520,
   "nbytes": 3072
  },
  "sem_head.3.norm.weight": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 45934592,
   "nbytes": 3072
  },
  "sem_head.3.norm.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 45937664,
   "nbytes": 3072
  },
  "sem_head.3.pw1.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    2048
   ],
   "shard": 30,
   "offset": 45940736,
   "nbytes": 3145728
  },
  "sem_head.3.pw1.bias": {
   "dtype": "f32",
   "shape": [
    2048
   ],
   "shard": 30,
   "offset": 49086464,
   "nbytes": 8192
  },
  "sem_head.3.pw2.weight": {
   "dtype": "bf16",
   "shape": [
    2048,
    768
   ],
   "shard": 30,
   "offset": 49094656,
   "nbytes": 3145728
  },
  "sem_head.3.pw2.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 52240384,
   "nbytes": 3072
  },
  "sem_head.out.weight": {
   "dtype": "bf16",
   "shape": [
    768,
    1024
   ],
   "shard": 30,
   "offset": 52243456,
   "nbytes": 1572864
  },
  "sem_head.out.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 53816320,
   "nbytes": 4096
  },
  "rvq.0.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    768,
    8
   ],
   "shard": 30,
   "offset": 53820416,
   "nbytes": 24576,
   "host": true
  },
  "rvq.0.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 53844992,
   "nbytes": 32,
   "host": true
  },
  "rvq.0.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    768
   ],
   "shard": 30,
   "offset": 53845248,
   "nbytes": 24576,
   "host": true
  },
  "rvq.0.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    768
   ],
   "shard": 30,
   "offset": 53869824,
   "nbytes": 3072,
   "host": true
  },
  "rvq.0.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 53872896,
   "nbytes": 131072,
   "host": true
  },
  "rvq.1.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 54003968,
   "nbytes": 32768,
   "host": true
  },
  "rvq.1.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 54036736,
   "nbytes": 32,
   "host": true
  },
  "rvq.1.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 54036992,
   "nbytes": 32768,
   "host": true
  },
  "rvq.1.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 54069760,
   "nbytes": 4096,
   "host": true
  },
  "rvq.1.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 54073856,
   "nbytes": 131072,
   "host": true
  },
  "rvq.2.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 54204928,
   "nbytes": 32768,
   "host": true
  },
  "rvq.2.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 54237696,
   "nbytes": 32,
   "host": true
  },
  "rvq.2.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 54237952,
   "nbytes": 32768,
   "host": true
  },
  "rvq.2.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 54270720,
   "nbytes": 4096,
   "host": true
  },
  "rvq.2.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 54274816,
   "nbytes": 131072,
   "host": true
  },
  "rvq.3.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 54405888,
   "nbytes": 32768,
   "host": true
  },
  "rvq.3.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 54438656,
   "nbytes": 32,
   "host": true
  },
  "rvq.3.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 54438912,
   "nbytes": 32768,
   "host": true
  },
  "rvq.3.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 54471680,
   "nbytes": 4096,
   "host": true
  },
  "rvq.3.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 54475776,
   "nbytes": 131072,
   "host": true
  },
  "rvq.4.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 54606848,
   "nbytes": 32768,
   "host": true
  },
  "rvq.4.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 54639616,
   "nbytes": 32,
   "host": true
  },
  "rvq.4.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 54639872,
   "nbytes": 32768,
   "host": true
  },
  "rvq.4.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 54672640,
   "nbytes": 4096,
   "host": true
  },
  "rvq.4.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 54676736,
   "nbytes": 131072,
   "host": true
  },
  "rvq.5.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 54807808,
   "nbytes": 32768,
   "host": true
  },
  "rvq.5.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 54840576,
   "nbytes": 32,
   "host": true
  },
  "rvq.5.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 54840832,
   "nbytes": 32768,
   "host": true
  },
  "rvq.5.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 54873600,
   "nbytes": 4096,
   "host": true
  },
  "rvq.5.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 54877696,
   "nbytes": 131072,
   "host": true
  },
  "rvq.6.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 55008768,
   "nbytes": 32768,
   "host": true
  },
  "rvq.6.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 55041536,
   "nbytes": 32,
   "host": true
  },
  "rvq.6.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 55041792,
   "nbytes": 32768,
   "host": true
  },
  "rvq.6.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 55074560,
   "nbytes": 4096,
   "host": true
  },
  "rvq.6.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 55078656,
   "nbytes": 131072,
   "host": true
  },
  "rvq.7.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 55209728,
   "nbytes": 32768,
   "host": true
  },
  "rvq.7.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 55242496,
   "nbytes": 32,
   "host": true
  },
  "rvq.7.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 55242752,
   "nbytes": 32768,
   "host": true
  },
  "rvq.7.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 55275520,
   "nbytes": 4096,
   "host": true
  },
  "rvq.7.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 55279616,
   "nbytes": 131072,
   "host": true
  },
  "rvq.8.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 55410688,
   "nbytes": 32768,
   "host": true
  },
  "rvq.8.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 55443456,
   "nbytes": 32,
   "host": true
  },
  "rvq.8.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 55443712,
   "nbytes": 32768,
   "host": true
  },
  "rvq.8.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 55476480,
   "nbytes": 4096,
   "host": true
  },
  "rvq.8.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 55480576,
   "nbytes": 131072,
   "host": true
  },
  "rvq.9.in_proj.weight": {
   "dtype": "f32",
   "shape": [
    1024,
    8
   ],
   "shard": 30,
   "offset": 55611648,
   "nbytes": 32768,
   "host": true
  },
  "rvq.9.in_proj.bias": {
   "dtype": "f32",
   "shape": [
    8
   ],
   "shard": 30,
   "offset": 55644416,
   "nbytes": 32,
   "host": true
  },
  "rvq.9.out_proj.weight": {
   "dtype": "f32",
   "shape": [
    8,
    1024
   ],
   "shard": 30,
   "offset": 55644672,
   "nbytes": 32768,
   "host": true
  },
  "rvq.9.out_proj.bias": {
   "dtype": "f32",
   "shape": [
    1024
   ],
   "shard": 30,
   "offset": 55677440,
   "nbytes": 4096,
   "host": true
  },
  "rvq.9.codebook": {
   "dtype": "f32",
   "shape": [
    8,
    4096
   ],
   "shard": 30,
   "offset": 55681536,
   "nbytes": 131072,
   "host": true
  }
 }
}