{
  "format": "carbon-compact-v1",
  "total_nonzero_params": 8985670530,
  "total_dense_params": 18208957504,
  "tensors": {
    "model.layers.0.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 0,
      "length": 8
    },
    "model.layers.0.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 8,
      "length": 8
    },
    "model.layers.0.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 51,
      "offset": 16,
      "length": 38
    },
    "model.layers.0.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.0.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 44,
      "offset": 54,
      "length": 43
    },
    "model.layers.0.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 12,
      "offset": 97,
      "length": 33
    },
    "model.layers.0.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 274,
      "offset": 130,
      "length": 119
    },
    "model.layers.0.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 21,
      "offset": 249,
      "length": 20
    },
    "model.layers.0.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.0.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 21,
      "offset": 269,
      "length": 24
    },
    "model.layers.0.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 357,
      "offset": 293,
      "length": 66
    },
    "model.layers.0.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 628,
      "offset": 359,
      "length": 90
    },
    "model.layers.0.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 2415,
      "offset": 449,
      "length": 320
    },
    "model.layers.0.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 769,
      "length": 8
    },
    "model.layers.1.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 777,
      "length": 8
    },
    "model.layers.1.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 785,
      "length": 8
    },
    "model.layers.1.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 7,
      "offset": 793,
      "length": 20
    },
    "model.layers.1.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.1.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 2,
      "offset": 813,
      "length": 16
    },
    "model.layers.1.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1,
      "offset": 829,
      "length": 12
    },
    "model.layers.1.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 13,
      "offset": 841,
      "length": 28
    },
    "model.layers.1.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1,
      "offset": 869,
      "length": 12
    },
    "model.layers.1.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.1.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 1,
      "offset": 881,
      "length": 12
    },
    "model.layers.1.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 1615,
      "offset": 893,
      "length": 129
    },
    "model.layers.1.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 95083,
      "offset": 1022,
      "length": 2118
    },
    "model.layers.1.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1775,
      "offset": 3140,
      "length": 373
    },
    "model.layers.1.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3513,
      "length": 8
    },
    "model.layers.10.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3521,
      "length": 8
    },
    "model.layers.10.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3529,
      "length": 8
    },
    "model.layers.10.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3537,
      "length": 8
    },
    "model.layers.10.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.10.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3545,
      "length": 8
    },
    "model.layers.10.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3553,
      "length": 8
    },
    "model.layers.10.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3561,
      "length": 8
    },
    "model.layers.10.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3569,
      "length": 8
    },
    "model.layers.10.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.10.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 3577,
      "length": 8
    },
    "model.layers.10.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 1713,
      "offset": 3585,
      "length": 109
    },
    "model.layers.10.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 14630,
      "offset": 3694,
      "length": 2646
    },
    "model.layers.10.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1713,
      "offset": 6340,
      "length": 61
    },
    "model.layers.10.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 6401,
      "length": 8
    },
    "model.layers.2.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 6409,
      "length": 8
    },
    "model.layers.2.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 6417,
      "length": 8
    },
    "model.layers.2.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 6425,
      "length": 34
    },
    "model.layers.2.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.2.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 6459,
      "length": 8
    },
    "model.layers.2.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 32,
      "offset": 6467,
      "length": 16
    },
    "model.layers.2.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 24576,
      "offset": 6483,
      "length": 327
    },
    "model.layers.2.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 11968,
      "offset": 6810,
      "length": 344
    },
    "model.layers.2.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.2.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 7154,
      "length": 74
    },
    "model.layers.2.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7228,
      "length": 8
    },
    "model.layers.2.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7236,
      "length": 8
    },
    "model.layers.2.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7244,
      "length": 8
    },
    "model.layers.2.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7252,
      "length": 8
    },
    "model.layers.3.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7260,
      "length": 8
    },
    "model.layers.3.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7268,
      "length": 8
    },
    "model.layers.3.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 7276,
      "length": 34
    },
    "model.layers.3.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.3.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 7310,
      "length": 8
    },
    "model.layers.3.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 32,
      "offset": 7318,
      "length": 16
    },
    "model.layers.3.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 24576,
      "offset": 7334,
      "length": 327
    },
    "model.layers.3.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 11968,
      "offset": 7661,
      "length": 344
    },
    "model.layers.3.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.3.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 8005,
      "length": 74
    },
    "model.layers.3.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 4159,
      "offset": 8079,
      "length": 141
    },
    "model.layers.3.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 8716,
      "offset": 8220,
      "length": 418
    },
    "model.layers.3.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4159,
      "offset": 8638,
      "length": 164
    },
    "model.layers.3.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 8802,
      "length": 8
    },
    "model.layers.4.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 8810,
      "length": 8
    },
    "model.layers.4.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 8818,
      "length": 8
    },
    "model.layers.4.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 780,
      "offset": 8826,
      "length": 106
    },
    "model.layers.4.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.4.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 8932,
      "length": 8
    },
    "model.layers.4.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 47,
      "offset": 8940,
      "length": 69
    },
    "model.layers.4.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1999,
      "offset": 9009,
      "length": 1015
    },
    "model.layers.4.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 429,
      "offset": 10024,
      "length": 74
    },
    "model.layers.4.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.4.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 429,
      "offset": 10098,
      "length": 77
    },
    "model.layers.4.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 8025,
      "offset": 10175,
      "length": 308
    },
    "model.layers.4.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 122002,
      "offset": 10483,
      "length": 5550
    },
    "model.layers.4.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 13849,
      "offset": 16033,
      "length": 1168
    },
    "model.layers.4.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 17201,
      "length": 8
    },
    "model.layers.5.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 17209,
      "length": 8
    },
    "model.layers.5.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 17217,
      "length": 8
    },
    "model.layers.5.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 1888,
      "offset": 17225,
      "length": 65
    },
    "model.layers.5.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.5.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 17290,
      "length": 8
    },
    "model.layers.5.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 48,
      "offset": 17298,
      "length": 64
    },
    "model.layers.5.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 3932,
      "offset": 17362,
      "length": 857
    },
    "model.layers.5.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 64,
      "offset": 18219,
      "length": 20
    },
    "model.layers.5.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.5.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 64,
      "offset": 18239,
      "length": 23
    },
    "model.layers.5.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 6992,
      "offset": 18262,
      "length": 359
    },
    "model.layers.5.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 29667,
      "offset": 18621,
      "length": 6801
    },
    "model.layers.5.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 9617,
      "offset": 25422,
      "length": 2172
    },
    "model.layers.5.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 27594,
      "length": 8
    },
    "model.layers.6.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 27602,
      "length": 8
    },
    "model.layers.6.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 27610,
      "length": 8
    },
    "model.layers.6.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 7232,
      "offset": 27618,
      "length": 134
    },
    "model.layers.6.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.6.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 27752,
      "length": 8
    },
    "model.layers.6.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 96,
      "offset": 27760,
      "length": 32
    },
    "model.layers.6.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 113728,
      "offset": 27792,
      "length": 1840
    },
    "model.layers.6.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 5120,
      "offset": 29632,
      "length": 132
    },
    "model.layers.6.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.6.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 29764,
      "length": 72
    },
    "model.layers.6.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 3896,
      "offset": 29836,
      "length": 255
    },
    "model.layers.6.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 27741,
      "offset": 30091,
      "length": 4009
    },
    "model.layers.6.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 3941,
      "offset": 34100,
      "length": 860
    },
    "model.layers.6.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 34960,
      "length": 8
    },
    "model.layers.7.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 34968,
      "length": 8
    },
    "model.layers.7.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 34976,
      "length": 8
    },
    "model.layers.7.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 7232,
      "offset": 34984,
      "length": 134
    },
    "model.layers.7.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.7.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 35118,
      "length": 8
    },
    "model.layers.7.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 96,
      "offset": 35126,
      "length": 32
    },
    "model.layers.7.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 113728,
      "offset": 35158,
      "length": 1840
    },
    "model.layers.7.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 5120,
      "offset": 36998,
      "length": 132
    },
    "model.layers.7.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.7.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 37130,
      "length": 72
    },
    "model.layers.7.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 8307,
      "offset": 37202,
      "length": 495
    },
    "model.layers.7.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 157966,
      "offset": 37697,
      "length": 7870
    },
    "model.layers.7.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 8307,
      "offset": 45567,
      "length": 2280
    },
    "model.layers.7.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 47847,
      "length": 8
    },
    "model.layers.8.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 47855,
      "length": 8
    },
    "model.layers.8.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 47863,
      "length": 8
    },
    "model.layers.8.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 47871,
      "length": 34
    },
    "model.layers.8.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.8.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 47905,
      "length": 8
    },
    "model.layers.8.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 32,
      "offset": 47913,
      "length": 16
    },
    "model.layers.8.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 38912,
      "offset": 47929,
      "length": 577
    },
    "model.layers.8.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 22208,
      "offset": 48506,
      "length": 554
    },
    "model.layers.8.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.8.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 2048,
      "offset": 49060,
      "length": 73
    },
    "model.layers.8.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 8558,
      "offset": 49133,
      "length": 326
    },
    "model.layers.8.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 60993,
      "offset": 49459,
      "length": 7871
    },
    "model.layers.8.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 8558,
      "offset": 57330,
      "length": 2234
    },
    "model.layers.8.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 59564,
      "length": 8
    },
    "model.layers.9.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 59572,
      "length": 8
    },
    "model.layers.9.linear_attn.A_log": {
      "mode": "sparse_delta_u32",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 59580,
      "length": 8
    },
    "model.layers.9.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 292,
      "offset": 59588,
      "length": 35
    },
    "model.layers.9.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.9.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 2,
      "offset": 59623,
      "length": 16
    },
    "model.layers.9.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4,
      "offset": 59639,
      "length": 19
    },
    "model.layers.9.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 2291,
      "offset": 59658,
      "length": 462
    },
    "model.layers.9.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 576,
      "offset": 60120,
      "length": 53
    },
    "model.layers.9.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.9.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 128,
      "offset": 60173,
      "length": 28
    },
    "model.layers.9.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 1407,
      "offset": 60201,
      "length": 102
    },
    "model.layers.9.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 9567,
      "offset": 60303,
      "length": 5420
    },
    "model.layers.9.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1407,
      "offset": 65723,
      "length": 287
    },
    "model.layers.9.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 0,
      "offset": 66010,
      "length": 8
    },
    "model.layers.11.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 66018,
      "length": 41
    },
    "model.layers.11.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.11.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32766,
      "offset": 66059,
      "length": 164
    },
    "model.layers.11.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.11.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 66223,
      "length": 630
    },
    "model.layers.11.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 66853,
      "length": 635
    },
    "model.layers.11.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554393,
      "offset": 67488,
      "length": 150114
    },
    "model.layers.11.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781289,
      "offset": 217602,
      "length": 76125
    },
    "model.layers.11.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.11.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777197,
      "offset": 293727,
      "length": 65370
    },
    "model.layers.11.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331556,
      "offset": 359097,
      "length": 196228
    },
    "model.layers.11.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331534,
      "offset": 555325,
      "length": 225535
    },
    "model.layers.11.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331542,
      "offset": 780860,
      "length": 225649
    },
    "model.layers.11.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 1006509,
      "length": 41
    },
    "model.layers.12.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 1006550,
      "length": 41
    },
    "model.layers.12.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.12.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.12.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.12.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 1006591,
      "length": 630
    },
    "model.layers.12.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 1007221,
      "length": 630
    },
    "model.layers.12.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554385,
      "offset": 1007851,
      "length": 150179
    },
    "model.layers.12.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781287,
      "offset": 1158030,
      "length": 76176
    },
    "model.layers.12.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.12.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777188,
      "offset": 1234206,
      "length": 65406
    },
    "model.layers.12.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331554,
      "offset": 1299612,
      "length": 196234
    },
    "model.layers.12.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331537,
      "offset": 1495846,
      "length": 225623
    },
    "model.layers.12.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331526,
      "offset": 1721469,
      "length": 225726
    },
    "model.layers.12.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 1947195,
      "length": 41
    },
    "model.layers.13.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 1947236,
      "length": 41
    },
    "model.layers.13.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.13.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.13.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.13.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 1947277,
      "length": 636
    },
    "model.layers.13.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 1947913,
      "length": 630
    },
    "model.layers.13.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554380,
      "offset": 1948543,
      "length": 150240
    },
    "model.layers.13.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781287,
      "offset": 2098783,
      "length": 76104
    },
    "model.layers.13.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.13.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 2174887,
      "length": 65365
    },
    "model.layers.13.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331530,
      "offset": 2240252,
      "length": 196345
    },
    "model.layers.13.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331544,
      "offset": 2436597,
      "length": 225599
    },
    "model.layers.13.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331520,
      "offset": 2662196,
      "length": 225725
    },
    "model.layers.13.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 2887921,
      "length": 41
    },
    "model.layers.14.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 2887962,
      "length": 41
    },
    "model.layers.14.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331543,
      "offset": 2888003,
      "length": 196294
    },
    "model.layers.14.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 3084297,
      "length": 225490
    },
    "model.layers.14.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331549,
      "offset": 3309787,
      "length": 225407
    },
    "model.layers.14.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 3535194,
      "length": 41
    },
    "model.layers.14.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.14.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194298,
      "offset": 3535235,
      "length": 18820
    },
    "model.layers.14.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777198,
      "offset": 3554055,
      "length": 65358
    },
    "model.layers.14.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.14.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558481,
      "offset": 3619413,
      "length": 151630
    },
    "model.layers.14.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194296,
      "offset": 3771043,
      "length": 18797
    },
    "model.layers.15.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 3789840,
      "length": 41
    },
    "model.layers.15.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.15.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.15.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.15.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 3789881,
      "length": 630
    },
    "model.layers.15.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 3790511,
      "length": 630
    },
    "model.layers.15.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554375,
      "offset": 3791141,
      "length": 150304
    },
    "model.layers.15.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781288,
      "offset": 3941445,
      "length": 76135
    },
    "model.layers.15.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.15.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777185,
      "offset": 4017580,
      "length": 65420
    },
    "model.layers.15.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 4083000,
      "length": 196290
    },
    "model.layers.15.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331555,
      "offset": 4279290,
      "length": 225437
    },
    "model.layers.15.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 4504727,
      "length": 225502
    },
    "model.layers.15.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 4730229,
      "length": 41
    },
    "model.layers.16.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 4730270,
      "length": 41
    },
    "model.layers.16.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.16.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.16.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.16.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 4730311,
      "length": 630
    },
    "model.layers.16.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 4730941,
      "length": 630
    },
    "model.layers.16.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554375,
      "offset": 4731571,
      "length": 150339
    },
    "model.layers.16.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781293,
      "offset": 4881910,
      "length": 76005
    },
    "model.layers.16.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.16.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777192,
      "offset": 4957915,
      "length": 65396
    },
    "model.layers.16.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331520,
      "offset": 5023311,
      "length": 196375
    },
    "model.layers.16.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331536,
      "offset": 5219686,
      "length": 225634
    },
    "model.layers.16.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331542,
      "offset": 5445320,
      "length": 225614
    },
    "model.layers.16.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 5670934,
      "length": 41
    },
    "model.layers.17.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 5670975,
      "length": 41
    },
    "model.layers.17.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.17.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.17.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.17.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 5671016,
      "length": 630
    },
    "model.layers.17.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 5671646,
      "length": 630
    },
    "model.layers.17.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554388,
      "offset": 5672276,
      "length": 150021
    },
    "model.layers.17.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781290,
      "offset": 5822297,
      "length": 76085
    },
    "model.layers.17.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.17.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777197,
      "offset": 5898382,
      "length": 65361
    },
    "model.layers.17.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331559,
      "offset": 5963743,
      "length": 196229
    },
    "model.layers.17.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331547,
      "offset": 6159972,
      "length": 225524
    },
    "model.layers.17.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331557,
      "offset": 6385496,
      "length": 225393
    },
    "model.layers.17.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 6610889,
      "length": 41
    },
    "model.layers.18.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 6610930,
      "length": 41
    },
    "model.layers.18.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331547,
      "offset": 6610971,
      "length": 196257
    },
    "model.layers.18.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331548,
      "offset": 6807228,
      "length": 225430
    },
    "model.layers.18.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331551,
      "offset": 7032658,
      "length": 225524
    },
    "model.layers.18.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 7258182,
      "length": 41
    },
    "model.layers.18.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.18.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194297,
      "offset": 7258223,
      "length": 18825
    },
    "model.layers.18.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777201,
      "offset": 7277048,
      "length": 65349
    },
    "model.layers.18.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.18.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558470,
      "offset": 7342397,
      "length": 151681
    },
    "model.layers.18.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194297,
      "offset": 7494078,
      "length": 18793
    },
    "model.layers.19.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 7512871,
      "length": 41
    },
    "model.layers.19.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.19.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.19.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.19.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 7512912,
      "length": 630
    },
    "model.layers.19.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 7513542,
      "length": 630
    },
    "model.layers.19.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554392,
      "offset": 7514172,
      "length": 150003
    },
    "model.layers.19.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781281,
      "offset": 7664175,
      "length": 76163
    },
    "model.layers.19.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.19.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 7740338,
      "length": 65372
    },
    "model.layers.19.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331558,
      "offset": 7805710,
      "length": 196223
    },
    "model.layers.19.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 8001933,
      "length": 225631
    },
    "model.layers.19.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331544,
      "offset": 8227564,
      "length": 225532
    },
    "model.layers.19.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 8453096,
      "length": 41
    },
    "model.layers.20.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 8453137,
      "length": 41
    },
    "model.layers.20.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.20.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.20.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.20.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 8453178,
      "length": 636
    },
    "model.layers.20.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 8453814,
      "length": 630
    },
    "model.layers.20.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554383,
      "offset": 8454444,
      "length": 150188
    },
    "model.layers.20.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781289,
      "offset": 8604632,
      "length": 76060
    },
    "model.layers.20.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.20.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777186,
      "offset": 8680692,
      "length": 65414
    },
    "model.layers.20.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331573,
      "offset": 8746106,
      "length": 196153
    },
    "model.layers.20.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331543,
      "offset": 8942259,
      "length": 225530
    },
    "model.layers.20.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331576,
      "offset": 9167789,
      "length": 225228
    },
    "model.layers.20.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 9393017,
      "length": 41
    },
    "model.layers.21.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 9393058,
      "length": 41
    },
    "model.layers.21.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.21.linear_attn.conv1d.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32767,
      "offset": 9393099,
      "length": 158
    },
    "model.layers.21.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.21.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 9393257,
      "length": 630
    },
    "model.layers.21.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 9393887,
      "length": 630
    },
    "model.layers.21.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554386,
      "offset": 9394517,
      "length": 150106
    },
    "model.layers.21.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781283,
      "offset": 9544623,
      "length": 76263
    },
    "model.layers.21.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.21.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 9620886,
      "length": 65363
    },
    "model.layers.21.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331562,
      "offset": 9686249,
      "length": 196195
    },
    "model.layers.21.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331533,
      "offset": 9882444,
      "length": 225606
    },
    "model.layers.21.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331549,
      "offset": 10108050,
      "length": 225576
    },
    "model.layers.21.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 10333626,
      "length": 41
    },
    "model.layers.22.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 10333667,
      "length": 41
    },
    "model.layers.22.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 10333708,
      "length": 196288
    },
    "model.layers.22.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331552,
      "offset": 10529996,
      "length": 225525
    },
    "model.layers.22.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331574,
      "offset": 10755521,
      "length": 225342
    },
    "model.layers.22.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 10980863,
      "length": 41
    },
    "model.layers.22.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.22.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194296,
      "offset": 10980904,
      "length": 18831
    },
    "model.layers.22.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777199,
      "offset": 10999735,
      "length": 65363
    },
    "model.layers.22.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.22.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558479,
      "offset": 11065098,
      "length": 151644
    },
    "model.layers.22.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194297,
      "offset": 11216742,
      "length": 18829
    },
    "model.layers.23.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 11235571,
      "length": 41
    },
    "model.layers.23.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.23.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.23.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.23.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 11235612,
      "length": 630
    },
    "model.layers.23.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 11236242,
      "length": 636
    },
    "model.layers.23.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554397,
      "offset": 11236878,
      "length": 150015
    },
    "model.layers.23.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781280,
      "offset": 11386893,
      "length": 76208
    },
    "model.layers.23.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.23.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777190,
      "offset": 11463101,
      "length": 65397
    },
    "model.layers.23.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331541,
      "offset": 11528498,
      "length": 196297
    },
    "model.layers.23.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331531,
      "offset": 11724795,
      "length": 225724
    },
    "model.layers.23.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331538,
      "offset": 11950519,
      "length": 225684
    },
    "model.layers.23.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 12176203,
      "length": 41
    },
    "model.layers.24.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 12176244,
      "length": 41
    },
    "model.layers.24.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.24.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.24.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.24.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 12176285,
      "length": 630
    },
    "model.layers.24.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 12176915,
      "length": 630
    },
    "model.layers.24.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554376,
      "offset": 12177545,
      "length": 150157
    },
    "model.layers.24.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781290,
      "offset": 12327702,
      "length": 76016
    },
    "model.layers.24.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.24.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777200,
      "offset": 12403718,
      "length": 65343
    },
    "model.layers.24.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331550,
      "offset": 12469061,
      "length": 196252
    },
    "model.layers.24.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331541,
      "offset": 12665313,
      "length": 225576
    },
    "model.layers.24.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331551,
      "offset": 12890889,
      "length": 225385
    },
    "model.layers.24.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 13116274,
      "length": 41
    },
    "model.layers.25.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 13116315,
      "length": 41
    },
    "model.layers.25.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.25.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.25.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.25.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 13116356,
      "length": 630
    },
    "model.layers.25.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 13116986,
      "length": 630
    },
    "model.layers.25.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554384,
      "offset": 13117616,
      "length": 150186
    },
    "model.layers.25.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781283,
      "offset": 13267802,
      "length": 76159
    },
    "model.layers.25.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.25.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777191,
      "offset": 13343961,
      "length": 65401
    },
    "model.layers.25.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331574,
      "offset": 13409362,
      "length": 196165
    },
    "model.layers.25.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331523,
      "offset": 13605527,
      "length": 225706
    },
    "model.layers.25.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331568,
      "offset": 13831233,
      "length": 225195
    },
    "model.layers.25.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 14056428,
      "length": 41
    },
    "model.layers.26.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 14056469,
      "length": 41
    },
    "model.layers.26.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331564,
      "offset": 14056510,
      "length": 196192
    },
    "model.layers.26.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331543,
      "offset": 14252702,
      "length": 225543
    },
    "model.layers.26.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331567,
      "offset": 14478245,
      "length": 225280
    },
    "model.layers.26.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 14703525,
      "length": 41
    },
    "model.layers.26.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.26.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194298,
      "offset": 14703566,
      "length": 18786
    },
    "model.layers.26.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777194,
      "offset": 14722352,
      "length": 65384
    },
    "model.layers.26.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.26.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558468,
      "offset": 14787736,
      "length": 151690
    },
    "model.layers.26.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194302,
      "offset": 14939426,
      "length": 18767
    },
    "model.layers.27.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 14958193,
      "length": 41
    },
    "model.layers.27.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.27.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.27.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.27.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 14958234,
      "length": 630
    },
    "model.layers.27.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 14958864,
      "length": 630
    },
    "model.layers.27.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554399,
      "offset": 14959494,
      "length": 149891
    },
    "model.layers.27.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781292,
      "offset": 15109385,
      "length": 76044
    },
    "model.layers.27.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.27.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 15185429,
      "length": 65374
    },
    "model.layers.27.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331558,
      "offset": 15250803,
      "length": 196232
    },
    "model.layers.27.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331538,
      "offset": 15447035,
      "length": 225537
    },
    "model.layers.27.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331554,
      "offset": 15672572,
      "length": 225506
    },
    "model.layers.27.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 15898078,
      "length": 41
    },
    "model.layers.28.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 15898119,
      "length": 41
    },
    "model.layers.28.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.28.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.28.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.28.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 15898160,
      "length": 635
    },
    "model.layers.28.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 15898795,
      "length": 630
    },
    "model.layers.28.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554381,
      "offset": 15899425,
      "length": 150194
    },
    "model.layers.28.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781277,
      "offset": 16049619,
      "length": 76252
    },
    "model.layers.28.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.28.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777205,
      "offset": 16125871,
      "length": 65325
    },
    "model.layers.28.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331556,
      "offset": 16191196,
      "length": 196228
    },
    "model.layers.28.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331532,
      "offset": 16387424,
      "length": 225522
    },
    "model.layers.28.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331563,
      "offset": 16612946,
      "length": 225514
    },
    "model.layers.28.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 16838460,
      "length": 41
    },
    "model.layers.29.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 16838501,
      "length": 41
    },
    "model.layers.29.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.29.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.29.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.29.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 16838542,
      "length": 630
    },
    "model.layers.29.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 16839172,
      "length": 630
    },
    "model.layers.29.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554397,
      "offset": 16839802,
      "length": 149907
    },
    "model.layers.29.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781296,
      "offset": 16989709,
      "length": 75954
    },
    "model.layers.29.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.29.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777199,
      "offset": 17065663,
      "length": 65352
    },
    "model.layers.29.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331566,
      "offset": 17131015,
      "length": 196194
    },
    "model.layers.29.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331539,
      "offset": 17327209,
      "length": 225550
    },
    "model.layers.29.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331545,
      "offset": 17552759,
      "length": 225524
    },
    "model.layers.29.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 17778283,
      "length": 41
    },
    "model.layers.30.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 17778324,
      "length": 41
    },
    "model.layers.30.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331578,
      "offset": 17778365,
      "length": 196127
    },
    "model.layers.30.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331541,
      "offset": 17974492,
      "length": 225640
    },
    "model.layers.30.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331558,
      "offset": 18200132,
      "length": 225389
    },
    "model.layers.30.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 18425521,
      "length": 41
    },
    "model.layers.30.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.30.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194298,
      "offset": 18425562,
      "length": 18824
    },
    "model.layers.30.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777206,
      "offset": 18444386,
      "length": 65317
    },
    "model.layers.30.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.30.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558479,
      "offset": 18509703,
      "length": 151643
    },
    "model.layers.30.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194296,
      "offset": 18661346,
      "length": 18834
    },
    "model.layers.31.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 18680180,
      "length": 41
    },
    "model.layers.31.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.31.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.31.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.31.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 18680221,
      "length": 630
    },
    "model.layers.31.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 18680851,
      "length": 630
    },
    "model.layers.31.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554392,
      "offset": 18681481,
      "length": 150044
    },
    "model.layers.31.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781282,
      "offset": 18831525,
      "length": 76164
    },
    "model.layers.31.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.31.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777197,
      "offset": 18907689,
      "length": 65369
    },
    "model.layers.31.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331558,
      "offset": 18973058,
      "length": 196233
    },
    "model.layers.31.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331547,
      "offset": 19169291,
      "length": 225522
    },
    "model.layers.31.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331560,
      "offset": 19394813,
      "length": 225417
    },
    "model.layers.31.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 19620230,
      "length": 41
    },
    "model.layers.32.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 19620271,
      "length": 41
    },
    "model.layers.32.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.32.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.32.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.32.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 19620312,
      "length": 634
    },
    "model.layers.32.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 19620946,
      "length": 635
    },
    "model.layers.32.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554385,
      "offset": 19621581,
      "length": 150144
    },
    "model.layers.32.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781290,
      "offset": 19771725,
      "length": 76160
    },
    "model.layers.32.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.32.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777198,
      "offset": 19847885,
      "length": 65358
    },
    "model.layers.32.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331573,
      "offset": 19913243,
      "length": 196145
    },
    "model.layers.32.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331566,
      "offset": 20109388,
      "length": 225308
    },
    "model.layers.32.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331571,
      "offset": 20334696,
      "length": 225213
    },
    "model.layers.32.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 20559909,
      "length": 41
    },
    "model.layers.33.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 20559950,
      "length": 41
    },
    "model.layers.33.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.33.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.33.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.33.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 20559991,
      "length": 630
    },
    "model.layers.33.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 20560621,
      "length": 635
    },
    "model.layers.33.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554397,
      "offset": 20561256,
      "length": 149877
    },
    "model.layers.33.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781287,
      "offset": 20711133,
      "length": 76142
    },
    "model.layers.33.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.33.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777187,
      "offset": 20787275,
      "length": 65406
    },
    "model.layers.33.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331574,
      "offset": 20852681,
      "length": 196155
    },
    "model.layers.33.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331560,
      "offset": 21048836,
      "length": 225314
    },
    "model.layers.33.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331586,
      "offset": 21274150,
      "length": 225142
    },
    "model.layers.33.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 21499292,
      "length": 41
    },
    "model.layers.34.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 21499333,
      "length": 41
    },
    "model.layers.34.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331572,
      "offset": 21499374,
      "length": 196151
    },
    "model.layers.34.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331552,
      "offset": 21695525,
      "length": 225381
    },
    "model.layers.34.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331546,
      "offset": 21920906,
      "length": 225550
    },
    "model.layers.34.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 22146456,
      "length": 41
    },
    "model.layers.34.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.34.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194297,
      "offset": 22146497,
      "length": 18826
    },
    "model.layers.34.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777202,
      "offset": 22165323,
      "length": 65344
    },
    "model.layers.34.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.34.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558466,
      "offset": 22230667,
      "length": 151701
    },
    "model.layers.34.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194298,
      "offset": 22382368,
      "length": 18823
    },
    "model.layers.35.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 22401191,
      "length": 41
    },
    "model.layers.35.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.35.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.35.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.35.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 22401232,
      "length": 630
    },
    "model.layers.35.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 22401862,
      "length": 630
    },
    "model.layers.35.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554383,
      "offset": 22402492,
      "length": 150126
    },
    "model.layers.35.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781282,
      "offset": 22552618,
      "length": 76129
    },
    "model.layers.35.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.35.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 22628747,
      "length": 65367
    },
    "model.layers.35.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331555,
      "offset": 22694114,
      "length": 196227
    },
    "model.layers.35.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331531,
      "offset": 22890341,
      "length": 225583
    },
    "model.layers.35.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331568,
      "offset": 23115924,
      "length": 225385
    },
    "model.layers.35.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 23341309,
      "length": 41
    },
    "model.layers.36.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 23341350,
      "length": 41
    },
    "model.layers.36.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.36.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.36.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.36.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 23341391,
      "length": 630
    },
    "model.layers.36.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 23342021,
      "length": 630
    },
    "model.layers.36.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554391,
      "offset": 23342651,
      "length": 150111
    },
    "model.layers.36.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781287,
      "offset": 23492762,
      "length": 76140
    },
    "model.layers.36.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.36.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777200,
      "offset": 23568902,
      "length": 65344
    },
    "model.layers.36.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331565,
      "offset": 23634246,
      "length": 196190
    },
    "model.layers.36.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331561,
      "offset": 23830436,
      "length": 225336
    },
    "model.layers.36.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331566,
      "offset": 24055772,
      "length": 225353
    },
    "model.layers.36.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 24281125,
      "length": 41
    },
    "model.layers.37.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 24281166,
      "length": 41
    },
    "model.layers.37.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.37.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.37.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.37.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 24281207,
      "length": 630
    },
    "model.layers.37.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131102,
      "offset": 24281837,
      "length": 641
    },
    "model.layers.37.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554388,
      "offset": 24282478,
      "length": 150170
    },
    "model.layers.37.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781290,
      "offset": 24432648,
      "length": 76126
    },
    "model.layers.37.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.37.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777186,
      "offset": 24508774,
      "length": 65411
    },
    "model.layers.37.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331559,
      "offset": 24574185,
      "length": 196217
    },
    "model.layers.37.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331564,
      "offset": 24770402,
      "length": 225319
    },
    "model.layers.37.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331560,
      "offset": 24995721,
      "length": 225384
    },
    "model.layers.37.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 25221105,
      "length": 41
    },
    "model.layers.38.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 25221146,
      "length": 41
    },
    "model.layers.38.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331563,
      "offset": 25221187,
      "length": 196184
    },
    "model.layers.38.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331576,
      "offset": 25417371,
      "length": 225167
    },
    "model.layers.38.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331569,
      "offset": 25642538,
      "length": 225375
    },
    "model.layers.38.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 25867913,
      "length": 41
    },
    "model.layers.38.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.38.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194301,
      "offset": 25867954,
      "length": 18809
    },
    "model.layers.38.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777201,
      "offset": 25886763,
      "length": 65341
    },
    "model.layers.38.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.38.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558470,
      "offset": 25952104,
      "length": 151686
    },
    "model.layers.38.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194299,
      "offset": 26103790,
      "length": 18785
    },
    "model.layers.39.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 26122575,
      "length": 41
    },
    "model.layers.39.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.39.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.39.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.39.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 26122616,
      "length": 630
    },
    "model.layers.39.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 26123246,
      "length": 630
    },
    "model.layers.39.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554381,
      "offset": 26123876,
      "length": 150276
    },
    "model.layers.39.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781290,
      "offset": 26274152,
      "length": 76051
    },
    "model.layers.39.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.39.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777195,
      "offset": 26350203,
      "length": 65378
    },
    "model.layers.39.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331572,
      "offset": 26415581,
      "length": 196162
    },
    "model.layers.39.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331556,
      "offset": 26611743,
      "length": 225294
    },
    "model.layers.39.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331568,
      "offset": 26837037,
      "length": 225239
    },
    "model.layers.39.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 27062276,
      "length": 41
    },
    "model.layers.40.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 27062317,
      "length": 41
    },
    "model.layers.40.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.40.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.40.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.40.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 27062358,
      "length": 630
    },
    "model.layers.40.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 27062988,
      "length": 630
    },
    "model.layers.40.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554390,
      "offset": 27063618,
      "length": 150049
    },
    "model.layers.40.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781289,
      "offset": 27213667,
      "length": 76096
    },
    "model.layers.40.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.40.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777193,
      "offset": 27289763,
      "length": 65384
    },
    "model.layers.40.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331558,
      "offset": 27355147,
      "length": 196219
    },
    "model.layers.40.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331538,
      "offset": 27551366,
      "length": 225560
    },
    "model.layers.40.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331575,
      "offset": 27776926,
      "length": 225234
    },
    "model.layers.40.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 28002160,
      "length": 41
    },
    "model.layers.41.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 28002201,
      "length": 41
    },
    "model.layers.41.linear_attn.A_log": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.41.linear_attn.conv1d.weight": {
      "mode": "dense",
      "shape": [
        8192,
        1,
        4
      ],
      "dtype": "bfloat16",
      "nnz": 32768
    },
    "model.layers.41.linear_attn.dt_bias": {
      "mode": "dense",
      "shape": [
        32
      ],
      "dtype": "bfloat16",
      "nnz": 32
    },
    "model.layers.41.linear_attn.in_proj_a.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131104,
      "offset": 28002242,
      "length": 630
    },
    "model.layers.41.linear_attn.in_proj_b.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        32,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 131103,
      "offset": 28002872,
      "length": 636
    },
    "model.layers.41.linear_attn.in_proj_qkv.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33554380,
      "offset": 28003508,
      "length": 150137
    },
    "model.layers.41.linear_attn.in_proj_z.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        4096,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 16781294,
      "offset": 28153645,
      "length": 75959
    },
    "model.layers.41.linear_attn.norm.weight": {
      "mode": "dense",
      "shape": [
        128
      ],
      "dtype": "bfloat16",
      "nnz": 128
    },
    "model.layers.41.linear_attn.out_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777196,
      "offset": 28229604,
      "length": 65366
    },
    "model.layers.41.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331573,
      "offset": 28294970,
      "length": 196170
    },
    "model.layers.41.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331565,
      "offset": 28491140,
      "length": 225210
    },
    "model.layers.41.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331595,
      "offset": 28716350,
      "length": 224926
    },
    "model.layers.41.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 28941276,
      "length": 41
    },
    "model.layers.42.input_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 28941317,
      "length": 41
    },
    "model.layers.42.mlp.down_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        12288
      ],
      "dtype": "bfloat16",
      "nnz": 50331600,
      "offset": 28941358,
      "length": 196022
    },
    "model.layers.42.mlp.gate_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331590,
      "offset": 29137380,
      "length": 224973
    },
    "model.layers.42.mlp.up_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        12288,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 50331593,
      "offset": 29362353,
      "length": 225040
    },
    "model.layers.42.post_attention_layernorm.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4096,
      "offset": 29587393,
      "length": 41
    },
    "model.layers.42.self_attn.k_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.42.self_attn.k_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194296,
      "offset": 29587434,
      "length": 18761
    },
    "model.layers.42.self_attn.o_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        5632,
        4096
      ],
      "dtype": "bfloat16",
      "nnz": 16777208,
      "offset": 29606195,
      "length": 65306
    },
    "model.layers.42.self_attn.q_norm.weight": {
      "mode": "dense",
      "shape": [
        256
      ],
      "dtype": "bfloat16",
      "nnz": 256
    },
    "model.layers.42.self_attn.q_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        8192,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 33558472,
      "offset": 29671501,
      "length": 151662
    },
    "model.layers.42.self_attn.v_proj.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        1024,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 4194303,
      "offset": 29823163,
      "length": 18727
    },
    "model.norm.weight": {
      "mode": "dense",
      "shape": [
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 5632
    },
    "model.embed_tokens.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        478656,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1042497871,
      "offset": 29841890,
      "length": 11390146
    },
    "lm_head.weight": {
      "mode": "sparse_delta_u32",
      "shape": [
        478656,
        5632
      ],
      "dtype": "bfloat16",
      "nnz": 1022438491,
      "offset": 41232036,
      "length": 4678381
    }
  }
}