{
"format": "carbon-compact-v1",
"total_nonzero_params": 8985670530,
"total_dense_params": 18208957504,
"tensors": {
"model.layers.0.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 0,
"length": 8
},
"model.layers.0.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8,
"length": 8
},
"model.layers.0.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 51,
"offset": 16,
"length": 38
},
"model.layers.0.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.0.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 44,
"offset": 54,
"length": 43
},
"model.layers.0.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 12,
"offset": 97,
"length": 33
},
"model.layers.0.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 274,
"offset": 130,
"length": 119
},
"model.layers.0.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 21,
"offset": 249,
"length": 20
},
"model.layers.0.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.0.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 21,
"offset": 269,
"length": 24
},
"model.layers.0.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 357,
"offset": 293,
"length": 66
},
"model.layers.0.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 628,
"offset": 359,
"length": 90
},
"model.layers.0.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 2415,
"offset": 449,
"length": 320
},
"model.layers.0.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 769,
"length": 8
},
"model.layers.1.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 777,
"length": 8
},
"model.layers.1.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 785,
"length": 8
},
"model.layers.1.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 7,
"offset": 793,
"length": 20
},
"model.layers.1.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.1.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 2,
"offset": 813,
"length": 16
},
"model.layers.1.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 1,
"offset": 829,
"length": 12
},
"model.layers.1.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 13,
"offset": 841,
"length": 28
},
"model.layers.1.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 1,
"offset": 869,
"length": 12
},
"model.layers.1.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.1.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 1,
"offset": 881,
"length": 12
},
"model.layers.1.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 1615,
"offset": 893,
"length": 129
},
"model.layers.1.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 95083,
"offset": 1022,
"length": 2118
},
"model.layers.1.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 1775,
"offset": 3140,
"length": 373
},
"model.layers.1.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3513,
"length": 8
},
"model.layers.10.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3521,
"length": 8
},
"model.layers.10.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3529,
"length": 8
},
"model.layers.10.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3537,
"length": 8
},
"model.layers.10.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.10.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3545,
"length": 8
},
"model.layers.10.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3553,
"length": 8
},
"model.layers.10.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3561,
"length": 8
},
"model.layers.10.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3569,
"length": 8
},
"model.layers.10.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.10.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 3577,
"length": 8
},
"model.layers.10.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 1713,
"offset": 3585,
"length": 109
},
"model.layers.10.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 14630,
"offset": 3694,
"length": 2646
},
"model.layers.10.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 1713,
"offset": 6340,
"length": 61
},
"model.layers.10.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 6401,
"length": 8
},
"model.layers.2.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 6409,
"length": 8
},
"model.layers.2.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 6417,
"length": 8
},
"model.layers.2.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 6425,
"length": 34
},
"model.layers.2.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.2.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 6459,
"length": 8
},
"model.layers.2.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 32,
"offset": 6467,
"length": 16
},
"model.layers.2.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 24576,
"offset": 6483,
"length": 327
},
"model.layers.2.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 11968,
"offset": 6810,
"length": 344
},
"model.layers.2.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.2.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 7154,
"length": 74
},
"model.layers.2.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7228,
"length": 8
},
"model.layers.2.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7236,
"length": 8
},
"model.layers.2.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7244,
"length": 8
},
"model.layers.2.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7252,
"length": 8
},
"model.layers.3.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7260,
"length": 8
},
"model.layers.3.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7268,
"length": 8
},
"model.layers.3.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 7276,
"length": 34
},
"model.layers.3.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.3.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 7310,
"length": 8
},
"model.layers.3.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 32,
"offset": 7318,
"length": 16
},
"model.layers.3.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 24576,
"offset": 7334,
"length": 327
},
"model.layers.3.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 11968,
"offset": 7661,
"length": 344
},
"model.layers.3.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.3.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 8005,
"length": 74
},
"model.layers.3.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 4159,
"offset": 8079,
"length": 141
},
"model.layers.3.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 8716,
"offset": 8220,
"length": 418
},
"model.layers.3.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 4159,
"offset": 8638,
"length": 164
},
"model.layers.3.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8802,
"length": 8
},
"model.layers.4.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8810,
"length": 8
},
"model.layers.4.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8818,
"length": 8
},
"model.layers.4.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 780,
"offset": 8826,
"length": 106
},
"model.layers.4.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.4.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8932,
"length": 8
},
"model.layers.4.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 47,
"offset": 8940,
"length": 69
},
"model.layers.4.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 1999,
"offset": 9009,
"length": 1015
},
"model.layers.4.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 429,
"offset": 10024,
"length": 74
},
"model.layers.4.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.4.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 429,
"offset": 10098,
"length": 77
},
"model.layers.4.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 8025,
"offset": 10175,
"length": 308
},
"model.layers.4.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 122002,
"offset": 10483,
"length": 5550
},
"model.layers.4.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 13849,
"offset": 16033,
"length": 1168
},
"model.layers.4.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 17201,
"length": 8
},
"model.layers.5.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 17209,
"length": 8
},
"model.layers.5.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 17217,
"length": 8
},
"model.layers.5.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 1888,
"offset": 17225,
"length": 65
},
"model.layers.5.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.5.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 17290,
"length": 8
},
"model.layers.5.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 48,
"offset": 17298,
"length": 64
},
"model.layers.5.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 3932,
"offset": 17362,
"length": 857
},
"model.layers.5.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 64,
"offset": 18219,
"length": 20
},
"model.layers.5.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.5.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 64,
"offset": 18239,
"length": 23
},
"model.layers.5.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 6992,
"offset": 18262,
"length": 359
},
"model.layers.5.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 29667,
"offset": 18621,
"length": 6801
},
"model.layers.5.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 9617,
"offset": 25422,
"length": 2172
},
"model.layers.5.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 27594,
"length": 8
},
"model.layers.6.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 27602,
"length": 8
},
"model.layers.6.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 27610,
"length": 8
},
"model.layers.6.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 7232,
"offset": 27618,
"length": 134
},
"model.layers.6.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.6.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 27752,
"length": 8
},
"model.layers.6.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 96,
"offset": 27760,
"length": 32
},
"model.layers.6.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 113728,
"offset": 27792,
"length": 1840
},
"model.layers.6.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 5120,
"offset": 29632,
"length": 132
},
"model.layers.6.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.6.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 29764,
"length": 72
},
"model.layers.6.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 3896,
"offset": 29836,
"length": 255
},
"model.layers.6.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 27741,
"offset": 30091,
"length": 4009
},
"model.layers.6.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 3941,
"offset": 34100,
"length": 860
},
"model.layers.6.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 34960,
"length": 8
},
"model.layers.7.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 34968,
"length": 8
},
"model.layers.7.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 34976,
"length": 8
},
"model.layers.7.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 7232,
"offset": 34984,
"length": 134
},
"model.layers.7.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.7.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 35118,
"length": 8
},
"model.layers.7.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 96,
"offset": 35126,
"length": 32
},
"model.layers.7.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 113728,
"offset": 35158,
"length": 1840
},
"model.layers.7.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 5120,
"offset": 36998,
"length": 132
},
"model.layers.7.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.7.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 37130,
"length": 72
},
"model.layers.7.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 8307,
"offset": 37202,
"length": 495
},
"model.layers.7.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 157966,
"offset": 37697,
"length": 7870
},
"model.layers.7.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 8307,
"offset": 45567,
"length": 2280
},
"model.layers.7.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 47847,
"length": 8
},
"model.layers.8.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 47855,
"length": 8
},
"model.layers.8.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 47863,
"length": 8
},
"model.layers.8.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 47871,
"length": 34
},
"model.layers.8.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.8.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 47905,
"length": 8
},
"model.layers.8.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 32,
"offset": 47913,
"length": 16
},
"model.layers.8.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 38912,
"offset": 47929,
"length": 577
},
"model.layers.8.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 22208,
"offset": 48506,
"length": 554
},
"model.layers.8.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.8.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 2048,
"offset": 49060,
"length": 73
},
"model.layers.8.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 8558,
"offset": 49133,
"length": 326
},
"model.layers.8.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 60993,
"offset": 49459,
"length": 7871
},
"model.layers.8.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 8558,
"offset": 57330,
"length": 2234
},
"model.layers.8.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 59564,
"length": 8
},
"model.layers.9.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 59572,
"length": 8
},
"model.layers.9.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 59580,
"length": 8
},
"model.layers.9.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 292,
"offset": 59588,
"length": 35
},
"model.layers.9.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.9.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 2,
"offset": 59623,
"length": 16
},
"model.layers.9.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 4,
"offset": 59639,
"length": 19
},
"model.layers.9.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 2291,
"offset": 59658,
"length": 462
},
"model.layers.9.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 576,
"offset": 60120,
"length": 53
},
"model.layers.9.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.9.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 128,
"offset": 60173,
"length": 28
},
"model.layers.9.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 1407,
"offset": 60201,
"length": 102
},
"model.layers.9.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 9567,
"offset": 60303,
"length": 5420
},
"model.layers.9.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 1407,
"offset": 65723,
"length": 287
},
"model.layers.9.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 66010,
"length": 8
},
"model.layers.11.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 66018,
"length": 41
},
"model.layers.11.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.11.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32766,
"offset": 66059,
"length": 164
},
"model.layers.11.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.11.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 66223,
"length": 630
},
"model.layers.11.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 66853,
"length": 635
},
"model.layers.11.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554393,
"offset": 67488,
"length": 150114
},
"model.layers.11.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781289,
"offset": 217602,
"length": 76125
},
"model.layers.11.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.11.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777197,
"offset": 293727,
"length": 65370
},
"model.layers.11.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331556,
"offset": 359097,
"length": 196228
},
"model.layers.11.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331534,
"offset": 555325,
"length": 225535
},
"model.layers.11.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331542,
"offset": 780860,
"length": 225649
},
"model.layers.11.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 1006509,
"length": 41
},
"model.layers.12.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 1006550,
"length": 41
},
"model.layers.12.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.12.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.12.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.12.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 1006591,
"length": 630
},
"model.layers.12.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 1007221,
"length": 630
},
"model.layers.12.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554385,
"offset": 1007851,
"length": 150179
},
"model.layers.12.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781287,
"offset": 1158030,
"length": 76176
},
"model.layers.12.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.12.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777188,
"offset": 1234206,
"length": 65406
},
"model.layers.12.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331554,
"offset": 1299612,
"length": 196234
},
"model.layers.12.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331537,
"offset": 1495846,
"length": 225623
},
"model.layers.12.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331526,
"offset": 1721469,
"length": 225726
},
"model.layers.12.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 1947195,
"length": 41
},
"model.layers.13.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 1947236,
"length": 41
},
"model.layers.13.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.13.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.13.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.13.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 1947277,
"length": 636
},
"model.layers.13.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 1947913,
"length": 630
},
"model.layers.13.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554380,
"offset": 1948543,
"length": 150240
},
"model.layers.13.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781287,
"offset": 2098783,
"length": 76104
},
"model.layers.13.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.13.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 2174887,
"length": 65365
},
"model.layers.13.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331530,
"offset": 2240252,
"length": 196345
},
"model.layers.13.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331544,
"offset": 2436597,
"length": 225599
},
"model.layers.13.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331520,
"offset": 2662196,
"length": 225725
},
"model.layers.13.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 2887921,
"length": 41
},
"model.layers.14.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 2887962,
"length": 41
},
"model.layers.14.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331543,
"offset": 2888003,
"length": 196294
},
"model.layers.14.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 3084297,
"length": 225490
},
"model.layers.14.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331549,
"offset": 3309787,
"length": 225407
},
"model.layers.14.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 3535194,
"length": 41
},
"model.layers.14.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.14.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194298,
"offset": 3535235,
"length": 18820
},
"model.layers.14.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777198,
"offset": 3554055,
"length": 65358
},
"model.layers.14.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.14.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558481,
"offset": 3619413,
"length": 151630
},
"model.layers.14.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194296,
"offset": 3771043,
"length": 18797
},
"model.layers.15.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 3789840,
"length": 41
},
"model.layers.15.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.15.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.15.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.15.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 3789881,
"length": 630
},
"model.layers.15.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 3790511,
"length": 630
},
"model.layers.15.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554375,
"offset": 3791141,
"length": 150304
},
"model.layers.15.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781288,
"offset": 3941445,
"length": 76135
},
"model.layers.15.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.15.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777185,
"offset": 4017580,
"length": 65420
},
"model.layers.15.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 4083000,
"length": 196290
},
"model.layers.15.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331555,
"offset": 4279290,
"length": 225437
},
"model.layers.15.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 4504727,
"length": 225502
},
"model.layers.15.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 4730229,
"length": 41
},
"model.layers.16.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 4730270,
"length": 41
},
"model.layers.16.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.16.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.16.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.16.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 4730311,
"length": 630
},
"model.layers.16.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 4730941,
"length": 630
},
"model.layers.16.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554375,
"offset": 4731571,
"length": 150339
},
"model.layers.16.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781293,
"offset": 4881910,
"length": 76005
},
"model.layers.16.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.16.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777192,
"offset": 4957915,
"length": 65396
},
"model.layers.16.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331520,
"offset": 5023311,
"length": 196375
},
"model.layers.16.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331536,
"offset": 5219686,
"length": 225634
},
"model.layers.16.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331542,
"offset": 5445320,
"length": 225614
},
"model.layers.16.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 5670934,
"length": 41
},
"model.layers.17.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 5670975,
"length": 41
},
"model.layers.17.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.17.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.17.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.17.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 5671016,
"length": 630
},
"model.layers.17.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 5671646,
"length": 630
},
"model.layers.17.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554388,
"offset": 5672276,
"length": 150021
},
"model.layers.17.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781290,
"offset": 5822297,
"length": 76085
},
"model.layers.17.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.17.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777197,
"offset": 5898382,
"length": 65361
},
"model.layers.17.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331559,
"offset": 5963743,
"length": 196229
},
"model.layers.17.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331547,
"offset": 6159972,
"length": 225524
},
"model.layers.17.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331557,
"offset": 6385496,
"length": 225393
},
"model.layers.17.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 6610889,
"length": 41
},
"model.layers.18.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 6610930,
"length": 41
},
"model.layers.18.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331547,
"offset": 6610971,
"length": 196257
},
"model.layers.18.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331548,
"offset": 6807228,
"length": 225430
},
"model.layers.18.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331551,
"offset": 7032658,
"length": 225524
},
"model.layers.18.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 7258182,
"length": 41
},
"model.layers.18.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.18.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194297,
"offset": 7258223,
"length": 18825
},
"model.layers.18.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777201,
"offset": 7277048,
"length": 65349
},
"model.layers.18.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.18.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558470,
"offset": 7342397,
"length": 151681
},
"model.layers.18.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194297,
"offset": 7494078,
"length": 18793
},
"model.layers.19.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 7512871,
"length": 41
},
"model.layers.19.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.19.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.19.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.19.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 7512912,
"length": 630
},
"model.layers.19.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 7513542,
"length": 630
},
"model.layers.19.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554392,
"offset": 7514172,
"length": 150003
},
"model.layers.19.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781281,
"offset": 7664175,
"length": 76163
},
"model.layers.19.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.19.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 7740338,
"length": 65372
},
"model.layers.19.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331558,
"offset": 7805710,
"length": 196223
},
"model.layers.19.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 8001933,
"length": 225631
},
"model.layers.19.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331544,
"offset": 8227564,
"length": 225532
},
"model.layers.19.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 8453096,
"length": 41
},
"model.layers.20.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 8453137,
"length": 41
},
"model.layers.20.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.20.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.20.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.20.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 8453178,
"length": 636
},
"model.layers.20.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 8453814,
"length": 630
},
"model.layers.20.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554383,
"offset": 8454444,
"length": 150188
},
"model.layers.20.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781289,
"offset": 8604632,
"length": 76060
},
"model.layers.20.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.20.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777186,
"offset": 8680692,
"length": 65414
},
"model.layers.20.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331573,
"offset": 8746106,
"length": 196153
},
"model.layers.20.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331543,
"offset": 8942259,
"length": 225530
},
"model.layers.20.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331576,
"offset": 9167789,
"length": 225228
},
"model.layers.20.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 9393017,
"length": 41
},
"model.layers.21.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 9393058,
"length": 41
},
"model.layers.21.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.21.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32767,
"offset": 9393099,
"length": 158
},
"model.layers.21.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.21.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 9393257,
"length": 630
},
"model.layers.21.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 9393887,
"length": 630
},
"model.layers.21.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554386,
"offset": 9394517,
"length": 150106
},
"model.layers.21.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781283,
"offset": 9544623,
"length": 76263
},
"model.layers.21.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.21.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 9620886,
"length": 65363
},
"model.layers.21.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331562,
"offset": 9686249,
"length": 196195
},
"model.layers.21.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331533,
"offset": 9882444,
"length": 225606
},
"model.layers.21.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331549,
"offset": 10108050,
"length": 225576
},
"model.layers.21.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 10333626,
"length": 41
},
"model.layers.22.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 10333667,
"length": 41
},
"model.layers.22.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 10333708,
"length": 196288
},
"model.layers.22.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331552,
"offset": 10529996,
"length": 225525
},
"model.layers.22.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331574,
"offset": 10755521,
"length": 225342
},
"model.layers.22.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 10980863,
"length": 41
},
"model.layers.22.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.22.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194296,
"offset": 10980904,
"length": 18831
},
"model.layers.22.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777199,
"offset": 10999735,
"length": 65363
},
"model.layers.22.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.22.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558479,
"offset": 11065098,
"length": 151644
},
"model.layers.22.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194297,
"offset": 11216742,
"length": 18829
},
"model.layers.23.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 11235571,
"length": 41
},
"model.layers.23.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.23.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.23.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.23.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 11235612,
"length": 630
},
"model.layers.23.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 11236242,
"length": 636
},
"model.layers.23.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554397,
"offset": 11236878,
"length": 150015
},
"model.layers.23.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781280,
"offset": 11386893,
"length": 76208
},
"model.layers.23.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.23.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777190,
"offset": 11463101,
"length": 65397
},
"model.layers.23.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331541,
"offset": 11528498,
"length": 196297
},
"model.layers.23.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331531,
"offset": 11724795,
"length": 225724
},
"model.layers.23.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331538,
"offset": 11950519,
"length": 225684
},
"model.layers.23.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 12176203,
"length": 41
},
"model.layers.24.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 12176244,
"length": 41
},
"model.layers.24.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.24.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.24.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.24.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 12176285,
"length": 630
},
"model.layers.24.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 12176915,
"length": 630
},
"model.layers.24.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554376,
"offset": 12177545,
"length": 150157
},
"model.layers.24.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781290,
"offset": 12327702,
"length": 76016
},
"model.layers.24.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.24.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777200,
"offset": 12403718,
"length": 65343
},
"model.layers.24.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331550,
"offset": 12469061,
"length": 196252
},
"model.layers.24.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331541,
"offset": 12665313,
"length": 225576
},
"model.layers.24.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331551,
"offset": 12890889,
"length": 225385
},
"model.layers.24.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 13116274,
"length": 41
},
"model.layers.25.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 13116315,
"length": 41
},
"model.layers.25.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.25.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.25.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.25.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 13116356,
"length": 630
},
"model.layers.25.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 13116986,
"length": 630
},
"model.layers.25.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554384,
"offset": 13117616,
"length": 150186
},
"model.layers.25.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781283,
"offset": 13267802,
"length": 76159
},
"model.layers.25.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.25.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777191,
"offset": 13343961,
"length": 65401
},
"model.layers.25.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331574,
"offset": 13409362,
"length": 196165
},
"model.layers.25.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331523,
"offset": 13605527,
"length": 225706
},
"model.layers.25.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331568,
"offset": 13831233,
"length": 225195
},
"model.layers.25.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 14056428,
"length": 41
},
"model.layers.26.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 14056469,
"length": 41
},
"model.layers.26.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331564,
"offset": 14056510,
"length": 196192
},
"model.layers.26.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331543,
"offset": 14252702,
"length": 225543
},
"model.layers.26.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331567,
"offset": 14478245,
"length": 225280
},
"model.layers.26.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 14703525,
"length": 41
},
"model.layers.26.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.26.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194298,
"offset": 14703566,
"length": 18786
},
"model.layers.26.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777194,
"offset": 14722352,
"length": 65384
},
"model.layers.26.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.26.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558468,
"offset": 14787736,
"length": 151690
},
"model.layers.26.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194302,
"offset": 14939426,
"length": 18767
},
"model.layers.27.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 14958193,
"length": 41
},
"model.layers.27.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.27.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.27.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.27.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 14958234,
"length": 630
},
"model.layers.27.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 14958864,
"length": 630
},
"model.layers.27.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554399,
"offset": 14959494,
"length": 149891
},
"model.layers.27.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781292,
"offset": 15109385,
"length": 76044
},
"model.layers.27.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.27.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 15185429,
"length": 65374
},
"model.layers.27.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331558,
"offset": 15250803,
"length": 196232
},
"model.layers.27.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331538,
"offset": 15447035,
"length": 225537
},
"model.layers.27.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331554,
"offset": 15672572,
"length": 225506
},
"model.layers.27.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 15898078,
"length": 41
},
"model.layers.28.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 15898119,
"length": 41
},
"model.layers.28.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.28.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.28.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.28.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 15898160,
"length": 635
},
"model.layers.28.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 15898795,
"length": 630
},
"model.layers.28.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554381,
"offset": 15899425,
"length": 150194
},
"model.layers.28.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781277,
"offset": 16049619,
"length": 76252
},
"model.layers.28.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.28.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777205,
"offset": 16125871,
"length": 65325
},
"model.layers.28.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331556,
"offset": 16191196,
"length": 196228
},
"model.layers.28.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331532,
"offset": 16387424,
"length": 225522
},
"model.layers.28.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331563,
"offset": 16612946,
"length": 225514
},
"model.layers.28.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 16838460,
"length": 41
},
"model.layers.29.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 16838501,
"length": 41
},
"model.layers.29.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.29.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.29.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.29.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 16838542,
"length": 630
},
"model.layers.29.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 16839172,
"length": 630
},
"model.layers.29.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554397,
"offset": 16839802,
"length": 149907
},
"model.layers.29.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781296,
"offset": 16989709,
"length": 75954
},
"model.layers.29.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.29.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777199,
"offset": 17065663,
"length": 65352
},
"model.layers.29.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331566,
"offset": 17131015,
"length": 196194
},
"model.layers.29.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331539,
"offset": 17327209,
"length": 225550
},
"model.layers.29.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331545,
"offset": 17552759,
"length": 225524
},
"model.layers.29.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 17778283,
"length": 41
},
"model.layers.30.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 17778324,
"length": 41
},
"model.layers.30.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331578,
"offset": 17778365,
"length": 196127
},
"model.layers.30.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331541,
"offset": 17974492,
"length": 225640
},
"model.layers.30.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331558,
"offset": 18200132,
"length": 225389
},
"model.layers.30.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 18425521,
"length": 41
},
"model.layers.30.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.30.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194298,
"offset": 18425562,
"length": 18824
},
"model.layers.30.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777206,
"offset": 18444386,
"length": 65317
},
"model.layers.30.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.30.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558479,
"offset": 18509703,
"length": 151643
},
"model.layers.30.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194296,
"offset": 18661346,
"length": 18834
},
"model.layers.31.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 18680180,
"length": 41
},
"model.layers.31.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.31.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.31.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.31.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 18680221,
"length": 630
},
"model.layers.31.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 18680851,
"length": 630
},
"model.layers.31.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554392,
"offset": 18681481,
"length": 150044
},
"model.layers.31.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781282,
"offset": 18831525,
"length": 76164
},
"model.layers.31.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.31.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777197,
"offset": 18907689,
"length": 65369
},
"model.layers.31.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331558,
"offset": 18973058,
"length": 196233
},
"model.layers.31.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331547,
"offset": 19169291,
"length": 225522
},
"model.layers.31.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331560,
"offset": 19394813,
"length": 225417
},
"model.layers.31.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 19620230,
"length": 41
},
"model.layers.32.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 19620271,
"length": 41
},
"model.layers.32.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.32.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.32.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.32.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 19620312,
"length": 634
},
"model.layers.32.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 19620946,
"length": 635
},
"model.layers.32.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554385,
"offset": 19621581,
"length": 150144
},
"model.layers.32.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781290,
"offset": 19771725,
"length": 76160
},
"model.layers.32.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.32.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777198,
"offset": 19847885,
"length": 65358
},
"model.layers.32.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331573,
"offset": 19913243,
"length": 196145
},
"model.layers.32.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331566,
"offset": 20109388,
"length": 225308
},
"model.layers.32.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331571,
"offset": 20334696,
"length": 225213
},
"model.layers.32.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 20559909,
"length": 41
},
"model.layers.33.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 20559950,
"length": 41
},
"model.layers.33.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.33.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.33.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.33.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 20559991,
"length": 630
},
"model.layers.33.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 20560621,
"length": 635
},
"model.layers.33.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554397,
"offset": 20561256,
"length": 149877
},
"model.layers.33.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781287,
"offset": 20711133,
"length": 76142
},
"model.layers.33.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.33.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777187,
"offset": 20787275,
"length": 65406
},
"model.layers.33.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331574,
"offset": 20852681,
"length": 196155
},
"model.layers.33.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331560,
"offset": 21048836,
"length": 225314
},
"model.layers.33.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331586,
"offset": 21274150,
"length": 225142
},
"model.layers.33.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 21499292,
"length": 41
},
"model.layers.34.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 21499333,
"length": 41
},
"model.layers.34.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331572,
"offset": 21499374,
"length": 196151
},
"model.layers.34.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331552,
"offset": 21695525,
"length": 225381
},
"model.layers.34.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331546,
"offset": 21920906,
"length": 225550
},
"model.layers.34.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 22146456,
"length": 41
},
"model.layers.34.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.34.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194297,
"offset": 22146497,
"length": 18826
},
"model.layers.34.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777202,
"offset": 22165323,
"length": 65344
},
"model.layers.34.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.34.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558466,
"offset": 22230667,
"length": 151701
},
"model.layers.34.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194298,
"offset": 22382368,
"length": 18823
},
"model.layers.35.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 22401191,
"length": 41
},
"model.layers.35.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.35.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.35.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.35.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 22401232,
"length": 630
},
"model.layers.35.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 22401862,
"length": 630
},
"model.layers.35.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554383,
"offset": 22402492,
"length": 150126
},
"model.layers.35.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781282,
"offset": 22552618,
"length": 76129
},
"model.layers.35.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.35.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 22628747,
"length": 65367
},
"model.layers.35.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331555,
"offset": 22694114,
"length": 196227
},
"model.layers.35.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331531,
"offset": 22890341,
"length": 225583
},
"model.layers.35.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331568,
"offset": 23115924,
"length": 225385
},
"model.layers.35.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 23341309,
"length": 41
},
"model.layers.36.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 23341350,
"length": 41
},
"model.layers.36.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.36.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.36.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.36.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 23341391,
"length": 630
},
"model.layers.36.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 23342021,
"length": 630
},
"model.layers.36.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554391,
"offset": 23342651,
"length": 150111
},
"model.layers.36.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781287,
"offset": 23492762,
"length": 76140
},
"model.layers.36.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.36.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777200,
"offset": 23568902,
"length": 65344
},
"model.layers.36.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331565,
"offset": 23634246,
"length": 196190
},
"model.layers.36.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331561,
"offset": 23830436,
"length": 225336
},
"model.layers.36.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331566,
"offset": 24055772,
"length": 225353
},
"model.layers.36.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 24281125,
"length": 41
},
"model.layers.37.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 24281166,
"length": 41
},
"model.layers.37.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.37.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.37.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.37.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 24281207,
"length": 630
},
"model.layers.37.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131102,
"offset": 24281837,
"length": 641
},
"model.layers.37.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554388,
"offset": 24282478,
"length": 150170
},
"model.layers.37.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781290,
"offset": 24432648,
"length": 76126
},
"model.layers.37.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.37.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777186,
"offset": 24508774,
"length": 65411
},
"model.layers.37.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331559,
"offset": 24574185,
"length": 196217
},
"model.layers.37.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331564,
"offset": 24770402,
"length": 225319
},
"model.layers.37.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331560,
"offset": 24995721,
"length": 225384
},
"model.layers.37.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 25221105,
"length": 41
},
"model.layers.38.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 25221146,
"length": 41
},
"model.layers.38.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331563,
"offset": 25221187,
"length": 196184
},
"model.layers.38.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331576,
"offset": 25417371,
"length": 225167
},
"model.layers.38.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331569,
"offset": 25642538,
"length": 225375
},
"model.layers.38.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 25867913,
"length": 41
},
"model.layers.38.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.38.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194301,
"offset": 25867954,
"length": 18809
},
"model.layers.38.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777201,
"offset": 25886763,
"length": 65341
},
"model.layers.38.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.38.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558470,
"offset": 25952104,
"length": 151686
},
"model.layers.38.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194299,
"offset": 26103790,
"length": 18785
},
"model.layers.39.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 26122575,
"length": 41
},
"model.layers.39.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.39.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.39.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.39.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 26122616,
"length": 630
},
"model.layers.39.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 26123246,
"length": 630
},
"model.layers.39.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554381,
"offset": 26123876,
"length": 150276
},
"model.layers.39.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781290,
"offset": 26274152,
"length": 76051
},
"model.layers.39.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.39.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777195,
"offset": 26350203,
"length": 65378
},
"model.layers.39.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331572,
"offset": 26415581,
"length": 196162
},
"model.layers.39.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331556,
"offset": 26611743,
"length": 225294
},
"model.layers.39.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331568,
"offset": 26837037,
"length": 225239
},
"model.layers.39.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 27062276,
"length": 41
},
"model.layers.40.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 27062317,
"length": 41
},
"model.layers.40.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.40.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.40.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.40.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 27062358,
"length": 630
},
"model.layers.40.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 27062988,
"length": 630
},
"model.layers.40.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554390,
"offset": 27063618,
"length": 150049
},
"model.layers.40.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781289,
"offset": 27213667,
"length": 76096
},
"model.layers.40.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.40.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777193,
"offset": 27289763,
"length": 65384
},
"model.layers.40.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331558,
"offset": 27355147,
"length": 196219
},
"model.layers.40.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331538,
"offset": 27551366,
"length": 225560
},
"model.layers.40.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331575,
"offset": 27776926,
"length": 225234
},
"model.layers.40.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 28002160,
"length": 41
},
"model.layers.41.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 28002201,
"length": 41
},
"model.layers.41.linear_attn.A_log": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.41.linear_attn.conv1d.weight": {
"mode": "dense",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 32768
},
"model.layers.41.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.41.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131104,
"offset": 28002242,
"length": 630
},
"model.layers.41.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 131103,
"offset": 28002872,
"length": 636
},
"model.layers.41.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33554380,
"offset": 28003508,
"length": 150137
},
"model.layers.41.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 16781294,
"offset": 28153645,
"length": 75959
},
"model.layers.41.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.41.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777196,
"offset": 28229604,
"length": 65366
},
"model.layers.41.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331573,
"offset": 28294970,
"length": 196170
},
"model.layers.41.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331565,
"offset": 28491140,
"length": 225210
},
"model.layers.41.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331595,
"offset": 28716350,
"length": 224926
},
"model.layers.41.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 28941276,
"length": 41
},
"model.layers.42.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 28941317,
"length": 41
},
"model.layers.42.mlp.down_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
12288
],
"dtype": "bfloat16",
"nnz": 50331600,
"offset": 28941358,
"length": 196022
},
"model.layers.42.mlp.gate_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331590,
"offset": 29137380,
"length": 224973
},
"model.layers.42.mlp.up_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
12288,
5632
],
"dtype": "bfloat16",
"nnz": 50331593,
"offset": 29362353,
"length": 225040
},
"model.layers.42.post_attention_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 4096,
"offset": 29587393,
"length": 41
},
"model.layers.42.self_attn.k_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.42.self_attn.k_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194296,
"offset": 29587434,
"length": 18761
},
"model.layers.42.self_attn.o_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632,
4096
],
"dtype": "bfloat16",
"nnz": 16777208,
"offset": 29606195,
"length": 65306
},
"model.layers.42.self_attn.q_norm.weight": {
"mode": "dense",
"shape": [
256
],
"dtype": "bfloat16",
"nnz": 256
},
"model.layers.42.self_attn.q_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 33558472,
"offset": 29671501,
"length": 151662
},
"model.layers.42.self_attn.v_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [
1024,
5632
],
"dtype": "bfloat16",
"nnz": 4194303,
"offset": 29823163,
"length": 18727
},
"model.norm.weight": {
"mode": "dense",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 5632
},
"model.embed_tokens.weight": {
"mode": "sparse_delta_u32",
"shape": [
478656,
5632
],
"dtype": "bfloat16",
"nnz": 1042497871,
"offset": 29841890,
"length": 11390146
},
"lm_head.weight": {
"mode": "sparse_delta_u32",
"shape": [
478656,
5632
],
"dtype": "bfloat16",
"nnz": 1022438491,
"offset": 41232036,
"length": 4678381
}
}
}
{
"format": "carbon-compact-v1",
"total_nonzero_params": 8985670530,
"total_dense_params": 18208957504,
"tensors": {
"model.layers.0.input_layernorm.weight": {
"mode": "sparse_delta_u32",
"shape": [
5632
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 0,
"length": 8
},
"model.layers.0.linear_attn.A_log": {
"mode": "sparse_delta_u32",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 0,
"offset": 8,
"length": 8
},
"model.layers.0.linear_attn.conv1d.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
1,
4
],
"dtype": "bfloat16",
"nnz": 51,
"offset": 16,
"length": 38
},
"model.layers.0.linear_attn.dt_bias": {
"mode": "dense",
"shape": [
32
],
"dtype": "bfloat16",
"nnz": 32
},
"model.layers.0.linear_attn.in_proj_a.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 44,
"offset": 54,
"length": 43
},
"model.layers.0.linear_attn.in_proj_b.weight": {
"mode": "sparse_delta_u32",
"shape": [
32,
5632
],
"dtype": "bfloat16",
"nnz": 12,
"offset": 97,
"length": 33
},
"model.layers.0.linear_attn.in_proj_qkv.weight": {
"mode": "sparse_delta_u32",
"shape": [
8192,
5632
],
"dtype": "bfloat16",
"nnz": 274,
"offset": 130,
"length": 119
},
"model.layers.0.linear_attn.in_proj_z.weight": {
"mode": "sparse_delta_u32",
"shape": [
4096,
5632
],
"dtype": "bfloat16",
"nnz": 21,
"offset": 249,
"length": 20
},
"model.layers.0.linear_attn.norm.weight": {
"mode": "dense",
"shape": [
128
],
"dtype": "bfloat16",
"nnz": 128
},
"model.layers.0.linear_attn.out_proj.weight": {
"mode": "sparse_delta_u32",
"shape": [