diff --git "a/chip_layout.json" "b/chip_layout.json" new file mode 100644--- /dev/null +++ "b/chip_layout.json" @@ -0,0 +1,5959 @@ +{ + "format": "carbon-compact-v1", + "total_nonzero_params": 8985670530, + "total_dense_params": 18208957504, + "tensors": { + "model.layers.0.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 0, + "length": 8 + }, + "model.layers.0.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 8, + "length": 8 + }, + "model.layers.0.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 51, + "offset": 16, + "length": 38 + }, + "model.layers.0.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.0.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 44, + "offset": 54, + "length": 43 + }, + "model.layers.0.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 12, + "offset": 97, + "length": 33 + }, + "model.layers.0.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 274, + "offset": 130, + "length": 119 + }, + "model.layers.0.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 21, + "offset": 249, + "length": 20 + }, + "model.layers.0.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.0.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 21, + "offset": 269, + "length": 24 + }, + "model.layers.0.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 357, + "offset": 293, + "length": 66 + }, + "model.layers.0.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 628, + "offset": 359, + "length": 90 + }, + "model.layers.0.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 2415, + "offset": 449, + "length": 320 + }, + "model.layers.0.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 769, + "length": 8 + }, + "model.layers.1.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 777, + "length": 8 + }, + "model.layers.1.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 785, + "length": 8 + }, + "model.layers.1.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 7, + "offset": 793, + "length": 20 + }, + "model.layers.1.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.1.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 2, + "offset": 813, + "length": 16 + }, + "model.layers.1.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1, + "offset": 829, + "length": 12 + }, + "model.layers.1.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 13, + "offset": 841, + "length": 28 + }, + "model.layers.1.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1, + "offset": 869, + "length": 12 + }, + "model.layers.1.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.1.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 1, + "offset": 881, + "length": 12 + }, + "model.layers.1.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 1615, + "offset": 893, + "length": 129 + }, + "model.layers.1.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 95083, + "offset": 1022, + "length": 2118 + }, + "model.layers.1.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1775, + "offset": 3140, + "length": 373 + }, + "model.layers.1.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3513, + "length": 8 + }, + "model.layers.10.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3521, + "length": 8 + }, + "model.layers.10.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3529, + "length": 8 + }, + "model.layers.10.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3537, + "length": 8 + }, + "model.layers.10.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.10.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3545, + "length": 8 + }, + "model.layers.10.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3553, + "length": 8 + }, + "model.layers.10.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3561, + "length": 8 + }, + "model.layers.10.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3569, + "length": 8 + }, + "model.layers.10.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.10.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 3577, + "length": 8 + }, + "model.layers.10.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 1713, + "offset": 3585, + "length": 109 + }, + "model.layers.10.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 14630, + "offset": 3694, + "length": 2646 + }, + "model.layers.10.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1713, + "offset": 6340, + "length": 61 + }, + "model.layers.10.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 6401, + "length": 8 + }, + "model.layers.2.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 6409, + "length": 8 + }, + "model.layers.2.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 6417, + "length": 8 + }, + "model.layers.2.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 6425, + "length": 34 + }, + "model.layers.2.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.2.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 6459, + "length": 8 + }, + "model.layers.2.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 32, + "offset": 6467, + "length": 16 + }, + "model.layers.2.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 24576, + "offset": 6483, + "length": 327 + }, + "model.layers.2.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 11968, + "offset": 6810, + "length": 344 + }, + "model.layers.2.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.2.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 7154, + "length": 74 + }, + "model.layers.2.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7228, + "length": 8 + }, + "model.layers.2.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7236, + "length": 8 + }, + "model.layers.2.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7244, + "length": 8 + }, + "model.layers.2.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7252, + "length": 8 + }, + "model.layers.3.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7260, + "length": 8 + }, + "model.layers.3.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7268, + "length": 8 + }, + "model.layers.3.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 7276, + "length": 34 + }, + "model.layers.3.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.3.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 7310, + "length": 8 + }, + "model.layers.3.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 32, + "offset": 7318, + "length": 16 + }, + "model.layers.3.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 24576, + "offset": 7334, + "length": 327 + }, + "model.layers.3.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 11968, + "offset": 7661, + "length": 344 + }, + "model.layers.3.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.3.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 8005, + "length": 74 + }, + "model.layers.3.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 4159, + "offset": 8079, + "length": 141 + }, + "model.layers.3.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 8716, + "offset": 8220, + "length": 418 + }, + "model.layers.3.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4159, + "offset": 8638, + "length": 164 + }, + "model.layers.3.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 8802, + "length": 8 + }, + "model.layers.4.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 8810, + "length": 8 + }, + "model.layers.4.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 8818, + "length": 8 + }, + "model.layers.4.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 780, + "offset": 8826, + "length": 106 + }, + "model.layers.4.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.4.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 8932, + "length": 8 + }, + "model.layers.4.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 47, + "offset": 8940, + "length": 69 + }, + "model.layers.4.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1999, + "offset": 9009, + "length": 1015 + }, + "model.layers.4.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 429, + "offset": 10024, + "length": 74 + }, + "model.layers.4.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.4.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 429, + "offset": 10098, + "length": 77 + }, + "model.layers.4.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 8025, + "offset": 10175, + "length": 308 + }, + "model.layers.4.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 122002, + "offset": 10483, + "length": 5550 + }, + "model.layers.4.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 13849, + "offset": 16033, + "length": 1168 + }, + "model.layers.4.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 17201, + "length": 8 + }, + "model.layers.5.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 17209, + "length": 8 + }, + "model.layers.5.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 17217, + "length": 8 + }, + "model.layers.5.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 1888, + "offset": 17225, + "length": 65 + }, + "model.layers.5.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.5.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 17290, + "length": 8 + }, + "model.layers.5.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 48, + "offset": 17298, + "length": 64 + }, + "model.layers.5.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 3932, + "offset": 17362, + "length": 857 + }, + "model.layers.5.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 64, + "offset": 18219, + "length": 20 + }, + "model.layers.5.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.5.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 64, + "offset": 18239, + "length": 23 + }, + "model.layers.5.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 6992, + "offset": 18262, + "length": 359 + }, + "model.layers.5.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 29667, + "offset": 18621, + "length": 6801 + }, + "model.layers.5.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 9617, + "offset": 25422, + "length": 2172 + }, + "model.layers.5.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 27594, + "length": 8 + }, + "model.layers.6.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 27602, + "length": 8 + }, + "model.layers.6.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 27610, + "length": 8 + }, + "model.layers.6.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 7232, + "offset": 27618, + "length": 134 + }, + "model.layers.6.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.6.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 27752, + "length": 8 + }, + "model.layers.6.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 96, + "offset": 27760, + "length": 32 + }, + "model.layers.6.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 113728, + "offset": 27792, + "length": 1840 + }, + "model.layers.6.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 5120, + "offset": 29632, + "length": 132 + }, + "model.layers.6.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.6.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 29764, + "length": 72 + }, + "model.layers.6.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 3896, + "offset": 29836, + "length": 255 + }, + "model.layers.6.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 27741, + "offset": 30091, + "length": 4009 + }, + "model.layers.6.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 3941, + "offset": 34100, + "length": 860 + }, + "model.layers.6.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 34960, + "length": 8 + }, + "model.layers.7.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 34968, + "length": 8 + }, + "model.layers.7.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 34976, + "length": 8 + }, + "model.layers.7.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 7232, + "offset": 34984, + "length": 134 + }, + "model.layers.7.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.7.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 35118, + "length": 8 + }, + "model.layers.7.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 96, + "offset": 35126, + "length": 32 + }, + "model.layers.7.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 113728, + "offset": 35158, + "length": 1840 + }, + "model.layers.7.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 5120, + "offset": 36998, + "length": 132 + }, + "model.layers.7.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.7.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 37130, + "length": 72 + }, + "model.layers.7.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 8307, + "offset": 37202, + "length": 495 + }, + "model.layers.7.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 157966, + "offset": 37697, + "length": 7870 + }, + "model.layers.7.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 8307, + "offset": 45567, + "length": 2280 + }, + "model.layers.7.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 47847, + "length": 8 + }, + "model.layers.8.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 47855, + "length": 8 + }, + "model.layers.8.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 47863, + "length": 8 + }, + "model.layers.8.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 47871, + "length": 34 + }, + "model.layers.8.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.8.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 47905, + "length": 8 + }, + "model.layers.8.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 32, + "offset": 47913, + "length": 16 + }, + "model.layers.8.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 38912, + "offset": 47929, + "length": 577 + }, + "model.layers.8.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 22208, + "offset": 48506, + "length": 554 + }, + "model.layers.8.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.8.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 2048, + "offset": 49060, + "length": 73 + }, + "model.layers.8.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 8558, + "offset": 49133, + "length": 326 + }, + "model.layers.8.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 60993, + "offset": 49459, + "length": 7871 + }, + "model.layers.8.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 8558, + "offset": 57330, + "length": 2234 + }, + "model.layers.8.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 59564, + "length": 8 + }, + "model.layers.9.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 59572, + "length": 8 + }, + "model.layers.9.linear_attn.A_log": { + "mode": "sparse_delta_u32", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 59580, + "length": 8 + }, + "model.layers.9.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 292, + "offset": 59588, + "length": 35 + }, + "model.layers.9.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.9.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 2, + "offset": 59623, + "length": 16 + }, + "model.layers.9.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4, + "offset": 59639, + "length": 19 + }, + "model.layers.9.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 2291, + "offset": 59658, + "length": 462 + }, + "model.layers.9.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 576, + "offset": 60120, + "length": 53 + }, + "model.layers.9.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.9.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 128, + "offset": 60173, + "length": 28 + }, + "model.layers.9.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 1407, + "offset": 60201, + "length": 102 + }, + "model.layers.9.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 9567, + "offset": 60303, + "length": 5420 + }, + "model.layers.9.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1407, + "offset": 65723, + "length": 287 + }, + "model.layers.9.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 0, + "offset": 66010, + "length": 8 + }, + "model.layers.11.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 66018, + "length": 41 + }, + "model.layers.11.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.11.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32766, + "offset": 66059, + "length": 164 + }, + "model.layers.11.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.11.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 66223, + "length": 630 + }, + "model.layers.11.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 66853, + "length": 635 + }, + "model.layers.11.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554393, + "offset": 67488, + "length": 150114 + }, + "model.layers.11.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781289, + "offset": 217602, + "length": 76125 + }, + "model.layers.11.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.11.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777197, + "offset": 293727, + "length": 65370 + }, + "model.layers.11.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331556, + "offset": 359097, + "length": 196228 + }, + "model.layers.11.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331534, + "offset": 555325, + "length": 225535 + }, + "model.layers.11.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331542, + "offset": 780860, + "length": 225649 + }, + "model.layers.11.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 1006509, + "length": 41 + }, + "model.layers.12.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 1006550, + "length": 41 + }, + "model.layers.12.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.12.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.12.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.12.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 1006591, + "length": 630 + }, + "model.layers.12.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 1007221, + "length": 630 + }, + "model.layers.12.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554385, + "offset": 1007851, + "length": 150179 + }, + "model.layers.12.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781287, + "offset": 1158030, + "length": 76176 + }, + "model.layers.12.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.12.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777188, + "offset": 1234206, + "length": 65406 + }, + "model.layers.12.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331554, + "offset": 1299612, + "length": 196234 + }, + "model.layers.12.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331537, + "offset": 1495846, + "length": 225623 + }, + "model.layers.12.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331526, + "offset": 1721469, + "length": 225726 + }, + "model.layers.12.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 1947195, + "length": 41 + }, + "model.layers.13.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 1947236, + "length": 41 + }, + "model.layers.13.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.13.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.13.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.13.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 1947277, + "length": 636 + }, + "model.layers.13.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 1947913, + "length": 630 + }, + "model.layers.13.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554380, + "offset": 1948543, + "length": 150240 + }, + "model.layers.13.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781287, + "offset": 2098783, + "length": 76104 + }, + "model.layers.13.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.13.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 2174887, + "length": 65365 + }, + "model.layers.13.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331530, + "offset": 2240252, + "length": 196345 + }, + "model.layers.13.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331544, + "offset": 2436597, + "length": 225599 + }, + "model.layers.13.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331520, + "offset": 2662196, + "length": 225725 + }, + "model.layers.13.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 2887921, + "length": 41 + }, + "model.layers.14.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 2887962, + "length": 41 + }, + "model.layers.14.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331543, + "offset": 2888003, + "length": 196294 + }, + "model.layers.14.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 3084297, + "length": 225490 + }, + "model.layers.14.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331549, + "offset": 3309787, + "length": 225407 + }, + "model.layers.14.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 3535194, + "length": 41 + }, + "model.layers.14.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.14.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194298, + "offset": 3535235, + "length": 18820 + }, + "model.layers.14.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777198, + "offset": 3554055, + "length": 65358 + }, + "model.layers.14.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.14.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558481, + "offset": 3619413, + "length": 151630 + }, + "model.layers.14.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194296, + "offset": 3771043, + "length": 18797 + }, + "model.layers.15.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 3789840, + "length": 41 + }, + "model.layers.15.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.15.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.15.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.15.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 3789881, + "length": 630 + }, + "model.layers.15.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 3790511, + "length": 630 + }, + "model.layers.15.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554375, + "offset": 3791141, + "length": 150304 + }, + "model.layers.15.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781288, + "offset": 3941445, + "length": 76135 + }, + "model.layers.15.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.15.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777185, + "offset": 4017580, + "length": 65420 + }, + "model.layers.15.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 4083000, + "length": 196290 + }, + "model.layers.15.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331555, + "offset": 4279290, + "length": 225437 + }, + "model.layers.15.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 4504727, + "length": 225502 + }, + "model.layers.15.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 4730229, + "length": 41 + }, + "model.layers.16.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 4730270, + "length": 41 + }, + "model.layers.16.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.16.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.16.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.16.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 4730311, + "length": 630 + }, + "model.layers.16.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 4730941, + "length": 630 + }, + "model.layers.16.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554375, + "offset": 4731571, + "length": 150339 + }, + "model.layers.16.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781293, + "offset": 4881910, + "length": 76005 + }, + "model.layers.16.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.16.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777192, + "offset": 4957915, + "length": 65396 + }, + "model.layers.16.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331520, + "offset": 5023311, + "length": 196375 + }, + "model.layers.16.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331536, + "offset": 5219686, + "length": 225634 + }, + "model.layers.16.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331542, + "offset": 5445320, + "length": 225614 + }, + "model.layers.16.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 5670934, + "length": 41 + }, + "model.layers.17.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 5670975, + "length": 41 + }, + "model.layers.17.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.17.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.17.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.17.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 5671016, + "length": 630 + }, + "model.layers.17.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 5671646, + "length": 630 + }, + "model.layers.17.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554388, + "offset": 5672276, + "length": 150021 + }, + "model.layers.17.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781290, + "offset": 5822297, + "length": 76085 + }, + "model.layers.17.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.17.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777197, + "offset": 5898382, + "length": 65361 + }, + "model.layers.17.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331559, + "offset": 5963743, + "length": 196229 + }, + "model.layers.17.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331547, + "offset": 6159972, + "length": 225524 + }, + "model.layers.17.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331557, + "offset": 6385496, + "length": 225393 + }, + "model.layers.17.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 6610889, + "length": 41 + }, + "model.layers.18.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 6610930, + "length": 41 + }, + "model.layers.18.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331547, + "offset": 6610971, + "length": 196257 + }, + "model.layers.18.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331548, + "offset": 6807228, + "length": 225430 + }, + "model.layers.18.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331551, + "offset": 7032658, + "length": 225524 + }, + "model.layers.18.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 7258182, + "length": 41 + }, + "model.layers.18.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.18.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194297, + "offset": 7258223, + "length": 18825 + }, + "model.layers.18.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777201, + "offset": 7277048, + "length": 65349 + }, + "model.layers.18.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.18.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558470, + "offset": 7342397, + "length": 151681 + }, + "model.layers.18.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194297, + "offset": 7494078, + "length": 18793 + }, + "model.layers.19.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 7512871, + "length": 41 + }, + "model.layers.19.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.19.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.19.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.19.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 7512912, + "length": 630 + }, + "model.layers.19.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 7513542, + "length": 630 + }, + "model.layers.19.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554392, + "offset": 7514172, + "length": 150003 + }, + "model.layers.19.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781281, + "offset": 7664175, + "length": 76163 + }, + "model.layers.19.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.19.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 7740338, + "length": 65372 + }, + "model.layers.19.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331558, + "offset": 7805710, + "length": 196223 + }, + "model.layers.19.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 8001933, + "length": 225631 + }, + "model.layers.19.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331544, + "offset": 8227564, + "length": 225532 + }, + "model.layers.19.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 8453096, + "length": 41 + }, + "model.layers.20.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 8453137, + "length": 41 + }, + "model.layers.20.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.20.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.20.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.20.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 8453178, + "length": 636 + }, + "model.layers.20.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 8453814, + "length": 630 + }, + "model.layers.20.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554383, + "offset": 8454444, + "length": 150188 + }, + "model.layers.20.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781289, + "offset": 8604632, + "length": 76060 + }, + "model.layers.20.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.20.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777186, + "offset": 8680692, + "length": 65414 + }, + "model.layers.20.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331573, + "offset": 8746106, + "length": 196153 + }, + "model.layers.20.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331543, + "offset": 8942259, + "length": 225530 + }, + "model.layers.20.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331576, + "offset": 9167789, + "length": 225228 + }, + "model.layers.20.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 9393017, + "length": 41 + }, + "model.layers.21.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 9393058, + "length": 41 + }, + "model.layers.21.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.21.linear_attn.conv1d.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32767, + "offset": 9393099, + "length": 158 + }, + "model.layers.21.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.21.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 9393257, + "length": 630 + }, + "model.layers.21.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 9393887, + "length": 630 + }, + "model.layers.21.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554386, + "offset": 9394517, + "length": 150106 + }, + "model.layers.21.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781283, + "offset": 9544623, + "length": 76263 + }, + "model.layers.21.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.21.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 9620886, + "length": 65363 + }, + "model.layers.21.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331562, + "offset": 9686249, + "length": 196195 + }, + "model.layers.21.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331533, + "offset": 9882444, + "length": 225606 + }, + "model.layers.21.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331549, + "offset": 10108050, + "length": 225576 + }, + "model.layers.21.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 10333626, + "length": 41 + }, + "model.layers.22.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 10333667, + "length": 41 + }, + "model.layers.22.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 10333708, + "length": 196288 + }, + "model.layers.22.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331552, + "offset": 10529996, + "length": 225525 + }, + "model.layers.22.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331574, + "offset": 10755521, + "length": 225342 + }, + "model.layers.22.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 10980863, + "length": 41 + }, + "model.layers.22.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.22.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194296, + "offset": 10980904, + "length": 18831 + }, + "model.layers.22.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777199, + "offset": 10999735, + "length": 65363 + }, + "model.layers.22.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.22.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558479, + "offset": 11065098, + "length": 151644 + }, + "model.layers.22.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194297, + "offset": 11216742, + "length": 18829 + }, + "model.layers.23.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 11235571, + "length": 41 + }, + "model.layers.23.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.23.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.23.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.23.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 11235612, + "length": 630 + }, + "model.layers.23.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 11236242, + "length": 636 + }, + "model.layers.23.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554397, + "offset": 11236878, + "length": 150015 + }, + "model.layers.23.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781280, + "offset": 11386893, + "length": 76208 + }, + "model.layers.23.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.23.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777190, + "offset": 11463101, + "length": 65397 + }, + "model.layers.23.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331541, + "offset": 11528498, + "length": 196297 + }, + "model.layers.23.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331531, + "offset": 11724795, + "length": 225724 + }, + "model.layers.23.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331538, + "offset": 11950519, + "length": 225684 + }, + "model.layers.23.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 12176203, + "length": 41 + }, + "model.layers.24.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 12176244, + "length": 41 + }, + "model.layers.24.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.24.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.24.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.24.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 12176285, + "length": 630 + }, + "model.layers.24.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 12176915, + "length": 630 + }, + "model.layers.24.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554376, + "offset": 12177545, + "length": 150157 + }, + "model.layers.24.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781290, + "offset": 12327702, + "length": 76016 + }, + "model.layers.24.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.24.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777200, + "offset": 12403718, + "length": 65343 + }, + "model.layers.24.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331550, + "offset": 12469061, + "length": 196252 + }, + "model.layers.24.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331541, + "offset": 12665313, + "length": 225576 + }, + "model.layers.24.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331551, + "offset": 12890889, + "length": 225385 + }, + "model.layers.24.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 13116274, + "length": 41 + }, + "model.layers.25.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 13116315, + "length": 41 + }, + "model.layers.25.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.25.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.25.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.25.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 13116356, + "length": 630 + }, + "model.layers.25.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 13116986, + "length": 630 + }, + "model.layers.25.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554384, + "offset": 13117616, + "length": 150186 + }, + "model.layers.25.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781283, + "offset": 13267802, + "length": 76159 + }, + "model.layers.25.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.25.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777191, + "offset": 13343961, + "length": 65401 + }, + "model.layers.25.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331574, + "offset": 13409362, + "length": 196165 + }, + "model.layers.25.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331523, + "offset": 13605527, + "length": 225706 + }, + "model.layers.25.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331568, + "offset": 13831233, + "length": 225195 + }, + "model.layers.25.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 14056428, + "length": 41 + }, + "model.layers.26.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 14056469, + "length": 41 + }, + "model.layers.26.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331564, + "offset": 14056510, + "length": 196192 + }, + "model.layers.26.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331543, + "offset": 14252702, + "length": 225543 + }, + "model.layers.26.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331567, + "offset": 14478245, + "length": 225280 + }, + "model.layers.26.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 14703525, + "length": 41 + }, + "model.layers.26.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.26.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194298, + "offset": 14703566, + "length": 18786 + }, + "model.layers.26.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777194, + "offset": 14722352, + "length": 65384 + }, + "model.layers.26.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.26.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558468, + "offset": 14787736, + "length": 151690 + }, + "model.layers.26.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194302, + "offset": 14939426, + "length": 18767 + }, + "model.layers.27.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 14958193, + "length": 41 + }, + "model.layers.27.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.27.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.27.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.27.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 14958234, + "length": 630 + }, + "model.layers.27.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 14958864, + "length": 630 + }, + "model.layers.27.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554399, + "offset": 14959494, + "length": 149891 + }, + "model.layers.27.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781292, + "offset": 15109385, + "length": 76044 + }, + "model.layers.27.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.27.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 15185429, + "length": 65374 + }, + "model.layers.27.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331558, + "offset": 15250803, + "length": 196232 + }, + "model.layers.27.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331538, + "offset": 15447035, + "length": 225537 + }, + "model.layers.27.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331554, + "offset": 15672572, + "length": 225506 + }, + "model.layers.27.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 15898078, + "length": 41 + }, + "model.layers.28.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 15898119, + "length": 41 + }, + "model.layers.28.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.28.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.28.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.28.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 15898160, + "length": 635 + }, + "model.layers.28.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 15898795, + "length": 630 + }, + "model.layers.28.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554381, + "offset": 15899425, + "length": 150194 + }, + "model.layers.28.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781277, + "offset": 16049619, + "length": 76252 + }, + "model.layers.28.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.28.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777205, + "offset": 16125871, + "length": 65325 + }, + "model.layers.28.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331556, + "offset": 16191196, + "length": 196228 + }, + "model.layers.28.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331532, + "offset": 16387424, + "length": 225522 + }, + "model.layers.28.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331563, + "offset": 16612946, + "length": 225514 + }, + "model.layers.28.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 16838460, + "length": 41 + }, + "model.layers.29.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 16838501, + "length": 41 + }, + "model.layers.29.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.29.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.29.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.29.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 16838542, + "length": 630 + }, + "model.layers.29.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 16839172, + "length": 630 + }, + "model.layers.29.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554397, + "offset": 16839802, + "length": 149907 + }, + "model.layers.29.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781296, + "offset": 16989709, + "length": 75954 + }, + "model.layers.29.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.29.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777199, + "offset": 17065663, + "length": 65352 + }, + "model.layers.29.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331566, + "offset": 17131015, + "length": 196194 + }, + "model.layers.29.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331539, + "offset": 17327209, + "length": 225550 + }, + "model.layers.29.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331545, + "offset": 17552759, + "length": 225524 + }, + "model.layers.29.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 17778283, + "length": 41 + }, + "model.layers.30.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 17778324, + "length": 41 + }, + "model.layers.30.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331578, + "offset": 17778365, + "length": 196127 + }, + "model.layers.30.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331541, + "offset": 17974492, + "length": 225640 + }, + "model.layers.30.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331558, + "offset": 18200132, + "length": 225389 + }, + "model.layers.30.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 18425521, + "length": 41 + }, + "model.layers.30.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.30.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194298, + "offset": 18425562, + "length": 18824 + }, + "model.layers.30.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777206, + "offset": 18444386, + "length": 65317 + }, + "model.layers.30.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.30.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558479, + "offset": 18509703, + "length": 151643 + }, + "model.layers.30.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194296, + "offset": 18661346, + "length": 18834 + }, + "model.layers.31.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 18680180, + "length": 41 + }, + "model.layers.31.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.31.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.31.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.31.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 18680221, + "length": 630 + }, + "model.layers.31.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 18680851, + "length": 630 + }, + "model.layers.31.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554392, + "offset": 18681481, + "length": 150044 + }, + "model.layers.31.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781282, + "offset": 18831525, + "length": 76164 + }, + "model.layers.31.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.31.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777197, + "offset": 18907689, + "length": 65369 + }, + "model.layers.31.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331558, + "offset": 18973058, + "length": 196233 + }, + "model.layers.31.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331547, + "offset": 19169291, + "length": 225522 + }, + "model.layers.31.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331560, + "offset": 19394813, + "length": 225417 + }, + "model.layers.31.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 19620230, + "length": 41 + }, + "model.layers.32.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 19620271, + "length": 41 + }, + "model.layers.32.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.32.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.32.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.32.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 19620312, + "length": 634 + }, + "model.layers.32.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 19620946, + "length": 635 + }, + "model.layers.32.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554385, + "offset": 19621581, + "length": 150144 + }, + "model.layers.32.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781290, + "offset": 19771725, + "length": 76160 + }, + "model.layers.32.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.32.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777198, + "offset": 19847885, + "length": 65358 + }, + "model.layers.32.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331573, + "offset": 19913243, + "length": 196145 + }, + "model.layers.32.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331566, + "offset": 20109388, + "length": 225308 + }, + "model.layers.32.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331571, + "offset": 20334696, + "length": 225213 + }, + "model.layers.32.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 20559909, + "length": 41 + }, + "model.layers.33.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 20559950, + "length": 41 + }, + "model.layers.33.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.33.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.33.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.33.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 20559991, + "length": 630 + }, + "model.layers.33.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 20560621, + "length": 635 + }, + "model.layers.33.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554397, + "offset": 20561256, + "length": 149877 + }, + "model.layers.33.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781287, + "offset": 20711133, + "length": 76142 + }, + "model.layers.33.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.33.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777187, + "offset": 20787275, + "length": 65406 + }, + "model.layers.33.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331574, + "offset": 20852681, + "length": 196155 + }, + "model.layers.33.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331560, + "offset": 21048836, + "length": 225314 + }, + "model.layers.33.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331586, + "offset": 21274150, + "length": 225142 + }, + "model.layers.33.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 21499292, + "length": 41 + }, + "model.layers.34.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 21499333, + "length": 41 + }, + "model.layers.34.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331572, + "offset": 21499374, + "length": 196151 + }, + "model.layers.34.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331552, + "offset": 21695525, + "length": 225381 + }, + "model.layers.34.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331546, + "offset": 21920906, + "length": 225550 + }, + "model.layers.34.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 22146456, + "length": 41 + }, + "model.layers.34.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.34.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194297, + "offset": 22146497, + "length": 18826 + }, + "model.layers.34.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777202, + "offset": 22165323, + "length": 65344 + }, + "model.layers.34.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.34.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558466, + "offset": 22230667, + "length": 151701 + }, + "model.layers.34.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194298, + "offset": 22382368, + "length": 18823 + }, + "model.layers.35.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 22401191, + "length": 41 + }, + "model.layers.35.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.35.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.35.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.35.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 22401232, + "length": 630 + }, + "model.layers.35.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 22401862, + "length": 630 + }, + "model.layers.35.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554383, + "offset": 22402492, + "length": 150126 + }, + "model.layers.35.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781282, + "offset": 22552618, + "length": 76129 + }, + "model.layers.35.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.35.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 22628747, + "length": 65367 + }, + "model.layers.35.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331555, + "offset": 22694114, + "length": 196227 + }, + "model.layers.35.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331531, + "offset": 22890341, + "length": 225583 + }, + "model.layers.35.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331568, + "offset": 23115924, + "length": 225385 + }, + "model.layers.35.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 23341309, + "length": 41 + }, + "model.layers.36.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 23341350, + "length": 41 + }, + "model.layers.36.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.36.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.36.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.36.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 23341391, + "length": 630 + }, + "model.layers.36.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 23342021, + "length": 630 + }, + "model.layers.36.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554391, + "offset": 23342651, + "length": 150111 + }, + "model.layers.36.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781287, + "offset": 23492762, + "length": 76140 + }, + "model.layers.36.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.36.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777200, + "offset": 23568902, + "length": 65344 + }, + "model.layers.36.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331565, + "offset": 23634246, + "length": 196190 + }, + "model.layers.36.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331561, + "offset": 23830436, + "length": 225336 + }, + "model.layers.36.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331566, + "offset": 24055772, + "length": 225353 + }, + "model.layers.36.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 24281125, + "length": 41 + }, + "model.layers.37.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 24281166, + "length": 41 + }, + "model.layers.37.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.37.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.37.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.37.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 24281207, + "length": 630 + }, + "model.layers.37.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131102, + "offset": 24281837, + "length": 641 + }, + "model.layers.37.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554388, + "offset": 24282478, + "length": 150170 + }, + "model.layers.37.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781290, + "offset": 24432648, + "length": 76126 + }, + "model.layers.37.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.37.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777186, + "offset": 24508774, + "length": 65411 + }, + "model.layers.37.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331559, + "offset": 24574185, + "length": 196217 + }, + "model.layers.37.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331564, + "offset": 24770402, + "length": 225319 + }, + "model.layers.37.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331560, + "offset": 24995721, + "length": 225384 + }, + "model.layers.37.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 25221105, + "length": 41 + }, + "model.layers.38.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 25221146, + "length": 41 + }, + "model.layers.38.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331563, + "offset": 25221187, + "length": 196184 + }, + "model.layers.38.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331576, + "offset": 25417371, + "length": 225167 + }, + "model.layers.38.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331569, + "offset": 25642538, + "length": 225375 + }, + "model.layers.38.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 25867913, + "length": 41 + }, + "model.layers.38.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.38.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194301, + "offset": 25867954, + "length": 18809 + }, + "model.layers.38.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777201, + "offset": 25886763, + "length": 65341 + }, + "model.layers.38.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.38.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558470, + "offset": 25952104, + "length": 151686 + }, + "model.layers.38.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194299, + "offset": 26103790, + "length": 18785 + }, + "model.layers.39.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 26122575, + "length": 41 + }, + "model.layers.39.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.39.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.39.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.39.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 26122616, + "length": 630 + }, + "model.layers.39.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 26123246, + "length": 630 + }, + "model.layers.39.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554381, + "offset": 26123876, + "length": 150276 + }, + "model.layers.39.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781290, + "offset": 26274152, + "length": 76051 + }, + "model.layers.39.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.39.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777195, + "offset": 26350203, + "length": 65378 + }, + "model.layers.39.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331572, + "offset": 26415581, + "length": 196162 + }, + "model.layers.39.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331556, + "offset": 26611743, + "length": 225294 + }, + "model.layers.39.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331568, + "offset": 26837037, + "length": 225239 + }, + "model.layers.39.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 27062276, + "length": 41 + }, + "model.layers.40.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 27062317, + "length": 41 + }, + "model.layers.40.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.40.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.40.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.40.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 27062358, + "length": 630 + }, + "model.layers.40.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 27062988, + "length": 630 + }, + "model.layers.40.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554390, + "offset": 27063618, + "length": 150049 + }, + "model.layers.40.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781289, + "offset": 27213667, + "length": 76096 + }, + "model.layers.40.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.40.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777193, + "offset": 27289763, + "length": 65384 + }, + "model.layers.40.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331558, + "offset": 27355147, + "length": 196219 + }, + "model.layers.40.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331538, + "offset": 27551366, + "length": 225560 + }, + "model.layers.40.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331575, + "offset": 27776926, + "length": 225234 + }, + "model.layers.40.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 28002160, + "length": 41 + }, + "model.layers.41.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 28002201, + "length": 41 + }, + "model.layers.41.linear_attn.A_log": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.41.linear_attn.conv1d.weight": { + "mode": "dense", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "bfloat16", + "nnz": 32768 + }, + "model.layers.41.linear_attn.dt_bias": { + "mode": "dense", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "nnz": 32 + }, + "model.layers.41.linear_attn.in_proj_a.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131104, + "offset": 28002242, + "length": 630 + }, + "model.layers.41.linear_attn.in_proj_b.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 32, + 5632 + ], + "dtype": "bfloat16", + "nnz": 131103, + "offset": 28002872, + "length": 636 + }, + "model.layers.41.linear_attn.in_proj_qkv.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33554380, + "offset": 28003508, + "length": 150137 + }, + "model.layers.41.linear_attn.in_proj_z.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 4096, + 5632 + ], + "dtype": "bfloat16", + "nnz": 16781294, + "offset": 28153645, + "length": 75959 + }, + "model.layers.41.linear_attn.norm.weight": { + "mode": "dense", + "shape": [ + 128 + ], + "dtype": "bfloat16", + "nnz": 128 + }, + "model.layers.41.linear_attn.out_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777196, + "offset": 28229604, + "length": 65366 + }, + "model.layers.41.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331573, + "offset": 28294970, + "length": 196170 + }, + "model.layers.41.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331565, + "offset": 28491140, + "length": 225210 + }, + "model.layers.41.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331595, + "offset": 28716350, + "length": 224926 + }, + "model.layers.41.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 28941276, + "length": 41 + }, + "model.layers.42.input_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 28941317, + "length": 41 + }, + "model.layers.42.mlp.down_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 12288 + ], + "dtype": "bfloat16", + "nnz": 50331600, + "offset": 28941358, + "length": 196022 + }, + "model.layers.42.mlp.gate_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331590, + "offset": 29137380, + "length": 224973 + }, + "model.layers.42.mlp.up_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 12288, + 5632 + ], + "dtype": "bfloat16", + "nnz": 50331593, + "offset": 29362353, + "length": 225040 + }, + "model.layers.42.post_attention_layernorm.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 4096, + "offset": 29587393, + "length": 41 + }, + "model.layers.42.self_attn.k_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.42.self_attn.k_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194296, + "offset": 29587434, + "length": 18761 + }, + "model.layers.42.self_attn.o_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 5632, + 4096 + ], + "dtype": "bfloat16", + "nnz": 16777208, + "offset": 29606195, + "length": 65306 + }, + "model.layers.42.self_attn.q_norm.weight": { + "mode": "dense", + "shape": [ + 256 + ], + "dtype": "bfloat16", + "nnz": 256 + }, + "model.layers.42.self_attn.q_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 8192, + 5632 + ], + "dtype": "bfloat16", + "nnz": 33558472, + "offset": 29671501, + "length": 151662 + }, + "model.layers.42.self_attn.v_proj.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 1024, + 5632 + ], + "dtype": "bfloat16", + "nnz": 4194303, + "offset": 29823163, + "length": 18727 + }, + "model.norm.weight": { + "mode": "dense", + "shape": [ + 5632 + ], + "dtype": "bfloat16", + "nnz": 5632 + }, + "model.embed_tokens.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 478656, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1042497871, + "offset": 29841890, + "length": 11390146 + }, + "lm_head.weight": { + "mode": "sparse_delta_u32", + "shape": [ + 478656, + 5632 + ], + "dtype": "bfloat16", + "nnz": 1022438491, + "offset": 41232036, + "length": 4678381 + } + } +} \ No newline at end of file