{ "bits": 8, "group_size": 32, "mode": "affine", "kept_16bit": [ "groups.0.0.f1.weight", "groups.0.0.f2.weight", "groups.0.0.o.weight", "groups.0.0.qkv.weight", "groups.0.1.f1.weight", "groups.0.1.f2.weight", "groups.0.1.o.weight", "groups.0.1.qkv.weight", "groups.0.2.f1.weight", "groups.0.2.f2.weight", "groups.0.2.o.weight", "groups.0.2.qkv.weight", "groups.0.3.f1.weight", "groups.0.3.f2.weight", "groups.0.3.o.weight", "groups.0.3.qkv.weight", "groups.1.0.f1.weight", "groups.1.0.f2.weight", "groups.1.0.o.weight", "groups.1.0.qkv.weight", "groups.1.1.f1.weight", "groups.1.1.f2.weight", "groups.1.1.o.weight", "groups.1.1.qkv.weight", "groups.1.2.f1.weight", "groups.1.2.f2.weight", "groups.1.2.o.weight", "groups.1.2.qkv.weight", "groups.1.3.f1.weight", "groups.1.3.f2.weight", "groups.1.3.o.weight", "groups.1.3.qkv.weight", "groups.10.0.f1.weight", "groups.10.0.f2.weight", "groups.10.0.o.weight", "groups.10.0.qkv.weight", "groups.10.1.f1.weight", "groups.10.1.f2.weight", "groups.10.1.o.weight", "groups.10.1.qkv.weight", "groups.10.2.f1.weight", "groups.10.2.f2.weight", "groups.10.2.o.weight", "groups.10.2.qkv.weight", "groups.10.3.f1.weight", "groups.10.3.f2.weight", "groups.10.3.o.weight", "groups.10.3.qkv.weight", "groups.2.0.f1.weight", "groups.2.0.f2.weight", "groups.2.0.o.weight", "groups.2.0.qkv.weight", "groups.2.1.f1.weight", "groups.2.1.f2.weight", "groups.2.1.o.weight", "groups.2.1.qkv.weight", "groups.2.2.f1.weight", "groups.2.2.f2.weight", "groups.2.2.o.weight", "groups.2.2.qkv.weight", "groups.2.3.f1.weight", "groups.2.3.f2.weight", "groups.2.3.o.weight", "groups.2.3.qkv.weight", "groups.3.0.f1.weight", "groups.3.0.f2.weight", "groups.3.0.o.weight", "groups.3.0.qkv.weight", "groups.3.1.f1.weight", "groups.3.1.f2.weight", "groups.3.1.o.weight", "groups.3.1.qkv.weight", "groups.3.2.f1.weight", "groups.3.2.f2.weight", "groups.3.2.o.weight", "groups.3.2.qkv.weight", "groups.3.3.f1.weight", "groups.3.3.f2.weight", "groups.3.3.o.weight", "groups.3.3.qkv.weight", "groups.4.0.f1.weight", "groups.4.0.f2.weight", "groups.4.0.o.weight", "groups.4.0.qkv.weight", "groups.4.1.f1.weight", "groups.4.1.f2.weight", "groups.4.1.o.weight", "groups.4.1.qkv.weight", "groups.4.2.f1.weight", "groups.4.2.f2.weight", "groups.4.2.o.weight", "groups.4.2.qkv.weight", "groups.4.3.f1.weight", "groups.4.3.f2.weight", "groups.4.3.o.weight", "groups.4.3.qkv.weight", "groups.5.0.f1.weight", "groups.5.0.f2.weight", "groups.5.0.o.weight", "groups.5.0.qkv.weight", "groups.5.1.f1.weight", "groups.5.1.f2.weight", "groups.5.1.o.weight", "groups.5.1.qkv.weight", "groups.5.2.f1.weight", "groups.5.2.f2.weight", "groups.5.2.o.weight", "groups.5.2.qkv.weight", "groups.5.3.f1.weight", "groups.5.3.f2.weight", "groups.5.3.o.weight", "groups.5.3.qkv.weight", "groups.6.0.f1.weight", "groups.6.0.f2.weight", "groups.6.0.o.weight", "groups.6.0.qkv.weight", "groups.6.1.f1.weight", "groups.6.1.f2.weight", "groups.6.1.o.weight", "groups.6.1.qkv.weight", "groups.6.2.f1.weight", "groups.6.2.f2.weight", "groups.6.2.o.weight", "groups.6.2.qkv.weight", "groups.6.3.f1.weight", "groups.6.3.f2.weight", "groups.6.3.o.weight", "groups.6.3.qkv.weight", "groups.7.0.f1.weight", "groups.7.0.f2.weight", "groups.7.0.o.weight", "groups.7.0.qkv.weight", "groups.7.1.f1.weight", "groups.7.1.f2.weight", "groups.7.1.o.weight", "groups.7.1.qkv.weight", "groups.7.2.f1.weight", "groups.7.2.f2.weight", "groups.7.2.o.weight", "groups.7.2.qkv.weight", "groups.7.3.f1.weight", "groups.7.3.f2.weight", "groups.7.3.o.weight", "groups.7.3.qkv.weight", "groups.8.0.f1.weight", "groups.8.0.f2.weight", "groups.8.0.o.weight", "groups.8.0.qkv.weight", "groups.8.1.f1.weight", "groups.8.1.f2.weight", "groups.8.1.o.weight", "groups.8.1.qkv.weight", "groups.8.2.f1.weight", "groups.8.2.f2.weight", "groups.8.2.o.weight", "groups.8.2.qkv.weight", "groups.8.3.f1.weight", "groups.8.3.f2.weight", "groups.8.3.o.weight", "groups.8.3.qkv.weight", "groups.9.0.f1.weight", "groups.9.0.f2.weight", "groups.9.0.o.weight", "groups.9.0.qkv.weight", "groups.9.1.f1.weight", "groups.9.1.f2.weight", "groups.9.1.o.weight", "groups.9.1.qkv.weight", "groups.9.2.f1.weight", "groups.9.2.f2.weight", "groups.9.2.o.weight", "groups.9.2.qkv.weight", "groups.9.3.f1.weight", "groups.9.3.f2.weight", "groups.9.3.o.weight", "groups.9.3.qkv.weight", "heads.0.weight", "heads.1.weight", "heads.10.weight", "heads.11.weight", "heads.12.weight", "heads.13.weight", "heads.14.weight", "heads.15.weight", "heads.16.weight", "heads.17.weight", "heads.18.weight", "heads.19.weight", "heads.2.weight", "heads.20.weight", "heads.21.weight", "heads.22.weight", "heads.23.weight", "heads.24.weight", "heads.25.weight", "heads.26.weight", "heads.27.weight", "heads.28.weight", "heads.29.weight", "heads.3.weight", "heads.30.weight", "heads.4.weight", "heads.5.weight", "heads.6.weight", "heads.7.weight", "heads.8.weight", "heads.9.weight", "in_proj.weight", "layers.2.mlp.down_proj.weight", "layers.27.mlp.down_proj.weight" ], "tensors": { "embed_tokens.weight": { "bits": 8, "group_size": 32, "shape": [ 194548, 2048 ] }, "layers.0.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.0.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.0.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.0.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.1.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.1.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.1.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.1.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.2.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.2.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.2.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.3.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.3.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.3.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.3.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.4.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.4.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.4.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.4.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.5.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.5.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.5.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.5.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.6.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.6.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.6.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.6.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.7.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.7.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.7.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.7.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.8.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.8.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.8.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.8.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.9.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.9.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.9.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.9.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.10.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.10.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.10.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.10.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.11.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.11.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.11.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.11.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.12.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.12.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.12.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.12.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.13.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.13.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.13.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.13.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.14.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.14.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.14.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.14.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.15.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.15.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.15.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.15.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.16.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.16.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.16.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.16.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.17.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.17.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.17.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.17.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.18.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.18.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.18.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.18.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.19.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.19.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.19.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.19.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.20.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.20.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.20.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.20.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.21.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.21.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.21.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.21.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.22.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.22.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.22.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.22.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.23.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.23.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.23.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.23.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.24.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.24.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.24.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.24.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.25.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.25.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.25.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.25.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.26.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.26.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.26.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "layers.26.mlp.down_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 6144 ] }, "layers.27.self_attn.qkv_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 4096, 2048 ] }, "layers.27.mlp.gate_up_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 12288, 2048 ] }, "layers.27.self_attn.o_proj.weight": { "bits": 8, "group_size": 32, "shape": [ 2048, 2048 ] }, "table.weight": { "bits": 8, "group_size": 32, "shape": [ 65536, 2048 ] } }, "scan": { "file": "tests/results/quant-scan-8b64.json", "keep_backbone": 2, "keep_depth": 999, "rule": "the N most sensitive tensors of each part stay 16-bit" } }