{ "schema": "dynquant_checkpoint_v2", "dynquant_core": "0.5.3", "quantization_config": { "quant_method": "dynquant", "checkpoint_format": "dynquant-packed", "schema_version": 1, "version": "0.5.3", "group_size": 128, "symmetric": false, "lm_head_quantized": false, "modules_to_not_convert": [], "modules": { "model.embed_tokens": { "bits": 4, "out_features": 151936 }, "model.layers.0.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.0.conv.out_proj": { "bits": 8, "out_features": 1024 }, "model.layers.0.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.0.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.0.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.0.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.0.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.0.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.1.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.1.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.1.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.1.moe.sw2": { "bits": 3, "out_features": 1024 }, "model.layers.1.moe.sw3": { "bits": 3, "out_features": 1152 }, "model.layers.1.moe.w1": { "bits": 2, "out_features": 16384 }, "model.layers.1.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.1.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.10.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.10.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.10.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.10.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.10.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.10.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.10.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.10.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.11.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.11.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.11.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.11.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.11.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.11.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.11.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.11.self_attn.o_proj": { "bits": 8, "out_features": 1024 }, "model.layers.11.self_attn.q_proj": { "bits": 8, "out_features": 1024 }, "model.layers.11.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.12.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.12.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.12.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.12.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.12.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.12.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.12.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.12.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.13.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.13.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.13.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.13.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.13.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.13.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.13.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.13.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.14.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.14.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.14.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.14.moe.sw2": { "bits": 8, "out_features": 1024 }, "model.layers.14.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.14.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.14.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.14.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.15.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.15.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.15.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.15.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.15.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.15.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.15.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.15.self_attn.o_proj": { "bits": 8, "out_features": 1024 }, "model.layers.15.self_attn.q_proj": { "bits": 8, "out_features": 1024 }, "model.layers.15.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.16.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.16.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.16.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.16.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.16.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.16.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.16.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.16.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.17.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.17.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.17.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.17.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.17.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.17.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.17.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.17.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.18.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.18.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.18.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.18.moe.sw2": { "bits": 8, "out_features": 1024 }, "model.layers.18.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.18.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.18.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.18.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.19.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.19.moe.sw2": { "bits": 8, "out_features": 1024 }, "model.layers.19.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.19.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.19.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.19.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.19.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.19.self_attn.o_proj": { "bits": 8, "out_features": 1024 }, "model.layers.19.self_attn.q_proj": { "bits": 8, "out_features": 1024 }, "model.layers.19.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.2.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.2.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.2.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.2.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.2.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.2.moe.w1": { "bits": 2, "out_features": 16384 }, "model.layers.2.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.2.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.20.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.20.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.20.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.20.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.20.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.20.moe.w1": { "bits": 2, "out_features": 16384 }, "model.layers.20.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.20.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.21.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.21.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.21.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.21.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.21.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.21.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.21.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.21.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.22.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.22.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.22.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.22.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.22.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.22.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.22.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.22.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.23.moe.sw1": { "bits": 8, "out_features": 1152 }, "model.layers.23.moe.sw2": { "bits": 8, "out_features": 1024 }, "model.layers.23.moe.sw3": { "bits": 8, "out_features": 1152 }, "model.layers.23.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.23.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.23.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.23.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "out_features": 1024 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "out_features": 1024 }, "model.layers.23.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.3.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.3.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.3.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.3.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.3.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.3.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.3.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "out_features": 1024 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "out_features": 1024 }, "model.layers.3.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.4.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.4.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.4.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.4.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.4.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.4.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.4.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.4.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.5.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.5.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.5.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.5.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.5.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.5.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.5.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.5.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.6.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.6.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.6.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.6.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.6.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.6.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.6.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.6.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.7.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.7.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.7.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.7.moe.w1": { "bits": 3, "out_features": 16384 }, "model.layers.7.moe.w2": { "bits": 2, "out_features": 18432 }, "model.layers.7.moe.w3": { "bits": 2, "out_features": 16384 }, "model.layers.7.self_attn.k_proj": { "bits": 8, "out_features": 256 }, "model.layers.7.self_attn.o_proj": { "bits": 8, "out_features": 1024 }, "model.layers.7.self_attn.q_proj": { "bits": 8, "out_features": 1024 }, "model.layers.7.self_attn.v_proj": { "bits": 8, "out_features": 256 }, "model.layers.8.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.8.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.8.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.8.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.8.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.8.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.8.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.8.moe.w3": { "bits": 3, "out_features": 16384 }, "model.layers.9.conv.in_proj": { "bits": 4, "out_features": 3072 }, "model.layers.9.conv.out_proj": { "bits": 4, "out_features": 1024 }, "model.layers.9.moe.sw1": { "bits": 4, "out_features": 1152 }, "model.layers.9.moe.sw2": { "bits": 4, "out_features": 1024 }, "model.layers.9.moe.sw3": { "bits": 4, "out_features": 1152 }, "model.layers.9.moe.w1": { "bits": 4, "out_features": 16384 }, "model.layers.9.moe.w2": { "bits": 3, "out_features": 18432 }, "model.layers.9.moe.w3": { "bits": 3, "out_features": 16384 } } }, "files": [ "model.safetensors" ], "accounting": { "packed_nbytes": 685944832, "dense_nbytes": 998912, "total_nbytes": 686943744, "quantized_elements": 1690697728, "average_bits": 3.245736 }, "tied": { "lm_head": "model.embed_tokens" }, "expert_banks": [ "model.layers.0.moe.w1", "model.layers.0.moe.w2", "model.layers.0.moe.w3", "model.layers.1.moe.w1", "model.layers.1.moe.w2", "model.layers.1.moe.w3", "model.layers.10.moe.w1", "model.layers.10.moe.w2", "model.layers.10.moe.w3", "model.layers.11.moe.w1", "model.layers.11.moe.w2", "model.layers.11.moe.w3", "model.layers.12.moe.w1", "model.layers.12.moe.w2", "model.layers.12.moe.w3", "model.layers.13.moe.w1", "model.layers.13.moe.w2", "model.layers.13.moe.w3", "model.layers.14.moe.w1", "model.layers.14.moe.w2", "model.layers.14.moe.w3", "model.layers.15.moe.w1", "model.layers.15.moe.w2", "model.layers.15.moe.w3", "model.layers.16.moe.w1", "model.layers.16.moe.w2", "model.layers.16.moe.w3", "model.layers.17.moe.w1", "model.layers.17.moe.w2", "model.layers.17.moe.w3", "model.layers.18.moe.w1", "model.layers.18.moe.w2", "model.layers.18.moe.w3", "model.layers.19.moe.w1", "model.layers.19.moe.w2", "model.layers.19.moe.w3", "model.layers.2.moe.w1", "model.layers.2.moe.w2", "model.layers.2.moe.w3", "model.layers.20.moe.w1", "model.layers.20.moe.w2", "model.layers.20.moe.w3", "model.layers.21.moe.w1", "model.layers.21.moe.w2", "model.layers.21.moe.w3", "model.layers.22.moe.w1", "model.layers.22.moe.w2", "model.layers.22.moe.w3", "model.layers.23.moe.w1", "model.layers.23.moe.w2", "model.layers.23.moe.w3", "model.layers.3.moe.w1", "model.layers.3.moe.w2", "model.layers.3.moe.w3", "model.layers.4.moe.w1", "model.layers.4.moe.w2", "model.layers.4.moe.w3", "model.layers.5.moe.w1", "model.layers.5.moe.w2", "model.layers.5.moe.w3", "model.layers.6.moe.w1", "model.layers.6.moe.w2", "model.layers.6.moe.w3", "model.layers.7.moe.w1", "model.layers.7.moe.w2", "model.layers.7.moe.w3", "model.layers.8.moe.w1", "model.layers.8.moe.w2", "model.layers.8.moe.w3", "model.layers.9.moe.w1", "model.layers.9.moe.w2", "model.layers.9.moe.w3" ], "provenance": { "dynquant_core": "0.5.3", "model": "runs/ft/model", "stats": null, "allocator": "sensitivity", "widths_from": "runs/ft/maps.json [3.25]", "predicted_nbytes": 686943744 }, "layers": { "model.embed_tokens": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 151936, 1024 ], "num_rows": 151936, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 82653184, "bits_per_weight": 4.25, "clipped_fraction": 0.958658, "clip_improvement": 0.118229 }, "model.layers.0.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.952189, "clip_improvement": 0.112491 }, "model.layers.0.conv.out_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.0.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954644, "clip_improvement": 0.112495 }, "model.layers.0.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.958767, "clip_improvement": 0.112372 }, "model.layers.0.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954861, "clip_improvement": 0.111493 }, "model.layers.0.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.953885, "clip_improvement": 0.11198 }, "model.layers.0.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999993, "clip_improvement": 0.353319 }, "model.layers.0.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.250049 }, "model.layers.1.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.953735, "clip_improvement": 0.11162 }, "model.layers.1.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954346, "clip_improvement": 0.111889 }, "model.layers.1.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953993, "clip_improvement": 0.112511 }, "model.layers.1.moe.sw2": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 479232, "bits_per_weight": 3.25, "clipped_fraction": 1.0, "clip_improvement": 0.248157 }, "model.layers.1.moe.sw3": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 479232, "bits_per_weight": 3.25, "clipped_fraction": 0.999457, "clip_improvement": 0.250475 }, "model.layers.1.moe.w1": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353228 }, "model.layers.1.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353234 }, "model.layers.1.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353331 }, "model.layers.10.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954997, "clip_improvement": 0.111521 }, "model.layers.10.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954468, "clip_improvement": 0.111474 }, "model.layers.10.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954102, "clip_improvement": 0.111773 }, "model.layers.10.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953234, "clip_improvement": 0.109561 }, "model.layers.10.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953342, "clip_improvement": 0.110917 }, "model.layers.10.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954163, "clip_improvement": 0.111924 }, "model.layers.10.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.249616 }, "model.layers.10.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.250139 }, "model.layers.11.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953342, "clip_improvement": 0.112209 }, "model.layers.11.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954102, "clip_improvement": 0.111731 }, "model.layers.11.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953125, "clip_improvement": 0.111907 }, "model.layers.11.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.952338, "clip_improvement": 0.111378 }, "model.layers.11.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999871, "clip_improvement": 0.249867 }, "model.layers.11.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.99979, "clip_improvement": 0.249888 }, "model.layers.11.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.self_attn.o_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.11.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.12.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.952799, "clip_improvement": 0.111045 }, "model.layers.12.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954834, "clip_improvement": 0.111137 }, "model.layers.12.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.950629, "clip_improvement": 0.110166 }, "model.layers.12.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955621, "clip_improvement": 0.109931 }, "model.layers.12.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954536, "clip_improvement": 0.111564 }, "model.layers.12.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.953545, "clip_improvement": 0.111491 }, "model.layers.12.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999824, "clip_improvement": 0.250105 }, "model.layers.12.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.99998, "clip_improvement": 0.353438 }, "model.layers.13.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.953939, "clip_improvement": 0.11168 }, "model.layers.13.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954712, "clip_improvement": 0.111625 }, "model.layers.13.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.959201, "clip_improvement": 0.113534 }, "model.layers.13.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.956489, "clip_improvement": 0.111952 }, "model.layers.13.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955187, "clip_improvement": 0.111205 }, "model.layers.13.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.953701, "clip_improvement": 0.111727 }, "model.layers.13.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999858, "clip_improvement": 0.249651 }, "model.layers.13.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.250087 }, "model.layers.14.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.953857, "clip_improvement": 0.111744 }, "model.layers.14.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.955811, "clip_improvement": 0.112209 }, "model.layers.14.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954861, "clip_improvement": 0.111702 }, "model.layers.14.moe.sw2": { "bits": 8, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.14.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.958008, "clip_improvement": 0.11119 }, "model.layers.14.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954027, "clip_improvement": 0.111455 }, "model.layers.14.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250077 }, "model.layers.14.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.249781 }, "model.layers.15.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955512, "clip_improvement": 0.112333 }, "model.layers.15.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954861, "clip_improvement": 0.110183 }, "model.layers.15.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953776, "clip_improvement": 0.111914 }, "model.layers.15.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954447, "clip_improvement": 0.111795 }, "model.layers.15.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.249892 }, "model.layers.15.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250124 }, "model.layers.15.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.self_attn.o_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.15.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.16.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.953735, "clip_improvement": 0.111251 }, "model.layers.16.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954834, "clip_improvement": 0.109889 }, "model.layers.16.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954102, "clip_improvement": 0.110828 }, "model.layers.16.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.956489, "clip_improvement": 0.110486 }, "model.layers.16.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.959852, "clip_improvement": 0.111776 }, "model.layers.16.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954197, "clip_improvement": 0.111497 }, "model.layers.16.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999993, "clip_improvement": 0.353362 }, "model.layers.16.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.35321 }, "model.layers.17.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954671, "clip_improvement": 0.111566 }, "model.layers.17.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.954834, "clip_improvement": 0.112852 }, "model.layers.17.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.951389, "clip_improvement": 0.110926 }, "model.layers.17.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955621, "clip_improvement": 0.114065 }, "model.layers.17.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.95638, "clip_improvement": 0.110832 }, "model.layers.17.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954603, "clip_improvement": 0.111657 }, "model.layers.17.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999973, "clip_improvement": 0.353656 }, "model.layers.17.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.99998, "clip_improvement": 0.35341 }, "model.layers.18.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.953735, "clip_improvement": 0.112103 }, "model.layers.18.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.9552, "clip_improvement": 0.111177 }, "model.layers.18.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953017, "clip_improvement": 0.110424 }, "model.layers.18.moe.sw2": { "bits": 8, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.18.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955729, "clip_improvement": 0.111512 }, "model.layers.18.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.954027, "clip_improvement": 0.111633 }, "model.layers.18.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.99998, "clip_improvement": 0.35331 }, "model.layers.18.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999986, "clip_improvement": 0.353614 }, "model.layers.19.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.952365, "clip_improvement": 0.11146 }, "model.layers.19.moe.sw2": { "bits": 8, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.957465, "clip_improvement": 0.112536 }, "model.layers.19.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.250005 }, "model.layers.19.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999993, "clip_improvement": 0.353536 }, "model.layers.19.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.99998, "clip_improvement": 0.353495 }, "model.layers.19.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.self_attn.o_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.19.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.2.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.951986, "clip_improvement": 0.111035 }, "model.layers.2.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.955322, "clip_improvement": 0.110804 }, "model.layers.2.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955187, "clip_improvement": 0.112738 }, "model.layers.2.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954427, "clip_improvement": 0.111498 }, "model.layers.2.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954753, "clip_improvement": 0.111969 }, "model.layers.2.moe.w1": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.35339 }, "model.layers.2.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.35348 }, "model.layers.2.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353625 }, "model.layers.20.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954712, "clip_improvement": 0.112217 }, "model.layers.20.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.955933, "clip_improvement": 0.111425 }, "model.layers.20.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.951063, "clip_improvement": 0.11131 }, "model.layers.20.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953559, "clip_improvement": 0.111117 }, "model.layers.20.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.952908, "clip_improvement": 0.111531 }, "model.layers.20.moe.w1": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353201 }, "model.layers.20.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.99998, "clip_improvement": 0.353628 }, "model.layers.20.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999986, "clip_improvement": 0.353323 }, "model.layers.21.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.952677, "clip_improvement": 0.111657 }, "model.layers.21.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.953369, "clip_improvement": 0.110681 }, "model.layers.21.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.952908, "clip_improvement": 0.111555 }, "model.layers.21.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.956055, "clip_improvement": 0.112092 }, "model.layers.21.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953234, "clip_improvement": 0.113501 }, "model.layers.21.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999844, "clip_improvement": 0.250563 }, "model.layers.21.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999966, "clip_improvement": 0.35342 }, "model.layers.21.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999973, "clip_improvement": 0.353488 }, "model.layers.22.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.955973, "clip_improvement": 0.111722 }, "model.layers.22.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.953735, "clip_improvement": 0.110481 }, "model.layers.22.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953559, "clip_improvement": 0.11164 }, "model.layers.22.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.952908, "clip_improvement": 0.111764 }, "model.layers.22.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953017, "clip_improvement": 0.111665 }, "model.layers.22.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999851, "clip_improvement": 0.249933 }, "model.layers.22.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999864, "clip_improvement": 0.353878 }, "model.layers.22.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999973, "clip_improvement": 0.353387 }, "model.layers.23.moe.sw1": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.moe.sw2": { "bits": 8, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.moe.sw3": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1216512, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.953844, "clip_improvement": 0.11135 }, "model.layers.23.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999464, "clip_improvement": 0.355379 }, "model.layers.23.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999932, "clip_improvement": 0.353554 }, "model.layers.23.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.23.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.953979, "clip_improvement": 0.112017 }, "model.layers.23.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.956055, "clip_improvement": 0.111754 }, "model.layers.23.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.3.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.958442, "clip_improvement": 0.111402 }, "model.layers.3.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.951931, "clip_improvement": 0.110399 }, "model.layers.3.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.950846, "clip_improvement": 0.110635 }, "model.layers.3.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999797, "clip_improvement": 0.249583 }, "model.layers.3.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353328 }, "model.layers.3.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353477 }, "model.layers.3.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.3.self_attn.o_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.95874, "clip_improvement": 0.112026 }, "model.layers.3.self_attn.q_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.957886, "clip_improvement": 0.111266 }, "model.layers.3.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.4.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.956177, "clip_improvement": 0.112503 }, "model.layers.4.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.957275, "clip_improvement": 0.110555 }, "model.layers.4.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955729, "clip_improvement": 0.110095 }, "model.layers.4.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955404, "clip_improvement": 0.112383 }, "model.layers.4.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953993, "clip_improvement": 0.111288 }, "model.layers.4.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.99983, "clip_improvement": 0.249834 }, "model.layers.4.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999993, "clip_improvement": 0.353369 }, "model.layers.4.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.3533 }, "model.layers.5.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954712, "clip_improvement": 0.111981 }, "model.layers.5.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.952271, "clip_improvement": 0.10894 }, "model.layers.5.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953342, "clip_improvement": 0.110832 }, "model.layers.5.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955404, "clip_improvement": 0.111948 }, "model.layers.5.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.958659, "clip_improvement": 0.110161 }, "model.layers.5.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999878, "clip_improvement": 0.249684 }, "model.layers.5.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.35354 }, "model.layers.5.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353415 }, "model.layers.6.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954224, "clip_improvement": 0.112098 }, "model.layers.6.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.953979, "clip_improvement": 0.112167 }, "model.layers.6.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953451, "clip_improvement": 0.110973 }, "model.layers.6.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.957031, "clip_improvement": 0.112692 }, "model.layers.6.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.957357, "clip_improvement": 0.112433 }, "model.layers.6.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.249893 }, "model.layers.6.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999986, "clip_improvement": 0.353306 }, "model.layers.6.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999993, "clip_improvement": 0.353463 }, "model.layers.7.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955078, "clip_improvement": 0.113294 }, "model.layers.7.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955512, "clip_improvement": 0.112441 }, "model.layers.7.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.955078, "clip_improvement": 0.111907 }, "model.layers.7.moe.w1": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999837, "clip_improvement": 0.250027 }, "model.layers.7.moe.w2": { "bits": 2, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 1.0, "clip_improvement": 0.353409 }, "model.layers.7.moe.w3": { "bits": 2, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 8, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 5308416, "bits_per_weight": 2.25, "clipped_fraction": 0.999986, "clip_improvement": 0.353434 }, "model.layers.7.self_attn.k_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.self_attn.o_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.self_attn.q_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1081344, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.7.self_attn.v_proj": { "bits": 8, "group_size": 128, "in_features": 1024, "logical_shape": [ 256, 1024 ], "num_rows": 256, "num_groups": 8, "words_per_group": 32, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 270336, "bits_per_weight": 8.25, "clipped_fraction": 0.0, "clip_improvement": 0.0 }, "model.layers.8.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954671, "clip_improvement": 0.111955 }, "model.layers.8.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.955566, "clip_improvement": 0.108682 }, "model.layers.8.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.956706, "clip_improvement": 0.110425 }, "model.layers.8.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954753, "clip_improvement": 0.111849 }, "model.layers.8.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.957031, "clip_improvement": 0.111082 }, "model.layers.8.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.952881, "clip_improvement": 0.111608 }, "model.layers.8.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999851, "clip_improvement": 0.250149 }, "model.layers.8.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999803, "clip_improvement": 0.249982 }, "model.layers.9.conv.in_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 3072, 1024 ], "num_rows": 3072, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 1671168, "bits_per_weight": 4.25, "clipped_fraction": 0.954915, "clip_improvement": 0.111643 }, "model.layers.9.conv.out_proj": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1024, 1024 ], "num_rows": 1024, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 557056, "bits_per_weight": 4.25, "clipped_fraction": 0.951782, "clip_improvement": 0.111073 }, "model.layers.9.moe.sw1": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.95497, "clip_improvement": 0.111141 }, "model.layers.9.moe.sw2": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 1024, 1152 ], "num_rows": 1024, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.954861, "clip_improvement": 0.113054 }, "model.layers.9.moe.sw3": { "bits": 4, "group_size": 128, "in_features": 1024, "logical_shape": [ 1152, 1024 ], "num_rows": 1152, "num_groups": 8, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 626688, "bits_per_weight": 4.25, "clipped_fraction": 0.953125, "clip_improvement": 0.112049 }, "model.layers.9.moe.w1": { "bits": 4, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 16, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 10027008, "bits_per_weight": 4.25, "clipped_fraction": 0.953722, "clip_improvement": 0.111856 }, "model.layers.9.moe.w2": { "bits": 3, "group_size": 128, "in_features": 1024, "logical_shape": [ 16, 1152, 1024 ], "num_rows": 18432, "num_groups": 8, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999864, "clip_improvement": 0.249884 }, "model.layers.9.moe.w3": { "bits": 3, "group_size": 128, "in_features": 1152, "logical_shape": [ 16, 1024, 1152 ], "num_rows": 16384, "num_groups": 9, "words_per_group": 12, "row_offset": 0, "layout": "linear", "symmetric": false, "has_offsets": true, "compute_dtype": "bfloat16", "nbytes": 7667712, "bits_per_weight": 3.25, "clipped_fraction": 0.999817, "clip_improvement": 0.250012 } } }