| { |
| "schema": "dynquant_checkpoint_v2", |
| "dynquant_core": "0.5.3", |
| "quantization_config": { |
| "quant_method": "dynquant", |
| "checkpoint_format": "dynquant-packed", |
| "schema_version": 1, |
| "version": "0.5.3", |
| "group_size": 128, |
| "symmetric": false, |
| "lm_head_quantized": false, |
| "modules_to_not_convert": [], |
| "modules": { |
| "model.embed_tokens": { |
| "bits": 4, |
| "out_features": 151936 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.0.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.0.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.0.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.0.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.0.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.0.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.1.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.1.moe.sw2": { |
| "bits": 3, |
| "out_features": 1024 |
| }, |
| "model.layers.1.moe.sw3": { |
| "bits": 3, |
| "out_features": 1152 |
| }, |
| "model.layers.1.moe.w1": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.1.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.1.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.10.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.10.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.10.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.10.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.10.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.10.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.10.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.10.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.11.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.11.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.11.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.11.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.11.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.11.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.12.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.12.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.12.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.12.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.13.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.13.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.13.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.13.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.13.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.13.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.14.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.14.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.14.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.14.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.14.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.14.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.14.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.14.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.15.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.15.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.15.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.15.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.15.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.15.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.16.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.16.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.16.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.16.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.16.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.16.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.17.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.17.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.17.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.17.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.18.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.18.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.18.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.18.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.18.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.18.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.18.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.18.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.19.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.19.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.19.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.19.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.19.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.2.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.2.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.2.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.2.moe.w1": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.2.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.2.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.20.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.20.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.20.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.20.moe.w1": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.20.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.20.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.21.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.21.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.21.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.21.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.21.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.21.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.22.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.22.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.22.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.22.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.22.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.22.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.23.moe.sw1": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.23.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.23.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.23.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.23.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.23.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.3.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.3.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.3.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.3.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.3.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.4.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.4.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.4.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.4.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.5.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.5.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.5.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.5.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.6.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.6.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.6.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.6.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.6.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.6.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.6.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.6.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.7.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.7.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.7.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.7.moe.w1": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.7.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.7.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.8.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.8.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.8.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.8.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.9.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.9.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.9.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.9.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| } |
| } |
| }, |
| "files": [ |
| "model.safetensors" |
| ], |
| "accounting": { |
| "packed_nbytes": 685944832, |
| "dense_nbytes": 998912, |
| "total_nbytes": 686943744, |
| "quantized_elements": 1690697728, |
| "average_bits": 3.245736 |
| }, |
| "tied": { |
| "lm_head": "model.embed_tokens" |
| }, |
| "expert_banks": [ |
| "model.layers.0.moe.w1", |
| "model.layers.0.moe.w2", |
| "model.layers.0.moe.w3", |
| "model.layers.1.moe.w1", |
| "model.layers.1.moe.w2", |
| "model.layers.1.moe.w3", |
| "model.layers.10.moe.w1", |
| "model.layers.10.moe.w2", |
| "model.layers.10.moe.w3", |
| "model.layers.11.moe.w1", |
| "model.layers.11.moe.w2", |
| "model.layers.11.moe.w3", |
| "model.layers.12.moe.w1", |
| "model.layers.12.moe.w2", |
| "model.layers.12.moe.w3", |
| "model.layers.13.moe.w1", |
| "model.layers.13.moe.w2", |
| "model.layers.13.moe.w3", |
| "model.layers.14.moe.w1", |
| "model.layers.14.moe.w2", |
| "model.layers.14.moe.w3", |
| "model.layers.15.moe.w1", |
| "model.layers.15.moe.w2", |
| "model.layers.15.moe.w3", |
| "model.layers.16.moe.w1", |
| "model.layers.16.moe.w2", |
| "model.layers.16.moe.w3", |
| "model.layers.17.moe.w1", |
| "model.layers.17.moe.w2", |
| "model.layers.17.moe.w3", |
| "model.layers.18.moe.w1", |
| "model.layers.18.moe.w2", |
| "model.layers.18.moe.w3", |
| "model.layers.19.moe.w1", |
| "model.layers.19.moe.w2", |
| "model.layers.19.moe.w3", |
| "model.layers.2.moe.w1", |
| "model.layers.2.moe.w2", |
| "model.layers.2.moe.w3", |
| "model.layers.20.moe.w1", |
| "model.layers.20.moe.w2", |
| "model.layers.20.moe.w3", |
| "model.layers.21.moe.w1", |
| "model.layers.21.moe.w2", |
| "model.layers.21.moe.w3", |
| "model.layers.22.moe.w1", |
| "model.layers.22.moe.w2", |
| "model.layers.22.moe.w3", |
| "model.layers.23.moe.w1", |
| "model.layers.23.moe.w2", |
| "model.layers.23.moe.w3", |
| "model.layers.3.moe.w1", |
| "model.layers.3.moe.w2", |
| "model.layers.3.moe.w3", |
| "model.layers.4.moe.w1", |
| "model.layers.4.moe.w2", |
| "model.layers.4.moe.w3", |
| "model.layers.5.moe.w1", |
| "model.layers.5.moe.w2", |
| "model.layers.5.moe.w3", |
| "model.layers.6.moe.w1", |
| "model.layers.6.moe.w2", |
| "model.layers.6.moe.w3", |
| "model.layers.7.moe.w1", |
| "model.layers.7.moe.w2", |
| "model.layers.7.moe.w3", |
| "model.layers.8.moe.w1", |
| "model.layers.8.moe.w2", |
| "model.layers.8.moe.w3", |
| "model.layers.9.moe.w1", |
| "model.layers.9.moe.w2", |
| "model.layers.9.moe.w3" |
| ], |
| "provenance": { |
| "dynquant_core": "0.5.3", |
| "model": "runs/ft/model", |
| "stats": null, |
| "allocator": "sensitivity", |
| "widths_from": "runs/ft/maps.json [3.25]", |
| "predicted_nbytes": 686943744 |
| }, |
| "layers": { |
| "model.embed_tokens": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 151936, |
| 1024 |
| ], |
| "num_rows": 151936, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 82653184, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958658, |
| "clip_improvement": 0.118229 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952189, |
| "clip_improvement": 0.112491 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954644, |
| "clip_improvement": 0.112495 |
| }, |
| "model.layers.0.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958767, |
| "clip_improvement": 0.112372 |
| }, |
| "model.layers.0.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.111493 |
| }, |
| "model.layers.0.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953885, |
| "clip_improvement": 0.11198 |
| }, |
| "model.layers.0.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353319 |
| }, |
| "model.layers.0.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.250049 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.11162 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954346, |
| "clip_improvement": 0.111889 |
| }, |
| "model.layers.1.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953993, |
| "clip_improvement": 0.112511 |
| }, |
| "model.layers.1.moe.sw2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 479232, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.248157 |
| }, |
| "model.layers.1.moe.sw3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 479232, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999457, |
| "clip_improvement": 0.250475 |
| }, |
| "model.layers.1.moe.w1": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353228 |
| }, |
| "model.layers.1.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353234 |
| }, |
| "model.layers.1.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353331 |
| }, |
| "model.layers.10.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954997, |
| "clip_improvement": 0.111521 |
| }, |
| "model.layers.10.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111474 |
| }, |
| "model.layers.10.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.111773 |
| }, |
| "model.layers.10.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953234, |
| "clip_improvement": 0.109561 |
| }, |
| "model.layers.10.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953342, |
| "clip_improvement": 0.110917 |
| }, |
| "model.layers.10.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954163, |
| "clip_improvement": 0.111924 |
| }, |
| "model.layers.10.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.249616 |
| }, |
| "model.layers.10.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.250139 |
| }, |
| "model.layers.11.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953342, |
| "clip_improvement": 0.112209 |
| }, |
| "model.layers.11.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.111731 |
| }, |
| "model.layers.11.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953125, |
| "clip_improvement": 0.111907 |
| }, |
| "model.layers.11.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952338, |
| "clip_improvement": 0.111378 |
| }, |
| "model.layers.11.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999871, |
| "clip_improvement": 0.249867 |
| }, |
| "model.layers.11.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99979, |
| "clip_improvement": 0.249888 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952799, |
| "clip_improvement": 0.111045 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.111137 |
| }, |
| "model.layers.12.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.950629, |
| "clip_improvement": 0.110166 |
| }, |
| "model.layers.12.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955621, |
| "clip_improvement": 0.109931 |
| }, |
| "model.layers.12.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954536, |
| "clip_improvement": 0.111564 |
| }, |
| "model.layers.12.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953545, |
| "clip_improvement": 0.111491 |
| }, |
| "model.layers.12.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999824, |
| "clip_improvement": 0.250105 |
| }, |
| "model.layers.12.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.353438 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953939, |
| "clip_improvement": 0.11168 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.111625 |
| }, |
| "model.layers.13.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959201, |
| "clip_improvement": 0.113534 |
| }, |
| "model.layers.13.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956489, |
| "clip_improvement": 0.111952 |
| }, |
| "model.layers.13.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955187, |
| "clip_improvement": 0.111205 |
| }, |
| "model.layers.13.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953701, |
| "clip_improvement": 0.111727 |
| }, |
| "model.layers.13.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999858, |
| "clip_improvement": 0.249651 |
| }, |
| "model.layers.13.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.250087 |
| }, |
| "model.layers.14.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953857, |
| "clip_improvement": 0.111744 |
| }, |
| "model.layers.14.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955811, |
| "clip_improvement": 0.112209 |
| }, |
| "model.layers.14.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.111702 |
| }, |
| "model.layers.14.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958008, |
| "clip_improvement": 0.11119 |
| }, |
| "model.layers.14.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954027, |
| "clip_improvement": 0.111455 |
| }, |
| "model.layers.14.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250077 |
| }, |
| "model.layers.14.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.249781 |
| }, |
| "model.layers.15.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955512, |
| "clip_improvement": 0.112333 |
| }, |
| "model.layers.15.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.110183 |
| }, |
| "model.layers.15.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953776, |
| "clip_improvement": 0.111914 |
| }, |
| "model.layers.15.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954447, |
| "clip_improvement": 0.111795 |
| }, |
| "model.layers.15.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.249892 |
| }, |
| "model.layers.15.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250124 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.111251 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.109889 |
| }, |
| "model.layers.16.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.110828 |
| }, |
| "model.layers.16.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956489, |
| "clip_improvement": 0.110486 |
| }, |
| "model.layers.16.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959852, |
| "clip_improvement": 0.111776 |
| }, |
| "model.layers.16.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954197, |
| "clip_improvement": 0.111497 |
| }, |
| "model.layers.16.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353362 |
| }, |
| "model.layers.16.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.35321 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954671, |
| "clip_improvement": 0.111566 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.112852 |
| }, |
| "model.layers.17.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951389, |
| "clip_improvement": 0.110926 |
| }, |
| "model.layers.17.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955621, |
| "clip_improvement": 0.114065 |
| }, |
| "model.layers.17.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95638, |
| "clip_improvement": 0.110832 |
| }, |
| "model.layers.17.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954603, |
| "clip_improvement": 0.111657 |
| }, |
| "model.layers.17.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999973, |
| "clip_improvement": 0.353656 |
| }, |
| "model.layers.17.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.35341 |
| }, |
| "model.layers.18.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.112103 |
| }, |
| "model.layers.18.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.9552, |
| "clip_improvement": 0.111177 |
| }, |
| "model.layers.18.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953017, |
| "clip_improvement": 0.110424 |
| }, |
| "model.layers.18.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955729, |
| "clip_improvement": 0.111512 |
| }, |
| "model.layers.18.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954027, |
| "clip_improvement": 0.111633 |
| }, |
| "model.layers.18.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.35331 |
| }, |
| "model.layers.18.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353614 |
| }, |
| "model.layers.19.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952365, |
| "clip_improvement": 0.11146 |
| }, |
| "model.layers.19.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957465, |
| "clip_improvement": 0.112536 |
| }, |
| "model.layers.19.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.250005 |
| }, |
| "model.layers.19.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353536 |
| }, |
| "model.layers.19.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.353495 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951986, |
| "clip_improvement": 0.111035 |
| }, |
| "model.layers.2.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955322, |
| "clip_improvement": 0.110804 |
| }, |
| "model.layers.2.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955187, |
| "clip_improvement": 0.112738 |
| }, |
| "model.layers.2.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954427, |
| "clip_improvement": 0.111498 |
| }, |
| "model.layers.2.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954753, |
| "clip_improvement": 0.111969 |
| }, |
| "model.layers.2.moe.w1": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.35339 |
| }, |
| "model.layers.2.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.35348 |
| }, |
| "model.layers.2.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353625 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.112217 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955933, |
| "clip_improvement": 0.111425 |
| }, |
| "model.layers.20.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951063, |
| "clip_improvement": 0.11131 |
| }, |
| "model.layers.20.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953559, |
| "clip_improvement": 0.111117 |
| }, |
| "model.layers.20.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952908, |
| "clip_improvement": 0.111531 |
| }, |
| "model.layers.20.moe.w1": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353201 |
| }, |
| "model.layers.20.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.353628 |
| }, |
| "model.layers.20.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353323 |
| }, |
| "model.layers.21.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952677, |
| "clip_improvement": 0.111657 |
| }, |
| "model.layers.21.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953369, |
| "clip_improvement": 0.110681 |
| }, |
| "model.layers.21.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952908, |
| "clip_improvement": 0.111555 |
| }, |
| "model.layers.21.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956055, |
| "clip_improvement": 0.112092 |
| }, |
| "model.layers.21.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953234, |
| "clip_improvement": 0.113501 |
| }, |
| "model.layers.21.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250563 |
| }, |
| "model.layers.21.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999966, |
| "clip_improvement": 0.35342 |
| }, |
| "model.layers.21.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999973, |
| "clip_improvement": 0.353488 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955973, |
| "clip_improvement": 0.111722 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.110481 |
| }, |
| "model.layers.22.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953559, |
| "clip_improvement": 0.11164 |
| }, |
| "model.layers.22.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952908, |
| "clip_improvement": 0.111764 |
| }, |
| "model.layers.22.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953017, |
| "clip_improvement": 0.111665 |
| }, |
| "model.layers.22.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999851, |
| "clip_improvement": 0.249933 |
| }, |
| "model.layers.22.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.353878 |
| }, |
| "model.layers.22.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999973, |
| "clip_improvement": 0.353387 |
| }, |
| "model.layers.23.moe.sw1": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953844, |
| "clip_improvement": 0.11135 |
| }, |
| "model.layers.23.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999464, |
| "clip_improvement": 0.355379 |
| }, |
| "model.layers.23.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999932, |
| "clip_improvement": 0.353554 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953979, |
| "clip_improvement": 0.112017 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956055, |
| "clip_improvement": 0.111754 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958442, |
| "clip_improvement": 0.111402 |
| }, |
| "model.layers.3.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951931, |
| "clip_improvement": 0.110399 |
| }, |
| "model.layers.3.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.950846, |
| "clip_improvement": 0.110635 |
| }, |
| "model.layers.3.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.249583 |
| }, |
| "model.layers.3.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353328 |
| }, |
| "model.layers.3.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353477 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95874, |
| "clip_improvement": 0.112026 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957886, |
| "clip_improvement": 0.111266 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956177, |
| "clip_improvement": 0.112503 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957275, |
| "clip_improvement": 0.110555 |
| }, |
| "model.layers.4.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955729, |
| "clip_improvement": 0.110095 |
| }, |
| "model.layers.4.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955404, |
| "clip_improvement": 0.112383 |
| }, |
| "model.layers.4.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953993, |
| "clip_improvement": 0.111288 |
| }, |
| "model.layers.4.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.249834 |
| }, |
| "model.layers.4.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353369 |
| }, |
| "model.layers.4.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.3533 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.111981 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952271, |
| "clip_improvement": 0.10894 |
| }, |
| "model.layers.5.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953342, |
| "clip_improvement": 0.110832 |
| }, |
| "model.layers.5.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955404, |
| "clip_improvement": 0.111948 |
| }, |
| "model.layers.5.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958659, |
| "clip_improvement": 0.110161 |
| }, |
| "model.layers.5.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.249684 |
| }, |
| "model.layers.5.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.35354 |
| }, |
| "model.layers.5.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353415 |
| }, |
| "model.layers.6.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954224, |
| "clip_improvement": 0.112098 |
| }, |
| "model.layers.6.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953979, |
| "clip_improvement": 0.112167 |
| }, |
| "model.layers.6.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953451, |
| "clip_improvement": 0.110973 |
| }, |
| "model.layers.6.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957031, |
| "clip_improvement": 0.112692 |
| }, |
| "model.layers.6.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957357, |
| "clip_improvement": 0.112433 |
| }, |
| "model.layers.6.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.249893 |
| }, |
| "model.layers.6.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353306 |
| }, |
| "model.layers.6.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353463 |
| }, |
| "model.layers.7.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.113294 |
| }, |
| "model.layers.7.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955512, |
| "clip_improvement": 0.112441 |
| }, |
| "model.layers.7.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.111907 |
| }, |
| "model.layers.7.moe.w1": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250027 |
| }, |
| "model.layers.7.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353409 |
| }, |
| "model.layers.7.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353434 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954671, |
| "clip_improvement": 0.111955 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955566, |
| "clip_improvement": 0.108682 |
| }, |
| "model.layers.8.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956706, |
| "clip_improvement": 0.110425 |
| }, |
| "model.layers.8.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954753, |
| "clip_improvement": 0.111849 |
| }, |
| "model.layers.8.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957031, |
| "clip_improvement": 0.111082 |
| }, |
| "model.layers.8.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952881, |
| "clip_improvement": 0.111608 |
| }, |
| "model.layers.8.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999851, |
| "clip_improvement": 0.250149 |
| }, |
| "model.layers.8.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999803, |
| "clip_improvement": 0.249982 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954915, |
| "clip_improvement": 0.111643 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951782, |
| "clip_improvement": 0.111073 |
| }, |
| "model.layers.9.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95497, |
| "clip_improvement": 0.111141 |
| }, |
| "model.layers.9.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.113054 |
| }, |
| "model.layers.9.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953125, |
| "clip_improvement": 0.112049 |
| }, |
| "model.layers.9.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953722, |
| "clip_improvement": 0.111856 |
| }, |
| "model.layers.9.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.249884 |
| }, |
| "model.layers.9.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.250012 |
| } |
| } |
| } |
