| { |
| "schema": "dynquant_checkpoint_v2", |
| "dynquant_core": "0.5.3", |
| "quantization_config": { |
| "quant_method": "dynquant", |
| "checkpoint_format": "dynquant-packed", |
| "schema_version": 1, |
| "version": "0.5.3", |
| "group_size": 128, |
| "symmetric": false, |
| "lm_head_quantized": false, |
| "modules_to_not_convert": [], |
| "modules": { |
| "model.embed_tokens": { |
| "bits": 8, |
| "out_features": 151936 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 8, |
| "out_features": 3072 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.0.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.0.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.0.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.0.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.0.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.0.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.1.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.1.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.1.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.1.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.1.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.1.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.10.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.10.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.10.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.10.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.10.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.10.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.10.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.10.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.11.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.11.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.11.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.11.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.11.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.11.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.12.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.12.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.12.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.12.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.12.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.13.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.13.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.13.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.13.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.13.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.13.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.14.conv.in_proj": { |
| "bits": 8, |
| "out_features": 3072 |
| }, |
| "model.layers.14.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.14.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.14.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.14.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.14.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.14.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.14.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.15.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.15.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.15.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.15.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.15.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.15.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 8, |
| "out_features": 3072 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.16.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.16.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.16.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.16.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.16.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.16.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.17.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.17.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.17.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.17.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.17.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.18.conv.in_proj": { |
| "bits": 8, |
| "out_features": 3072 |
| }, |
| "model.layers.18.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.18.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.18.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.18.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.18.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.18.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.18.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.19.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.19.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.19.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.19.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.19.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.2.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.2.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.2.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.2.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.2.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.2.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.2.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.20.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.20.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.20.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.20.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.20.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.20.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.21.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.21.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.21.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.21.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.21.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.21.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.21.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.22.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.22.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.22.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.22.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.22.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.22.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.23.moe.sw1": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.23.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.23.moe.sw3": { |
| "bits": 8, |
| "out_features": 1152 |
| }, |
| "model.layers.23.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.23.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.23.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.3.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.3.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.3.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.3.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.3.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.3.moe.w3": { |
| "bits": 2, |
| "out_features": 16384 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.4.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.4.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.4.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.4.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.4.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.5.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.5.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.5.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.5.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.5.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.6.conv.in_proj": { |
| "bits": 8, |
| "out_features": 3072 |
| }, |
| "model.layers.6.conv.out_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.6.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.6.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.6.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.6.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.6.moe.w2": { |
| "bits": 2, |
| "out_features": 18432 |
| }, |
| "model.layers.6.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.7.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.7.moe.sw2": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.7.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.7.moe.w2": { |
| "bits": 3, |
| "out_features": 18432 |
| }, |
| "model.layers.7.moe.w3": { |
| "bits": 3, |
| "out_features": 16384 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 8, |
| "out_features": 1024 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "out_features": 256 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.8.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.8.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.8.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.8.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.8.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 4, |
| "out_features": 3072 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.moe.sw1": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.9.moe.sw2": { |
| "bits": 4, |
| "out_features": 1024 |
| }, |
| "model.layers.9.moe.sw3": { |
| "bits": 4, |
| "out_features": 1152 |
| }, |
| "model.layers.9.moe.w1": { |
| "bits": 4, |
| "out_features": 16384 |
| }, |
| "model.layers.9.moe.w2": { |
| "bits": 4, |
| "out_features": 18432 |
| }, |
| "model.layers.9.moe.w3": { |
| "bits": 4, |
| "out_features": 16384 |
| } |
| } |
| }, |
| "files": [ |
| "model.safetensors" |
| ], |
| "accounting": { |
| "packed_nbytes": 897003520, |
| "dense_nbytes": 998912, |
| "total_nbytes": 898002432, |
| "quantized_elements": 1690697728, |
| "average_bits": 4.244418 |
| }, |
| "tied": { |
| "lm_head": "model.embed_tokens" |
| }, |
| "expert_banks": [ |
| "model.layers.0.moe.w1", |
| "model.layers.0.moe.w2", |
| "model.layers.0.moe.w3", |
| "model.layers.1.moe.w1", |
| "model.layers.1.moe.w2", |
| "model.layers.1.moe.w3", |
| "model.layers.10.moe.w1", |
| "model.layers.10.moe.w2", |
| "model.layers.10.moe.w3", |
| "model.layers.11.moe.w1", |
| "model.layers.11.moe.w2", |
| "model.layers.11.moe.w3", |
| "model.layers.12.moe.w1", |
| "model.layers.12.moe.w2", |
| "model.layers.12.moe.w3", |
| "model.layers.13.moe.w1", |
| "model.layers.13.moe.w2", |
| "model.layers.13.moe.w3", |
| "model.layers.14.moe.w1", |
| "model.layers.14.moe.w2", |
| "model.layers.14.moe.w3", |
| "model.layers.15.moe.w1", |
| "model.layers.15.moe.w2", |
| "model.layers.15.moe.w3", |
| "model.layers.16.moe.w1", |
| "model.layers.16.moe.w2", |
| "model.layers.16.moe.w3", |
| "model.layers.17.moe.w1", |
| "model.layers.17.moe.w2", |
| "model.layers.17.moe.w3", |
| "model.layers.18.moe.w1", |
| "model.layers.18.moe.w2", |
| "model.layers.18.moe.w3", |
| "model.layers.19.moe.w1", |
| "model.layers.19.moe.w2", |
| "model.layers.19.moe.w3", |
| "model.layers.2.moe.w1", |
| "model.layers.2.moe.w2", |
| "model.layers.2.moe.w3", |
| "model.layers.20.moe.w1", |
| "model.layers.20.moe.w2", |
| "model.layers.20.moe.w3", |
| "model.layers.21.moe.w1", |
| "model.layers.21.moe.w2", |
| "model.layers.21.moe.w3", |
| "model.layers.22.moe.w1", |
| "model.layers.22.moe.w2", |
| "model.layers.22.moe.w3", |
| "model.layers.23.moe.w1", |
| "model.layers.23.moe.w2", |
| "model.layers.23.moe.w3", |
| "model.layers.3.moe.w1", |
| "model.layers.3.moe.w2", |
| "model.layers.3.moe.w3", |
| "model.layers.4.moe.w1", |
| "model.layers.4.moe.w2", |
| "model.layers.4.moe.w3", |
| "model.layers.5.moe.w1", |
| "model.layers.5.moe.w2", |
| "model.layers.5.moe.w3", |
| "model.layers.6.moe.w1", |
| "model.layers.6.moe.w2", |
| "model.layers.6.moe.w3", |
| "model.layers.7.moe.w1", |
| "model.layers.7.moe.w2", |
| "model.layers.7.moe.w3", |
| "model.layers.8.moe.w1", |
| "model.layers.8.moe.w2", |
| "model.layers.8.moe.w3", |
| "model.layers.9.moe.w1", |
| "model.layers.9.moe.w2", |
| "model.layers.9.moe.w3" |
| ], |
| "provenance": { |
| "dynquant_core": "0.5.3", |
| "model": "runs/ft/model", |
| "stats": null, |
| "allocator": "sensitivity", |
| "widths_from": "runs/ft/maps.json [4.25]", |
| "predicted_nbytes": 898002432 |
| }, |
| "layers": { |
| "model.embed_tokens": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 151936, |
| 1024 |
| ], |
| "num_rows": 151936, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 160444416, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 3244032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954644, |
| "clip_improvement": 0.112495 |
| }, |
| "model.layers.0.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.0.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953885, |
| "clip_improvement": 0.11198 |
| }, |
| "model.layers.0.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999993, |
| "clip_improvement": 0.353319 |
| }, |
| "model.layers.0.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111985 |
| }, |
| "model.layers.1.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.11162 |
| }, |
| "model.layers.1.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954346, |
| "clip_improvement": 0.111889 |
| }, |
| "model.layers.1.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953993, |
| "clip_improvement": 0.112511 |
| }, |
| "model.layers.1.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958767, |
| "clip_improvement": 0.112735 |
| }, |
| "model.layers.1.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956055, |
| "clip_improvement": 0.111175 |
| }, |
| "model.layers.1.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953301, |
| "clip_improvement": 0.111398 |
| }, |
| "model.layers.1.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353234 |
| }, |
| "model.layers.1.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353331 |
| }, |
| "model.layers.10.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954997, |
| "clip_improvement": 0.111521 |
| }, |
| "model.layers.10.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954468, |
| "clip_improvement": 0.111474 |
| }, |
| "model.layers.10.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.111773 |
| }, |
| "model.layers.10.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.10.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954163, |
| "clip_improvement": 0.111924 |
| }, |
| "model.layers.10.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952053, |
| "clip_improvement": 0.111265 |
| }, |
| "model.layers.10.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.250139 |
| }, |
| "model.layers.11.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953342, |
| "clip_improvement": 0.112209 |
| }, |
| "model.layers.11.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.111731 |
| }, |
| "model.layers.11.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953125, |
| "clip_improvement": 0.111907 |
| }, |
| "model.layers.11.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952338, |
| "clip_improvement": 0.111378 |
| }, |
| "model.layers.11.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953803, |
| "clip_improvement": 0.112095 |
| }, |
| "model.layers.11.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953552, |
| "clip_improvement": 0.111801 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.12.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952799, |
| "clip_improvement": 0.111045 |
| }, |
| "model.layers.12.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.111137 |
| }, |
| "model.layers.12.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.950629, |
| "clip_improvement": 0.110166 |
| }, |
| "model.layers.12.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955621, |
| "clip_improvement": 0.109931 |
| }, |
| "model.layers.12.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954536, |
| "clip_improvement": 0.111564 |
| }, |
| "model.layers.12.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953545, |
| "clip_improvement": 0.111491 |
| }, |
| "model.layers.12.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999824, |
| "clip_improvement": 0.250105 |
| }, |
| "model.layers.12.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999824, |
| "clip_improvement": 0.25004 |
| }, |
| "model.layers.13.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953939, |
| "clip_improvement": 0.11168 |
| }, |
| "model.layers.13.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.13.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959201, |
| "clip_improvement": 0.113534 |
| }, |
| "model.layers.13.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956489, |
| "clip_improvement": 0.111952 |
| }, |
| "model.layers.13.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955187, |
| "clip_improvement": 0.111205 |
| }, |
| "model.layers.13.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953701, |
| "clip_improvement": 0.111727 |
| }, |
| "model.layers.13.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954, |
| "clip_improvement": 0.111756 |
| }, |
| "model.layers.13.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953715, |
| "clip_improvement": 0.111681 |
| }, |
| "model.layers.14.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 3244032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.111702 |
| }, |
| "model.layers.14.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.14.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954027, |
| "clip_improvement": 0.111455 |
| }, |
| "model.layers.14.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250077 |
| }, |
| "model.layers.14.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954624, |
| "clip_improvement": 0.112067 |
| }, |
| "model.layers.15.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955512, |
| "clip_improvement": 0.112333 |
| }, |
| "model.layers.15.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.110183 |
| }, |
| "model.layers.15.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953776, |
| "clip_improvement": 0.111914 |
| }, |
| "model.layers.15.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954447, |
| "clip_improvement": 0.111795 |
| }, |
| "model.layers.15.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953254, |
| "clip_improvement": 0.111632 |
| }, |
| "model.layers.15.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250124 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 3244032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.16.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954102, |
| "clip_improvement": 0.110828 |
| }, |
| "model.layers.16.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956489, |
| "clip_improvement": 0.110486 |
| }, |
| "model.layers.16.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.959852, |
| "clip_improvement": 0.111776 |
| }, |
| "model.layers.16.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954197, |
| "clip_improvement": 0.111497 |
| }, |
| "model.layers.16.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999803, |
| "clip_improvement": 0.250038 |
| }, |
| "model.layers.16.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999817, |
| "clip_improvement": 0.249726 |
| }, |
| "model.layers.17.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954671, |
| "clip_improvement": 0.111566 |
| }, |
| "model.layers.17.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954834, |
| "clip_improvement": 0.112852 |
| }, |
| "model.layers.17.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951389, |
| "clip_improvement": 0.110926 |
| }, |
| "model.layers.17.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955621, |
| "clip_improvement": 0.114065 |
| }, |
| "model.layers.17.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95638, |
| "clip_improvement": 0.110832 |
| }, |
| "model.layers.17.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954603, |
| "clip_improvement": 0.111657 |
| }, |
| "model.layers.17.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999871, |
| "clip_improvement": 0.249902 |
| }, |
| "model.layers.17.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99981, |
| "clip_improvement": 0.249763 |
| }, |
| "model.layers.18.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 3244032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953017, |
| "clip_improvement": 0.110424 |
| }, |
| "model.layers.18.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.18.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954027, |
| "clip_improvement": 0.111633 |
| }, |
| "model.layers.18.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.249999 |
| }, |
| "model.layers.18.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999871, |
| "clip_improvement": 0.250365 |
| }, |
| "model.layers.19.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952365, |
| "clip_improvement": 0.11146 |
| }, |
| "model.layers.19.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957465, |
| "clip_improvement": 0.112536 |
| }, |
| "model.layers.19.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955349, |
| "clip_improvement": 0.111617 |
| }, |
| "model.layers.19.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.249911 |
| }, |
| "model.layers.19.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999885, |
| "clip_improvement": 0.250217 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.2.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951986, |
| "clip_improvement": 0.111035 |
| }, |
| "model.layers.2.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955322, |
| "clip_improvement": 0.110804 |
| }, |
| "model.layers.2.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955187, |
| "clip_improvement": 0.112738 |
| }, |
| "model.layers.2.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954427, |
| "clip_improvement": 0.111498 |
| }, |
| "model.layers.2.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954753, |
| "clip_improvement": 0.111969 |
| }, |
| "model.layers.2.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953918, |
| "clip_improvement": 0.111989 |
| }, |
| "model.layers.2.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.35348 |
| }, |
| "model.layers.2.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353625 |
| }, |
| "model.layers.20.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.112217 |
| }, |
| "model.layers.20.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955933, |
| "clip_improvement": 0.111425 |
| }, |
| "model.layers.20.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951063, |
| "clip_improvement": 0.11131 |
| }, |
| "model.layers.20.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953559, |
| "clip_improvement": 0.111117 |
| }, |
| "model.layers.20.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952908, |
| "clip_improvement": 0.111531 |
| }, |
| "model.layers.20.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953715, |
| "clip_improvement": 0.111884 |
| }, |
| "model.layers.20.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.99998, |
| "clip_improvement": 0.353628 |
| }, |
| "model.layers.20.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353323 |
| }, |
| "model.layers.21.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952677, |
| "clip_improvement": 0.111657 |
| }, |
| "model.layers.21.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953369, |
| "clip_improvement": 0.110681 |
| }, |
| "model.layers.21.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952908, |
| "clip_improvement": 0.111555 |
| }, |
| "model.layers.21.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956055, |
| "clip_improvement": 0.112092 |
| }, |
| "model.layers.21.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953234, |
| "clip_improvement": 0.113501 |
| }, |
| "model.layers.21.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953695, |
| "clip_improvement": 0.111866 |
| }, |
| "model.layers.21.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999803, |
| "clip_improvement": 0.250011 |
| }, |
| "model.layers.21.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999844, |
| "clip_improvement": 0.250283 |
| }, |
| "model.layers.22.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955973, |
| "clip_improvement": 0.111722 |
| }, |
| "model.layers.22.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953735, |
| "clip_improvement": 0.110481 |
| }, |
| "model.layers.22.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953559, |
| "clip_improvement": 0.11164 |
| }, |
| "model.layers.22.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.22.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953017, |
| "clip_improvement": 0.111665 |
| }, |
| "model.layers.22.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954203, |
| "clip_improvement": 0.111655 |
| }, |
| "model.layers.22.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999885, |
| "clip_improvement": 0.250033 |
| }, |
| "model.layers.22.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999837, |
| "clip_improvement": 0.250374 |
| }, |
| "model.layers.23.moe.sw1": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.sw3": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953844, |
| "clip_improvement": 0.11135 |
| }, |
| "model.layers.23.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999797, |
| "clip_improvement": 0.251224 |
| }, |
| "model.layers.23.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999803, |
| "clip_improvement": 0.250249 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953979, |
| "clip_improvement": 0.112017 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958442, |
| "clip_improvement": 0.111402 |
| }, |
| "model.layers.3.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951931, |
| "clip_improvement": 0.110399 |
| }, |
| "model.layers.3.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.950846, |
| "clip_improvement": 0.110635 |
| }, |
| "model.layers.3.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953179, |
| "clip_improvement": 0.111499 |
| }, |
| "model.layers.3.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999878, |
| "clip_improvement": 0.250279 |
| }, |
| "model.layers.3.moe.w3": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 1.0, |
| "clip_improvement": 0.353477 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.4.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956177, |
| "clip_improvement": 0.112503 |
| }, |
| "model.layers.4.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957275, |
| "clip_improvement": 0.110555 |
| }, |
| "model.layers.4.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955729, |
| "clip_improvement": 0.110095 |
| }, |
| "model.layers.4.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955404, |
| "clip_improvement": 0.112383 |
| }, |
| "model.layers.4.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953993, |
| "clip_improvement": 0.111288 |
| }, |
| "model.layers.4.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953627, |
| "clip_improvement": 0.11169 |
| }, |
| "model.layers.4.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999851, |
| "clip_improvement": 0.250145 |
| }, |
| "model.layers.4.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999851, |
| "clip_improvement": 0.249635 |
| }, |
| "model.layers.5.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954712, |
| "clip_improvement": 0.111981 |
| }, |
| "model.layers.5.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952271, |
| "clip_improvement": 0.10894 |
| }, |
| "model.layers.5.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953342, |
| "clip_improvement": 0.110832 |
| }, |
| "model.layers.5.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955404, |
| "clip_improvement": 0.111948 |
| }, |
| "model.layers.5.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.958659, |
| "clip_improvement": 0.110161 |
| }, |
| "model.layers.5.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953179, |
| "clip_improvement": 0.111496 |
| }, |
| "model.layers.5.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.99983, |
| "clip_improvement": 0.250085 |
| }, |
| "model.layers.5.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999851, |
| "clip_improvement": 0.250076 |
| }, |
| "model.layers.6.conv.in_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 3244032, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.conv.out_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953451, |
| "clip_improvement": 0.110973 |
| }, |
| "model.layers.6.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.6.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957357, |
| "clip_improvement": 0.112433 |
| }, |
| "model.layers.6.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952806, |
| "clip_improvement": 0.111867 |
| }, |
| "model.layers.6.moe.w2": { |
| "bits": 2, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 8, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 5308416, |
| "bits_per_weight": 2.25, |
| "clipped_fraction": 0.999986, |
| "clip_improvement": 0.353306 |
| }, |
| "model.layers.6.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999864, |
| "clip_improvement": 0.249861 |
| }, |
| "model.layers.7.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.113294 |
| }, |
| "model.layers.7.moe.sw2": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1216512, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955078, |
| "clip_improvement": 0.111907 |
| }, |
| "model.layers.7.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953891, |
| "clip_improvement": 0.111827 |
| }, |
| "model.layers.7.moe.w2": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999858, |
| "clip_improvement": 0.24996 |
| }, |
| "model.layers.7.moe.w3": { |
| "bits": 3, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 12, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 7667712, |
| "bits_per_weight": 3.25, |
| "clipped_fraction": 0.999885, |
| "clip_improvement": 0.249583 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1081344, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "bits": 8, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 256, |
| 1024 |
| ], |
| "num_rows": 256, |
| "num_groups": 8, |
| "words_per_group": 32, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 270336, |
| "bits_per_weight": 8.25, |
| "clipped_fraction": 0.0, |
| "clip_improvement": 0.0 |
| }, |
| "model.layers.8.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954671, |
| "clip_improvement": 0.111955 |
| }, |
| "model.layers.8.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.955566, |
| "clip_improvement": 0.108682 |
| }, |
| "model.layers.8.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.956706, |
| "clip_improvement": 0.110425 |
| }, |
| "model.layers.8.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954753, |
| "clip_improvement": 0.111849 |
| }, |
| "model.layers.8.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.957031, |
| "clip_improvement": 0.111082 |
| }, |
| "model.layers.8.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.952881, |
| "clip_improvement": 0.111608 |
| }, |
| "model.layers.8.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953891, |
| "clip_improvement": 0.11176 |
| }, |
| "model.layers.8.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953729, |
| "clip_improvement": 0.111612 |
| }, |
| "model.layers.9.conv.in_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 3072, |
| 1024 |
| ], |
| "num_rows": 3072, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 1671168, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954915, |
| "clip_improvement": 0.111643 |
| }, |
| "model.layers.9.conv.out_proj": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1024, |
| 1024 |
| ], |
| "num_rows": 1024, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 557056, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.951782, |
| "clip_improvement": 0.111073 |
| }, |
| "model.layers.9.moe.sw1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.95497, |
| "clip_improvement": 0.111141 |
| }, |
| "model.layers.9.moe.sw2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 1024, |
| 1152 |
| ], |
| "num_rows": 1024, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.954861, |
| "clip_improvement": 0.113054 |
| }, |
| "model.layers.9.moe.sw3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 1152, |
| 1024 |
| ], |
| "num_rows": 1152, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 626688, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953125, |
| "clip_improvement": 0.112049 |
| }, |
| "model.layers.9.moe.w1": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953722, |
| "clip_improvement": 0.111856 |
| }, |
| "model.layers.9.moe.w2": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1024, |
| "logical_shape": [ |
| 16, |
| 1152, |
| 1024 |
| ], |
| "num_rows": 18432, |
| "num_groups": 8, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953749, |
| "clip_improvement": 0.111389 |
| }, |
| "model.layers.9.moe.w3": { |
| "bits": 4, |
| "group_size": 128, |
| "in_features": 1152, |
| "logical_shape": [ |
| 16, |
| 1024, |
| 1152 |
| ], |
| "num_rows": 16384, |
| "num_groups": 9, |
| "words_per_group": 16, |
| "row_offset": 0, |
| "layout": "linear", |
| "symmetric": false, |
| "has_offsets": true, |
| "compute_dtype": "bfloat16", |
| "nbytes": 10027008, |
| "bits_per_weight": 4.25, |
| "clipped_fraction": 0.953451, |
| "clip_improvement": 0.111851 |
| } |
| } |
| } |
