kev-27b-W4A16 / quantization_config.json
bowmanslayer's picture
Add quantization_config.json
5945bd9 verified
Raw History Blame Contribute Delete
8.92 kB
{
"bits": 4,
"data_type": "int",
"group_size": 128,
"sym": true,
"batch_size": 4,
"low_gpu_mem_usage": true,
"nsamples": 256,
"seqlen": 512,
"autoround_version": "0.12.3",
"quant_method": "auto-round",
"packing_format": "auto_round:auto_gptq",
"extra_config": {
"layers.0.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.0.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.1.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.1.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.2.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.2.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.4.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.4.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.5.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.5.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.6.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.6.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.8.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.8.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.9.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.9.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.10.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.10.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.12.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.12.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.13.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.13.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.14.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.14.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.16.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.16.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.17.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.17.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.18.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.18.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.20.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.20.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.21.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.21.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.22.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.22.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.24.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.24.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.25.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.25.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.26.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.26.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.28.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.28.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.29.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.29.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.30.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.30.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.32.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.32.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.33.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.33.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.34.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.34.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.36.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.36.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.37.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.37.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.38.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.38.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.40.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.40.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.41.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.41.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.42.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.42.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.44.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.44.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.45.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.45.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.46.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.46.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.48.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.48.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.49.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.49.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.50.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.50.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.52.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.52.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.53.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.53.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.54.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.54.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.56.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.56.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.57.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.57.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.58.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.58.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.60.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.60.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.61.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.61.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
},
"layers.62.linear_attn.in_proj_b": {
"bits": 16,
"data_type": "fp"
},
"layers.62.linear_attn.in_proj_a": {
"bits": 16,
"data_type": "fp"
}
}
}