| { |
| "format": "auto-quant-v1", |
| "bits": 4, |
| "group_size": 128, |
| "symmetric": true, |
| "q_range": [ |
| -8, |
| 7 |
| ], |
| "scale_dtype": "float16", |
| "granularity": "per 128 input weights", |
| "modules": [ |
| "model.embeddings.tok_embeddings", |
| "model.layers.0.attn.Wqkv", |
| "model.layers.0.attn.Wo", |
| "model.layers.0.mlp.Wi", |
| "model.layers.0.mlp.Wo", |
| "model.layers.1.attn.Wqkv", |
| "model.layers.1.attn.Wo", |
| "model.layers.1.mlp.Wi", |
| "model.layers.1.mlp.Wo", |
| "model.layers.2.attn.Wqkv", |
| "model.layers.2.attn.Wo", |
| "model.layers.2.mlp.Wi", |
| "model.layers.2.mlp.Wo", |
| "model.layers.3.attn.Wqkv", |
| "model.layers.3.attn.Wo", |
| "model.layers.3.mlp.Wi", |
| "model.layers.3.mlp.Wo", |
| "model.layers.4.attn.Wqkv", |
| "model.layers.4.attn.Wo", |
| "model.layers.4.mlp.Wi", |
| "model.layers.4.mlp.Wo", |
| "model.layers.5.attn.Wqkv", |
| "model.layers.5.attn.Wo", |
| "model.layers.5.mlp.Wi", |
| "model.layers.5.mlp.Wo", |
| "model.layers.6.attn.Wqkv", |
| "model.layers.6.attn.Wo", |
| "model.layers.6.mlp.Wi", |
| "model.layers.6.mlp.Wo", |
| "model.layers.7.attn.Wqkv", |
| "model.layers.7.attn.Wo", |
| "model.layers.7.mlp.Wi", |
| "model.layers.7.mlp.Wo", |
| "model.layers.8.attn.Wqkv", |
| "model.layers.8.attn.Wo", |
| "model.layers.8.mlp.Wi", |
| "model.layers.8.mlp.Wo", |
| "model.layers.9.attn.Wqkv", |
| "model.layers.9.attn.Wo", |
| "model.layers.9.mlp.Wi", |
| "model.layers.9.mlp.Wo", |
| "model.layers.10.attn.Wqkv", |
| "model.layers.10.attn.Wo", |
| "model.layers.10.mlp.Wi", |
| "model.layers.10.mlp.Wo", |
| "model.layers.11.attn.Wqkv", |
| "model.layers.11.attn.Wo", |
| "model.layers.11.mlp.Wi", |
| "model.layers.11.mlp.Wo", |
| "model.layers.12.attn.Wqkv", |
| "model.layers.12.attn.Wo", |
| "model.layers.12.mlp.Wi", |
| "model.layers.12.mlp.Wo", |
| "model.layers.13.attn.Wqkv", |
| "model.layers.13.attn.Wo", |
| "model.layers.13.mlp.Wi", |
| "model.layers.13.mlp.Wo", |
| "model.layers.14.attn.Wqkv", |
| "model.layers.14.attn.Wo", |
| "model.layers.14.mlp.Wi", |
| "model.layers.14.mlp.Wo", |
| "model.layers.15.attn.Wqkv", |
| "model.layers.15.attn.Wo", |
| "model.layers.15.mlp.Wi", |
| "model.layers.15.mlp.Wo", |
| "model.layers.16.attn.Wqkv", |
| "model.layers.16.attn.Wo", |
| "model.layers.16.mlp.Wi", |
| "model.layers.16.mlp.Wo", |
| "model.layers.17.attn.Wqkv", |
| "model.layers.17.attn.Wo", |
| "model.layers.17.mlp.Wi", |
| "model.layers.17.mlp.Wo", |
| "model.layers.18.attn.Wqkv", |
| "model.layers.18.attn.Wo", |
| "model.layers.18.mlp.Wi", |
| "model.layers.18.mlp.Wo", |
| "model.layers.19.attn.Wqkv", |
| "model.layers.19.attn.Wo", |
| "model.layers.19.mlp.Wi", |
| "model.layers.19.mlp.Wo", |
| "model.layers.20.attn.Wqkv", |
| "model.layers.20.attn.Wo", |
| "model.layers.20.mlp.Wi", |
| "model.layers.20.mlp.Wo", |
| "model.layers.21.attn.Wqkv", |
| "model.layers.21.attn.Wo", |
| "model.layers.21.mlp.Wi", |
| "model.layers.21.mlp.Wo", |
| "head.dense" |
| ], |
| "weights": "auto_quant_int4.safetensors", |
| "activations": "unquantized (device float dtype)" |
| } |
|
|
