File size: 3,082 Bytes
c675b8d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | import argparse, json, sys
def main(argv=None):
p = argparse.ArgumentParser("shapequant", description="Data-free, training-free quantization of Hugging Face causal LMs")
sub = p.add_subparsers(dest="cmd", required=True)
q = sub.add_parser("quantize", help="quantize a model (hub id or local dir) to a GPTQ-format checkpoint")
q.add_argument("model"); q.add_argument("out")
q.add_argument("--bits", type=int, default=4, choices=[4, 8])
q.add_argument("--group-size", type=int, default=128)
q.add_argument("--method", default="shaped", choices=["shaped", "rtn"])
q.add_argument("--budget", type=float, default=None, help="average bits/weight for mixed INT4/INT8/FP16 (e.g. 4.75)")
q.add_argument("--lam", type=float, default=1.0); q.add_argument("--kappa", type=float, default=1.0)
q.add_argument("--samples", type=int, default=16384); q.add_argument("--device", default=None)
q.add_argument("--revision", default=None); q.add_argument("--shard-gb", type=float, default=4.0)
q.add_argument("--static-groups", action="store_true", help="fit group grids to the original weights instead of the error-fed ones")
e = sub.add_parser("eval", help="perplexity of a quantized checkpoint on WikiText-2 (or a text file)")
e.add_argument("ckpt"); e.add_argument("--text", default=None); e.add_argument("--seq-len", type=int, default=1024)
e.add_argument("--windows", type=int, default=32); e.add_argument("--device", default="cpu"); e.add_argument("--bos", action="store_true")
m = sub.add_parser("memory", help="peak-memory estimate from a config (hub id, dir or config.json)")
m.add_argument("model"); m.add_argument("--method", default="shaped", choices=["shaped", "rtn"])
a = p.parse_args(argv)
if a.cmd == "quantize":
from .quantize import quantize_model
s = quantize_model(a.model, a.out, bits=a.bits, group_size=a.group_size, method=a.method, budget=a.budget, lam=a.lam,
kappa=a.kappa, n_samples=a.samples, device=a.device, revision=a.revision, max_shard_size_gb=a.shard_gb,
dynamic_groups=not a.static_groups)
print(json.dumps({k: v for k, v in s.items() if k != "bits_per_module"}, indent=2))
elif a.cmd == "eval":
from transformers import AutoTokenizer
from .qlinear import load_quantized
from .evaluate import wikitext2_windows, perplexity
tok = AutoTokenizer.from_pretrained(a.ckpt)
model = load_quantized(a.ckpt, device=a.device)
W = wikitext2_windows(tok, a.seq_len, a.windows, a.text)
print(f"perplexity {perplexity(model, W, tok.bos_token_id if a.bos else None):.3f} on {W.numel()} tokens")
elif a.cmd == "memory":
import os
from .memory import estimate
src = a.model
if not os.path.exists(src):
from huggingface_hub import hf_hub_download
src = hf_hub_download(a.model, "config.json")
print(json.dumps(estimate(src, method=a.method), indent=2))
if __name__ == "__main__":
sys.exit(main())
|