Download tools/benchmark_quantization.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 5.48 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/tools/benchmark_quantization.py
- Command line
-
hf download hf://RANITBAG/on_the_way/tools/benchmark_quantization.py
-
curl -L -o benchmark_quantization.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/tools/benchmark_quantization.py
5.48 kB
| #!/usr/bin/env python3 | |
| """benchmark_quantization.py -- reference vs simulated-low-bit comparison. | |
| Fixed evaluation set (seeded), CPU decode timing (the Q4/Q5 path must be | |
| measured on CPU per spec). Reports: loss/perplexity proxy, greedy response | |
| regression (match rate vs reference), memory estimate, decode speed. | |
| SIMULATION DISCLAIMER: the quantized side uses inference.qat fake-quant | |
| weights. It is NOT llama.cpp Q4_K_M; real-format numbers come only from the | |
| Phase-8 GGUF conversion. | |
| Usage (repo root): | |
| python tools/benchmark_quantization.py --checkpoint ckpt --tokenizer-dir \ | |
| tokenizer_out/Rivinity-T48 --out reports/quant_bench.json --decode-tokens 32 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import copy | |
| import json | |
| import statistics | |
| import sys | |
| import time | |
| from pathlib import Path | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(REPO_ROOT)) | |
| def eval_loss(model, batches) -> dict: | |
| import torch | |
| model.eval() | |
| losses = [] | |
| with torch.no_grad(): | |
| for b in batches: | |
| ids = b["input_ids"] | |
| losses.append(model(input_ids=ids, labels=ids).loss.item()) | |
| mean = statistics.mean(losses) | |
| return {"mean_loss": round(mean, 6), "perplexity_proxy": round(2.718281828 ** mean, 4), | |
| "n_batches": len(batches)} | |
| def response_regression(model_a, model_b, prompts_ids, gen_cfg) -> dict: | |
| from rivinity.generation import generate | |
| matches = 0 | |
| for ids in prompts_ids: | |
| ra = generate(model_a, ids, gen_cfg) | |
| rb = generate(model_b, ids, gen_cfg) | |
| matches += int(ra.token_ids == rb.token_ids) | |
| return {"greedy_match_rate": round(matches / max(1, len(prompts_ids)), 4), | |
| "n_prompts": len(prompts_ids)} | |
| def memory_estimate(model, bits_default: int, bits_sensitive: int) -> dict: | |
| from inference.qat import _is_sensitive | |
| total = 0 | |
| for name, p in model.named_parameters(): | |
| b = bits_sensitive if _is_sensitive(name) else bits_default | |
| total += p.numel() * b | |
| return {"estimated_bytes": total, "estimated_mb": round(total / 8 / 1e6, 2)} | |
| def decode_speed(model, prompt_ids, gen_cfg, iters=3) -> dict: | |
| import torch | |
| torch.set_num_threads(gen_cfg.threads) | |
| times = [] | |
| for _ in range(iters): | |
| t0 = time.perf_counter() | |
| from rivinity.generation import generate | |
| generate(model, prompt_ids, gen_cfg) | |
| times.append(time.perf_counter() - t0) | |
| med = statistics.median(times) | |
| return {"median_decode_seconds": round(med, 4), | |
| "tokens_per_second": round(gen_cfg.max_new_tokens / med, 2)} | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| parser.add_argument("--checkpoint", type=Path, required=True) | |
| parser.add_argument("--tokenizer-dir", type=Path, default=None) | |
| parser.add_argument("--out", type=Path, default=Path("reports/quant_bench.json")) | |
| parser.add_argument("--eval-batches", type=int, default=8) | |
| parser.add_argument("--decode-tokens", type=int, default=32) | |
| parser.add_argument("--threads", type=int, default=4) | |
| parser.add_argument("--policy", nargs="*", default=["default:4:64", "sensitive:8:32"]) | |
| args = parser.parse_args() | |
| import torch | |
| from rivinity import RivinityForCausalLM | |
| from rivinity.generation import RivinityGenerationConfig | |
| from inference.qat import apply_qat | |
| from inference.quantize_reference import parse_policy | |
| policy = parse_policy(args.policy) | |
| reference = RivinityForCausalLM.from_pretrained(args.checkpoint).eval() | |
| quantized = apply_qat(copy.deepcopy(reference), policy) | |
| quantized.eval() | |
| g = torch.Generator().manual_seed(0) | |
| vocab = reference.config.vocab_size | |
| batches = [{"input_ids": torch.randint(0, vocab, (1, 64), generator=g)} | |
| for _ in range(args.eval_batches)] | |
| prompts = [torch.randint(0, vocab, (1, 16), generator=g) for _ in range(4)] | |
| gen_cfg = RivinityGenerationConfig(do_sample=False, repetition_penalty=1.0, | |
| max_new_tokens=args.decode_tokens) | |
| report = { | |
| "simulation_disclaimer": ("quantized side = inference.qat fake-quant; " | |
| "NOT llama.cpp Q4_K_M. Real-format numbers in Phase 8."), | |
| "policy": {k: {"bits": v[0], "group_size": v[1]} for k, v in policy.items()}, | |
| "environment": {"threads": args.threads, | |
| "torch": torch.__version__, | |
| "device": "cpu"}, | |
| "reference": { | |
| "eval": eval_loss(reference, batches), | |
| "memory_fp32_mb": round(sum(p.numel() for p in reference.parameters()) * 4 / 1e6, 2), | |
| "decode": decode_speed(reference, prompts[0], gen_cfg), | |
| }, | |
| "simulated_quantized": { | |
| "eval": eval_loss(quantized, batches), | |
| "memory": memory_estimate(quantized, policy["default"][0], policy["sensitive"][0]), | |
| "decode": decode_speed(quantized, prompts[0], gen_cfg), | |
| "response_regression_vs_reference": response_regression( | |
| reference, quantized, prompts, gen_cfg), | |
| }, | |
| } | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") | |
| print(f"benchmark -> {args.out}") | |
| print(json.dumps(report["simulated_quantized"]["response_regression_vs_reference"])) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |