on_the_way / tools /benchmark_quantization.py
RANITBAG's picture
2nd run
1a1fc1f verified
Raw History Blame Contribute Delete
5.48 kB
#!/usr/bin/env python3
"""benchmark_quantization.py -- reference vs simulated-low-bit comparison.
Fixed evaluation set (seeded), CPU decode timing (the Q4/Q5 path must be
measured on CPU per spec). Reports: loss/perplexity proxy, greedy response
regression (match rate vs reference), memory estimate, decode speed.
SIMULATION DISCLAIMER: the quantized side uses inference.qat fake-quant
weights. It is NOT llama.cpp Q4_K_M; real-format numbers come only from the
Phase-8 GGUF conversion.
Usage (repo root):
python tools/benchmark_quantization.py --checkpoint ckpt --tokenizer-dir \
tokenizer_out/Rivinity-T48 --out reports/quant_bench.json --decode-tokens 32
"""
from __future__ import annotations
import argparse
import copy
import json
import statistics
import sys
import time
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT))
def eval_loss(model, batches) -> dict:
import torch
model.eval()
losses = []
with torch.no_grad():
for b in batches:
ids = b["input_ids"]
losses.append(model(input_ids=ids, labels=ids).loss.item())
mean = statistics.mean(losses)
return {"mean_loss": round(mean, 6), "perplexity_proxy": round(2.718281828 ** mean, 4),
"n_batches": len(batches)}
def response_regression(model_a, model_b, prompts_ids, gen_cfg) -> dict:
from rivinity.generation import generate
matches = 0
for ids in prompts_ids:
ra = generate(model_a, ids, gen_cfg)
rb = generate(model_b, ids, gen_cfg)
matches += int(ra.token_ids == rb.token_ids)
return {"greedy_match_rate": round(matches / max(1, len(prompts_ids)), 4),
"n_prompts": len(prompts_ids)}
def memory_estimate(model, bits_default: int, bits_sensitive: int) -> dict:
from inference.qat import _is_sensitive
total = 0
for name, p in model.named_parameters():
b = bits_sensitive if _is_sensitive(name) else bits_default
total += p.numel() * b
return {"estimated_bytes": total, "estimated_mb": round(total / 8 / 1e6, 2)}
def decode_speed(model, prompt_ids, gen_cfg, iters=3) -> dict:
import torch
torch.set_num_threads(gen_cfg.threads)
times = []
for _ in range(iters):
t0 = time.perf_counter()
from rivinity.generation import generate
generate(model, prompt_ids, gen_cfg)
times.append(time.perf_counter() - t0)
med = statistics.median(times)
return {"median_decode_seconds": round(med, 4),
"tokens_per_second": round(gen_cfg.max_new_tokens / med, 2)}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--tokenizer-dir", type=Path, default=None)
parser.add_argument("--out", type=Path, default=Path("reports/quant_bench.json"))
parser.add_argument("--eval-batches", type=int, default=8)
parser.add_argument("--decode-tokens", type=int, default=32)
parser.add_argument("--threads", type=int, default=4)
parser.add_argument("--policy", nargs="*", default=["default:4:64", "sensitive:8:32"])
args = parser.parse_args()
import torch
from rivinity import RivinityForCausalLM
from rivinity.generation import RivinityGenerationConfig
from inference.qat import apply_qat
from inference.quantize_reference import parse_policy
policy = parse_policy(args.policy)
reference = RivinityForCausalLM.from_pretrained(args.checkpoint).eval()
quantized = apply_qat(copy.deepcopy(reference), policy)
quantized.eval()
g = torch.Generator().manual_seed(0)
vocab = reference.config.vocab_size
batches = [{"input_ids": torch.randint(0, vocab, (1, 64), generator=g)}
for _ in range(args.eval_batches)]
prompts = [torch.randint(0, vocab, (1, 16), generator=g) for _ in range(4)]
gen_cfg = RivinityGenerationConfig(do_sample=False, repetition_penalty=1.0,
max_new_tokens=args.decode_tokens)
report = {
"simulation_disclaimer": ("quantized side = inference.qat fake-quant; "
"NOT llama.cpp Q4_K_M. Real-format numbers in Phase 8."),
"policy": {k: {"bits": v[0], "group_size": v[1]} for k, v in policy.items()},
"environment": {"threads": args.threads,
"torch": torch.__version__,
"device": "cpu"},
"reference": {
"eval": eval_loss(reference, batches),
"memory_fp32_mb": round(sum(p.numel() for p in reference.parameters()) * 4 / 1e6, 2),
"decode": decode_speed(reference, prompts[0], gen_cfg),
},
"simulated_quantized": {
"eval": eval_loss(quantized, batches),
"memory": memory_estimate(quantized, policy["default"][0], policy["sensitive"][0]),
"decode": decode_speed(quantized, prompts[0], gen_cfg),
"response_regression_vs_reference": response_regression(
reference, quantized, prompts, gen_cfg),
},
}
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
print(f"benchmark -> {args.out}")
print(json.dumps(report["simulated_quantized"]["response_regression_vs_reference"]))
return 0
if __name__ == "__main__":
raise SystemExit(main())