#!/usr/bin/env python3 """benchmark_quantization.py -- reference vs simulated-low-bit comparison. Fixed evaluation set (seeded), CPU decode timing (the Q4/Q5 path must be measured on CPU per spec). Reports: loss/perplexity proxy, greedy response regression (match rate vs reference), memory estimate, decode speed. SIMULATION DISCLAIMER: the quantized side uses inference.qat fake-quant weights. It is NOT llama.cpp Q4_K_M; real-format numbers come only from the Phase-8 GGUF conversion. Usage (repo root): python tools/benchmark_quantization.py --checkpoint ckpt --tokenizer-dir \ tokenizer_out/Rivinity-T48 --out reports/quant_bench.json --decode-tokens 32 """ from __future__ import annotations import argparse import copy import json import statistics import sys import time from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT)) def eval_loss(model, batches) -> dict: import torch model.eval() losses = [] with torch.no_grad(): for b in batches: ids = b["input_ids"] losses.append(model(input_ids=ids, labels=ids).loss.item()) mean = statistics.mean(losses) return {"mean_loss": round(mean, 6), "perplexity_proxy": round(2.718281828 ** mean, 4), "n_batches": len(batches)} def response_regression(model_a, model_b, prompts_ids, gen_cfg) -> dict: from rivinity.generation import generate matches = 0 for ids in prompts_ids: ra = generate(model_a, ids, gen_cfg) rb = generate(model_b, ids, gen_cfg) matches += int(ra.token_ids == rb.token_ids) return {"greedy_match_rate": round(matches / max(1, len(prompts_ids)), 4), "n_prompts": len(prompts_ids)} def memory_estimate(model, bits_default: int, bits_sensitive: int) -> dict: from inference.qat import _is_sensitive total = 0 for name, p in model.named_parameters(): b = bits_sensitive if _is_sensitive(name) else bits_default total += p.numel() * b return {"estimated_bytes": total, "estimated_mb": round(total / 8 / 1e6, 2)} def decode_speed(model, prompt_ids, gen_cfg, iters=3) -> dict: import torch torch.set_num_threads(gen_cfg.threads) times = [] for _ in range(iters): t0 = time.perf_counter() from rivinity.generation import generate generate(model, prompt_ids, gen_cfg) times.append(time.perf_counter() - t0) med = statistics.median(times) return {"median_decode_seconds": round(med, 4), "tokens_per_second": round(gen_cfg.max_new_tokens / med, 2)} def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--checkpoint", type=Path, required=True) parser.add_argument("--tokenizer-dir", type=Path, default=None) parser.add_argument("--out", type=Path, default=Path("reports/quant_bench.json")) parser.add_argument("--eval-batches", type=int, default=8) parser.add_argument("--decode-tokens", type=int, default=32) parser.add_argument("--threads", type=int, default=4) parser.add_argument("--policy", nargs="*", default=["default:4:64", "sensitive:8:32"]) args = parser.parse_args() import torch from rivinity import RivinityForCausalLM from rivinity.generation import RivinityGenerationConfig from inference.qat import apply_qat from inference.quantize_reference import parse_policy policy = parse_policy(args.policy) reference = RivinityForCausalLM.from_pretrained(args.checkpoint).eval() quantized = apply_qat(copy.deepcopy(reference), policy) quantized.eval() g = torch.Generator().manual_seed(0) vocab = reference.config.vocab_size batches = [{"input_ids": torch.randint(0, vocab, (1, 64), generator=g)} for _ in range(args.eval_batches)] prompts = [torch.randint(0, vocab, (1, 16), generator=g) for _ in range(4)] gen_cfg = RivinityGenerationConfig(do_sample=False, repetition_penalty=1.0, max_new_tokens=args.decode_tokens) report = { "simulation_disclaimer": ("quantized side = inference.qat fake-quant; " "NOT llama.cpp Q4_K_M. Real-format numbers in Phase 8."), "policy": {k: {"bits": v[0], "group_size": v[1]} for k, v in policy.items()}, "environment": {"threads": args.threads, "torch": torch.__version__, "device": "cpu"}, "reference": { "eval": eval_loss(reference, batches), "memory_fp32_mb": round(sum(p.numel() for p in reference.parameters()) * 4 / 1e6, 2), "decode": decode_speed(reference, prompts[0], gen_cfg), }, "simulated_quantized": { "eval": eval_loss(quantized, batches), "memory": memory_estimate(quantized, policy["default"][0], policy["sensitive"][0]), "decode": decode_speed(quantized, prompts[0], gen_cfg), "response_regression_vs_reference": response_regression( reference, quantized, prompts, gen_cfg), }, } args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") print(f"benchmark -> {args.out}") print(json.dumps(report["simulated_quantized"]["response_regression_vs_reference"])) return 0 if __name__ == "__main__": raise SystemExit(main())