Download runtime/evaluate.py from groxaxo/ZONOS3-Mini-GPTQ-Pro-INT4: direct link, hf CLI and curl.
- Browser
- Download file 3.61 kB
-
https://huggingface.co/groxaxo/ZONOS3-Mini-GPTQ-Pro-INT4/resolve/main/runtime/evaluate.py
- Command line
-
hf download hf://groxaxo/ZONOS3-Mini-GPTQ-Pro-INT4/runtime/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/groxaxo/ZONOS3-Mini-GPTQ-Pro-INT4/resolve/main/runtime/evaluate.py
3.61 kB
| #!/usr/bin/env python3 | |
| """Paired teacher/INT4 held-out token-logit comparison (no codec decode).""" | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from overlay_runtime import load_overlay | |
| from quantize import load_calibration | |
| def compare(model, samples): | |
| from zonos3.reference.lm import LMCache | |
| outputs = [] | |
| for ids, speakers, conditions in samples: | |
| logits = model(ids.to(model.device), LMCache(), | |
| speaker_rows=speakers, condition_rows=conditions).logits | |
| outputs.append(logits.detach().float().cpu()) | |
| return outputs | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument("--base", type=Path, required=True) | |
| p.add_argument("--overlay", type=Path, required=True) | |
| p.add_argument("--holdout", type=Path, required=True, help="Separate speaker/utterance JSONL; never fit set") | |
| p.add_argument("--device", default="cuda:0", help="CUDA device holding native packed modules") | |
| p.add_argument("--model-device", default=None, help="Original BF16 model device; CPU for low-VRAM eval") | |
| p.add_argument("--allow-cpu-offload", action="store_true", help="Explicitly copy activations CPU/CUDA for native modules") | |
| p.add_argument("--allow-incomplete", action="store_true", help="Evaluate partial overlays, not for final release") | |
| p.add_argument("--max-nmse", type=float, default=0.10) | |
| p.add_argument("--max-kl", type=float, default=0.10) | |
| args = p.parse_args() | |
| from zonos3.reference.lm import Zonos3LM | |
| model_device = args.model_device or args.device | |
| if model_device != args.device and not (model_device == "cpu" and args.allow_cpu_offload): | |
| p.error("Different source/native devices require --model-device cpu --allow-cpu-offload") | |
| base = Zonos3LM.load(args.base, device=model_device, dtype=torch.bfloat16).eval() | |
| samples = load_calibration(args.holdout, base.config, limit=0) | |
| reference = compare(base, samples) | |
| load_overlay(base, args.overlay, device=args.device, | |
| require_complete=not args.allow_incomplete, | |
| allow_cpu_offload=args.allow_cpu_offload) | |
| candidate = compare(base, samples) | |
| total_error = total_signal = total_kl = total_argmax = total_entries = 0.0 | |
| for teacher, student in zip(reference, candidate): | |
| if not torch.isfinite(student).all(): | |
| raise RuntimeError("Non-finite quantized logits") | |
| total_error += float((teacher - student).square().sum()) | |
| total_signal += float(teacher.square().sum()) | |
| total_kl += float(F.kl_div( | |
| F.log_softmax(student, dim=-1), | |
| F.softmax(teacher, dim=-1), | |
| reduction="sum", | |
| )) | |
| total_argmax += float((teacher.argmax(-1) == student.argmax(-1)).sum()) | |
| total_entries += teacher.numel() / teacher.shape[-1] | |
| nmse = total_error / max(total_signal, 1e-9) | |
| kl = total_kl / max(total_entries, 1) | |
| argmax = total_argmax / max(total_entries, 1) | |
| print(f"heldout_logit_nmse={nmse:.6f} heldout_KL_nats={kl:.6f} argmax_agreement={argmax:.4%}") | |
| if args.allow_incomplete: | |
| print("INCOMPLETE OVERLAY PILOT: these numbers are NOT full-model quantization results") | |
| print("This is a numeric gate only; run matched blind listening and codec end-to-end tests before shipping.") | |
| if not math.isfinite(nmse + kl + argmax) or nmse > args.max_nmse or kl > args.max_kl: | |
| raise SystemExit("FAILED: held-out teacher/INT4 gate") | |
| if __name__ == "__main__": | |
| main() | |