groxaxo's picture
Release experimental ZONOS3-Mini GPTQ-Pro INT4/BF16 overlay, 11-layer checkpoints, code, English/Spanish audio A/B tests and quality report
53b79a5 verified
Raw History Blame Contribute Delete
3.61 kB
#!/usr/bin/env python3
"""Paired teacher/INT4 held-out token-logit comparison (no codec decode)."""
from __future__ import annotations
import argparse
import math
from pathlib import Path
import torch
import torch.nn.functional as F
from overlay_runtime import load_overlay
from quantize import load_calibration
@torch.inference_mode()
def compare(model, samples):
from zonos3.reference.lm import LMCache
outputs = []
for ids, speakers, conditions in samples:
logits = model(ids.to(model.device), LMCache(),
speaker_rows=speakers, condition_rows=conditions).logits
outputs.append(logits.detach().float().cpu())
return outputs
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--base", type=Path, required=True)
p.add_argument("--overlay", type=Path, required=True)
p.add_argument("--holdout", type=Path, required=True, help="Separate speaker/utterance JSONL; never fit set")
p.add_argument("--device", default="cuda:0", help="CUDA device holding native packed modules")
p.add_argument("--model-device", default=None, help="Original BF16 model device; CPU for low-VRAM eval")
p.add_argument("--allow-cpu-offload", action="store_true", help="Explicitly copy activations CPU/CUDA for native modules")
p.add_argument("--allow-incomplete", action="store_true", help="Evaluate partial overlays, not for final release")
p.add_argument("--max-nmse", type=float, default=0.10)
p.add_argument("--max-kl", type=float, default=0.10)
args = p.parse_args()
from zonos3.reference.lm import Zonos3LM
model_device = args.model_device or args.device
if model_device != args.device and not (model_device == "cpu" and args.allow_cpu_offload):
p.error("Different source/native devices require --model-device cpu --allow-cpu-offload")
base = Zonos3LM.load(args.base, device=model_device, dtype=torch.bfloat16).eval()
samples = load_calibration(args.holdout, base.config, limit=0)
reference = compare(base, samples)
load_overlay(base, args.overlay, device=args.device,
require_complete=not args.allow_incomplete,
allow_cpu_offload=args.allow_cpu_offload)
candidate = compare(base, samples)
total_error = total_signal = total_kl = total_argmax = total_entries = 0.0
for teacher, student in zip(reference, candidate):
if not torch.isfinite(student).all():
raise RuntimeError("Non-finite quantized logits")
total_error += float((teacher - student).square().sum())
total_signal += float(teacher.square().sum())
total_kl += float(F.kl_div(
F.log_softmax(student, dim=-1),
F.softmax(teacher, dim=-1),
reduction="sum",
))
total_argmax += float((teacher.argmax(-1) == student.argmax(-1)).sum())
total_entries += teacher.numel() / teacher.shape[-1]
nmse = total_error / max(total_signal, 1e-9)
kl = total_kl / max(total_entries, 1)
argmax = total_argmax / max(total_entries, 1)
print(f"heldout_logit_nmse={nmse:.6f} heldout_KL_nats={kl:.6f} argmax_agreement={argmax:.4%}")
if args.allow_incomplete:
print("INCOMPLETE OVERLAY PILOT: these numbers are NOT full-model quantization results")
print("This is a numeric gate only; run matched blind listening and codec end-to-end tests before shipping.")
if not math.isfinite(nmse + kl + argmax) or nmse > args.max_nmse or kl > args.max_kl:
raise SystemExit("FAILED: held-out teacher/INT4 gate")
if __name__ == "__main__":
main()