"""Quantisation shared by QAT (torch) and the exporter, so both agree exactly. Modes int8_row : symmetric int8, one f16 scale per row (linear layers) int4_row : symmetric int4 (-7..7), one f16 scale per row (token embeddings) int4_tensor: symmetric int4, one f16 scale per tensor, scale = CLIP_STD * std / 7 (large hashed n-gram table - per-row scales would cost 2 B/row) """ from __future__ import annotations import torch CLIP_STD = 4.0 MODES = {"int8_row": (8, True), "int4_row": (4, True), "int4_tensor": (4, False)} def _scale(w: torch.Tensor, mode: str) -> torch.Tensor: bits, per_row = MODES[mode] qmax = 2 ** (bits - 1) - 1 if per_row: s = w.abs().amax(dim=1, keepdim=True) / qmax else: s = (w.std() * CLIP_STD / qmax).reshape(1, 1) # scales are stored as f16 return s.clamp(min=1e-5).half().float() def quantize(w: torch.Tensor, mode: str) -> tuple[torch.Tensor, torch.Tensor]: """-> (integer tensor, scale tensor) with `w ~= q * scale`.""" bits, _ = MODES[mode] qmax = 2 ** (bits - 1) - 1 s = _scale(w, mode) q = torch.clamp(torch.round(w / s), -qmax, qmax) return q, s def fake_quant(w: torch.Tensor, mode: str) -> torch.Tensor: """Straight-through fake quantisation.""" q, s = quantize(w.detach(), mode) return w + (q * s - w).detach()