Download quant.py from anudit/tinyemo: direct link, hf CLI and curl.
- Browser
- Download file 1.37 kB
-
https://huggingface.co/anudit/tinyemo/resolve/main/quant.py
- Command line
-
hf download hf://anudit/tinyemo/quant.py
-
curl -L -o quant.py https://huggingface.co/anudit/tinyemo/resolve/main/quant.py
1.37 kB
| """Quantisation shared by QAT (torch) and the exporter, so both agree exactly. | |
| Modes | |
| int8_row : symmetric int8, one f16 scale per row (linear layers) | |
| int4_row : symmetric int4 (-7..7), one f16 scale per row (token embeddings) | |
| int4_tensor: symmetric int4, one f16 scale per tensor, scale = CLIP_STD * std / 7 | |
| (large hashed n-gram table - per-row scales would cost 2 B/row) | |
| """ | |
| from __future__ import annotations | |
| import torch | |
| CLIP_STD = 4.0 | |
| MODES = {"int8_row": (8, True), "int4_row": (4, True), "int4_tensor": (4, False)} | |
| def _scale(w: torch.Tensor, mode: str) -> torch.Tensor: | |
| bits, per_row = MODES[mode] | |
| qmax = 2 ** (bits - 1) - 1 | |
| if per_row: | |
| s = w.abs().amax(dim=1, keepdim=True) / qmax | |
| else: | |
| s = (w.std() * CLIP_STD / qmax).reshape(1, 1) | |
| # scales are stored as f16 | |
| return s.clamp(min=1e-5).half().float() | |
| def quantize(w: torch.Tensor, mode: str) -> tuple[torch.Tensor, torch.Tensor]: | |
| """-> (integer tensor, scale tensor) with `w ~= q * scale`.""" | |
| bits, _ = MODES[mode] | |
| qmax = 2 ** (bits - 1) - 1 | |
| s = _scale(w, mode) | |
| q = torch.clamp(torch.round(w / s), -qmax, qmax) | |
| return q, s | |
| def fake_quant(w: torch.Tensor, mode: str) -> torch.Tensor: | |
| """Straight-through fake quantisation.""" | |
| q, s = quantize(w.detach(), mode) | |
| return w + (q * s - w).detach() | |