File size: 1,366 Bytes
c394b65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
"""Quantisation shared by QAT (torch) and the exporter, so both agree exactly.

Modes
  int8_row   : symmetric int8, one f16 scale per row        (linear layers)
  int4_row   : symmetric int4 (-7..7), one f16 scale per row (token embeddings)
  int4_tensor: symmetric int4, one f16 scale per tensor, scale = CLIP_STD * std / 7
               (large hashed n-gram table - per-row scales would cost 2 B/row)
"""

from __future__ import annotations

import torch

CLIP_STD = 4.0
MODES = {"int8_row": (8, True), "int4_row": (4, True), "int4_tensor": (4, False)}


def _scale(w: torch.Tensor, mode: str) -> torch.Tensor:
    bits, per_row = MODES[mode]
    qmax = 2 ** (bits - 1) - 1
    if per_row:
        s = w.abs().amax(dim=1, keepdim=True) / qmax
    else:
        s = (w.std() * CLIP_STD / qmax).reshape(1, 1)
    # scales are stored as f16
    return s.clamp(min=1e-5).half().float()


def quantize(w: torch.Tensor, mode: str) -> tuple[torch.Tensor, torch.Tensor]:
    """-> (integer tensor, scale tensor) with `w ~= q * scale`."""
    bits, _ = MODES[mode]
    qmax = 2 ** (bits - 1) - 1
    s = _scale(w, mode)
    q = torch.clamp(torch.round(w / s), -qmax, qmax)
    return q, s


def fake_quant(w: torch.Tensor, mode: str) -> torch.Tensor:
    """Straight-through fake quantisation."""
    q, s = quantize(w.detach(), mode)
    return w + (q * s - w).detach()