smoothquant-phi35-mini-vastrtx3090 / quant_agent_inference_adapter.py
chris320211's picture
Upload folder using huggingface_hub
99a8c10 verified
Raw History Blame Contribute Delete
18.8 kB
"""Reload a SmoothQuant W8A8 int8 artifact (``w8a8_int8`` format) and run it on int8.
The artifact (written by smoothquant/int8_export.py) stores, for every quantized
decoder linear, an int8 ``weight`` [out, in] and an fp16 per-output-channel
``weight_scale`` [out, 1], plus fp16 smoothed norms, embeddings, final norm and
(untied) lm_head. Smoothing is folded into those weights offline, so no
transform runs at load or inference time and there are no eval-only flags.
The kernels below are copied verbatim from the shared library entry
``agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py`` (``w8a8_int8_triton``,
matched by this artifact's quantization_config.json): a Triton per-token absmax
int8 activation quant, then a Triton int8 x int8 -> int32 tensor-core GEMM with a
fused ``acc * act_scale[m] * weight_scale[n] (+ bias[n])`` epilogue. On CPU, or
with ``QUANT_AGENT_W8A8_REFERENCE=1``, each linear falls back to the dense fp16
dequant reference.
The model is always built with the native transformers class and SDPA
attention: the artifact has no ``auto_map`` and no modeling code, so remote code
is never loaded whatever ``trust_remote_code`` says.
"""
import glob
import json
import os
import time
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from torch import nn
# --- kernels: verbatim copy of agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py ---
@triton.jit
def _act_quant_kernel(
x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm,
BLOCK_K: tl.constexpr,
):
"""Per-token absmax int8 quantization of one row of x (fp16/bf16)."""
row = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLOCK_K)
mask = offs < K
x_raw = tl.load(x_ptr + row * stride_xm + offs, mask=mask, other=0.0)
act_ty = x_raw.dtype
x = x_raw.to(tl.float32)
amax = tl.max(tl.abs(x), axis=0)
# Mirror torch on the activation dtype: clamp, cast, divide, cast.
amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32)
scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32)
v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32)
# Round half to even, like torch.round; |v| <= ~127.5 so v + 0.5 is exact.
r = tl.floor(v + 0.5)
tie = (r - v) == 0.5
odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0
r = tl.where(tie & odd, r - 1.0, r)
r = tl.minimum(tl.maximum(r, -127.0), 127.0)
tl.store(q_ptr + row * stride_qm + offs, r.to(tl.int8), mask=mask)
tl.store(s_ptr + row, scale)
@triton.jit
def _act_quant_looped_kernel(
x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm,
BLOCK_K: tl.constexpr,
):
"""Same math as _act_quant_kernel for long rows: absmax pass, then quantize pass."""
row = tl.program_id(0).to(tl.int64)
x_row = x_ptr + row * stride_xm
q_row = q_ptr + row * stride_qm
offs = tl.arange(0, BLOCK_K)
run_max = tl.zeros((BLOCK_K,), dtype=tl.float32)
for k0 in range(0, K, BLOCK_K):
mask = (k0 + offs) < K
x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32)
run_max = tl.maximum(run_max, tl.abs(x))
act_ty = x_ptr.dtype.element_ty
amax = tl.max(run_max, axis=0)
amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32)
scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32)
for k0 in range(0, K, BLOCK_K):
mask = (k0 + offs) < K
x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32)
v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32)
r = tl.floor(v + 0.5)
tie = (r - v) == 0.5
odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0
r = tl.where(tie & odd, r - 1.0, r)
r = tl.minimum(tl.maximum(r, -127.0), 127.0)
tl.store(q_row + k0 + offs, r.to(tl.int8), mask=mask)
tl.store(s_ptr + row, scale)
@triton.jit
def _w8a8_gemm_kernel(
a_ptr, b_ptr, c_ptr, as_ptr, bs_ptr, bias_ptr,
M, N, K,
stride_am, stride_ak, stride_bn, stride_bk, stride_cm, stride_cn,
HAS_BIAS: tl.constexpr,
EVEN_K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_M: tl.constexpr,
):
"""C[M, N] = (A_int8[M, K] @ B_int8[N, K]^T) * a_scale[M] * b_scale[N] + bias[N]."""
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
num_pid_n = tl.cdiv(N, BLOCK_N)
num_pid_in_group = GROUP_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_M)
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
# Wrap out-of-range rows/cols onto valid memory; the store is masked.
offs_am = offs_m % M
offs_bn = offs_n % N
offs_k = tl.arange(0, BLOCK_K)
a_ptrs = a_ptr + offs_am[:, None].to(tl.int64) * stride_am + offs_k[None, :] * stride_ak
b_ptrs = b_ptr + offs_bn[None, :].to(tl.int64) * stride_bn + offs_k[:, None] * stride_bk
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.int32)
for k in range(0, tl.cdiv(K, BLOCK_K)):
if EVEN_K:
a = tl.load(a_ptrs)
b = tl.load(b_ptrs)
else:
k_left = K - k * BLOCK_K
a = tl.load(a_ptrs, mask=offs_k[None, :] < k_left, other=0)
b = tl.load(b_ptrs, mask=offs_k[:, None] < k_left, other=0)
acc = tl.dot(a, b, acc, out_dtype=tl.int32)
a_ptrs += BLOCK_K * stride_ak
b_ptrs += BLOCK_K * stride_bk
a_scale = tl.load(as_ptr + offs_am).to(tl.float32)
b_scale = tl.load(bs_ptr + offs_bn).to(tl.float32)
out = acc.to(tl.float32) * a_scale[:, None] * b_scale[None, :]
if HAS_BIAS:
bias = tl.load(bias_ptr + offs_bn).to(tl.float32)
out = out + bias[None, :]
c_ptrs = c_ptr + offs_m[:, None].to(tl.int64) * stride_cm + offs_n[None, :] * stride_cn
c_mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, out.to(c_ptr.dtype.element_ty), mask=c_mask)
def quantize_activation_int8(x):
"""x [M, K] fp16/bf16 contiguous -> (int8 codes [M, K], fp32 per-token scale [M])."""
m, k = x.shape
q = torch.empty((m, k), dtype=torch.int8, device=x.device)
scale = torch.empty((m,), dtype=torch.float32, device=x.device)
if m == 0:
return q, scale
block_k = triton.next_power_of_2(int(k))
args = (x, q, scale, int(k), int(x.stride(0)), int(q.stride(0)))
if block_k <= 4096:
# Whole row in registers: one read of x.
_act_quant_kernel[(int(m),)](*args, BLOCK_K=block_k, num_warps=8)
else:
# Long rows: absmax pass + quantize pass over 4096-wide chunks; the
# second read hits L2. Measured at copy bandwidth on sm_86.
_act_quant_looped_kernel[(int(m),)](*args, BLOCK_K=4096, num_warps=8)
return q, scale
def _gemm_config(m, n, k):
"""Default (BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, num_warps, num_stages).
Tuned on sm_86 (RTX 3090, see kernel.json `tuned_on`). On another compute
capability, run `tune_config` on the model's shapes and pass the result as
`config` to `w8a8_matmul`.
"""
if m <= 16:
return 16, 64, 256, 1, 4, 3
if m <= 64:
return 64, 64, 128, 8, 4, 4
return 128, 128, 64, 8, 4, 4
def w8a8_matmul(q, a_scale, weight, weight_scale, bias, out_dtype, config=None):
"""int8 q [M, K] x int8 weight [N, K] -> out_dtype [M, N] with the dequant epilogue."""
m, k = q.shape
n = weight.shape[0]
out = torch.empty((m, n), dtype=out_dtype, device=q.device)
if m == 0:
return out
block_m, block_n, block_k, group_m, warps, stages = config or _gemm_config(m, n, k)
grid = (triton.cdiv(int(m), block_m) * triton.cdiv(int(n), block_n),)
_w8a8_gemm_kernel[grid](
q, weight, out, a_scale, weight_scale, bias if bias is not None else a_scale,
int(m), int(n), int(k),
int(q.stride(0)), int(q.stride(1)),
int(weight.stride(0)), int(weight.stride(1)),
int(out.stride(0)), int(out.stride(1)),
HAS_BIAS=bias is not None,
EVEN_K=(int(k) % block_k == 0),
BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, GROUP_M=group_m,
num_warps=warps, num_stages=stages,
)
return out
def w8a8_linear(x2d, weight, weight_scale, bias):
"""Real int8 W8A8 linear: Triton per-token quant, then int8 GEMM + epilogue."""
q, a_scale = quantize_activation_int8(x2d)
return w8a8_matmul(q, a_scale, weight, weight_scale, bias, x2d.dtype)
def w8a8_linear_reference(x, weight, weight_scale, bias):
"""fp16 dequant reference: same quantization, dense fp16 matmul. CPU-safe."""
x_scale = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-5) / 127
x_q = torch.round(x / x_scale).clamp_(-127, 127).mul_(x_scale)
w = weight.to(x.dtype).mul_(weight_scale.reshape(-1, 1).to(x.dtype))
b = None if bias is None else bias.to(x.dtype)
return F.linear(x_q, w, b)
CONFIG_CANDIDATES = (
(128, 128, 64, 8, 4, 4),
(128, 128, 64, 8, 8, 3),
(128, 256, 64, 8, 8, 3),
(64, 128, 128, 8, 4, 4),
(64, 64, 128, 8, 4, 4),
(16, 64, 256, 1, 4, 3),
)
def tune_config(m, n, k, *, device="cuda", candidates=CONFIG_CANDIDATES, iters=20):
"""Time each candidate tile config on random int8 inputs of this shape; return the fastest.
Use once per (M, N, K) on a GPU whose compute capability is not in kernel.json
`tuned_on`, then pass the result as `config` to `w8a8_matmul`.
"""
q = torch.randint(-127, 128, (m, k), dtype=torch.int8, device=device)
w = torch.randint(-127, 128, (n, k), dtype=torch.int8, device=device)
a_scale = torch.rand(m, dtype=torch.float32, device=device)
w_scale = torch.rand(n, 1, dtype=torch.float16, device=device)
best, best_t = None, float("inf")
for config in candidates:
try:
w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config)
except Exception: # a tile the GPU cannot fit (shared memory, registers)
continue
torch.cuda.synchronize(device)
start = time.perf_counter()
for _ in range(iters):
w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config)
torch.cuda.synchronize(device)
elapsed = (time.perf_counter() - start) / iters
if elapsed < best_t:
best, best_t = config, elapsed
if best is None:
raise RuntimeError(f"no candidate config ran for M={m} N={n} K={k}")
return best
# --- loader (port-specific) ----------------------------------------------------
QUANT_AGENT_ADAPTER_API = 1
QUANT_CONFIG_NAME = "quantization_config.json"
EXPECTED_FORMAT = "w8a8_int8"
SCALE_SUFFIX = ".weight_scale"
REFERENCE_ENV = "QUANT_AGENT_W8A8_REFERENCE"
def _w8a8_int8_linear_class():
class W8A8Int8Linear(nn.Module):
"""int8 weight + per-channel scale; per-token dynamic int8 activations; int8 GEMM."""
def __init__(self, in_features, out_features, has_bias, dtype):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.register_buffer(
"weight", torch.empty(out_features, in_features, dtype=torch.int8)
)
self.register_buffer(
"weight_scale", torch.empty(out_features, 1, dtype=dtype)
)
if has_bias:
self.register_buffer("bias", torch.empty(out_features, dtype=dtype))
else:
self.bias = None
self.use_reference = os.environ.get(REFERENCE_ENV) == "1"
@torch.no_grad()
def forward(self, x):
if (
self.use_reference
or not x.is_cuda
or x.dtype not in (torch.float16, torch.bfloat16)
):
return w8a8_linear_reference(x, self.weight, self.weight_scale, self.bias)
lead = x.shape[:-1]
x2d = x.reshape(-1, self.in_features)
if not x2d.is_contiguous():
x2d = x2d.contiguous()
out = w8a8_linear(x2d, self.weight, self.weight_scale, self.bias)
return out.reshape(*lead, self.out_features)
def extra_repr(self):
return (
f"in_features={self.in_features}, out_features={self.out_features}, "
f"bias={self.bias is not None}, weight=int8 per_channel, "
"act=int8 per_token dynamic, kernel=triton int8 gemm"
)
return W8A8Int8Linear
def _resolve_dtype(dtype):
if dtype in (torch.bfloat16, "bfloat16", "bf16"):
return torch.bfloat16
if dtype in (None, "auto", torch.float16, "float16", "fp16", "half"):
return torch.float16
raise ValueError(f"unsupported dtype for the W8A8 int8 artifact: {dtype!r}")
def _read_quant_config(model_path):
path = os.path.join(model_path, QUANT_CONFIG_NAME)
if not os.path.isfile(path):
raise FileNotFoundError(f"{QUANT_CONFIG_NAME} is missing in {model_path}")
with open(path) as handle:
qcfg = json.load(handle)
if qcfg.get("format") != EXPECTED_FORMAT:
raise ValueError(f"expected format {EXPECTED_FORMAT!r}, got {qcfg.get('format')!r}")
if str(qcfg.get("act_quant")) != "per_token" or str(qcfg.get("weight_quant")) != "per_channel":
raise ValueError(
"this adapter handles per_channel int8 weights with per_token dynamic activations only"
)
return qcfg
def _read_state(model_path):
from safetensors.torch import load_file
files = sorted(glob.glob(os.path.join(model_path, "*.safetensors")))
if not files:
raise FileNotFoundError(f"no .safetensors weights in {model_path}")
state = {}
for path in files:
part = load_file(path, device="cpu")
overlap = set(part) & set(state)
if overlap:
raise ValueError(f"tensor(s) {sorted(overlap)[:3]} appear in two weight files")
state.update(part)
return state
def _set_submodule(model, name, module):
parent_name, _, leaf = name.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
setattr(parent, leaf, module)
def load_model_and_tokenizer(model_path, model_id, dtype, device, trust_remote_code):
from accelerate import init_empty_weights
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig
del model_id # the saved artifact is the only source of weights
# The artifact carries no modeling code (int8_export strips auto_map); the
# architecture is native to transformers, so remote code is never used.
del trust_remote_code
torch_dtype = _resolve_dtype(dtype)
qcfg = _read_quant_config(model_path)
state = _read_state(model_path)
config = AutoConfig.from_pretrained(
model_path, trust_remote_code=False, local_files_only=True
)
if getattr(config, "quantization_config", None):
raise ValueError("config.json carries a quantization_config; not a SmoothQuant int8 artifact")
# Parameters on meta (no fp16 copy of the linears is ever allocated); buffers
# such as the rotary inv_freq stay real.
with init_empty_weights(include_buffers=False):
model = AutoModelForCausalLM.from_config(
config,
torch_dtype=torch_dtype,
trust_remote_code=False,
attn_implementation="sdpa",
)
linear_cls = _w8a8_int8_linear_class()
modules = dict(model.named_modules())
quantized = []
for key in sorted(state):
if not key.endswith(SCALE_SUFFIX):
continue
name = key[: -len(SCALE_SUFFIX)]
weight = state.get(f"{name}.weight")
if weight is None or weight.dtype != torch.int8 or weight.dim() != 2:
raise ValueError(f"{key} has no int8 [out, in] weight next to it")
original = modules.get(name)
if not isinstance(original, nn.Linear):
raise ValueError(f"{name} is not an nn.Linear in {type(model).__name__}")
out_features, in_features = weight.shape
if (original.out_features, original.in_features) != (out_features, in_features):
raise ValueError(f"{name}: saved shape {tuple(weight.shape)} does not match the config")
if tuple(state[key].shape) not in ((out_features, 1), (out_features,)):
raise ValueError(f"{key}: expected [{out_features}, 1], got {tuple(state[key].shape)}")
state[key] = state[key].reshape(out_features, 1)
# The GEMM reads B as [N, K] with K contiguous; make sure that holds.
state[f"{name}.weight"] = weight.contiguous()
has_bias = f"{name}.bias" in state
if has_bias != (original.bias is not None):
raise ValueError(f"{name}: bias presence differs between artifact and config")
_set_submodule(model, name, linear_cls(in_features, out_features, has_bias, torch_dtype))
quantized.append(name)
if not quantized:
raise ValueError("no int8 weight + weight_scale pairs in the artifact")
expected = qcfg.get("num_quantized_linears")
if isinstance(expected, int) and expected != len(quantized):
raise ValueError(
f"{QUANT_CONFIG_NAME} lists {expected} quantized linears, artifact has {len(quantized)}"
)
for key, tensor in list(state.items()):
if tensor.is_floating_point() and tensor.dtype != torch_dtype:
state[key] = tensor.to(torch_dtype)
tied = bool(getattr(config, "tie_word_embeddings", False))
result = model.load_state_dict(state, strict=False, assign=True)
missing = set(result.missing_keys)
if tied:
missing.discard("lm_head.weight")
if missing or result.unexpected_keys:
raise ValueError(
f"artifact does not match the model: missing={sorted(missing)[:8]} "
f"unexpected={sorted(result.unexpected_keys)[:8]}"
)
if tied:
model.tie_weights()
leftover = [n for n, t in list(model.named_parameters()) + list(model.named_buffers()) if t.is_meta]
if leftover:
raise ValueError(f"tensors left uninitialised after load: {leftover[:8]}")
try:
model.generation_config = GenerationConfig.from_pretrained(
model_path, local_files_only=True
)
except OSError:
pass
model.to(device)
model.eval()
attn_impl = getattr(model.config, "_attn_implementation", None)
if attn_impl != "sdpa":
raise RuntimeError(f"expected SDPA attention for prefill, got {attn_impl!r}")
tokenizer = AutoTokenizer.from_pretrained(
model_path, trust_remote_code=False, local_files_only=True
)
return model, tokenizer