chris320211's picture
Upload folder using huggingface_hub
99a8c10 verified
Raw History Blame Contribute Delete
32.8 kB
diff --git a/quant_agent_inference_adapter.py b/quant_agent_inference_adapter.py
new file mode 100644
index 0000000..392f991
--- /dev/null
+++ b/quant_agent_inference_adapter.py
@@ -0,0 +1,456 @@
+"""Reload a SmoothQuant W8A8 int8 artifact (``w8a8_int8`` format) and run it on int8.
+
+The artifact (written by smoothquant/int8_export.py) stores, for every quantized
+decoder linear, an int8 ``weight`` [out, in] and an fp16 per-output-channel
+``weight_scale`` [out, 1], plus fp16 smoothed norms, embeddings, final norm and
+(untied) lm_head. Smoothing is folded into those weights offline, so no
+transform runs at load or inference time and there are no eval-only flags.
+
+The kernels below are copied verbatim from the shared library entry
+``agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py`` (``w8a8_int8_triton``,
+matched by this artifact's quantization_config.json): a Triton per-token absmax
+int8 activation quant, then a Triton int8 x int8 -> int32 tensor-core GEMM with a
+fused ``acc * act_scale[m] * weight_scale[n] (+ bias[n])`` epilogue. On CPU, or
+with ``QUANT_AGENT_W8A8_REFERENCE=1``, each linear falls back to the dense fp16
+dequant reference.
+
+The model is always built with the native transformers class and SDPA
+attention: the artifact has no ``auto_map`` and no modeling code, so remote code
+is never loaded whatever ``trust_remote_code`` says.
+"""
+import glob
+import json
+import os
+import time
+
+import torch
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from torch import nn
+
+
+# --- kernels: verbatim copy of agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py ---
+
+@triton.jit
+def _act_quant_kernel(
+ x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm,
+ BLOCK_K: tl.constexpr,
+):
+ """Per-token absmax int8 quantization of one row of x (fp16/bf16)."""
+ row = tl.program_id(0).to(tl.int64)
+ offs = tl.arange(0, BLOCK_K)
+ mask = offs < K
+ x_raw = tl.load(x_ptr + row * stride_xm + offs, mask=mask, other=0.0)
+ act_ty = x_raw.dtype
+ x = x_raw.to(tl.float32)
+ amax = tl.max(tl.abs(x), axis=0)
+ # Mirror torch on the activation dtype: clamp, cast, divide, cast.
+ amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32)
+ scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32)
+ v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32)
+ # Round half to even, like torch.round; |v| <= ~127.5 so v + 0.5 is exact.
+ r = tl.floor(v + 0.5)
+ tie = (r - v) == 0.5
+ odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0
+ r = tl.where(tie & odd, r - 1.0, r)
+ r = tl.minimum(tl.maximum(r, -127.0), 127.0)
+ tl.store(q_ptr + row * stride_qm + offs, r.to(tl.int8), mask=mask)
+ tl.store(s_ptr + row, scale)
+
+
+@triton.jit
+def _act_quant_looped_kernel(
+ x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm,
+ BLOCK_K: tl.constexpr,
+):
+ """Same math as _act_quant_kernel for long rows: absmax pass, then quantize pass."""
+ row = tl.program_id(0).to(tl.int64)
+ x_row = x_ptr + row * stride_xm
+ q_row = q_ptr + row * stride_qm
+ offs = tl.arange(0, BLOCK_K)
+ run_max = tl.zeros((BLOCK_K,), dtype=tl.float32)
+ for k0 in range(0, K, BLOCK_K):
+ mask = (k0 + offs) < K
+ x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32)
+ run_max = tl.maximum(run_max, tl.abs(x))
+ act_ty = x_ptr.dtype.element_ty
+ amax = tl.max(run_max, axis=0)
+ amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32)
+ scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32)
+ for k0 in range(0, K, BLOCK_K):
+ mask = (k0 + offs) < K
+ x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32)
+ v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32)
+ r = tl.floor(v + 0.5)
+ tie = (r - v) == 0.5
+ odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0
+ r = tl.where(tie & odd, r - 1.0, r)
+ r = tl.minimum(tl.maximum(r, -127.0), 127.0)
+ tl.store(q_row + k0 + offs, r.to(tl.int8), mask=mask)
+ tl.store(s_ptr + row, scale)
+
+
+@triton.jit
+def _w8a8_gemm_kernel(
+ a_ptr, b_ptr, c_ptr, as_ptr, bs_ptr, bias_ptr,
+ M, N, K,
+ stride_am, stride_ak, stride_bn, stride_bk, stride_cm, stride_cn,
+ HAS_BIAS: tl.constexpr,
+ EVEN_K: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ GROUP_M: tl.constexpr,
+):
+ """C[M, N] = (A_int8[M, K] @ B_int8[N, K]^T) * a_scale[M] * b_scale[N] + bias[N]."""
+ pid = tl.program_id(0)
+ num_pid_m = tl.cdiv(M, BLOCK_M)
+ num_pid_n = tl.cdiv(N, BLOCK_N)
+ num_pid_in_group = GROUP_M * num_pid_n
+ group_id = pid // num_pid_in_group
+ first_pid_m = group_id * GROUP_M
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_M)
+ pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
+ pid_n = (pid % num_pid_in_group) // group_size_m
+
+ offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ # Wrap out-of-range rows/cols onto valid memory; the store is masked.
+ offs_am = offs_m % M
+ offs_bn = offs_n % N
+ offs_k = tl.arange(0, BLOCK_K)
+ a_ptrs = a_ptr + offs_am[:, None].to(tl.int64) * stride_am + offs_k[None, :] * stride_ak
+ b_ptrs = b_ptr + offs_bn[None, :].to(tl.int64) * stride_bn + offs_k[:, None] * stride_bk
+
+ acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.int32)
+ for k in range(0, tl.cdiv(K, BLOCK_K)):
+ if EVEN_K:
+ a = tl.load(a_ptrs)
+ b = tl.load(b_ptrs)
+ else:
+ k_left = K - k * BLOCK_K
+ a = tl.load(a_ptrs, mask=offs_k[None, :] < k_left, other=0)
+ b = tl.load(b_ptrs, mask=offs_k[:, None] < k_left, other=0)
+ acc = tl.dot(a, b, acc, out_dtype=tl.int32)
+ a_ptrs += BLOCK_K * stride_ak
+ b_ptrs += BLOCK_K * stride_bk
+
+ a_scale = tl.load(as_ptr + offs_am).to(tl.float32)
+ b_scale = tl.load(bs_ptr + offs_bn).to(tl.float32)
+ out = acc.to(tl.float32) * a_scale[:, None] * b_scale[None, :]
+ if HAS_BIAS:
+ bias = tl.load(bias_ptr + offs_bn).to(tl.float32)
+ out = out + bias[None, :]
+ c_ptrs = c_ptr + offs_m[:, None].to(tl.int64) * stride_cm + offs_n[None, :] * stride_cn
+ c_mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
+ tl.store(c_ptrs, out.to(c_ptr.dtype.element_ty), mask=c_mask)
+
+
+def quantize_activation_int8(x):
+ """x [M, K] fp16/bf16 contiguous -> (int8 codes [M, K], fp32 per-token scale [M])."""
+ m, k = x.shape
+ q = torch.empty((m, k), dtype=torch.int8, device=x.device)
+ scale = torch.empty((m,), dtype=torch.float32, device=x.device)
+ if m == 0:
+ return q, scale
+ block_k = triton.next_power_of_2(int(k))
+ args = (x, q, scale, int(k), int(x.stride(0)), int(q.stride(0)))
+ if block_k <= 4096:
+ # Whole row in registers: one read of x.
+ _act_quant_kernel[(int(m),)](*args, BLOCK_K=block_k, num_warps=8)
+ else:
+ # Long rows: absmax pass + quantize pass over 4096-wide chunks; the
+ # second read hits L2. Measured at copy bandwidth on sm_86.
+ _act_quant_looped_kernel[(int(m),)](*args, BLOCK_K=4096, num_warps=8)
+ return q, scale
+
+
+def _gemm_config(m, n, k):
+ """Default (BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, num_warps, num_stages).
+
+ Tuned on sm_86 (RTX 3090, see kernel.json `tuned_on`). On another compute
+ capability, run `tune_config` on the model's shapes and pass the result as
+ `config` to `w8a8_matmul`.
+ """
+ if m <= 16:
+ return 16, 64, 256, 1, 4, 3
+ if m <= 64:
+ return 64, 64, 128, 8, 4, 4
+ return 128, 128, 64, 8, 4, 4
+
+
+def w8a8_matmul(q, a_scale, weight, weight_scale, bias, out_dtype, config=None):
+ """int8 q [M, K] x int8 weight [N, K] -> out_dtype [M, N] with the dequant epilogue."""
+ m, k = q.shape
+ n = weight.shape[0]
+ out = torch.empty((m, n), dtype=out_dtype, device=q.device)
+ if m == 0:
+ return out
+ block_m, block_n, block_k, group_m, warps, stages = config or _gemm_config(m, n, k)
+ grid = (triton.cdiv(int(m), block_m) * triton.cdiv(int(n), block_n),)
+ _w8a8_gemm_kernel[grid](
+ q, weight, out, a_scale, weight_scale, bias if bias is not None else a_scale,
+ int(m), int(n), int(k),
+ int(q.stride(0)), int(q.stride(1)),
+ int(weight.stride(0)), int(weight.stride(1)),
+ int(out.stride(0)), int(out.stride(1)),
+ HAS_BIAS=bias is not None,
+ EVEN_K=(int(k) % block_k == 0),
+ BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, GROUP_M=group_m,
+ num_warps=warps, num_stages=stages,
+ )
+ return out
+
+
+def w8a8_linear(x2d, weight, weight_scale, bias):
+ """Real int8 W8A8 linear: Triton per-token quant, then int8 GEMM + epilogue."""
+ q, a_scale = quantize_activation_int8(x2d)
+ return w8a8_matmul(q, a_scale, weight, weight_scale, bias, x2d.dtype)
+
+
+def w8a8_linear_reference(x, weight, weight_scale, bias):
+ """fp16 dequant reference: same quantization, dense fp16 matmul. CPU-safe."""
+ x_scale = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-5) / 127
+ x_q = torch.round(x / x_scale).clamp_(-127, 127).mul_(x_scale)
+ w = weight.to(x.dtype).mul_(weight_scale.reshape(-1, 1).to(x.dtype))
+ b = None if bias is None else bias.to(x.dtype)
+ return F.linear(x_q, w, b)
+
+
+CONFIG_CANDIDATES = (
+ (128, 128, 64, 8, 4, 4),
+ (128, 128, 64, 8, 8, 3),
+ (128, 256, 64, 8, 8, 3),
+ (64, 128, 128, 8, 4, 4),
+ (64, 64, 128, 8, 4, 4),
+ (16, 64, 256, 1, 4, 3),
+)
+
+
+def tune_config(m, n, k, *, device="cuda", candidates=CONFIG_CANDIDATES, iters=20):
+ """Time each candidate tile config on random int8 inputs of this shape; return the fastest.
+
+ Use once per (M, N, K) on a GPU whose compute capability is not in kernel.json
+ `tuned_on`, then pass the result as `config` to `w8a8_matmul`.
+ """
+ q = torch.randint(-127, 128, (m, k), dtype=torch.int8, device=device)
+ w = torch.randint(-127, 128, (n, k), dtype=torch.int8, device=device)
+ a_scale = torch.rand(m, dtype=torch.float32, device=device)
+ w_scale = torch.rand(n, 1, dtype=torch.float16, device=device)
+ best, best_t = None, float("inf")
+ for config in candidates:
+ try:
+ w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config)
+ except Exception: # a tile the GPU cannot fit (shared memory, registers)
+ continue
+ torch.cuda.synchronize(device)
+ start = time.perf_counter()
+ for _ in range(iters):
+ w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config)
+ torch.cuda.synchronize(device)
+ elapsed = (time.perf_counter() - start) / iters
+ if elapsed < best_t:
+ best, best_t = config, elapsed
+ if best is None:
+ raise RuntimeError(f"no candidate config ran for M={m} N={n} K={k}")
+ return best
+
+
+# --- loader (port-specific) ----------------------------------------------------
+
+QUANT_AGENT_ADAPTER_API = 1
+QUANT_CONFIG_NAME = "quantization_config.json"
+EXPECTED_FORMAT = "w8a8_int8"
+SCALE_SUFFIX = ".weight_scale"
+REFERENCE_ENV = "QUANT_AGENT_W8A8_REFERENCE"
+
+
+def _w8a8_int8_linear_class():
+ class W8A8Int8Linear(nn.Module):
+ """int8 weight + per-channel scale; per-token dynamic int8 activations; int8 GEMM."""
+
+ def __init__(self, in_features, out_features, has_bias, dtype):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.register_buffer(
+ "weight", torch.empty(out_features, in_features, dtype=torch.int8)
+ )
+ self.register_buffer(
+ "weight_scale", torch.empty(out_features, 1, dtype=dtype)
+ )
+ if has_bias:
+ self.register_buffer("bias", torch.empty(out_features, dtype=dtype))
+ else:
+ self.bias = None
+ self.use_reference = os.environ.get(REFERENCE_ENV) == "1"
+
+ @torch.no_grad()
+ def forward(self, x):
+ if (
+ self.use_reference
+ or not x.is_cuda
+ or x.dtype not in (torch.float16, torch.bfloat16)
+ ):
+ return w8a8_linear_reference(x, self.weight, self.weight_scale, self.bias)
+ lead = x.shape[:-1]
+ x2d = x.reshape(-1, self.in_features)
+ if not x2d.is_contiguous():
+ x2d = x2d.contiguous()
+ out = w8a8_linear(x2d, self.weight, self.weight_scale, self.bias)
+ return out.reshape(*lead, self.out_features)
+
+ def extra_repr(self):
+ return (
+ f"in_features={self.in_features}, out_features={self.out_features}, "
+ f"bias={self.bias is not None}, weight=int8 per_channel, "
+ "act=int8 per_token dynamic, kernel=triton int8 gemm"
+ )
+
+ return W8A8Int8Linear
+
+
+def _resolve_dtype(dtype):
+ if dtype in (torch.bfloat16, "bfloat16", "bf16"):
+ return torch.bfloat16
+ if dtype in (None, "auto", torch.float16, "float16", "fp16", "half"):
+ return torch.float16
+ raise ValueError(f"unsupported dtype for the W8A8 int8 artifact: {dtype!r}")
+
+
+def _read_quant_config(model_path):
+ path = os.path.join(model_path, QUANT_CONFIG_NAME)
+ if not os.path.isfile(path):
+ raise FileNotFoundError(f"{QUANT_CONFIG_NAME} is missing in {model_path}")
+ with open(path) as handle:
+ qcfg = json.load(handle)
+ if qcfg.get("format") != EXPECTED_FORMAT:
+ raise ValueError(f"expected format {EXPECTED_FORMAT!r}, got {qcfg.get('format')!r}")
+ if str(qcfg.get("act_quant")) != "per_token" or str(qcfg.get("weight_quant")) != "per_channel":
+ raise ValueError(
+ "this adapter handles per_channel int8 weights with per_token dynamic activations only"
+ )
+ return qcfg
+
+
+def _read_state(model_path):
+ from safetensors.torch import load_file
+
+ files = sorted(glob.glob(os.path.join(model_path, "*.safetensors")))
+ if not files:
+ raise FileNotFoundError(f"no .safetensors weights in {model_path}")
+ state = {}
+ for path in files:
+ part = load_file(path, device="cpu")
+ overlap = set(part) & set(state)
+ if overlap:
+ raise ValueError(f"tensor(s) {sorted(overlap)[:3]} appear in two weight files")
+ state.update(part)
+ return state
+
+
+def _set_submodule(model, name, module):
+ parent_name, _, leaf = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ setattr(parent, leaf, module)
+
+
+def load_model_and_tokenizer(model_path, model_id, dtype, device, trust_remote_code):
+ from accelerate import init_empty_weights
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+
+ del model_id # the saved artifact is the only source of weights
+ # The artifact carries no modeling code (int8_export strips auto_map); the
+ # architecture is native to transformers, so remote code is never used.
+ del trust_remote_code
+ torch_dtype = _resolve_dtype(dtype)
+ qcfg = _read_quant_config(model_path)
+ state = _read_state(model_path)
+
+ config = AutoConfig.from_pretrained(
+ model_path, trust_remote_code=False, local_files_only=True
+ )
+ if getattr(config, "quantization_config", None):
+ raise ValueError("config.json carries a quantization_config; not a SmoothQuant int8 artifact")
+
+ # Parameters on meta (no fp16 copy of the linears is ever allocated); buffers
+ # such as the rotary inv_freq stay real.
+ with init_empty_weights(include_buffers=False):
+ model = AutoModelForCausalLM.from_config(
+ config,
+ torch_dtype=torch_dtype,
+ trust_remote_code=False,
+ attn_implementation="sdpa",
+ )
+
+ linear_cls = _w8a8_int8_linear_class()
+ modules = dict(model.named_modules())
+ quantized = []
+ for key in sorted(state):
+ if not key.endswith(SCALE_SUFFIX):
+ continue
+ name = key[: -len(SCALE_SUFFIX)]
+ weight = state.get(f"{name}.weight")
+ if weight is None or weight.dtype != torch.int8 or weight.dim() != 2:
+ raise ValueError(f"{key} has no int8 [out, in] weight next to it")
+ original = modules.get(name)
+ if not isinstance(original, nn.Linear):
+ raise ValueError(f"{name} is not an nn.Linear in {type(model).__name__}")
+ out_features, in_features = weight.shape
+ if (original.out_features, original.in_features) != (out_features, in_features):
+ raise ValueError(f"{name}: saved shape {tuple(weight.shape)} does not match the config")
+ if tuple(state[key].shape) not in ((out_features, 1), (out_features,)):
+ raise ValueError(f"{key}: expected [{out_features}, 1], got {tuple(state[key].shape)}")
+ state[key] = state[key].reshape(out_features, 1)
+ # The GEMM reads B as [N, K] with K contiguous; make sure that holds.
+ state[f"{name}.weight"] = weight.contiguous()
+ has_bias = f"{name}.bias" in state
+ if has_bias != (original.bias is not None):
+ raise ValueError(f"{name}: bias presence differs between artifact and config")
+ _set_submodule(model, name, linear_cls(in_features, out_features, has_bias, torch_dtype))
+ quantized.append(name)
+ if not quantized:
+ raise ValueError("no int8 weight + weight_scale pairs in the artifact")
+ expected = qcfg.get("num_quantized_linears")
+ if isinstance(expected, int) and expected != len(quantized):
+ raise ValueError(
+ f"{QUANT_CONFIG_NAME} lists {expected} quantized linears, artifact has {len(quantized)}"
+ )
+
+ for key, tensor in list(state.items()):
+ if tensor.is_floating_point() and tensor.dtype != torch_dtype:
+ state[key] = tensor.to(torch_dtype)
+
+ tied = bool(getattr(config, "tie_word_embeddings", False))
+ result = model.load_state_dict(state, strict=False, assign=True)
+ missing = set(result.missing_keys)
+ if tied:
+ missing.discard("lm_head.weight")
+ if missing or result.unexpected_keys:
+ raise ValueError(
+ f"artifact does not match the model: missing={sorted(missing)[:8]} "
+ f"unexpected={sorted(result.unexpected_keys)[:8]}"
+ )
+ if tied:
+ model.tie_weights()
+ leftover = [n for n, t in list(model.named_parameters()) + list(model.named_buffers()) if t.is_meta]
+ if leftover:
+ raise ValueError(f"tensors left uninitialised after load: {leftover[:8]}")
+
+ try:
+ model.generation_config = GenerationConfig.from_pretrained(
+ model_path, local_files_only=True
+ )
+ except OSError:
+ pass
+ model.to(device)
+ model.eval()
+ attn_impl = getattr(model.config, "_attn_implementation", None)
+ if attn_impl != "sdpa":
+ raise RuntimeError(f"expected SDPA attention for prefill, got {attn_impl!r}")
+
+ tokenizer = AutoTokenizer.from_pretrained(
+ model_path, trust_remote_code=False, local_files_only=True
+ )
+ return model, tokenizer
diff --git a/smoothquant/fake_quant.py b/smoothquant/fake_quant.py
index 24b62b5..bb7c175 100644
--- a/smoothquant/fake_quant.py
+++ b/smoothquant/fake_quant.py
@@ -13,6 +13,23 @@ def quantize_weight_per_channel_absmax(w, n_bits=8):
return w
+@torch.no_grad()
+def quantize_weight_per_channel_absmax_int8(w, n_bits=8):
+ """Same math as quantize_weight_per_channel_absmax, but also returns the integers.
+
+ Returns (q, scales, w_dq): q is int8 [out, in], scales is [out, 1] in w.dtype, and
+ w_dq = q * scales is bit-identical to what quantize_weight_per_channel_absmax
+ computes. w itself is not modified.
+ """
+ assert n_bits == 8, "int8 storage only"
+ scales = w.abs().max(dim=-1, keepdim=True)[0]
+ q_max = 2 ** (n_bits - 1) - 1
+ scales.clamp_(min=1e-5).div_(q_max)
+ q = w.div(scales).round_().clamp_(-q_max, q_max)
+ w_dq = q.mul(scales)
+ return q.to(torch.int8), scales, w_dq
+
+
@torch.no_grad()
def quantize_weight_per_tensor_absmax(w, n_bits=8):
# w: (out_features, in_features)
@@ -109,8 +126,15 @@ class W8A8Linear(nn.Module):
@staticmethod
def from_float(
- module, weight_quant="per_channel", act_quant="per_token", quantize_output=False
+ module,
+ weight_quant="per_channel",
+ act_quant="per_token",
+ quantize_output=False,
+ keep_int8=False,
):
+ """keep_int8=True also keeps the int8 weight and its per-channel scale as
+ buffers (weight_int8 [out, in], weight_scale [out, 1]) so the model can be
+ saved as real int8. The floating-point weight used in forward is unchanged."""
assert isinstance(module, torch.nn.Linear)
new_module = W8A8Linear(
module.in_features,
@@ -119,7 +143,16 @@ class W8A8Linear(nn.Module):
act_quant=act_quant,
quantize_output=quantize_output,
)
- if weight_quant == "per_channel":
+ if weight_quant == "per_channel" and keep_int8:
+ q, scales, w_dq = quantize_weight_per_channel_absmax_int8(
+ module.weight, n_bits=8
+ )
+ new_module.weight = w_dq
+ new_module.register_buffer("weight_int8", q)
+ new_module.register_buffer("weight_scale", scales)
+ elif keep_int8:
+ raise ValueError("keep_int8 is only supported with per_channel weights")
+ elif weight_quant == "per_channel":
new_module.weight = quantize_weight_per_channel_absmax(
module.weight, n_bits=8
) # use 8-bit integer for weight
@@ -310,16 +343,60 @@ def quantize_falcon(
return model
+def quantize_phi3(
+ model,
+ weight_quant="per_channel",
+ act_quant="per_token",
+ quantize_bmm_input=False,
+ keep_int8=False,
+):
+ """Phi-3: the Llama-like scheme on fused linears (qkv_proj, gate_up_proj)."""
+ # Phi3SdpaAttention / Phi3FlashAttention2 subclass Phi3Attention.
+ from transformers.models.phi3.modeling_phi3 import Phi3Attention, Phi3MLP
+
+ from_float = partial(
+ W8A8Linear.from_float,
+ weight_quant=weight_quant,
+ act_quant=act_quant,
+ keep_int8=keep_int8,
+ )
+ for name, m in model.model.named_modules():
+ if isinstance(m, Phi3MLP):
+ m.gate_up_proj = from_float(m.gate_up_proj)
+ m.down_proj = from_float(m.down_proj)
+ elif isinstance(m, Phi3Attention):
+ # q/k/v are one linear; quantize_output simulates int8 BMM inputs as the
+ # Llama path does for q_proj/k_proj/v_proj.
+ m.qkv_proj = from_float(m.qkv_proj, quantize_output=quantize_bmm_input)
+ m.o_proj = from_float(m.o_proj)
+ return model
+
+
def quantize_model(
- model, weight_quant="per_channel", act_quant="per_token", quantize_bmm_input=False
+ model,
+ weight_quant="per_channel",
+ act_quant="per_token",
+ quantize_bmm_input=False,
+ keep_int8=False,
):
from transformers.models.opt.modeling_opt import OPTPreTrainedModel
from transformers.models.llama.modeling_llama import LlamaPreTrainedModel
from transformers.models.mistral.modeling_mistral import MistralPreTrainedModel
from transformers.models.mixtral.modeling_mixtral import MixtralPreTrainedModel
from transformers.models.falcon.modeling_falcon import FalconPreTrainedModel
+ from transformers.models.phi3.modeling_phi3 import Phi3PreTrainedModel
- if isinstance(model, OPTPreTrainedModel):
+ if keep_int8 and not isinstance(model, Phi3PreTrainedModel):
+ raise ValueError("keep_int8 is only implemented for the Phi-3 path")
+ if isinstance(model, Phi3PreTrainedModel):
+ return quantize_phi3(
+ model,
+ weight_quant=weight_quant,
+ act_quant=act_quant,
+ quantize_bmm_input=quantize_bmm_input,
+ keep_int8=keep_int8,
+ )
+ elif isinstance(model, OPTPreTrainedModel):
return quantize_opt(
model,
weight_quant=weight_quant,
diff --git a/smoothquant/int8_export.py b/smoothquant/int8_export.py
new file mode 100644
index 0000000..7d1f5bf
--- /dev/null
+++ b/smoothquant/int8_export.py
@@ -0,0 +1,138 @@
+"""Save a smoothed, W8A8-quantized model as real int8 (the ``w8a8_int8`` format).
+
+quantize_model(..., keep_int8=True) leaves W8A8Linear modules that carry the
+int8 weight and its per-output-channel scale next to the floating-point weight
+they use in forward. This writes:
+
+- ``<linear>.weight`` int8 [out, in]
+- ``<linear>.weight_scale`` fp16 [out, 1] (w_fp16 = weight * weight_scale)
+- ``<linear>.bias`` fp16 [out] (unchanged by smoothing; when present)
+- every other tensor (smoothed norms, embeddings, final norm, an untied lm_head)
+ in fp16
+- no ``lm_head.weight`` when it is tied to the input embeddings
+
+plus config.json (no ``quantization_config`` and no ``auto_map``: the artifact
+carries no remote code and loads with the native transformers class),
+generation_config.json, the tokenizer, and ``quantization_config.json``
+describing the scheme. Activations are quantized per token, dynamically, at
+runtime; nothing about them is stored.
+"""
+import json
+import os
+
+import torch
+
+from smoothquant.fake_quant import W8A8Linear
+
+QUANT_CONFIG_NAME = "quantization_config.json"
+WEIGHTS_NAME = "model.safetensors"
+CONFIG_NAME = "config.json"
+
+
+@torch.no_grad()
+def w8a8_int8_state_dict(model, dtype=torch.float16):
+ """Return (state_dict, quantized module names) for the int8 artifact."""
+ int8_modules = {}
+ for name, module in model.named_modules():
+ if isinstance(module, W8A8Linear):
+ if getattr(module, "weight_int8", None) is None:
+ raise ValueError(
+ f"{name} was quantized without keep_int8=True; cannot save int8"
+ )
+ if getattr(module, "output_quant_name", "None") != "None":
+ raise ValueError(
+ f"{name} quantizes its output (quantize_bmm_input=True); "
+ "the w8a8_int8 format cannot represent that"
+ )
+ int8_modules[name] = module
+
+ state = {}
+ for key, tensor in model.state_dict().items():
+ owner = key.rpartition(".")[0]
+ if owner in int8_modules:
+ continue
+ tensor = tensor.detach()
+ if tensor.is_floating_point():
+ tensor = tensor.to(dtype)
+ state[key] = tensor.to("cpu", copy=True).contiguous()
+
+ for name, module in int8_modules.items():
+ weight = module.weight_int8.detach()
+ scale = module.weight_scale.detach()
+ out_features, in_features = module.out_features, module.in_features
+ if weight.dtype != torch.int8 or tuple(weight.shape) != (out_features, in_features):
+ raise ValueError(f"{name}: bad int8 weight {weight.dtype} {tuple(weight.shape)}")
+ if tuple(scale.shape) != (out_features, 1):
+ raise ValueError(f"{name}: bad weight_scale shape {tuple(scale.shape)}")
+ if not torch.isfinite(scale).all():
+ raise ValueError(f"{name}: non-finite weight_scale")
+ state[f"{name}.weight"] = weight.to("cpu", copy=True).contiguous()
+ state[f"{name}.weight_scale"] = scale.to("cpu", dtype=dtype, copy=True).contiguous()
+ if module.bias is not None:
+ bias = module.bias.detach().reshape(-1)
+ state[f"{name}.bias"] = bias.to("cpu", dtype=dtype, copy=True).contiguous()
+
+ for key, tensor in state.items():
+ if tensor.is_floating_point() and not torch.isfinite(tensor).all():
+ raise ValueError(f"{key}: non-finite values after casting to {dtype}")
+
+ if getattr(model.config, "tie_word_embeddings", False):
+ head = model.get_output_embeddings()
+ embed = model.get_input_embeddings()
+ if head is not None and head.weight is not embed.weight:
+ raise ValueError("config ties lm_head to embed_tokens but the weights differ")
+ for key in [k for k in state if k.endswith("lm_head.weight")]:
+ del state[key]
+ return state, sorted(int8_modules)
+
+
+def _strip_remote_code(output_dir):
+ """Drop auto_map: the artifact ships no modeling code and loads natively."""
+ path = os.path.join(output_dir, CONFIG_NAME)
+ with open(path) as handle:
+ config = json.load(handle)
+ if config.pop("auto_map", None) is None:
+ return False
+ with open(path, "w") as handle:
+ json.dump(config, handle, indent=2, sort_keys=True)
+ handle.write("\n")
+ return True
+
+
+@torch.no_grad()
+def save_w8a8_int8(model, tokenizer, output_dir, quant_config, dtype=torch.float16):
+ """Write the int8 artifact to output_dir. Returns a short summary dict."""
+ from safetensors.torch import save_file
+
+ os.makedirs(output_dir, exist_ok=True)
+ state, modules = w8a8_int8_state_dict(model, dtype=dtype)
+ if not modules:
+ raise ValueError("no W8A8Linear modules found; nothing was quantized")
+ save_file(state, os.path.join(output_dir, WEIGHTS_NAME), metadata={"format": "pt"})
+
+ config = model.config
+ if getattr(config, "quantization_config", None) is not None:
+ raise ValueError("model config already carries a quantization_config")
+ config.torch_dtype = dtype
+ config.save_pretrained(output_dir)
+ stripped = _strip_remote_code(output_dir)
+ generation_config = getattr(model, "generation_config", None)
+ if generation_config is not None:
+ generation_config.save_pretrained(output_dir)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+
+ record = dict(quant_config)
+ record["quantized_modules"] = modules
+ record["num_quantized_linears"] = len(modules)
+ with open(os.path.join(output_dir, QUANT_CONFIG_NAME), "w") as handle:
+ json.dump(record, handle, indent=2)
+ handle.write("\n")
+ int8_params = sum(t.numel() for t in state.values() if t.dtype == torch.int8)
+ return {
+ "output_dir": output_dir,
+ "num_quantized_linears": len(modules),
+ "int8_params": int8_params,
+ "tensors": len(state),
+ "auto_map_stripped": stripped,
+ }
diff --git a/smoothquant/smooth.py b/smoothquant/smooth.py
index c2a70e7..2ec010e 100644
--- a/smoothquant/smooth.py
+++ b/smoothquant/smooth.py
@@ -13,6 +13,7 @@ from transformers.models.mixtral.modeling_mixtral import (
MixtralRMSNorm,
)
from transformers.models.falcon.modeling_falcon import FalconDecoderLayer
+from transformers.models.phi3.modeling_phi3 import Phi3DecoderLayer, Phi3RMSNorm
@torch.no_grad()
@@ -49,7 +50,7 @@ def smooth_ln_fcs(ln, fcs, act_scales, alpha=0.5):
def smooth_ln_fcs_llama_like(ln, fcs, act_scales, alpha=0.5):
if not isinstance(fcs, list):
fcs = [fcs]
- assert isinstance(ln, (LlamaRMSNorm, MistralRMSNorm, MixtralRMSNorm))
+ assert isinstance(ln, (LlamaRMSNorm, MistralRMSNorm, MixtralRMSNorm, Phi3RMSNorm))
for fc in fcs:
assert isinstance(fc, nn.Linear)
assert ln.weight.numel() == fc.in_features == act_scales.numel()
@@ -158,3 +159,20 @@ def smooth_lm(model, scales, alpha=0.5):
fcs_input_scales = scales[name + ".block_sparse_moe.gate"]
smooth_ln_fcs_llama_like(ffn_ln, fcs, fcs_input_scales, alpha)
+ elif isinstance(module, Phi3DecoderLayer):
+ # Phi-3 fuses q/k/v into one qkv_proj and gate/up into one gate_up_proj.
+ # Each fused linear is the only reader of its norm, so it takes the whole
+ # smoothing scale; the column max over its rows equals the max over the
+ # separate q/k/v (gate/up) weights the Llama branch concatenates.
+ smooth_ln_fcs_llama_like(
+ module.input_layernorm,
+ [module.self_attn.qkv_proj],
+ scales[name + ".self_attn.qkv_proj"],
+ alpha,
+ )
+ smooth_ln_fcs_llama_like(
+ module.post_attention_layernorm,
+ [module.mlp.gate_up_proj],
+ scales[name + ".mlp.gate_up_proj"],
+ alpha,
+ )