diff --git a/quant_agent_inference_adapter.py b/quant_agent_inference_adapter.py new file mode 100644 index 0000000..392f991 --- /dev/null +++ b/quant_agent_inference_adapter.py @@ -0,0 +1,456 @@ +"""Reload a SmoothQuant W8A8 int8 artifact (``w8a8_int8`` format) and run it on int8. + +The artifact (written by smoothquant/int8_export.py) stores, for every quantized +decoder linear, an int8 ``weight`` [out, in] and an fp16 per-output-channel +``weight_scale`` [out, 1], plus fp16 smoothed norms, embeddings, final norm and +(untied) lm_head. Smoothing is folded into those weights offline, so no +transform runs at load or inference time and there are no eval-only flags. + +The kernels below are copied verbatim from the shared library entry +``agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py`` (``w8a8_int8_triton``, +matched by this artifact's quantization_config.json): a Triton per-token absmax +int8 activation quant, then a Triton int8 x int8 -> int32 tensor-core GEMM with a +fused ``acc * act_scale[m] * weight_scale[n] (+ bias[n])`` epilogue. On CPU, or +with ``QUANT_AGENT_W8A8_REFERENCE=1``, each linear falls back to the dense fp16 +dequant reference. + +The model is always built with the native transformers class and SDPA +attention: the artifact has no ``auto_map`` and no modeling code, so remote code +is never loaded whatever ``trust_remote_code`` says. +""" +import glob +import json +import os +import time + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from torch import nn + + +# --- kernels: verbatim copy of agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py --- + +@triton.jit +def _act_quant_kernel( + x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm, + BLOCK_K: tl.constexpr, +): + """Per-token absmax int8 quantization of one row of x (fp16/bf16).""" + row = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, BLOCK_K) + mask = offs < K + x_raw = tl.load(x_ptr + row * stride_xm + offs, mask=mask, other=0.0) + act_ty = x_raw.dtype + x = x_raw.to(tl.float32) + amax = tl.max(tl.abs(x), axis=0) + # Mirror torch on the activation dtype: clamp, cast, divide, cast. + amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32) + scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32) + v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32) + # Round half to even, like torch.round; |v| <= ~127.5 so v + 0.5 is exact. + r = tl.floor(v + 0.5) + tie = (r - v) == 0.5 + odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0 + r = tl.where(tie & odd, r - 1.0, r) + r = tl.minimum(tl.maximum(r, -127.0), 127.0) + tl.store(q_ptr + row * stride_qm + offs, r.to(tl.int8), mask=mask) + tl.store(s_ptr + row, scale) + + +@triton.jit +def _act_quant_looped_kernel( + x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm, + BLOCK_K: tl.constexpr, +): + """Same math as _act_quant_kernel for long rows: absmax pass, then quantize pass.""" + row = tl.program_id(0).to(tl.int64) + x_row = x_ptr + row * stride_xm + q_row = q_ptr + row * stride_qm + offs = tl.arange(0, BLOCK_K) + run_max = tl.zeros((BLOCK_K,), dtype=tl.float32) + for k0 in range(0, K, BLOCK_K): + mask = (k0 + offs) < K + x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32) + run_max = tl.maximum(run_max, tl.abs(x)) + act_ty = x_ptr.dtype.element_ty + amax = tl.max(run_max, axis=0) + amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32) + scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32) + for k0 in range(0, K, BLOCK_K): + mask = (k0 + offs) < K + x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32) + v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32) + r = tl.floor(v + 0.5) + tie = (r - v) == 0.5 + odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0 + r = tl.where(tie & odd, r - 1.0, r) + r = tl.minimum(tl.maximum(r, -127.0), 127.0) + tl.store(q_row + k0 + offs, r.to(tl.int8), mask=mask) + tl.store(s_ptr + row, scale) + + +@triton.jit +def _w8a8_gemm_kernel( + a_ptr, b_ptr, c_ptr, as_ptr, bs_ptr, bias_ptr, + M, N, K, + stride_am, stride_ak, stride_bn, stride_bk, stride_cm, stride_cn, + HAS_BIAS: tl.constexpr, + EVEN_K: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, +): + """C[M, N] = (A_int8[M, K] @ B_int8[N, K]^T) * a_scale[M] * b_scale[N] + bias[N].""" + pid = tl.program_id(0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + # Wrap out-of-range rows/cols onto valid memory; the store is masked. + offs_am = offs_m % M + offs_bn = offs_n % N + offs_k = tl.arange(0, BLOCK_K) + a_ptrs = a_ptr + offs_am[:, None].to(tl.int64) * stride_am + offs_k[None, :] * stride_ak + b_ptrs = b_ptr + offs_bn[None, :].to(tl.int64) * stride_bn + offs_k[:, None] * stride_bk + + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.int32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + if EVEN_K: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + else: + k_left = K - k * BLOCK_K + a = tl.load(a_ptrs, mask=offs_k[None, :] < k_left, other=0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < k_left, other=0) + acc = tl.dot(a, b, acc, out_dtype=tl.int32) + a_ptrs += BLOCK_K * stride_ak + b_ptrs += BLOCK_K * stride_bk + + a_scale = tl.load(as_ptr + offs_am).to(tl.float32) + b_scale = tl.load(bs_ptr + offs_bn).to(tl.float32) + out = acc.to(tl.float32) * a_scale[:, None] * b_scale[None, :] + if HAS_BIAS: + bias = tl.load(bias_ptr + offs_bn).to(tl.float32) + out = out + bias[None, :] + c_ptrs = c_ptr + offs_m[:, None].to(tl.int64) * stride_cm + offs_n[None, :] * stride_cn + c_mask = (offs_m[:, None] < M) & (offs_n[None, :] < N) + tl.store(c_ptrs, out.to(c_ptr.dtype.element_ty), mask=c_mask) + + +def quantize_activation_int8(x): + """x [M, K] fp16/bf16 contiguous -> (int8 codes [M, K], fp32 per-token scale [M]).""" + m, k = x.shape + q = torch.empty((m, k), dtype=torch.int8, device=x.device) + scale = torch.empty((m,), dtype=torch.float32, device=x.device) + if m == 0: + return q, scale + block_k = triton.next_power_of_2(int(k)) + args = (x, q, scale, int(k), int(x.stride(0)), int(q.stride(0))) + if block_k <= 4096: + # Whole row in registers: one read of x. + _act_quant_kernel[(int(m),)](*args, BLOCK_K=block_k, num_warps=8) + else: + # Long rows: absmax pass + quantize pass over 4096-wide chunks; the + # second read hits L2. Measured at copy bandwidth on sm_86. + _act_quant_looped_kernel[(int(m),)](*args, BLOCK_K=4096, num_warps=8) + return q, scale + + +def _gemm_config(m, n, k): + """Default (BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, num_warps, num_stages). + + Tuned on sm_86 (RTX 3090, see kernel.json `tuned_on`). On another compute + capability, run `tune_config` on the model's shapes and pass the result as + `config` to `w8a8_matmul`. + """ + if m <= 16: + return 16, 64, 256, 1, 4, 3 + if m <= 64: + return 64, 64, 128, 8, 4, 4 + return 128, 128, 64, 8, 4, 4 + + +def w8a8_matmul(q, a_scale, weight, weight_scale, bias, out_dtype, config=None): + """int8 q [M, K] x int8 weight [N, K] -> out_dtype [M, N] with the dequant epilogue.""" + m, k = q.shape + n = weight.shape[0] + out = torch.empty((m, n), dtype=out_dtype, device=q.device) + if m == 0: + return out + block_m, block_n, block_k, group_m, warps, stages = config or _gemm_config(m, n, k) + grid = (triton.cdiv(int(m), block_m) * triton.cdiv(int(n), block_n),) + _w8a8_gemm_kernel[grid]( + q, weight, out, a_scale, weight_scale, bias if bias is not None else a_scale, + int(m), int(n), int(k), + int(q.stride(0)), int(q.stride(1)), + int(weight.stride(0)), int(weight.stride(1)), + int(out.stride(0)), int(out.stride(1)), + HAS_BIAS=bias is not None, + EVEN_K=(int(k) % block_k == 0), + BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, GROUP_M=group_m, + num_warps=warps, num_stages=stages, + ) + return out + + +def w8a8_linear(x2d, weight, weight_scale, bias): + """Real int8 W8A8 linear: Triton per-token quant, then int8 GEMM + epilogue.""" + q, a_scale = quantize_activation_int8(x2d) + return w8a8_matmul(q, a_scale, weight, weight_scale, bias, x2d.dtype) + + +def w8a8_linear_reference(x, weight, weight_scale, bias): + """fp16 dequant reference: same quantization, dense fp16 matmul. CPU-safe.""" + x_scale = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-5) / 127 + x_q = torch.round(x / x_scale).clamp_(-127, 127).mul_(x_scale) + w = weight.to(x.dtype).mul_(weight_scale.reshape(-1, 1).to(x.dtype)) + b = None if bias is None else bias.to(x.dtype) + return F.linear(x_q, w, b) + + +CONFIG_CANDIDATES = ( + (128, 128, 64, 8, 4, 4), + (128, 128, 64, 8, 8, 3), + (128, 256, 64, 8, 8, 3), + (64, 128, 128, 8, 4, 4), + (64, 64, 128, 8, 4, 4), + (16, 64, 256, 1, 4, 3), +) + + +def tune_config(m, n, k, *, device="cuda", candidates=CONFIG_CANDIDATES, iters=20): + """Time each candidate tile config on random int8 inputs of this shape; return the fastest. + + Use once per (M, N, K) on a GPU whose compute capability is not in kernel.json + `tuned_on`, then pass the result as `config` to `w8a8_matmul`. + """ + q = torch.randint(-127, 128, (m, k), dtype=torch.int8, device=device) + w = torch.randint(-127, 128, (n, k), dtype=torch.int8, device=device) + a_scale = torch.rand(m, dtype=torch.float32, device=device) + w_scale = torch.rand(n, 1, dtype=torch.float16, device=device) + best, best_t = None, float("inf") + for config in candidates: + try: + w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config) + except Exception: # a tile the GPU cannot fit (shared memory, registers) + continue + torch.cuda.synchronize(device) + start = time.perf_counter() + for _ in range(iters): + w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config) + torch.cuda.synchronize(device) + elapsed = (time.perf_counter() - start) / iters + if elapsed < best_t: + best, best_t = config, elapsed + if best is None: + raise RuntimeError(f"no candidate config ran for M={m} N={n} K={k}") + return best + + +# --- loader (port-specific) ---------------------------------------------------- + +QUANT_AGENT_ADAPTER_API = 1 +QUANT_CONFIG_NAME = "quantization_config.json" +EXPECTED_FORMAT = "w8a8_int8" +SCALE_SUFFIX = ".weight_scale" +REFERENCE_ENV = "QUANT_AGENT_W8A8_REFERENCE" + + +def _w8a8_int8_linear_class(): + class W8A8Int8Linear(nn.Module): + """int8 weight + per-channel scale; per-token dynamic int8 activations; int8 GEMM.""" + + def __init__(self, in_features, out_features, has_bias, dtype): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.register_buffer( + "weight", torch.empty(out_features, in_features, dtype=torch.int8) + ) + self.register_buffer( + "weight_scale", torch.empty(out_features, 1, dtype=dtype) + ) + if has_bias: + self.register_buffer("bias", torch.empty(out_features, dtype=dtype)) + else: + self.bias = None + self.use_reference = os.environ.get(REFERENCE_ENV) == "1" + + @torch.no_grad() + def forward(self, x): + if ( + self.use_reference + or not x.is_cuda + or x.dtype not in (torch.float16, torch.bfloat16) + ): + return w8a8_linear_reference(x, self.weight, self.weight_scale, self.bias) + lead = x.shape[:-1] + x2d = x.reshape(-1, self.in_features) + if not x2d.is_contiguous(): + x2d = x2d.contiguous() + out = w8a8_linear(x2d, self.weight, self.weight_scale, self.bias) + return out.reshape(*lead, self.out_features) + + def extra_repr(self): + return ( + f"in_features={self.in_features}, out_features={self.out_features}, " + f"bias={self.bias is not None}, weight=int8 per_channel, " + "act=int8 per_token dynamic, kernel=triton int8 gemm" + ) + + return W8A8Int8Linear + + +def _resolve_dtype(dtype): + if dtype in (torch.bfloat16, "bfloat16", "bf16"): + return torch.bfloat16 + if dtype in (None, "auto", torch.float16, "float16", "fp16", "half"): + return torch.float16 + raise ValueError(f"unsupported dtype for the W8A8 int8 artifact: {dtype!r}") + + +def _read_quant_config(model_path): + path = os.path.join(model_path, QUANT_CONFIG_NAME) + if not os.path.isfile(path): + raise FileNotFoundError(f"{QUANT_CONFIG_NAME} is missing in {model_path}") + with open(path) as handle: + qcfg = json.load(handle) + if qcfg.get("format") != EXPECTED_FORMAT: + raise ValueError(f"expected format {EXPECTED_FORMAT!r}, got {qcfg.get('format')!r}") + if str(qcfg.get("act_quant")) != "per_token" or str(qcfg.get("weight_quant")) != "per_channel": + raise ValueError( + "this adapter handles per_channel int8 weights with per_token dynamic activations only" + ) + return qcfg + + +def _read_state(model_path): + from safetensors.torch import load_file + + files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) + if not files: + raise FileNotFoundError(f"no .safetensors weights in {model_path}") + state = {} + for path in files: + part = load_file(path, device="cpu") + overlap = set(part) & set(state) + if overlap: + raise ValueError(f"tensor(s) {sorted(overlap)[:3]} appear in two weight files") + state.update(part) + return state + + +def _set_submodule(model, name, module): + parent_name, _, leaf = name.rpartition(".") + parent = model.get_submodule(parent_name) if parent_name else model + setattr(parent, leaf, module) + + +def load_model_and_tokenizer(model_path, model_id, dtype, device, trust_remote_code): + from accelerate import init_empty_weights + from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig + + del model_id # the saved artifact is the only source of weights + # The artifact carries no modeling code (int8_export strips auto_map); the + # architecture is native to transformers, so remote code is never used. + del trust_remote_code + torch_dtype = _resolve_dtype(dtype) + qcfg = _read_quant_config(model_path) + state = _read_state(model_path) + + config = AutoConfig.from_pretrained( + model_path, trust_remote_code=False, local_files_only=True + ) + if getattr(config, "quantization_config", None): + raise ValueError("config.json carries a quantization_config; not a SmoothQuant int8 artifact") + + # Parameters on meta (no fp16 copy of the linears is ever allocated); buffers + # such as the rotary inv_freq stay real. + with init_empty_weights(include_buffers=False): + model = AutoModelForCausalLM.from_config( + config, + torch_dtype=torch_dtype, + trust_remote_code=False, + attn_implementation="sdpa", + ) + + linear_cls = _w8a8_int8_linear_class() + modules = dict(model.named_modules()) + quantized = [] + for key in sorted(state): + if not key.endswith(SCALE_SUFFIX): + continue + name = key[: -len(SCALE_SUFFIX)] + weight = state.get(f"{name}.weight") + if weight is None or weight.dtype != torch.int8 or weight.dim() != 2: + raise ValueError(f"{key} has no int8 [out, in] weight next to it") + original = modules.get(name) + if not isinstance(original, nn.Linear): + raise ValueError(f"{name} is not an nn.Linear in {type(model).__name__}") + out_features, in_features = weight.shape + if (original.out_features, original.in_features) != (out_features, in_features): + raise ValueError(f"{name}: saved shape {tuple(weight.shape)} does not match the config") + if tuple(state[key].shape) not in ((out_features, 1), (out_features,)): + raise ValueError(f"{key}: expected [{out_features}, 1], got {tuple(state[key].shape)}") + state[key] = state[key].reshape(out_features, 1) + # The GEMM reads B as [N, K] with K contiguous; make sure that holds. + state[f"{name}.weight"] = weight.contiguous() + has_bias = f"{name}.bias" in state + if has_bias != (original.bias is not None): + raise ValueError(f"{name}: bias presence differs between artifact and config") + _set_submodule(model, name, linear_cls(in_features, out_features, has_bias, torch_dtype)) + quantized.append(name) + if not quantized: + raise ValueError("no int8 weight + weight_scale pairs in the artifact") + expected = qcfg.get("num_quantized_linears") + if isinstance(expected, int) and expected != len(quantized): + raise ValueError( + f"{QUANT_CONFIG_NAME} lists {expected} quantized linears, artifact has {len(quantized)}" + ) + + for key, tensor in list(state.items()): + if tensor.is_floating_point() and tensor.dtype != torch_dtype: + state[key] = tensor.to(torch_dtype) + + tied = bool(getattr(config, "tie_word_embeddings", False)) + result = model.load_state_dict(state, strict=False, assign=True) + missing = set(result.missing_keys) + if tied: + missing.discard("lm_head.weight") + if missing or result.unexpected_keys: + raise ValueError( + f"artifact does not match the model: missing={sorted(missing)[:8]} " + f"unexpected={sorted(result.unexpected_keys)[:8]}" + ) + if tied: + model.tie_weights() + leftover = [n for n, t in list(model.named_parameters()) + list(model.named_buffers()) if t.is_meta] + if leftover: + raise ValueError(f"tensors left uninitialised after load: {leftover[:8]}") + + try: + model.generation_config = GenerationConfig.from_pretrained( + model_path, local_files_only=True + ) + except OSError: + pass + model.to(device) + model.eval() + attn_impl = getattr(model.config, "_attn_implementation", None) + if attn_impl != "sdpa": + raise RuntimeError(f"expected SDPA attention for prefill, got {attn_impl!r}") + + tokenizer = AutoTokenizer.from_pretrained( + model_path, trust_remote_code=False, local_files_only=True + ) + return model, tokenizer diff --git a/smoothquant/fake_quant.py b/smoothquant/fake_quant.py index 24b62b5..bb7c175 100644 --- a/smoothquant/fake_quant.py +++ b/smoothquant/fake_quant.py @@ -13,6 +13,23 @@ def quantize_weight_per_channel_absmax(w, n_bits=8): return w +@torch.no_grad() +def quantize_weight_per_channel_absmax_int8(w, n_bits=8): + """Same math as quantize_weight_per_channel_absmax, but also returns the integers. + + Returns (q, scales, w_dq): q is int8 [out, in], scales is [out, 1] in w.dtype, and + w_dq = q * scales is bit-identical to what quantize_weight_per_channel_absmax + computes. w itself is not modified. + """ + assert n_bits == 8, "int8 storage only" + scales = w.abs().max(dim=-1, keepdim=True)[0] + q_max = 2 ** (n_bits - 1) - 1 + scales.clamp_(min=1e-5).div_(q_max) + q = w.div(scales).round_().clamp_(-q_max, q_max) + w_dq = q.mul(scales) + return q.to(torch.int8), scales, w_dq + + @torch.no_grad() def quantize_weight_per_tensor_absmax(w, n_bits=8): # w: (out_features, in_features) @@ -109,8 +126,15 @@ class W8A8Linear(nn.Module): @staticmethod def from_float( - module, weight_quant="per_channel", act_quant="per_token", quantize_output=False + module, + weight_quant="per_channel", + act_quant="per_token", + quantize_output=False, + keep_int8=False, ): + """keep_int8=True also keeps the int8 weight and its per-channel scale as + buffers (weight_int8 [out, in], weight_scale [out, 1]) so the model can be + saved as real int8. The floating-point weight used in forward is unchanged.""" assert isinstance(module, torch.nn.Linear) new_module = W8A8Linear( module.in_features, @@ -119,7 +143,16 @@ class W8A8Linear(nn.Module): act_quant=act_quant, quantize_output=quantize_output, ) - if weight_quant == "per_channel": + if weight_quant == "per_channel" and keep_int8: + q, scales, w_dq = quantize_weight_per_channel_absmax_int8( + module.weight, n_bits=8 + ) + new_module.weight = w_dq + new_module.register_buffer("weight_int8", q) + new_module.register_buffer("weight_scale", scales) + elif keep_int8: + raise ValueError("keep_int8 is only supported with per_channel weights") + elif weight_quant == "per_channel": new_module.weight = quantize_weight_per_channel_absmax( module.weight, n_bits=8 ) # use 8-bit integer for weight @@ -310,16 +343,60 @@ def quantize_falcon( return model +def quantize_phi3( + model, + weight_quant="per_channel", + act_quant="per_token", + quantize_bmm_input=False, + keep_int8=False, +): + """Phi-3: the Llama-like scheme on fused linears (qkv_proj, gate_up_proj).""" + # Phi3SdpaAttention / Phi3FlashAttention2 subclass Phi3Attention. + from transformers.models.phi3.modeling_phi3 import Phi3Attention, Phi3MLP + + from_float = partial( + W8A8Linear.from_float, + weight_quant=weight_quant, + act_quant=act_quant, + keep_int8=keep_int8, + ) + for name, m in model.model.named_modules(): + if isinstance(m, Phi3MLP): + m.gate_up_proj = from_float(m.gate_up_proj) + m.down_proj = from_float(m.down_proj) + elif isinstance(m, Phi3Attention): + # q/k/v are one linear; quantize_output simulates int8 BMM inputs as the + # Llama path does for q_proj/k_proj/v_proj. + m.qkv_proj = from_float(m.qkv_proj, quantize_output=quantize_bmm_input) + m.o_proj = from_float(m.o_proj) + return model + + def quantize_model( - model, weight_quant="per_channel", act_quant="per_token", quantize_bmm_input=False + model, + weight_quant="per_channel", + act_quant="per_token", + quantize_bmm_input=False, + keep_int8=False, ): from transformers.models.opt.modeling_opt import OPTPreTrainedModel from transformers.models.llama.modeling_llama import LlamaPreTrainedModel from transformers.models.mistral.modeling_mistral import MistralPreTrainedModel from transformers.models.mixtral.modeling_mixtral import MixtralPreTrainedModel from transformers.models.falcon.modeling_falcon import FalconPreTrainedModel + from transformers.models.phi3.modeling_phi3 import Phi3PreTrainedModel - if isinstance(model, OPTPreTrainedModel): + if keep_int8 and not isinstance(model, Phi3PreTrainedModel): + raise ValueError("keep_int8 is only implemented for the Phi-3 path") + if isinstance(model, Phi3PreTrainedModel): + return quantize_phi3( + model, + weight_quant=weight_quant, + act_quant=act_quant, + quantize_bmm_input=quantize_bmm_input, + keep_int8=keep_int8, + ) + elif isinstance(model, OPTPreTrainedModel): return quantize_opt( model, weight_quant=weight_quant, diff --git a/smoothquant/int8_export.py b/smoothquant/int8_export.py new file mode 100644 index 0000000..7d1f5bf --- /dev/null +++ b/smoothquant/int8_export.py @@ -0,0 +1,138 @@ +"""Save a smoothed, W8A8-quantized model as real int8 (the ``w8a8_int8`` format). + +quantize_model(..., keep_int8=True) leaves W8A8Linear modules that carry the +int8 weight and its per-output-channel scale next to the floating-point weight +they use in forward. This writes: + +- ``.weight`` int8 [out, in] +- ``.weight_scale`` fp16 [out, 1] (w_fp16 = weight * weight_scale) +- ``.bias`` fp16 [out] (unchanged by smoothing; when present) +- every other tensor (smoothed norms, embeddings, final norm, an untied lm_head) + in fp16 +- no ``lm_head.weight`` when it is tied to the input embeddings + +plus config.json (no ``quantization_config`` and no ``auto_map``: the artifact +carries no remote code and loads with the native transformers class), +generation_config.json, the tokenizer, and ``quantization_config.json`` +describing the scheme. Activations are quantized per token, dynamically, at +runtime; nothing about them is stored. +""" +import json +import os + +import torch + +from smoothquant.fake_quant import W8A8Linear + +QUANT_CONFIG_NAME = "quantization_config.json" +WEIGHTS_NAME = "model.safetensors" +CONFIG_NAME = "config.json" + + +@torch.no_grad() +def w8a8_int8_state_dict(model, dtype=torch.float16): + """Return (state_dict, quantized module names) for the int8 artifact.""" + int8_modules = {} + for name, module in model.named_modules(): + if isinstance(module, W8A8Linear): + if getattr(module, "weight_int8", None) is None: + raise ValueError( + f"{name} was quantized without keep_int8=True; cannot save int8" + ) + if getattr(module, "output_quant_name", "None") != "None": + raise ValueError( + f"{name} quantizes its output (quantize_bmm_input=True); " + "the w8a8_int8 format cannot represent that" + ) + int8_modules[name] = module + + state = {} + for key, tensor in model.state_dict().items(): + owner = key.rpartition(".")[0] + if owner in int8_modules: + continue + tensor = tensor.detach() + if tensor.is_floating_point(): + tensor = tensor.to(dtype) + state[key] = tensor.to("cpu", copy=True).contiguous() + + for name, module in int8_modules.items(): + weight = module.weight_int8.detach() + scale = module.weight_scale.detach() + out_features, in_features = module.out_features, module.in_features + if weight.dtype != torch.int8 or tuple(weight.shape) != (out_features, in_features): + raise ValueError(f"{name}: bad int8 weight {weight.dtype} {tuple(weight.shape)}") + if tuple(scale.shape) != (out_features, 1): + raise ValueError(f"{name}: bad weight_scale shape {tuple(scale.shape)}") + if not torch.isfinite(scale).all(): + raise ValueError(f"{name}: non-finite weight_scale") + state[f"{name}.weight"] = weight.to("cpu", copy=True).contiguous() + state[f"{name}.weight_scale"] = scale.to("cpu", dtype=dtype, copy=True).contiguous() + if module.bias is not None: + bias = module.bias.detach().reshape(-1) + state[f"{name}.bias"] = bias.to("cpu", dtype=dtype, copy=True).contiguous() + + for key, tensor in state.items(): + if tensor.is_floating_point() and not torch.isfinite(tensor).all(): + raise ValueError(f"{key}: non-finite values after casting to {dtype}") + + if getattr(model.config, "tie_word_embeddings", False): + head = model.get_output_embeddings() + embed = model.get_input_embeddings() + if head is not None and head.weight is not embed.weight: + raise ValueError("config ties lm_head to embed_tokens but the weights differ") + for key in [k for k in state if k.endswith("lm_head.weight")]: + del state[key] + return state, sorted(int8_modules) + + +def _strip_remote_code(output_dir): + """Drop auto_map: the artifact ships no modeling code and loads natively.""" + path = os.path.join(output_dir, CONFIG_NAME) + with open(path) as handle: + config = json.load(handle) + if config.pop("auto_map", None) is None: + return False + with open(path, "w") as handle: + json.dump(config, handle, indent=2, sort_keys=True) + handle.write("\n") + return True + + +@torch.no_grad() +def save_w8a8_int8(model, tokenizer, output_dir, quant_config, dtype=torch.float16): + """Write the int8 artifact to output_dir. Returns a short summary dict.""" + from safetensors.torch import save_file + + os.makedirs(output_dir, exist_ok=True) + state, modules = w8a8_int8_state_dict(model, dtype=dtype) + if not modules: + raise ValueError("no W8A8Linear modules found; nothing was quantized") + save_file(state, os.path.join(output_dir, WEIGHTS_NAME), metadata={"format": "pt"}) + + config = model.config + if getattr(config, "quantization_config", None) is not None: + raise ValueError("model config already carries a quantization_config") + config.torch_dtype = dtype + config.save_pretrained(output_dir) + stripped = _strip_remote_code(output_dir) + generation_config = getattr(model, "generation_config", None) + if generation_config is not None: + generation_config.save_pretrained(output_dir) + if tokenizer is not None: + tokenizer.save_pretrained(output_dir) + + record = dict(quant_config) + record["quantized_modules"] = modules + record["num_quantized_linears"] = len(modules) + with open(os.path.join(output_dir, QUANT_CONFIG_NAME), "w") as handle: + json.dump(record, handle, indent=2) + handle.write("\n") + int8_params = sum(t.numel() for t in state.values() if t.dtype == torch.int8) + return { + "output_dir": output_dir, + "num_quantized_linears": len(modules), + "int8_params": int8_params, + "tensors": len(state), + "auto_map_stripped": stripped, + } diff --git a/smoothquant/smooth.py b/smoothquant/smooth.py index c2a70e7..2ec010e 100644 --- a/smoothquant/smooth.py +++ b/smoothquant/smooth.py @@ -13,6 +13,7 @@ from transformers.models.mixtral.modeling_mixtral import ( MixtralRMSNorm, ) from transformers.models.falcon.modeling_falcon import FalconDecoderLayer +from transformers.models.phi3.modeling_phi3 import Phi3DecoderLayer, Phi3RMSNorm @torch.no_grad() @@ -49,7 +50,7 @@ def smooth_ln_fcs(ln, fcs, act_scales, alpha=0.5): def smooth_ln_fcs_llama_like(ln, fcs, act_scales, alpha=0.5): if not isinstance(fcs, list): fcs = [fcs] - assert isinstance(ln, (LlamaRMSNorm, MistralRMSNorm, MixtralRMSNorm)) + assert isinstance(ln, (LlamaRMSNorm, MistralRMSNorm, MixtralRMSNorm, Phi3RMSNorm)) for fc in fcs: assert isinstance(fc, nn.Linear) assert ln.weight.numel() == fc.in_features == act_scales.numel() @@ -158,3 +159,20 @@ def smooth_lm(model, scales, alpha=0.5): fcs_input_scales = scales[name + ".block_sparse_moe.gate"] smooth_ln_fcs_llama_like(ffn_ln, fcs, fcs_input_scales, alpha) + elif isinstance(module, Phi3DecoderLayer): + # Phi-3 fuses q/k/v into one qkv_proj and gate/up into one gate_up_proj. + # Each fused linear is the only reader of its norm, so it takes the whole + # smoothing scale; the column max over its rows equals the max over the + # separate q/k/v (gate/up) weights the Llama branch concatenates. + smooth_ln_fcs_llama_like( + module.input_layernorm, + [module.self_attn.qkv_proj], + scales[name + ".self_attn.qkv_proj"], + alpha, + ) + smooth_ln_fcs_llama_like( + module.post_attention_layernorm, + [module.mlp.gate_up_proj], + scales[name + ".mlp.gate_up_proj"], + alpha, + )