"""Reload a SmoothQuant W8A8 int8 artifact (``w8a8_int8`` format) and run it on int8. The artifact (written by smoothquant/int8_export.py) stores, for every quantized decoder linear, an int8 ``weight`` [out, in] and an fp16 per-output-channel ``weight_scale`` [out, 1], plus fp16 smoothed norms, embeddings, final norm and (untied) lm_head. Smoothing is folded into those weights offline, so no transform runs at load or inference time and there are no eval-only flags. The kernels below are copied verbatim from the shared library entry ``agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py`` (``w8a8_int8_triton``, matched by this artifact's quantization_config.json): a Triton per-token absmax int8 activation quant, then a Triton int8 x int8 -> int32 tensor-core GEMM with a fused ``acc * act_scale[m] * weight_scale[n] (+ bias[n])`` epilogue. On CPU, or with ``QUANT_AGENT_W8A8_REFERENCE=1``, each linear falls back to the dense fp16 dequant reference. The model is always built with the native transformers class and SDPA attention: the artifact has no ``auto_map`` and no modeling code, so remote code is never loaded whatever ``trust_remote_code`` says. """ import glob import json import os import time import torch import torch.nn.functional as F import triton import triton.language as tl from torch import nn # --- kernels: verbatim copy of agents/skills/_shared/kernels/w8a8_int8/triton_w8a8.py --- @triton.jit def _act_quant_kernel( x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm, BLOCK_K: tl.constexpr, ): """Per-token absmax int8 quantization of one row of x (fp16/bf16).""" row = tl.program_id(0).to(tl.int64) offs = tl.arange(0, BLOCK_K) mask = offs < K x_raw = tl.load(x_ptr + row * stride_xm + offs, mask=mask, other=0.0) act_ty = x_raw.dtype x = x_raw.to(tl.float32) amax = tl.max(tl.abs(x), axis=0) # Mirror torch on the activation dtype: clamp, cast, divide, cast. amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32) scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32) v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32) # Round half to even, like torch.round; |v| <= ~127.5 so v + 0.5 is exact. r = tl.floor(v + 0.5) tie = (r - v) == 0.5 odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0 r = tl.where(tie & odd, r - 1.0, r) r = tl.minimum(tl.maximum(r, -127.0), 127.0) tl.store(q_ptr + row * stride_qm + offs, r.to(tl.int8), mask=mask) tl.store(s_ptr + row, scale) @triton.jit def _act_quant_looped_kernel( x_ptr, q_ptr, s_ptr, K, stride_xm, stride_qm, BLOCK_K: tl.constexpr, ): """Same math as _act_quant_kernel for long rows: absmax pass, then quantize pass.""" row = tl.program_id(0).to(tl.int64) x_row = x_ptr + row * stride_xm q_row = q_ptr + row * stride_qm offs = tl.arange(0, BLOCK_K) run_max = tl.zeros((BLOCK_K,), dtype=tl.float32) for k0 in range(0, K, BLOCK_K): mask = (k0 + offs) < K x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32) run_max = tl.maximum(run_max, tl.abs(x)) act_ty = x_ptr.dtype.element_ty amax = tl.max(run_max, axis=0) amax = tl.maximum(amax, 1e-5).to(act_ty).to(tl.float32) scale = tl.math.div_rn(amax, 127.0).to(act_ty).to(tl.float32) for k0 in range(0, K, BLOCK_K): mask = (k0 + offs) < K x = tl.load(x_row + k0 + offs, mask=mask, other=0.0).to(tl.float32) v = tl.math.div_rn(x, scale).to(act_ty).to(tl.float32) r = tl.floor(v + 0.5) tie = (r - v) == 0.5 odd = (r - 2.0 * tl.floor(r * 0.5)) != 0.0 r = tl.where(tie & odd, r - 1.0, r) r = tl.minimum(tl.maximum(r, -127.0), 127.0) tl.store(q_row + k0 + offs, r.to(tl.int8), mask=mask) tl.store(s_ptr + row, scale) @triton.jit def _w8a8_gemm_kernel( a_ptr, b_ptr, c_ptr, as_ptr, bs_ptr, bias_ptr, M, N, K, stride_am, stride_ak, stride_bn, stride_bk, stride_cm, stride_cn, HAS_BIAS: tl.constexpr, EVEN_K: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, GROUP_M: tl.constexpr, ): """C[M, N] = (A_int8[M, K] @ B_int8[N, K]^T) * a_scale[M] * b_scale[N] + bias[N].""" pid = tl.program_id(0) num_pid_m = tl.cdiv(M, BLOCK_M) num_pid_n = tl.cdiv(N, BLOCK_N) num_pid_in_group = GROUP_M * num_pid_n group_id = pid // num_pid_in_group first_pid_m = group_id * GROUP_M group_size_m = min(num_pid_m - first_pid_m, GROUP_M) pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) pid_n = (pid % num_pid_in_group) // group_size_m offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) # Wrap out-of-range rows/cols onto valid memory; the store is masked. offs_am = offs_m % M offs_bn = offs_n % N offs_k = tl.arange(0, BLOCK_K) a_ptrs = a_ptr + offs_am[:, None].to(tl.int64) * stride_am + offs_k[None, :] * stride_ak b_ptrs = b_ptr + offs_bn[None, :].to(tl.int64) * stride_bn + offs_k[:, None] * stride_bk acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.int32) for k in range(0, tl.cdiv(K, BLOCK_K)): if EVEN_K: a = tl.load(a_ptrs) b = tl.load(b_ptrs) else: k_left = K - k * BLOCK_K a = tl.load(a_ptrs, mask=offs_k[None, :] < k_left, other=0) b = tl.load(b_ptrs, mask=offs_k[:, None] < k_left, other=0) acc = tl.dot(a, b, acc, out_dtype=tl.int32) a_ptrs += BLOCK_K * stride_ak b_ptrs += BLOCK_K * stride_bk a_scale = tl.load(as_ptr + offs_am).to(tl.float32) b_scale = tl.load(bs_ptr + offs_bn).to(tl.float32) out = acc.to(tl.float32) * a_scale[:, None] * b_scale[None, :] if HAS_BIAS: bias = tl.load(bias_ptr + offs_bn).to(tl.float32) out = out + bias[None, :] c_ptrs = c_ptr + offs_m[:, None].to(tl.int64) * stride_cm + offs_n[None, :] * stride_cn c_mask = (offs_m[:, None] < M) & (offs_n[None, :] < N) tl.store(c_ptrs, out.to(c_ptr.dtype.element_ty), mask=c_mask) def quantize_activation_int8(x): """x [M, K] fp16/bf16 contiguous -> (int8 codes [M, K], fp32 per-token scale [M]).""" m, k = x.shape q = torch.empty((m, k), dtype=torch.int8, device=x.device) scale = torch.empty((m,), dtype=torch.float32, device=x.device) if m == 0: return q, scale block_k = triton.next_power_of_2(int(k)) args = (x, q, scale, int(k), int(x.stride(0)), int(q.stride(0))) if block_k <= 4096: # Whole row in registers: one read of x. _act_quant_kernel[(int(m),)](*args, BLOCK_K=block_k, num_warps=8) else: # Long rows: absmax pass + quantize pass over 4096-wide chunks; the # second read hits L2. Measured at copy bandwidth on sm_86. _act_quant_looped_kernel[(int(m),)](*args, BLOCK_K=4096, num_warps=8) return q, scale def _gemm_config(m, n, k): """Default (BLOCK_M, BLOCK_N, BLOCK_K, GROUP_M, num_warps, num_stages). Tuned on sm_86 (RTX 3090, see kernel.json `tuned_on`). On another compute capability, run `tune_config` on the model's shapes and pass the result as `config` to `w8a8_matmul`. """ if m <= 16: return 16, 64, 256, 1, 4, 3 if m <= 64: return 64, 64, 128, 8, 4, 4 return 128, 128, 64, 8, 4, 4 def w8a8_matmul(q, a_scale, weight, weight_scale, bias, out_dtype, config=None): """int8 q [M, K] x int8 weight [N, K] -> out_dtype [M, N] with the dequant epilogue.""" m, k = q.shape n = weight.shape[0] out = torch.empty((m, n), dtype=out_dtype, device=q.device) if m == 0: return out block_m, block_n, block_k, group_m, warps, stages = config or _gemm_config(m, n, k) grid = (triton.cdiv(int(m), block_m) * triton.cdiv(int(n), block_n),) _w8a8_gemm_kernel[grid]( q, weight, out, a_scale, weight_scale, bias if bias is not None else a_scale, int(m), int(n), int(k), int(q.stride(0)), int(q.stride(1)), int(weight.stride(0)), int(weight.stride(1)), int(out.stride(0)), int(out.stride(1)), HAS_BIAS=bias is not None, EVEN_K=(int(k) % block_k == 0), BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, GROUP_M=group_m, num_warps=warps, num_stages=stages, ) return out def w8a8_linear(x2d, weight, weight_scale, bias): """Real int8 W8A8 linear: Triton per-token quant, then int8 GEMM + epilogue.""" q, a_scale = quantize_activation_int8(x2d) return w8a8_matmul(q, a_scale, weight, weight_scale, bias, x2d.dtype) def w8a8_linear_reference(x, weight, weight_scale, bias): """fp16 dequant reference: same quantization, dense fp16 matmul. CPU-safe.""" x_scale = x.abs().amax(dim=-1, keepdim=True).clamp(min=1e-5) / 127 x_q = torch.round(x / x_scale).clamp_(-127, 127).mul_(x_scale) w = weight.to(x.dtype).mul_(weight_scale.reshape(-1, 1).to(x.dtype)) b = None if bias is None else bias.to(x.dtype) return F.linear(x_q, w, b) CONFIG_CANDIDATES = ( (128, 128, 64, 8, 4, 4), (128, 128, 64, 8, 8, 3), (128, 256, 64, 8, 8, 3), (64, 128, 128, 8, 4, 4), (64, 64, 128, 8, 4, 4), (16, 64, 256, 1, 4, 3), ) def tune_config(m, n, k, *, device="cuda", candidates=CONFIG_CANDIDATES, iters=20): """Time each candidate tile config on random int8 inputs of this shape; return the fastest. Use once per (M, N, K) on a GPU whose compute capability is not in kernel.json `tuned_on`, then pass the result as `config` to `w8a8_matmul`. """ q = torch.randint(-127, 128, (m, k), dtype=torch.int8, device=device) w = torch.randint(-127, 128, (n, k), dtype=torch.int8, device=device) a_scale = torch.rand(m, dtype=torch.float32, device=device) w_scale = torch.rand(n, 1, dtype=torch.float16, device=device) best, best_t = None, float("inf") for config in candidates: try: w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config) except Exception: # a tile the GPU cannot fit (shared memory, registers) continue torch.cuda.synchronize(device) start = time.perf_counter() for _ in range(iters): w8a8_matmul(q, a_scale, w, w_scale, None, torch.float16, config) torch.cuda.synchronize(device) elapsed = (time.perf_counter() - start) / iters if elapsed < best_t: best, best_t = config, elapsed if best is None: raise RuntimeError(f"no candidate config ran for M={m} N={n} K={k}") return best # --- loader (port-specific) ---------------------------------------------------- QUANT_AGENT_ADAPTER_API = 1 QUANT_CONFIG_NAME = "quantization_config.json" EXPECTED_FORMAT = "w8a8_int8" SCALE_SUFFIX = ".weight_scale" REFERENCE_ENV = "QUANT_AGENT_W8A8_REFERENCE" def _w8a8_int8_linear_class(): class W8A8Int8Linear(nn.Module): """int8 weight + per-channel scale; per-token dynamic int8 activations; int8 GEMM.""" def __init__(self, in_features, out_features, has_bias, dtype): super().__init__() self.in_features = in_features self.out_features = out_features self.register_buffer( "weight", torch.empty(out_features, in_features, dtype=torch.int8) ) self.register_buffer( "weight_scale", torch.empty(out_features, 1, dtype=dtype) ) if has_bias: self.register_buffer("bias", torch.empty(out_features, dtype=dtype)) else: self.bias = None self.use_reference = os.environ.get(REFERENCE_ENV) == "1" @torch.no_grad() def forward(self, x): if ( self.use_reference or not x.is_cuda or x.dtype not in (torch.float16, torch.bfloat16) ): return w8a8_linear_reference(x, self.weight, self.weight_scale, self.bias) lead = x.shape[:-1] x2d = x.reshape(-1, self.in_features) if not x2d.is_contiguous(): x2d = x2d.contiguous() out = w8a8_linear(x2d, self.weight, self.weight_scale, self.bias) return out.reshape(*lead, self.out_features) def extra_repr(self): return ( f"in_features={self.in_features}, out_features={self.out_features}, " f"bias={self.bias is not None}, weight=int8 per_channel, " "act=int8 per_token dynamic, kernel=triton int8 gemm" ) return W8A8Int8Linear def _resolve_dtype(dtype): if dtype in (torch.bfloat16, "bfloat16", "bf16"): return torch.bfloat16 if dtype in (None, "auto", torch.float16, "float16", "fp16", "half"): return torch.float16 raise ValueError(f"unsupported dtype for the W8A8 int8 artifact: {dtype!r}") def _read_quant_config(model_path): path = os.path.join(model_path, QUANT_CONFIG_NAME) if not os.path.isfile(path): raise FileNotFoundError(f"{QUANT_CONFIG_NAME} is missing in {model_path}") with open(path) as handle: qcfg = json.load(handle) if qcfg.get("format") != EXPECTED_FORMAT: raise ValueError(f"expected format {EXPECTED_FORMAT!r}, got {qcfg.get('format')!r}") if str(qcfg.get("act_quant")) != "per_token" or str(qcfg.get("weight_quant")) != "per_channel": raise ValueError( "this adapter handles per_channel int8 weights with per_token dynamic activations only" ) return qcfg def _read_state(model_path): from safetensors.torch import load_file files = sorted(glob.glob(os.path.join(model_path, "*.safetensors"))) if not files: raise FileNotFoundError(f"no .safetensors weights in {model_path}") state = {} for path in files: part = load_file(path, device="cpu") overlap = set(part) & set(state) if overlap: raise ValueError(f"tensor(s) {sorted(overlap)[:3]} appear in two weight files") state.update(part) return state def _set_submodule(model, name, module): parent_name, _, leaf = name.rpartition(".") parent = model.get_submodule(parent_name) if parent_name else model setattr(parent, leaf, module) def load_model_and_tokenizer(model_path, model_id, dtype, device, trust_remote_code): from accelerate import init_empty_weights from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig del model_id # the saved artifact is the only source of weights # The artifact carries no modeling code (int8_export strips auto_map); the # architecture is native to transformers, so remote code is never used. del trust_remote_code torch_dtype = _resolve_dtype(dtype) qcfg = _read_quant_config(model_path) state = _read_state(model_path) config = AutoConfig.from_pretrained( model_path, trust_remote_code=False, local_files_only=True ) if getattr(config, "quantization_config", None): raise ValueError("config.json carries a quantization_config; not a SmoothQuant int8 artifact") # Parameters on meta (no fp16 copy of the linears is ever allocated); buffers # such as the rotary inv_freq stay real. with init_empty_weights(include_buffers=False): model = AutoModelForCausalLM.from_config( config, torch_dtype=torch_dtype, trust_remote_code=False, attn_implementation="sdpa", ) linear_cls = _w8a8_int8_linear_class() modules = dict(model.named_modules()) quantized = [] for key in sorted(state): if not key.endswith(SCALE_SUFFIX): continue name = key[: -len(SCALE_SUFFIX)] weight = state.get(f"{name}.weight") if weight is None or weight.dtype != torch.int8 or weight.dim() != 2: raise ValueError(f"{key} has no int8 [out, in] weight next to it") original = modules.get(name) if not isinstance(original, nn.Linear): raise ValueError(f"{name} is not an nn.Linear in {type(model).__name__}") out_features, in_features = weight.shape if (original.out_features, original.in_features) != (out_features, in_features): raise ValueError(f"{name}: saved shape {tuple(weight.shape)} does not match the config") if tuple(state[key].shape) not in ((out_features, 1), (out_features,)): raise ValueError(f"{key}: expected [{out_features}, 1], got {tuple(state[key].shape)}") state[key] = state[key].reshape(out_features, 1) # The GEMM reads B as [N, K] with K contiguous; make sure that holds. state[f"{name}.weight"] = weight.contiguous() has_bias = f"{name}.bias" in state if has_bias != (original.bias is not None): raise ValueError(f"{name}: bias presence differs between artifact and config") _set_submodule(model, name, linear_cls(in_features, out_features, has_bias, torch_dtype)) quantized.append(name) if not quantized: raise ValueError("no int8 weight + weight_scale pairs in the artifact") expected = qcfg.get("num_quantized_linears") if isinstance(expected, int) and expected != len(quantized): raise ValueError( f"{QUANT_CONFIG_NAME} lists {expected} quantized linears, artifact has {len(quantized)}" ) for key, tensor in list(state.items()): if tensor.is_floating_point() and tensor.dtype != torch_dtype: state[key] = tensor.to(torch_dtype) tied = bool(getattr(config, "tie_word_embeddings", False)) result = model.load_state_dict(state, strict=False, assign=True) missing = set(result.missing_keys) if tied: missing.discard("lm_head.weight") if missing or result.unexpected_keys: raise ValueError( f"artifact does not match the model: missing={sorted(missing)[:8]} " f"unexpected={sorted(result.unexpected_keys)[:8]}" ) if tied: model.tie_weights() leftover = [n for n, t in list(model.named_parameters()) + list(model.named_buffers()) if t.is_meta] if leftover: raise ValueError(f"tensors left uninitialised after load: {leftover[:8]}") try: model.generation_config = GenerationConfig.from_pretrained( model_path, local_files_only=True ) except OSError: pass model.to(device) model.eval() attn_impl = getattr(model.config, "_attn_implementation", None) if attn_impl != "sdpa": raise RuntimeError(f"expected SDPA attention for prefill, got {attn_impl!r}") tokenizer = AutoTokenizer.from_pretrained( model_path, trust_remote_code=False, local_files_only=True ) return model, tokenizer