Spaces:
Running on Zero
Running on Zero
Download third_party/musubi_tuner/modules/nvfp4_utils.py from hugging-apps/mpeg-diffusion: direct link, hf CLI and curl.
- Browser
- Download file 25.1 kB
-
https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/third_party/musubi_tuner/modules/nvfp4_utils.py
- Command line
-
hf download hf://spaces/hugging-apps/mpeg-diffusion/third_party/musubi_tuner/modules/nvfp4_utils.py
-
curl -L -o nvfp4_utils.py https://huggingface.co/spaces/hugging-apps/mpeg-diffusion/resolve/main/third_party/musubi_tuner/modules/nvfp4_utils.py
25.1 kB
| # Low-level NVFP4 routines (E2M1 encode/decode, cuBLAS swizzle, scaled_mm call) are | |
| # adapted from the exp-nvfp4-support-for-torch-2-10 branch, which was inspired by | |
| # comfy-kitchen (https://github.com/Comfy-Org/comfy-kitchen, Apache License 2.0). | |
| """Loading ComfyUI pre-quantized NVFP4 (+AWQ) checkpoints for frozen text encoders. | |
| Layout of a ComfyUI NVFP4 artifact (verified bit-level against the BF16 reference of | |
| the MiniMax-H3 Qwen3-VL text encoder): | |
| - NVFP4 Linear: ``.weight`` U8 [N, K/2] with two E2M1 codes per byte (element 0 in the | |
| HIGH nibble), ``.weight_scale`` F8_E4M3 per-16-block scales stored in the cuBLAS | |
| 128x4 tiled ("swizzled") layout, ``.weight_scale_2`` F32 per-tensor scale, and a | |
| ``.comfy_quant`` spec ``{"format": "nvfp4", ...}``. Modules quantized with AWQ carry | |
| a ``.pre_quant_scale`` [K] tensor that is multiplied into the *input* at runtime; | |
| for the remaining modules the AWQ scale is folded into the preceding norm weights, | |
| so the checkpoint is self-consistent and needs no special handling here. | |
| - INT8 embedding: ``.weight`` I8 [V, D] + ``.weight_scale`` F32 [V, 1] | |
| (``{"format": "int8_tensorwise"}``, effectively per-row); dequant = weight * scale. | |
| - Everything else (norms, vision tower, biases) stays BF16. | |
| The state dict is converted to the Musubi runtime layout: ``.weight_scale`` becomes | |
| ``.nvfp4_block_scale`` (kept swizzled: ``torch.nn.functional.scaled_mm`` consumes the | |
| swizzled layout directly, the dequantizing fallback unswizzles on the fly) and | |
| ``.weight_scale_2`` becomes ``.nvfp4_scale``; the embedding scale becomes | |
| ``.scale_weight`` (same dequant semantics as the ConvRot INT8 layout). | |
| Inference only: NVFP4 modules are frozen, the patched forwards have no autograd | |
| support. Dynamic (on-the-fly) NVFP4 quantization is deliberately not offered — the | |
| published artifacts are AWQ-calibrated, which cannot be reproduced without | |
| calibration data, and dynamically quantizing BF16 weights would silently produce a | |
| lower-quality model than ConvRot INT8. | |
| """ | |
| import os | |
| from typing import Dict, List, Optional, Tuple, Union | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import logging | |
| from tqdm import tqdm | |
| from musubi_tuner.modules.comfy_quant_utils import ( | |
| COMFY_QUANT_SUFFIX, | |
| COMFY_WEIGHT_SCALE_SUFFIX, | |
| FORMAT_INT8_TENSORWISE, | |
| FORMAT_NVFP4, | |
| classify_comfy_quant_spec, | |
| decode_comfy_quant_spec, | |
| ) | |
| from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, TensorWeightAdapter, WeightTransformHooks | |
| logger = logging.getLogger(__name__) | |
| NVFP4_BLOCK_SIZE = 16 | |
| F4_E2M1_MAX = 6.0 | |
| F8_E4M3_MAX = 448.0 | |
| COMFY_WEIGHT_SCALE_2_SUFFIX = ".weight_scale_2" | |
| COMFY_PRE_QUANT_SCALE_SUFFIX = ".pre_quant_scale" | |
| # E2M1 code -> value (codes 0..7 positive, 8..15 negative) | |
| _E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0) | |
| # byte -> (high nibble value, low nibble value), cached per (device, dtype) | |
| _BYTE_PAIR_LUT_CACHE: Dict[Tuple[torch.device, torch.dtype], torch.Tensor] = {} | |
| def _byte_pair_lut(device: torch.device, dtype: torch.dtype) -> torch.Tensor: | |
| lut = _BYTE_PAIR_LUT_CACHE.get((device, dtype)) | |
| if lut is None: | |
| values = torch.tensor(_E2M1_VALUES, dtype=torch.float32) | |
| codes = torch.arange(256, dtype=torch.int64) | |
| lut = torch.stack([values[codes >> 4], values[codes & 0x0F]], dim=1).to(device=device, dtype=dtype) | |
| _BYTE_PAIR_LUT_CACHE[(device, dtype)] = lut | |
| return lut | |
| def _ceil_div(a: int, b: int) -> int: | |
| return (a + b - 1) // b | |
| def _roundup(value: int, multiple: int) -> int: | |
| return _ceil_div(value, multiple) * multiple | |
| # region cuBLAS 128x4 tiled block-scale layout | |
| # https://docs.nvidia.com/cuda/cublas/index.html#d-block-scaling-factors-layout | |
| def to_blocked(input_matrix: torch.Tensor) -> torch.Tensor: | |
| """Rearrange (H, W) block scales into the cuBLAS tiled layout (padded to 128x4).""" | |
| rows, cols = input_matrix.shape | |
| n_row_blocks = _ceil_div(rows, 128) | |
| n_col_blocks = _ceil_div(cols, 4) | |
| padded_rows = n_row_blocks * 128 | |
| padded_cols = n_col_blocks * 4 | |
| padded = input_matrix | |
| if (rows, cols) != (padded_rows, padded_cols): | |
| padded = torch.zeros((padded_rows, padded_cols), device=input_matrix.device, dtype=input_matrix.dtype) | |
| padded[:rows, :cols] = input_matrix | |
| blocks = padded.view(n_row_blocks, 128, n_col_blocks, 4).permute(0, 2, 1, 3) | |
| rearranged = blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1, 32, 16) | |
| return rearranged.reshape(padded_rows, padded_cols) | |
| def from_blocked(blocked_matrix: torch.Tensor, num_rows: int, num_cols: int) -> torch.Tensor: | |
| """Reverse the cuBLAS tiled layout back to a row-major (num_rows, num_cols) matrix.""" | |
| n_row_blocks = _ceil_div(num_rows, 128) | |
| n_col_blocks = _ceil_div(num_cols, 4) | |
| padded_rows = n_row_blocks * 128 | |
| padded_cols = n_col_blocks * 4 | |
| step1 = blocked_matrix.reshape(-1, 32, 16) | |
| step2 = step1.reshape(-1, 32, 4, 4).transpose(1, 2) | |
| step3 = step2.reshape(n_row_blocks, n_col_blocks, 4, 32, 4) | |
| step4 = step3.reshape(n_row_blocks, n_col_blocks, 128, 4) | |
| step5 = step4.permute(0, 2, 1, 3) | |
| unblocked = step5.reshape(padded_rows, padded_cols) | |
| return unblocked[:num_rows, :num_cols] | |
| # endregion | |
| def dequantize_nvfp4( | |
| packed: torch.Tensor, | |
| block_scale: torch.Tensor, | |
| per_tensor_scale: torch.Tensor, | |
| orig_shape: Tuple[int, int], | |
| out_dtype: torch.dtype, | |
| ) -> torch.Tensor: | |
| """Dequantize packed NVFP4 data (block_scale in the swizzled layout) to ``orig_shape``.""" | |
| stored_rows = packed.shape[0] | |
| stored_cols = packed.shape[1] * 2 | |
| lut = _byte_pair_lut(packed.device, out_dtype) | |
| # one embedding lookup decodes both nibbles of each byte: [R, C/2] -> [R, C/2, 2] -> [R, C] | |
| weight = F.embedding(packed.reshape(-1).int(), lut).reshape(stored_rows, stored_cols) | |
| scales = from_blocked(block_scale, stored_rows, stored_cols // NVFP4_BLOCK_SIZE) | |
| total = (per_tensor_scale.float() * scales.float()).to(out_dtype) | |
| weight = (weight.reshape(stored_rows, -1, NVFP4_BLOCK_SIZE) * total.unsqueeze(-1)).reshape(stored_rows, stored_cols) | |
| return weight[: orig_shape[0], : orig_shape[1]] | |
| # region runtime activation quantization for the scaled_mm (W4A4) path | |
| def _n_ones(n: int) -> int: | |
| return (1 << n) - 1 | |
| _EBITS_F32, _MBITS_F32 = 8, 23 | |
| _F32_EXP_BIAS = _n_ones(_EBITS_F32 - 1) | |
| def _f32_to_e2m1_unpacked(x: torch.Tensor) -> torch.Tensor: | |
| """Convert FP32 to E2M1 codes stored one-per-byte in uint8 (round-to-nearest-even).""" | |
| ebits, mbits = 2, 1 | |
| assert x.dtype == torch.float | |
| exp_bias = _n_ones(ebits - 1) | |
| max_int = _n_ones(ebits + mbits) | |
| sign_mask = 1 << (ebits + mbits) | |
| magic_adder = _n_ones(_MBITS_F32 - mbits - 1) | |
| max_normal = 2 ** (_n_ones(ebits) - exp_bias) * (_n_ones(mbits + 1) / (2**mbits)) | |
| min_normal = 2 ** (1 - exp_bias) | |
| denorm_exp = (_F32_EXP_BIAS - exp_bias) + (_MBITS_F32 - mbits) + 1 | |
| denorm_mask_int = denorm_exp << _MBITS_F32 | |
| denorm_mask_float = torch.tensor(denorm_mask_int, dtype=torch.int32, device=x.device).view(torch.float32) | |
| x = x.view(torch.int32) | |
| sign = x & 0x80000000 | |
| x = (x ^ sign).view(torch.float) | |
| saturate_mask = x >= max_normal | |
| denormal_mask = torch.logical_and(torch.logical_not(saturate_mask), x < min_normal) | |
| normal_mask = torch.logical_not(torch.logical_or(saturate_mask, denormal_mask)) | |
| denormal_x = (x + denorm_mask_float).view(torch.int32) - denorm_mask_int | |
| denormal_x = denormal_x.to(torch.uint8) | |
| normal_x = x.view(torch.int32) | |
| mant_odd = (normal_x >> (_MBITS_F32 - mbits)) & 1 | |
| normal_x = normal_x + (((exp_bias - _F32_EXP_BIAS) << _MBITS_F32) + magic_adder) + mant_odd | |
| normal_x = (normal_x >> (_MBITS_F32 - mbits)).to(torch.uint8) | |
| result = torch.full_like(x, max_int, dtype=torch.uint8) | |
| result = torch.where(denormal_mask, denormal_x, result) | |
| result = torch.where(normal_mask, normal_x, result) | |
| sign_lp = (sign >> (_MBITS_F32 + _EBITS_F32 - mbits - ebits)).to(torch.uint8) & sign_mask | |
| return result | sign_lp | |
| def pack_uint4(codes: torch.Tensor) -> torch.Tensor: | |
| """Pack pairs of 4-bit codes (one per byte) into bytes, element 0 in the HIGH nibble.""" | |
| shape = codes.shape | |
| assert shape[-1] % 2 == 0 | |
| codes = codes.contiguous().view(-1) | |
| return (codes[::2] << 4 | codes[1::2]).view(*shape[:-1], shape[-1] // 2) | |
| def quantize_nvfp4_activation(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: | |
| """Quantize a 2D activation to NVFP4 for scaled_mm. | |
| Returns (packed uint8 [Mp, K/2], swizzled block scales F8_E4M3, per-tensor scale F32, | |
| original row count). Rows are padded to a multiple of 16 as scaled_mm requires. | |
| """ | |
| orig_rows, cols = x.shape | |
| if cols % NVFP4_BLOCK_SIZE != 0: | |
| raise ValueError(f"NVFP4 activation width must be a multiple of {NVFP4_BLOCK_SIZE}, got {cols}") | |
| padded_rows = _roundup(orig_rows, 16) | |
| if padded_rows != orig_rows: | |
| x = F.pad(x, (0, 0, 0, padded_rows - orig_rows)) | |
| per_tensor_scale = (torch.amax(x.abs()).float() / (F8_E4M3_MAX * F4_E2M1_MAX)).reshape(()) | |
| blocks = x.reshape(padded_rows, -1, NVFP4_BLOCK_SIZE) | |
| block_scale = torch.amax(blocks.abs(), dim=-1).float() / F4_E2M1_MAX | |
| scaled = torch.clamp(block_scale / torch.clamp(per_tensor_scale, min=torch.finfo(torch.float32).tiny), max=F8_E4M3_MAX) | |
| scaled_f8 = scaled.to(torch.float8_e4m3fn) | |
| total = per_tensor_scale * scaled_f8.float() | |
| total_safe = torch.where(total == 0, torch.ones_like(total), total) | |
| data = blocks.float() / total_safe.unsqueeze(-1) | |
| data = torch.where((total == 0).unsqueeze(-1), torch.zeros_like(data), data) | |
| data = torch.clamp(data, -F4_E2M1_MAX, F4_E2M1_MAX).reshape(padded_rows, cols) | |
| packed = pack_uint4(_f32_to_e2m1_unpacked(data)) | |
| return packed, to_blocked(scaled_f8), per_tensor_scale, orig_rows | |
| def nvfp4_scaled_mm_available() -> bool: | |
| return hasattr(torch, "float4_e2m1fn_x2") and hasattr(torch.nn.functional, "scaled_mm") | |
| def nvfp4_scaled_mm_linear( | |
| x: torch.Tensor, | |
| weight_packed: torch.Tensor, | |
| weight_block_scale: torch.Tensor, | |
| weight_scale: torch.Tensor, | |
| bias: Optional[torch.Tensor], | |
| orig_out_features: int, | |
| ) -> torch.Tensor: | |
| """W4A4 linear via torch.nn.functional.scaled_mm (torch 2.10+, Blackwell).""" | |
| from torch.nn.functional import ScalingType, SwizzleType | |
| x_packed, x_block_scale, x_scale, orig_rows = quantize_nvfp4_activation(x) | |
| result = torch.nn.functional.scaled_mm( | |
| x_packed.view(torch.float4_e2m1fn_x2), | |
| weight_packed.view(torch.float4_e2m1fn_x2).t(), | |
| scale_a=[x_block_scale.view(-1), x_scale], | |
| scale_b=[weight_block_scale.view(-1), weight_scale], | |
| bias=bias, | |
| output_dtype=x.dtype, | |
| scale_recipe_a=[ScalingType.BlockWise1x16, ScalingType.TensorWise], | |
| scale_recipe_b=[ScalingType.BlockWise1x16, ScalingType.TensorWise], | |
| swizzle_a=[SwizzleType.SWIZZLE_32_4_4, SwizzleType.NO_SWIZZLE], | |
| swizzle_b=[SwizzleType.SWIZZLE_32_4_4, SwizzleType.NO_SWIZZLE], | |
| ) | |
| return result[:orig_rows, :orig_out_features] | |
| # endregion | |
| # region pre-quantized state dict loading | |
| class NvFp4Quantizer: | |
| """Streams a ComfyUI pre-quantized NVFP4 (+ INT8 embedding) checkpoint. | |
| Same protocol as ``ConvRotInt8Quantizer``: passed as ``quantizer`` to | |
| ``load_safetensors_with_lora_and_fp8``. Loading only converts key names and | |
| validates the tensors — there is no dynamic quantization, the file dictates the | |
| quantized layers. ``nvfp4_module_shapes`` maps module paths to their original | |
| (out_features, in_features) after loading; ``int8_embedding_modules`` lists the | |
| per-row INT8 modules (embeddings). Pass both to ``apply_nvfp4_monkey_patch``. | |
| """ | |
| def __init__(self): | |
| self.nvfp4_module_shapes: Dict[str, Tuple[int, int]] = {} | |
| self.int8_embedding_modules: List[str] = [] | |
| def load_and_quantize( | |
| self, | |
| model_files: List[str], | |
| calc_device: Union[str, torch.device, None], | |
| move_to_device: bool = False, | |
| weight_hook: Optional[callable] = None, | |
| disable_numpy_memmap: bool = False, | |
| weight_transform_hooks: Optional[WeightTransformHooks] = None, | |
| ) -> dict: | |
| state_dict = {} | |
| module_formats: Dict[str, str] = {} # spans all shards | |
| for model_file in model_files: | |
| with MemoryEfficientSafeOpen(model_file, disable_numpy_memmap=disable_numpy_memmap) as original_f: | |
| f = TensorWeightAdapter(weight_transform_hooks, original_f) if weight_transform_hooks is not None else original_f | |
| keys = f.keys() | |
| # pre-scan the tiny spec tensors so each module's format is known before | |
| # the (possibly earlier-iterated) weight/scale keys arrive | |
| for key in keys: | |
| if key.endswith(COMFY_QUANT_SUFFIX): | |
| module_path = key[: -len(COMFY_QUANT_SUFFIX)] | |
| spec_format = classify_comfy_quant_spec(decode_comfy_quant_spec(key, f.get_tensor(key))) | |
| if spec_format not in (FORMAT_NVFP4, FORMAT_INT8_TENSORWISE): | |
| raise ValueError( | |
| f"Unsupported comfy_quant format for {key}: {spec_format}. The NVFP4 loader supports" | |
| ' "nvfp4" Linear layers and "int8_tensorwise" embeddings only.' | |
| f" / {key} の comfy_quant 形式 {spec_format} はNVFP4ローダーではサポートされていません。" | |
| ) | |
| module_formats[module_path] = spec_format | |
| if module_formats and weight_hook is not None: | |
| raise ValueError( | |
| f"Cannot merge LoRA weights into pre-quantized NVFP4 checkpoint {model_file}." | |
| " Use the original BF16 weights instead." | |
| f" / 事前量子化済みNVFP4チェックポイント {model_file} にはLoRAをマージできません。" | |
| "BF16の元重みを使用してください。" | |
| ) | |
| for key in tqdm(keys, desc=f"Loading {os.path.basename(model_file)}", unit="key"): | |
| if key.endswith(COMFY_QUANT_SUFFIX): | |
| continue # consumed in the pre-scan, not a model tensor | |
| value = f.get_tensor(key) | |
| original_device = value.device # usually cpu | |
| passthrough_device = calc_device if (calc_device is not None and move_to_device) else original_device | |
| converted_key = self._convert_key(key, value, module_formats) | |
| state_dict[converted_key] = value.to(passthrough_device) | |
| self._validate_completeness(state_dict, module_formats) | |
| logger.info( | |
| f"Number of pre-quantized layers: {len(self.nvfp4_module_shapes)} NVFP4 Linear," | |
| f" {len(self.int8_embedding_modules)} INT8 embedding" | |
| ) | |
| return state_dict | |
| def _convert_key(self, key: str, value: torch.Tensor, module_formats: Dict[str, str]) -> str: | |
| """Validate a tensor against its module's declared format and return the Musubi key.""" | |
| for suffix in (COMFY_WEIGHT_SCALE_SUFFIX, COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX, ".weight"): | |
| if key.endswith(suffix): | |
| module_path = key[: -len(suffix)] | |
| break | |
| else: | |
| return key # bias, norm, etc.: passthrough | |
| spec_format = module_formats.get(module_path) | |
| if spec_format is None: | |
| if key.endswith(".weight") and value.dtype.itemsize == 1: | |
| raise ValueError( | |
| f"Layer {key} is already in {value.dtype} format but has no {COMFY_QUANT_SUFFIX} spec." | |
| f" / レイヤー {key} は既に{value.dtype}形式ですが {COMFY_QUANT_SUFFIX} がありません。" | |
| ) | |
| if key.endswith((COMFY_WEIGHT_SCALE_SUFFIX, COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX)): | |
| raise ValueError(f"Found {key} without a matching {module_path}{COMFY_QUANT_SUFFIX} spec") | |
| return key | |
| if spec_format == FORMAT_NVFP4: | |
| if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX): | |
| if value.dtype is not torch.float8_e4m3fn: | |
| raise ValueError(f"NVFP4 block scale {key} must be F8_E4M3, got {value.dtype}") | |
| return module_path + ".nvfp4_block_scale" | |
| if key.endswith(COMFY_WEIGHT_SCALE_2_SUFFIX): | |
| if value.dtype is not torch.float32 or value.ndim != 0: | |
| raise ValueError(f"NVFP4 per-tensor scale {key} must be a F32 scalar, got {value.dtype} {tuple(value.shape)}") | |
| return module_path + ".nvfp4_scale" | |
| if key.endswith(COMFY_PRE_QUANT_SCALE_SUFFIX): | |
| if not value.is_floating_point() or value.ndim != 1: | |
| raise ValueError(f"AWQ pre_quant_scale {key} must be a 1D float tensor, got {value.dtype} {tuple(value.shape)}") | |
| return key | |
| # .weight | |
| if value.dtype is not torch.uint8 or value.ndim != 2: | |
| raise ValueError(f"NVFP4 weight {key} must be 2D uint8 (packed), got {value.dtype} ndim={value.ndim}") | |
| in_features = value.shape[1] * 2 | |
| if in_features % NVFP4_BLOCK_SIZE != 0: | |
| raise ValueError(f"NVFP4 weight {key}: in_features {in_features} not a multiple of {NVFP4_BLOCK_SIZE}") | |
| self.nvfp4_module_shapes[module_path] = (value.shape[0], in_features) | |
| return key | |
| # FORMAT_INT8_TENSORWISE: per-row INT8, embeddings only (validated at patch time) | |
| if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX): | |
| if value.dtype is not torch.float32: | |
| raise ValueError(f"INT8 per-row scale {key} must be F32, got {value.dtype}") | |
| return module_path + ".scale_weight" | |
| if key.endswith((COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX)): | |
| raise ValueError(f"Unexpected tensor {key} for int8_tensorwise module {module_path}") | |
| if value.dtype is not torch.int8: | |
| raise ValueError(f"INT8 weight {key} must be int8, got {value.dtype}") | |
| self.int8_embedding_modules.append(module_path) | |
| return key | |
| def _validate_completeness(self, state_dict: dict, module_formats: Dict[str, str]) -> None: | |
| for module_path, spec_format in module_formats.items(): | |
| if spec_format == FORMAT_NVFP4: | |
| required = (".weight", ".nvfp4_block_scale", ".nvfp4_scale") | |
| else: | |
| required = (".weight", ".scale_weight") | |
| missing = [module_path + suffix for suffix in required if module_path + suffix not in state_dict] | |
| if missing: | |
| raise ValueError(f"Pre-quantized module {module_path} is missing tensors {missing}") | |
| if spec_format == FORMAT_NVFP4: | |
| rows, in_features = self.nvfp4_module_shapes[module_path] | |
| expected_scale_numel = _roundup(rows, 128) * _roundup(in_features // NVFP4_BLOCK_SIZE, 4) | |
| block_scale = state_dict[module_path + ".nvfp4_block_scale"] | |
| if block_scale.numel() != expected_scale_numel: | |
| raise ValueError( | |
| f"NVFP4 module {module_path}: block scale has {block_scale.numel()} elements," | |
| f" expected {expected_scale_numel} for weight shape ({rows}, {in_features})" | |
| ) | |
| pre_quant_scale = state_dict.get(module_path + COMFY_PRE_QUANT_SCALE_SUFFIX) | |
| if pre_quant_scale is not None and pre_quant_scale.shape[0] != in_features: | |
| raise ValueError( | |
| f"NVFP4 module {module_path}: pre_quant_scale has {pre_quant_scale.shape[0]} elements," | |
| f" expected in_features {in_features}" | |
| ) | |
| else: | |
| weight = state_dict[module_path + ".weight"] | |
| scale = state_dict[module_path + ".scale_weight"] | |
| expected_scale_shape = (weight.shape[0], 1) | |
| if tuple(scale.shape) != expected_scale_shape: | |
| raise ValueError( | |
| f"INT8 module {module_path}: scale shape must be {expected_scale_shape}, got {tuple(scale.shape)}" | |
| ) | |
| # endregion | |
| # region monkey patch | |
| def nvfp4_linear_forward_patch(self: nn.Linear, x: torch.Tensor) -> torch.Tensor: | |
| pre_quant_scale = getattr(self, "pre_quant_scale", None) | |
| if pre_quant_scale is not None: | |
| x = x * pre_quant_scale | |
| if self._nvfp4_use_scaled_mm: | |
| x_2d = x.reshape(-1, x.shape[-1]) | |
| out = nvfp4_scaled_mm_linear( | |
| x_2d, self.weight, self.nvfp4_block_scale, self.nvfp4_scale, self.bias, self._nvfp4_orig_shape[0] | |
| ) | |
| return out.reshape(*x.shape[:-1], out.shape[-1]) | |
| weight = dequantize_nvfp4(self.weight, self.nvfp4_block_scale, self.nvfp4_scale, self._nvfp4_orig_shape, x.dtype) | |
| return F.linear(x, weight, self.bias) | |
| def int8_embedding_forward_patch(self: nn.Embedding, input: torch.Tensor) -> torch.Tensor: | |
| rows = self.weight[input] # index_select works on int8; padding_idx etc. only affect training | |
| return (rows.float() * self.scale_weight[input]).to(self._int8_dequant_dtype) | |
| def apply_nvfp4_monkey_patch( | |
| model: nn.Module, | |
| optimized_state_dict: dict, | |
| nvfp4_module_shapes: Dict[str, Tuple[int, int]], | |
| int8_embedding_modules: List[str], | |
| use_scaled_mm: bool = False, | |
| embedding_dtype: torch.dtype = torch.bfloat16, | |
| ) -> nn.Module: | |
| """Patch NVFP4 Linear and INT8 embedding modules so a strict assign load can follow. | |
| The patched modules get placeholder parameters/buffers with the quantized shapes and | |
| dtypes (on the meta device); ``model.load_state_dict(state_dict, strict=True, | |
| assign=True)`` then installs the real tensors. Modules stay ``nn.Linear`` / | |
| ``nn.Embedding`` (patched forward), mirroring the ConvRot INT8 approach. | |
| """ | |
| if use_scaled_mm and not nvfp4_scaled_mm_available(): | |
| raise ValueError( | |
| "NVFP4 scaled_mm requires PyTorch 2.10+ (torch.float4_e2m1fn_x2 and torch.nn.functional.scaled_mm)." | |
| " Omit the scaled_mm option to use the dequantize fallback." | |
| " / NVFP4 scaled_mm には PyTorch 2.10 以降が必要です。scaled_mm オプションを外すと dequantize フォールバックで動作します。" | |
| ) | |
| modules_by_name = dict(model.named_modules()) | |
| patched_count = 0 | |
| for name, (out_features, in_features) in nvfp4_module_shapes.items(): | |
| module = modules_by_name.get(name) | |
| if not isinstance(module, nn.Linear): | |
| raise ValueError(f"NVFP4 state dict declares {name}, which is not an nn.Linear in the model") | |
| weight_key = name + ".weight" | |
| module.weight = nn.Parameter(torch.empty_like(optimized_state_dict[weight_key], device="meta"), requires_grad=False) | |
| module.register_buffer( | |
| "nvfp4_block_scale", torch.empty_like(optimized_state_dict[name + ".nvfp4_block_scale"], device="meta") | |
| ) | |
| module.register_buffer("nvfp4_scale", torch.empty((), dtype=torch.float32, device="meta")) | |
| pre_quant_key = name + COMFY_PRE_QUANT_SCALE_SUFFIX | |
| if pre_quant_key in optimized_state_dict: | |
| module.register_buffer("pre_quant_scale", torch.empty_like(optimized_state_dict[pre_quant_key], device="meta")) | |
| module._nvfp4_orig_shape = (out_features, in_features) | |
| module._nvfp4_use_scaled_mm = use_scaled_mm | |
| module.forward = nvfp4_linear_forward_patch.__get__(module, type(module)) | |
| patched_count += 1 | |
| for name in int8_embedding_modules: | |
| module = modules_by_name.get(name) | |
| if not isinstance(module, nn.Embedding): | |
| raise ValueError( | |
| f"int8_tensorwise module {name} is not an nn.Embedding; INT8 per-row quantization is only" | |
| " supported for embeddings (use ConvRot INT8 checkpoints for Linear layers)" | |
| ) | |
| module.weight = nn.Parameter(torch.empty_like(optimized_state_dict[name + ".weight"], device="meta"), requires_grad=False) | |
| module.register_buffer("scale_weight", torch.empty_like(optimized_state_dict[name + ".scale_weight"], device="meta")) | |
| module._int8_dequant_dtype = embedding_dtype | |
| module.forward = int8_embedding_forward_patch.__get__(module, type(module)) | |
| patched_count += 1 | |
| if not use_scaled_mm and nvfp4_module_shapes: | |
| logger.info("NVFP4 runs in weight-only mode (transient dequantization per forward)") | |
| model.is_nvfp4 = True | |
| model.nvfp4_layer_count = len(nvfp4_module_shapes) | |
| logger.info(f"Number of NVFP4/INT8 monkey-patched modules: {patched_count}") | |
| return model | |
| # endregion | |