File size: 5,475 Bytes
d07994b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
"""Evaluation-only shims so stock vLLM 0.31 can load NVIDIA's Qwen3.8-Flash-Next-FP4-Local layout:
  - `<proj>.weight_scale.nvfp4_csf_fixed` + `..._exceptions` (b12x lossless CSF code) -> `<proj>.weight_scale` (E4M3);
  - NVFP4 PLE n-gram table shards (`shard_k.weight` u8 + `shard_k.weight_scale` E4M3 + one `weight_scale_2`) ->
    BF16 `shard_k.weight` (stock vLLM keeps an unquantized PLE table in pinned host memory)."""
import importlib.abc
import importlib.util
import os
import sys

TARGETS = ("vllm.model_executor.model_loader.weight_utils", "vllm.model_executor.model_loader.default_loader",
           "vllm.v1.worker.gpu.spec_decode.mtp.speculator", "vllm.model_executor.models.deepseek_v4.nvidia.model")
DRAFT_HOME = os.environ.get("Q38_DRAFT_HOME", "/root/q38prune/draft")


def _csf_decode(fixed, exceptions):
    import torch
    rows = fixed.shape[0] * 16
    cols = (fixed.shape[1] // 16 - 1) * 2
    slab = fixed.view(rows // 16, 16 * (1 + cols // 2))
    base = slab[:, :16].reshape(rows, 1).to(torch.int32)
    codes = slab[:, 16:].reshape(rows, cols // 2)
    codes = torch.stack([codes & 0xF, codes >> 4], dim=-1).reshape(rows, cols).to(torch.int32)
    out = (base + codes).view(-1)
    if exceptions.numel():
        ex = exceptions.to(torch.int64) & 0xFFFFFFFF
        out[ex & 0xFFFFFF] = (ex >> 24).to(torch.int32)
    return out.to(torch.uint8).view(rows, cols).view(torch.float8_e4m3fn)


_E2M1 = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0]


def _ple_global(files):
    from safetensors import safe_open
    for f in files:
        with safe_open(f, "pt") as h:
            for n in h.keys():
                if n.endswith("ngram_embedding.weight_scale_2"):
                    return float(h.get_tensor(n).float())
    return None


def _ple_dequant(w, s, g):
    import torch
    dev = "cuda" if torch.cuda.is_available() else "cpu"
    w, s = w.to(dev), s.to(dev).view(torch.float8_e4m3fn).float()
    lut = torch.tensor(_E2M1, device=dev)
    x = torch.stack([lut[(w & 0xF).long()], lut[(w >> 4).long()]], -1).flatten(-2)
    x = x.view(*x.shape[:-1], -1, 16) * s.unsqueeze(-1) * g
    return x.flatten(-2).to(torch.bfloat16).cpu()


def _wrap(it_fn):
    def wrapped(*a, **k):
        pending = {}
        ple = {}
        files = a[0] if a and isinstance(a[0], (list, tuple)) else k.get("hf_weights_files", [])
        g = _ple_global(files)
        for name, t in it_fn(*a, **k):
            if g is not None and ".ngram_embedding." in name and (".shard_" in name or name.endswith("weight_scale_2")):
                if name.endswith("weight_scale_2"):
                    continue
                base, leaf = name.rsplit(".", 1)
                if leaf == "weight_scale":
                    ple.setdefault(base, {})["s"] = t.clone()
                elif leaf == "weight":
                    ple.setdefault(base, {})["w"] = t.clone()
                else:
                    yield name, t
                    continue
                d = ple[base]
                if len(d) == 2:
                    del ple[base]
                    yield base + ".weight", _ple_dequant(d["w"], d["s"], g)
                continue
            for suf in (".nvfp4_csf_fixed", ".nvfp4_csf_exceptions"):
                if name.endswith(suf):
                    base = name[: -len(suf)]
                    d = pending.setdefault(base, {})
                    d[suf] = t.clone() if hasattr(t, "clone") else t
                    if len(d) == 2:
                        del pending[base]
                        yield base, _csf_decode(d[".nvfp4_csf_fixed"].cpu(), d[".nvfp4_csf_exceptions"].cpu())
                    break
            else:
                yield name, t
        assert not pending, f"unpaired CSF tensors: {list(pending)[:4]}"
        assert not ple, f"unpaired PLE shards: {list(ple)[:4]}"
    wrapped._csf = True
    return wrapped


def _patch(mod):
    for attr in ("safetensors_weights_iterator", "multi_thread_safetensors_weights_iterator"):
        fn = getattr(mod, attr, None)
        if fn is not None and not getattr(fn, "_csf", False):
            setattr(mod, attr, _wrap(fn))


class _Finder(importlib.abc.MetaPathFinder):
    def find_spec(self, fullname, path, target=None):
        if fullname not in TARGETS:
            return None
        sys.meta_path.remove(self)
        try:
            spec = importlib.util.find_spec(fullname)
        finally:
            sys.meta_path.insert(0, self)
        if spec is None or spec.loader is None:
            return spec
        orig = spec.loader.exec_module

        def exec_module(module):
            orig(module)
            if fullname.endswith("mtp.speculator"):
                if os.environ.get("Q38_DRAFT_VOCAB"):          # cheap MTP drafter (draft/q38_draft.py)
                    if DRAFT_HOME not in sys.path:
                        sys.path.insert(0, DRAFT_HOME)
                    import q38_draft
                    q38_draft.install_from_env()
            elif fullname.endswith("deepseek_v4.nvidia.model"):
                if os.environ.get("DS_STATS_DIR"):             # routed-expert usage statistics
                    sys.path.insert(0, "/root/ds4prune")
                    import ds4_stats_patch
                    ds4_stats_patch.install()
            else:
                _patch(module)
        spec.loader.exec_module = exec_module
        return spec


sys.meta_path.insert(0, _Finder())