File size: 4,626 Bytes
fbb094f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | """Quantize Index-Translate-9B (Qwen3.5 hybrid GDN) to NVFP4 for a 12 GB card with llm-compressor 0.15.
python quantize.py --layout allfp4 # the published checkpoint: all LM linears NVFP4
python quantize.py --layout hybrid # GDN projections FP8-dynamic instead (same quality, bigger, slower)
python quantize.py --layout split --split 16 # layers < 16 FP8-dynamic, rest NVFP4 (not evaluated)
lm_head, embed_tokens, the GDN in_proj_a/in_proj_b gates, the vision tower and the MTP head stay BF16.
Calibration: calib_opus.jsonl (576 OPUS-100 pairs, EN->8 EU langs + PL->EN), rendered with the
Index-Translate prompt and the reference translation as the assistant turn.
"""
import argparse, json, shutil, sys, pathlib
import torch
from datasets import Dataset
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.observers import fused_mappings as fm
from compressed_tensors.quantization import preset_name_to_scheme
from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration
HERE = pathlib.Path(__file__).resolve().parent
def p_index(r):
# Index-Translate prompt from bilibili/Index-Translate inference/llm/translate.py (enable_thinking=False).
return [{"role": "user", "content":
f"请将以下{r['src_name']}文本翻译为{r['tgt_name']},直接输出翻译结果,不要进行任何解释。\n\n{r['source']}"}]
ap = argparse.ArgumentParser()
ap.add_argument("--src", default="IndexTeam/Index-Translate-9B")
ap.add_argument("--layout", required=True, choices=["hybrid", "allfp4", "split"])
ap.add_argument("--split", type=int, default=16)
ap.add_argument("--calib", default=str(HERE / "calib_opus.jsonl"))
ap.add_argument("--out", default=None)
ap.add_argument("--num-samples", type=int, default=576)
args = ap.parse_args()
MLP = r"mlp\.(gate|up|down)_proj$"
ATTN = r"self_attn\.(q|k|v|o)_proj$"
GDN = r"linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$"
IGNORE = ["lm_head", "re:.*visual.*", "re:^mtp.*",
"re:.*linear_attn.in_proj_a$", "re:.*linear_attn.in_proj_b$"] # 32-wide GDN gates
if args.layout == "hybrid":
groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}"]), "fp8": ("FP8_DYNAMIC", [f"re:.*{GDN}"])}
elif args.layout == "allfp4":
groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}", f"re:.*{GDN}"])}
else:
lo = "|".join(str(i) for i in range(args.split))
groups = {"fp8": ("FP8_DYNAMIC", [rf"re:.*layers\.({lo})\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"]),
"nvfp4": ("NVFP4", [rf"re:.*layers\.(?!({lo})\.)\d+\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"])}
config_groups = {k: preset_name_to_scheme(p, t) for k, (p, t) in groups.items()}
# vLLM fuses GDN in_proj_qkv + in_proj_z into one GEMM; NVFP4 needs them to share the weight global scale
# (llm-compressor 0.15's FUSED_MODULE_MAPPINGS has no Qwen3.5 GDN entry).
fm.FUSED_MODULE_MAPPINGS += (fm.FusedMapping("in_proj_qkvz", ("in_proj_qkv", "in_proj_z")),)
fm.FUSED_MODULE_NAMES = fm.FUSED_MODULE_NAMES | {"in_proj_qkv", "in_proj_z"}
tokenizer = AutoTokenizer.from_pretrained(args.src)
model = Qwen3_5ForConditionalGeneration.from_pretrained(args.src, dtype=torch.bfloat16)
rows = [json.loads(l) for l in open(args.calib)][: args.num_samples]
texts = [tokenizer.apply_chat_template(
p_index(r) + [{"role": "assistant", "content": r["reference"]}], tokenize=False,
enable_thinking=False) for r in rows]
enc = [tokenizer(t, add_special_tokens=False).input_ids for t in texts]
print("calib samples", len(enc), "tokens", sum(map(len, enc)), "example:", repr(texts[0][:300]))
dataset = Dataset.from_dict({"input_ids": enc, "attention_mask": [[1] * len(x) for x in enc]})
oneshot(
model=model, dataset=dataset, processor=tokenizer,
recipe=QuantizationModifier(config_groups=config_groups, ignore=IGNORE),
max_seq_length=2048, num_calibration_samples=len(enc),
pipeline="sequential", sequential_targets=["Qwen3_5DecoderLayer"],
)
name = f"Index-Translate-9B-{args.layout}" + (f"{args.split}" if args.layout == "split" else "")
out = pathlib.Path(args.out or name)
model.save_pretrained(out, save_compressed=True)
from huggingface_hub import snapshot_download
for f in pathlib.Path(snapshot_download(args.src, allow_patterns=["*.json", "*.jinja", "*.txt"])).iterdir():
if f.suffix in (".json", ".jinja", ".txt") and "safetensors" not in f.name and not (out / f.name).exists():
shutil.copy(f, out / f.name)
print("saved", out)
|