"""Quantize Index-Translate-9B (Qwen3.5 hybrid GDN) to NVFP4 for a 12 GB card with llm-compressor 0.15. python quantize.py --layout allfp4 # the published checkpoint: all LM linears NVFP4 python quantize.py --layout hybrid # GDN projections FP8-dynamic instead (same quality, bigger, slower) python quantize.py --layout split --split 16 # layers < 16 FP8-dynamic, rest NVFP4 (not evaluated) lm_head, embed_tokens, the GDN in_proj_a/in_proj_b gates, the vision tower and the MTP head stay BF16. Calibration: calib_opus.jsonl (576 OPUS-100 pairs, EN->8 EU langs + PL->EN), rendered with the Index-Translate prompt and the reference translation as the assistant turn. """ import argparse, json, shutil, sys, pathlib import torch from datasets import Dataset from llmcompressor import oneshot from llmcompressor.modifiers.quantization import QuantizationModifier from llmcompressor.observers import fused_mappings as fm from compressed_tensors.quantization import preset_name_to_scheme from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration HERE = pathlib.Path(__file__).resolve().parent def p_index(r): # Index-Translate prompt from bilibili/Index-Translate inference/llm/translate.py (enable_thinking=False). return [{"role": "user", "content": f"请将以下{r['src_name']}文本翻译为{r['tgt_name']},直接输出翻译结果,不要进行任何解释。\n\n{r['source']}"}] ap = argparse.ArgumentParser() ap.add_argument("--src", default="IndexTeam/Index-Translate-9B") ap.add_argument("--layout", required=True, choices=["hybrid", "allfp4", "split"]) ap.add_argument("--split", type=int, default=16) ap.add_argument("--calib", default=str(HERE / "calib_opus.jsonl")) ap.add_argument("--out", default=None) ap.add_argument("--num-samples", type=int, default=576) args = ap.parse_args() MLP = r"mlp\.(gate|up|down)_proj$" ATTN = r"self_attn\.(q|k|v|o)_proj$" GDN = r"linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$" IGNORE = ["lm_head", "re:.*visual.*", "re:^mtp.*", "re:.*linear_attn.in_proj_a$", "re:.*linear_attn.in_proj_b$"] # 32-wide GDN gates if args.layout == "hybrid": groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}"]), "fp8": ("FP8_DYNAMIC", [f"re:.*{GDN}"])} elif args.layout == "allfp4": groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}", f"re:.*{GDN}"])} else: lo = "|".join(str(i) for i in range(args.split)) groups = {"fp8": ("FP8_DYNAMIC", [rf"re:.*layers\.({lo})\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"]), "nvfp4": ("NVFP4", [rf"re:.*layers\.(?!({lo})\.)\d+\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"])} config_groups = {k: preset_name_to_scheme(p, t) for k, (p, t) in groups.items()} # vLLM fuses GDN in_proj_qkv + in_proj_z into one GEMM; NVFP4 needs them to share the weight global scale # (llm-compressor 0.15's FUSED_MODULE_MAPPINGS has no Qwen3.5 GDN entry). fm.FUSED_MODULE_MAPPINGS += (fm.FusedMapping("in_proj_qkvz", ("in_proj_qkv", "in_proj_z")),) fm.FUSED_MODULE_NAMES = fm.FUSED_MODULE_NAMES | {"in_proj_qkv", "in_proj_z"} tokenizer = AutoTokenizer.from_pretrained(args.src) model = Qwen3_5ForConditionalGeneration.from_pretrained(args.src, dtype=torch.bfloat16) rows = [json.loads(l) for l in open(args.calib)][: args.num_samples] texts = [tokenizer.apply_chat_template( p_index(r) + [{"role": "assistant", "content": r["reference"]}], tokenize=False, enable_thinking=False) for r in rows] enc = [tokenizer(t, add_special_tokens=False).input_ids for t in texts] print("calib samples", len(enc), "tokens", sum(map(len, enc)), "example:", repr(texts[0][:300])) dataset = Dataset.from_dict({"input_ids": enc, "attention_mask": [[1] * len(x) for x in enc]}) oneshot( model=model, dataset=dataset, processor=tokenizer, recipe=QuantizationModifier(config_groups=config_groups, ignore=IGNORE), max_seq_length=2048, num_calibration_samples=len(enc), pipeline="sequential", sequential_targets=["Qwen3_5DecoderLayer"], ) name = f"Index-Translate-9B-{args.layout}" + (f"{args.split}" if args.layout == "split" else "") out = pathlib.Path(args.out or name) model.save_pretrained(out, save_compressed=True) from huggingface_hub import snapshot_download for f in pathlib.Path(snapshot_download(args.src, allow_patterns=["*.json", "*.jinja", "*.txt"])).iterdir(): if f.suffix in (".json", ".jinja", ".txt") and "safetensors" not in f.name and not (out / f.name).exists(): shutil.copy(f, out / f.name) print("saved", out)