Download quantization/quantize.py from kurcontko/Index-Translate-9B-NVFP4: direct link, hf CLI and curl.
- Browser
- Download file 4.63 kB
-
https://huggingface.co/kurcontko/Index-Translate-9B-NVFP4/resolve/main/quantization/quantize.py
- Command line
-
hf download hf://kurcontko/Index-Translate-9B-NVFP4/quantization/quantize.py
-
curl -L -o quantize.py https://huggingface.co/kurcontko/Index-Translate-9B-NVFP4/resolve/main/quantization/quantize.py
4.63 kB
| """Quantize Index-Translate-9B (Qwen3.5 hybrid GDN) to NVFP4 for a 12 GB card with llm-compressor 0.15. | |
| python quantize.py --layout allfp4 # the published checkpoint: all LM linears NVFP4 | |
| python quantize.py --layout hybrid # GDN projections FP8-dynamic instead (same quality, bigger, slower) | |
| python quantize.py --layout split --split 16 # layers < 16 FP8-dynamic, rest NVFP4 (not evaluated) | |
| lm_head, embed_tokens, the GDN in_proj_a/in_proj_b gates, the vision tower and the MTP head stay BF16. | |
| Calibration: calib_opus.jsonl (576 OPUS-100 pairs, EN->8 EU langs + PL->EN), rendered with the | |
| Index-Translate prompt and the reference translation as the assistant turn. | |
| """ | |
| import argparse, json, shutil, sys, pathlib | |
| import torch | |
| from datasets import Dataset | |
| from llmcompressor import oneshot | |
| from llmcompressor.modifiers.quantization import QuantizationModifier | |
| from llmcompressor.observers import fused_mappings as fm | |
| from compressed_tensors.quantization import preset_name_to_scheme | |
| from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration | |
| HERE = pathlib.Path(__file__).resolve().parent | |
| def p_index(r): | |
| # Index-Translate prompt from bilibili/Index-Translate inference/llm/translate.py (enable_thinking=False). | |
| return [{"role": "user", "content": | |
| f"请将以下{r['src_name']}文本翻译为{r['tgt_name']},直接输出翻译结果,不要进行任何解释。\n\n{r['source']}"}] | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--src", default="IndexTeam/Index-Translate-9B") | |
| ap.add_argument("--layout", required=True, choices=["hybrid", "allfp4", "split"]) | |
| ap.add_argument("--split", type=int, default=16) | |
| ap.add_argument("--calib", default=str(HERE / "calib_opus.jsonl")) | |
| ap.add_argument("--out", default=None) | |
| ap.add_argument("--num-samples", type=int, default=576) | |
| args = ap.parse_args() | |
| MLP = r"mlp\.(gate|up|down)_proj$" | |
| ATTN = r"self_attn\.(q|k|v|o)_proj$" | |
| GDN = r"linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$" | |
| IGNORE = ["lm_head", "re:.*visual.*", "re:^mtp.*", | |
| "re:.*linear_attn.in_proj_a$", "re:.*linear_attn.in_proj_b$"] # 32-wide GDN gates | |
| if args.layout == "hybrid": | |
| groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}"]), "fp8": ("FP8_DYNAMIC", [f"re:.*{GDN}"])} | |
| elif args.layout == "allfp4": | |
| groups = {"nvfp4": ("NVFP4", [f"re:.*{MLP}", f"re:.*{ATTN}", f"re:.*{GDN}"])} | |
| else: | |
| lo = "|".join(str(i) for i in range(args.split)) | |
| groups = {"fp8": ("FP8_DYNAMIC", [rf"re:.*layers\.({lo})\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"]), | |
| "nvfp4": ("NVFP4", [rf"re:.*layers\.(?!({lo})\.)\d+\.(mlp|self_attn|linear_attn)\.(gate|up|down|q|k|v|o|in_proj_qkv|in_proj_z|out)_proj$"])} | |
| config_groups = {k: preset_name_to_scheme(p, t) for k, (p, t) in groups.items()} | |
| # vLLM fuses GDN in_proj_qkv + in_proj_z into one GEMM; NVFP4 needs them to share the weight global scale | |
| # (llm-compressor 0.15's FUSED_MODULE_MAPPINGS has no Qwen3.5 GDN entry). | |
| fm.FUSED_MODULE_MAPPINGS += (fm.FusedMapping("in_proj_qkvz", ("in_proj_qkv", "in_proj_z")),) | |
| fm.FUSED_MODULE_NAMES = fm.FUSED_MODULE_NAMES | {"in_proj_qkv", "in_proj_z"} | |
| tokenizer = AutoTokenizer.from_pretrained(args.src) | |
| model = Qwen3_5ForConditionalGeneration.from_pretrained(args.src, dtype=torch.bfloat16) | |
| rows = [json.loads(l) for l in open(args.calib)][: args.num_samples] | |
| texts = [tokenizer.apply_chat_template( | |
| p_index(r) + [{"role": "assistant", "content": r["reference"]}], tokenize=False, | |
| enable_thinking=False) for r in rows] | |
| enc = [tokenizer(t, add_special_tokens=False).input_ids for t in texts] | |
| print("calib samples", len(enc), "tokens", sum(map(len, enc)), "example:", repr(texts[0][:300])) | |
| dataset = Dataset.from_dict({"input_ids": enc, "attention_mask": [[1] * len(x) for x in enc]}) | |
| oneshot( | |
| model=model, dataset=dataset, processor=tokenizer, | |
| recipe=QuantizationModifier(config_groups=config_groups, ignore=IGNORE), | |
| max_seq_length=2048, num_calibration_samples=len(enc), | |
| pipeline="sequential", sequential_targets=["Qwen3_5DecoderLayer"], | |
| ) | |
| name = f"Index-Translate-9B-{args.layout}" + (f"{args.split}" if args.layout == "split" else "") | |
| out = pathlib.Path(args.out or name) | |
| model.save_pretrained(out, save_compressed=True) | |
| from huggingface_hub import snapshot_download | |
| for f in pathlib.Path(snapshot_download(args.src, allow_patterns=["*.json", "*.jinja", "*.txt"])).iterdir(): | |
| if f.suffix in (".json", ".jinja", ".txt") and "safetensors" not in f.name and not (out / f.name).exists(): | |
| shutil.copy(f, out / f.name) | |
| print("saved", out) | |