Download _infer.py from braindeck/text2text: direct link, hf CLI and curl.
- Browser
- Download file 12.9 kB
-
https://huggingface.co/spaces/braindeck/text2text/resolve/main/_infer.py
- Command line
-
hf download hf://spaces/braindeck/text2text/_infer.py
-
curl -L -o _infer.py https://huggingface.co/spaces/braindeck/text2text/resolve/main/_infer.py
12.9 kB
| #!/usr/bin/env python | |
| # Copyright 2024 Bytedance | |
| # Apache-2.0 | |
| # | |
| # VERL + vLLM inference with runtime LoRA (no merge). | |
| # - Wraps a LoRA .pt into a PEFT adapter and attaches via rollout.lora_modules | |
| # - Mixed precision defaults for H100: dtype=bf16, kv_cache_dtype=fp8_e5m2 | |
| # - Pins max_model_len, max_num_batched_tokens, sets swap_space | |
| # - Uses OmegaConf.open_dict to add keys safely (no "not in struct" errors) | |
| # - Prevents FSDP from trying to load LoRA .pt as a full model | |
| import os | |
| import ast | |
| import json | |
| import hydra | |
| import numpy as np | |
| import ray | |
| import torch | |
| from pathlib import Path | |
| from pprint import pprint | |
| # Quiet logs | |
| os.environ["NCCL_DEBUG"] = os.environ.get("NCCL_DEBUG", "WARN") | |
| os.environ["TOKENIZERS_PARALLELISM"] = os.environ.get("TOKENIZERS_PARALLELISM", "true") | |
| # vLLM CuMem allocator is incompatible with expandable_segments | |
| _bad = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") | |
| if "expandable_segments:True" in _bad: | |
| print(f"[fix] Removing incompatible PYTORCH_CUDA_ALLOC_CONF={_bad}") | |
| os.environ.pop("PYTORCH_CUDA_ALLOC_CONF", None) | |
| import pandas as pd | |
| from omegaconf import OmegaConf, open_dict | |
| from verl import DataProto | |
| from verl.protocol import pad_dataproto_to_divisor, unpad_dataproto | |
| from verl.single_controller.ray import RayClassWithInitArgs, RayResourcePool, RayWorkerGroup | |
| from verl.utils import hf_tokenizer | |
| from verl.utils.fs import copy_to_local | |
| from verl.utils.hdfs_io import makedirs | |
| from verl.utils.model import compute_position_id_with_mask | |
| from verl.workers.fsdp_workers import ActorRolloutRefWorker | |
| # ---------------- LoRA helpers ---------------- | |
| DEFAULT_TARGET_MODULES = [ | |
| "q_proj","k_proj","v_proj","o_proj", | |
| "up_proj","gate_proj","down_proj", | |
| ] | |
| def _infer_lengths_and_defaults(config): | |
| """Ensure rollout/data keys exist and set reasonable H100 defaults.""" | |
| # Ensure nested structs exist | |
| with open_dict(config): | |
| if "rollout" not in config: | |
| config["rollout"] = OmegaConf.create() | |
| if "data" not in config: | |
| config["data"] = OmegaConf.create() | |
| if "trainer" not in config: | |
| config["trainer"] = OmegaConf.create() | |
| if "ray_init" not in config: | |
| config["ray_init"] = OmegaConf.create() | |
| # Defaults that work on a single H100 | |
| with open_dict(config.rollout): | |
| # If user didn't set these, choose H100-friendly defaults | |
| config.rollout.setdefault("dtype", "bfloat16") # weights/activations | |
| config.rollout.setdefault("kv_cache_dtype", "fp8_e5m2") # KV cache precision | |
| config.rollout.setdefault("tensor_model_parallel_size", 1) | |
| config.rollout.setdefault("enable_chunked_prefill", True) | |
| config.rollout.setdefault("swap_space", 8) # GB of host swap for KV | |
| config.rollout.setdefault("gpu_memory_utilization", 0.62) # adjust 0.60~0.75 if needed | |
| # Pin lengths to avoid vLLM over-reserving KV cache | |
| pl = int(config.rollout.get("prompt_length", 1024)) | |
| rl = int(config.rollout.get("response_length", 128)) | |
| need = int(pl + rl) | |
| config.rollout.setdefault("max_model_len", need) | |
| config.rollout.setdefault("max_num_batched_tokens", need) | |
| # Users may pass +rollout.quantization={fp8|awq|gptq} to shrink weights further | |
| # We don't force it here. | |
| with open_dict(config.data): | |
| config.data.setdefault("batch_size", 1) | |
| config.data.setdefault("n_samples", 1) | |
| config.data.setdefault("prompt_key", "prompt") | |
| with open_dict(config.trainer): | |
| config.trainer.setdefault("n_gpus_per_node", 1) | |
| config.trainer.setdefault("nnodes", 1) | |
| with open_dict(config.ray_init): | |
| config.ray_init.setdefault("num_cpus", 4) | |
| def _infer_lora_rank_from_state(sd): | |
| for k, v in sd.items(): | |
| if k.endswith("lora_A.weight") and hasattr(v, "dim") and v.dim() == 2: | |
| return int(v.shape[0]) | |
| return None | |
| def _list_target_modules_from_state(sd): | |
| found = set() | |
| for k in sd.keys(): | |
| if "lora_A.weight" in k or "lora_B.weight" in k: | |
| if ".q_proj." in k: found.add("q_proj") | |
| if ".k_proj." in k: found.add("k_proj") | |
| if ".v_proj." in k: found.add("v_proj") | |
| if ".o_proj." in k: found.add("o_proj") | |
| if ".up_proj." in k: found.add("up_proj") | |
| if ".gate_proj." in k: found.add("gate_proj") | |
| if ".down_proj." in k: found.add("down_proj") | |
| return sorted(found) | |
| def _write_adapter_config(adapter_dir, r, alpha, target_modules, dropout=0.0): | |
| cfg = { | |
| "peft_type": "LORA", | |
| "auto_mapping": None, | |
| "base_model_name_or_path": "", | |
| "bias": "none", | |
| "inference_mode": True, | |
| "lora_alpha": int(alpha), | |
| "lora_dropout": float(dropout), | |
| "r": int(r), | |
| "target_modules": target_modules, | |
| "task_type": "CAUSAL_LM", | |
| } | |
| with open(os.path.join(adapter_dir, "adapter_config.json"), "w", encoding="utf-8") as f: | |
| json.dump(cfg, f, ensure_ascii=False, indent=2) | |
| def _wrap_lora_pt_as_peft(adapter_pt_path: str, out_dir: str, | |
| fallback_rank=32, fallback_alpha=16): | |
| os.makedirs(out_dir, exist_ok=True) | |
| print(f"[lora] Loading LoRA state from: {adapter_pt_path}") | |
| sd = torch.load(adapter_pt_path, map_location="cpu") | |
| if isinstance(sd, dict) and "state_dict" in sd: | |
| sd = sd["state_dict"] | |
| r = _infer_lora_rank_from_state(sd) or int(fallback_rank) | |
| tmods = _list_target_modules_from_state(sd) or DEFAULT_TARGET_MODULES | |
| print(f"[lora] inferred rank={r}, target_modules={tmods}") | |
| _write_adapter_config(out_dir, r=r, alpha=fallback_alpha, target_modules=tmods) | |
| torch.save(sd, os.path.join(out_dir, "adapter_model.bin")) | |
| return r, tmods | |
| def _maybe_attach_lora_adapter(config): | |
| """Attach LoRA adapter directory to vLLM rollout (runtime LoRA).""" | |
| # Accept either +lora.pt_path or model.load_param_path as a hint | |
| lora_pt = None | |
| if "lora" in config and getattr(config.lora, "pt_path", ""): | |
| lora_pt = config.lora.pt_path | |
| elif getattr(config.model, "load_param_path", ""): | |
| lora_pt = config.model.load_param_path | |
| if not lora_pt or not Path(lora_pt).is_file(): | |
| print("[lora] No LoRA .pt provided; running base model only.") | |
| return | |
| adapter_dir = os.path.join("/tmp", "lora_adapter_vllm") | |
| r, _ = _wrap_lora_pt_as_peft(lora_pt, adapter_dir, fallback_rank=32, fallback_alpha=16) | |
| # Ensure rollout keys exist and add LoRA knobs required by vLLM | |
| with open_dict(config): | |
| if "rollout" not in config: | |
| config["rollout"] = OmegaConf.create() | |
| with open_dict(config.rollout): | |
| config.rollout.setdefault("max_loras", 1) | |
| config.rollout.setdefault("max_lora_rank", int(r)) | |
| config.rollout["lora_modules"] = [{"path": adapter_dir, "scale": 1.0}] | |
| print(f"[lora] Attached PEFT adapter: {adapter_dir} (rank={r})") | |
| # CRITICAL: don't let FSDP try to load the LoRA .pt as a full state dict | |
| with open_dict(config.model): | |
| if getattr(config.model, "load_param", False): | |
| print("[lora] Disabling model.load_param to avoid FSDP load_state_dict mismatch.") | |
| config.model["load_param"] = False | |
| # ---------------- Hydra entry ---------------- | |
| def main(config): | |
| _infer_lengths_and_defaults(config) | |
| # Ray env for workers | |
| if not ray.is_initialized(): | |
| ray.init( | |
| runtime_env={"env_vars": { | |
| "TOKENIZERS_PARALLELISM": "true", | |
| "NCCL_DEBUG": "WARN", | |
| "PYTORCH_CUDA_ALLOC_CONF": "", # keep allocator happy for vLLM | |
| }}, | |
| num_cpus=config.ray_init.num_cpus, | |
| ) | |
| ray.get(main_task.remote(config)) | |
| def main_task(config): | |
| print("[worker] PYTORCH_CUDA_ALLOC_CONF =", os.environ.get("PYTORCH_CUDA_ALLOC_CONF")) | |
| pprint(OmegaConf.to_container(config, resolve=True)) | |
| OmegaConf.resolve(config) | |
| # Build LoRA adapter if provided | |
| _maybe_attach_lora_adapter(config) | |
| # Optionally pre-gen dataset schema if your repo provides it | |
| try: | |
| from prompts.infer_prompt import infer_dataset | |
| infer_dataset( | |
| model_name=config.model.path, | |
| data_path=os.path.dirname(os.path.dirname(config.data.path)), | |
| ) | |
| except Exception as e: | |
| print(f"[info] infer_dataset() skipped: {e}") | |
| # ---- Tokenizer from base model | |
| local_path = copy_to_local(config.model.path) | |
| trust_remote_code = getattr(config.model, "trust_remote_code", False) | |
| tokenizer = hf_tokenizer(local_path, trust_remote_code=trust_remote_code) | |
| tokenizer.padding_side = "left" | |
| if tokenizer.pad_token is None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| # ---- Sampling checks | |
| if float(config.rollout.temperature) == 0.0: | |
| assert int(config.data.n_samples) == 1, "When temperature=0, n_samples must be 1." | |
| assert int(config.data.n_samples) >= 1, "n_samples should always >= 1" | |
| # ---- Load dataset | |
| dataset = pd.read_parquet(config.data.path) | |
| prompt_key = getattr(config.data, "prompt_key", "prompt") | |
| if prompt_key not in dataset.columns: | |
| raise KeyError(f"Dataset missing column '{prompt_key}'") | |
| chat_lst = dataset[prompt_key].tolist() | |
| chat_lst = [chat.tolist() if hasattr(chat, "tolist") else chat for chat in chat_lst] | |
| # ---- Worker group (vLLM inside Rollout) | |
| ray_cls_with_init = RayClassWithInitArgs(cls=ray.remote(ActorRolloutRefWorker), config=config, role="rollout") | |
| resource_pool = RayResourcePool(process_on_nodes=[config.trainer.n_gpus_per_node] * config.trainer.nnodes) | |
| print("[debug] rollout.lora_modules =", config.rollout.get("lora_modules", None)) | |
| wg = RayWorkerGroup(resource_pool=resource_pool, ray_cls_with_init=ray_cls_with_init) | |
| wg.init_model() # vLLM spins up; adapter used if set in rollout.lora_modules | |
| total = len(dataset) | |
| bs = int(config.data.batch_size) | |
| num_batch = -(-total // bs) | |
| slots = [[] for _ in range(int(config.data.n_samples))] | |
| for b in range(num_batch): | |
| print(f"[{b+1}/{num_batch}] Start to process.") | |
| batch_chat = chat_lst[b * bs : (b + 1) * bs] | |
| inputs = tokenizer.apply_chat_template( | |
| batch_chat, | |
| add_generation_prompt=True, | |
| padding=True, | |
| truncation=True, | |
| max_length=int(config.rollout.prompt_length), | |
| return_tensors="pt", | |
| return_dict=True, | |
| tokenize=True, | |
| ) | |
| input_ids = inputs["input_ids"] | |
| attention_mask = inputs["attention_mask"] | |
| position_ids = compute_position_id_with_mask(attention_mask) | |
| batch_dict = {"input_ids": input_ids, "attention_mask": attention_mask, "position_ids": position_ids} | |
| data = DataProto.from_dict(batch_dict) | |
| data_padded, pad_size = pad_dataproto_to_divisor(data, wg.world_size) | |
| print(f"[{b+1}/{num_batch}] Start to generate.") | |
| for n in range(int(config.data.n_samples)): | |
| output_padded = wg.generate_sequences(data_padded) | |
| output = unpad_dataproto(output_padded, pad_size=pad_size) | |
| texts = [] | |
| for i in range(len(output)): | |
| item = output[i] | |
| pl = item.batch["prompts"].shape[-1] | |
| valid_len = item.batch["attention_mask"][pl:].sum() | |
| resp_ids = item.batch["responses"][:valid_len] | |
| s = tokenizer.decode(resp_ids, skip_special_tokens=True) | |
| print(f"[raw] Response {i}: {s!r}") | |
| ix = s.find("</think>") | |
| if ix != -1: | |
| s = s[ix + len("</think>") :].lstrip() | |
| print(f"Response {i}: {s!r}") | |
| try: | |
| texts.append(ast.literal_eval(s)) | |
| except Exception: | |
| texts.append(s) | |
| slots[n].extend(texts) | |
| outputs = np.array(slots, dtype=object) | |
| outputs = np.transpose(outputs, (1, 0)).tolist() | |
| dataset["response"] = outputs | |
| keep = ["file_id", "vt", "gt", "response"] | |
| cols = [c for c in keep if c in dataset.columns] | |
| if cols: | |
| dataset = dataset[cols] | |
| out_path = config.data.output_path | |
| makedirs(os.path.dirname(out_path), exist_ok=True) | |
| dataset.to_json(out_path, orient="records", lines=True, force_ascii=False) | |
| print(f"[done] Wrote: {out_path}") | |
| if __name__ == "__main__": | |
| main() | |