Buckets:
| """Generate answers for every prompt in prompts.jsonl. | |
| Model: Qwen/Qwen3.5-4B (Feb 2026; 4.66B params with its vision tower), loaded in | |
| 4-bit NF4 with bitsandbytes so it fits the 4 GB GPU this was run on (RTX 2050). | |
| Thinking mode is off, so the model answers as a chat assistant would by | |
| default. Decoding is greedy, so a rerun gives the same answers. | |
| Resumable: answers are appended to outputs.jsonl after each batch, and ids | |
| already there are skipped. | |
| python run_eval.py [--model Qwen/Qwen3.5-4B] [--batch 4] [--max-new 512] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import time | |
| from pathlib import Path | |
| import torch | |
| from huggingface_hub import model_info | |
| from transformers import AutoTokenizer, BitsAndBytesConfig | |
| HERE = Path(__file__).parent | |
| def load(model_id: str): | |
| quant = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", | |
| bnb_4bit_compute_dtype=torch.float16, | |
| bnb_4bit_use_double_quant=True) | |
| tok = AutoTokenizer.from_pretrained(model_id) | |
| tok.padding_side = "left" | |
| if tok.pad_token is None: | |
| tok.pad_token = tok.eos_token | |
| last = None | |
| # Qwen3.5 checkpoints are vision-language; try the text-only head first. | |
| from transformers import AutoModelForCausalLM, AutoModelForImageTextToText | |
| for cls in (AutoModelForCausalLM, AutoModelForImageTextToText): | |
| try: | |
| model = cls.from_pretrained(model_id, quantization_config=quant, device_map="cuda:0", | |
| dtype=torch.float16, low_cpu_mem_usage=True) | |
| model.eval() | |
| return tok, model, cls.__name__ | |
| except Exception as e: # noqa: BLE001 — fall through to the next class | |
| last = e | |
| raise RuntimeError(f"could not load {model_id}: {last}") | |
| def render(tok, messages): | |
| try: | |
| return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) | |
| except TypeError: | |
| return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", default="Qwen/Qwen3.5-4B") | |
| ap.add_argument("--batch", type=int, default=4) | |
| ap.add_argument("--max-new", type=int, default=512) | |
| ap.add_argument("--out", default="outputs.jsonl") | |
| ap.add_argument("--limit", type=int, default=0, help="only the first N prompts (smoke test)") | |
| args = ap.parse_args() | |
| prompts = [json.loads(l) for l in (HERE / "prompts.jsonl").read_text(encoding="utf-8").splitlines() if l] | |
| out_path = HERE / args.out | |
| done = set() | |
| if out_path.exists(): | |
| done = {json.loads(l)["id"] for l in out_path.read_text(encoding="utf-8").splitlines() if l} | |
| todo = [p for p in prompts if p["id"] not in done] | |
| if args.limit: | |
| todo = todo[:args.limit] | |
| print(f"{len(done)} done, {len(todo)} to generate", flush=True) | |
| if not todo: | |
| return | |
| revision = model_info(args.model).sha | |
| tok, model, cls_name = load(args.model) | |
| print(f"loaded {args.model}@{revision[:10]} via {cls_name}; " | |
| f"GPU mem {torch.cuda.memory_allocated() / 1e9:.2f} GB", flush=True) | |
| for p in todo: | |
| p["text"] = render(tok, p["messages"]) | |
| todo.sort(key=lambda p: len(p["text"])) # similar lengths batch with less padding | |
| t0, n = time.time(), 0 | |
| batch = args.batch | |
| i = 0 | |
| while i < len(todo): | |
| chunk = todo[i:i + batch] | |
| enc = tok([p["text"] for p in chunk], return_tensors="pt", padding=True).to("cuda:0") | |
| try: | |
| with torch.no_grad(): | |
| gen = model.generate(**enc, max_new_tokens=args.max_new, do_sample=False, | |
| temperature=None, top_p=None, top_k=None, | |
| pad_token_id=tok.pad_token_id) | |
| except torch.cuda.OutOfMemoryError: | |
| torch.cuda.empty_cache() | |
| if batch == 1: | |
| raise | |
| batch = max(1, batch // 2) | |
| print(f"OOM, batch -> {batch}", flush=True) | |
| continue | |
| new = gen[:, enc["input_ids"].shape[1]:] | |
| with out_path.open("a", encoding="utf-8") as fh: | |
| for p, ids in zip(chunk, new): | |
| ids = ids[ids != tok.pad_token_id] | |
| text = tok.decode(ids, skip_special_tokens=True).strip() | |
| fh.write(json.dumps({ | |
| "id": p["id"], "scenario": p["scenario"], "family": p["family"], | |
| "condition": p["condition"], "response": text, | |
| "new_tokens": int(len(ids)), "truncated": bool(len(ids) >= args.max_new), | |
| "model": args.model, "revision": revision, "quantization": "bnb-nf4-4bit", | |
| "decoding": f"greedy, max_new_tokens={args.max_new}, thinking off", | |
| }, ensure_ascii=False) + "\n") | |
| i += len(chunk) | |
| n += len(chunk) | |
| rate = (time.time() - t0) / n | |
| print(f"{i}/{len(todo)} {rate:.1f}s/prompt eta {rate * (len(todo) - i) / 60:.0f} min", | |
| flush=True) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.2 kB
- Xet hash:
- 79903f71dcaa9b475ca7f8c84d7d5c7860e826fd10d1e4ea32ce4a440d62c6be
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.