Ahmed5's picture
download
raw
5.2 kB
"""Generate answers for every prompt in prompts.jsonl.
Model: Qwen/Qwen3.5-4B (Feb 2026; 4.66B params with its vision tower), loaded in
4-bit NF4 with bitsandbytes so it fits the 4 GB GPU this was run on (RTX 2050).
Thinking mode is off, so the model answers as a chat assistant would by
default. Decoding is greedy, so a rerun gives the same answers.
Resumable: answers are appended to outputs.jsonl after each batch, and ids
already there are skipped.
python run_eval.py [--model Qwen/Qwen3.5-4B] [--batch 4] [--max-new 512]
"""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
import torch
from huggingface_hub import model_info
from transformers import AutoTokenizer, BitsAndBytesConfig
HERE = Path(__file__).parent
def load(model_id: str):
quant = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.float16,
bnb_4bit_use_double_quant=True)
tok = AutoTokenizer.from_pretrained(model_id)
tok.padding_side = "left"
if tok.pad_token is None:
tok.pad_token = tok.eos_token
last = None
# Qwen3.5 checkpoints are vision-language; try the text-only head first.
from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
for cls in (AutoModelForCausalLM, AutoModelForImageTextToText):
try:
model = cls.from_pretrained(model_id, quantization_config=quant, device_map="cuda:0",
dtype=torch.float16, low_cpu_mem_usage=True)
model.eval()
return tok, model, cls.__name__
except Exception as e: # noqa: BLE001 — fall through to the next class
last = e
raise RuntimeError(f"could not load {model_id}: {last}")
def render(tok, messages):
try:
return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True,
enable_thinking=False)
except TypeError:
return tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--model", default="Qwen/Qwen3.5-4B")
ap.add_argument("--batch", type=int, default=4)
ap.add_argument("--max-new", type=int, default=512)
ap.add_argument("--out", default="outputs.jsonl")
ap.add_argument("--limit", type=int, default=0, help="only the first N prompts (smoke test)")
args = ap.parse_args()
prompts = [json.loads(l) for l in (HERE / "prompts.jsonl").read_text(encoding="utf-8").splitlines() if l]
out_path = HERE / args.out
done = set()
if out_path.exists():
done = {json.loads(l)["id"] for l in out_path.read_text(encoding="utf-8").splitlines() if l}
todo = [p for p in prompts if p["id"] not in done]
if args.limit:
todo = todo[:args.limit]
print(f"{len(done)} done, {len(todo)} to generate", flush=True)
if not todo:
return
revision = model_info(args.model).sha
tok, model, cls_name = load(args.model)
print(f"loaded {args.model}@{revision[:10]} via {cls_name}; "
f"GPU mem {torch.cuda.memory_allocated() / 1e9:.2f} GB", flush=True)
for p in todo:
p["text"] = render(tok, p["messages"])
todo.sort(key=lambda p: len(p["text"])) # similar lengths batch with less padding
t0, n = time.time(), 0
batch = args.batch
i = 0
while i < len(todo):
chunk = todo[i:i + batch]
enc = tok([p["text"] for p in chunk], return_tensors="pt", padding=True).to("cuda:0")
try:
with torch.no_grad():
gen = model.generate(**enc, max_new_tokens=args.max_new, do_sample=False,
temperature=None, top_p=None, top_k=None,
pad_token_id=tok.pad_token_id)
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
if batch == 1:
raise
batch = max(1, batch // 2)
print(f"OOM, batch -> {batch}", flush=True)
continue
new = gen[:, enc["input_ids"].shape[1]:]
with out_path.open("a", encoding="utf-8") as fh:
for p, ids in zip(chunk, new):
ids = ids[ids != tok.pad_token_id]
text = tok.decode(ids, skip_special_tokens=True).strip()
fh.write(json.dumps({
"id": p["id"], "scenario": p["scenario"], "family": p["family"],
"condition": p["condition"], "response": text,
"new_tokens": int(len(ids)), "truncated": bool(len(ids) >= args.max_new),
"model": args.model, "revision": revision, "quantization": "bnb-nf4-4bit",
"decoding": f"greedy, max_new_tokens={args.max_new}, thinking off",
}, ensure_ascii=False) + "\n")
i += len(chunk)
n += len(chunk)
rate = (time.time() - t0) / n
print(f"{i}/{len(todo)} {rate:.1f}s/prompt eta {rate * (len(todo) - i) / 60:.0f} min",
flush=True)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
5.2 kB
·
Xet hash:
79903f71dcaa9b475ca7f8c84d7d5c7860e826fd10d1e4ea32ce4a440d62c6be

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.