Buckets:

AsianPlayer
/

TriDrive

Files

xet

AsianPlayer/TriDrive / code /tools /vlm_handson /score_vlm_handson.py

AsianPlayer

16 days ago

download

raw

13.4 kB

#!/usr/bin/env python3
"""tools/vlm_handson/score_vlm_handson.py — local VLM hands-on scoring of the validation frames (pre-registration docs/decisions.md
2026-09-16T23:26Z "vision-based hands-on gold").  NEW files only; frames never leave the machine (local HF models; no API).
Per frame the model is asked EXACTLY
    "Look at the driver. Is at least one of the driver's hands touching the steering wheel? Answer with yes or no."
and P(yes) = softmax over the FIRST answer-token logits restricted to the single-token case variants of 'yes' vs 'no' (variants summed).
Candidates (FIXED order, first pass is used, all evaluated ones reported):  qwen3vl2b = Qwen/Qwen3-VL-2B-Instruct (cached),
  qwen25vl3b = Qwen/Qwen2.5-VL-3B-Instruct (cached weights; processor / tokenizer files fetched from the hub if absent — text metadata only),
  qwen3vl4b = Qwen/Qwen3-VL-4B-Instruct (download).
Input variants (both reported):
  crop     the causal driver box expanded x1.6 horizontally toward the wheel side (+0.5 w toward the driver's outer side, +0.1 w inward) and
           x1.3 vertically (+0.25 h down, +0.05 h up), clipped, longer side resized to <= 896 px;
  full896  the whole frame resized to 896 px wide;
  overlay  the owner's review jpg (640 px wide, red box + text) as-is — reports whether the overlay changes the scores (fallback check).
GPU gate before loading: free VRAM >= need + 4 GB and MemAvailable >= 28 GB (else wait, 60 s per loop); <= 4 CPU threads; bf16; sdpa; batch 8.
Output: results/vlm_handson/scores_<cand>.json (per-frame P(yes) per variant, timing, VRAM, token bookkeeping) + scores_<cand>.log.
    python tools/vlm_handson/score_vlm_handson.py --cand qwen3vl2b [--variants crop,full896,overlay] [--batch 8] [--limit N]
"""
from __future__ import annotations
import os
os.environ.setdefault("OMP_NUM_THREADS", "4"); os.environ.setdefault("MKL_NUM_THREADS", "4"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
import argparse, json, subprocess, sys, time
from pathlib import Path
import numpy as np

CWM3 = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(CWM3 / "common"))
from runmeta import write_run_json  # noqa: E402

OUT = CWM3 / "results/vlm_handson"
VAL = OUT / "validation_frames.json"
PROMPT = "Look at the driver. Is at least one of the driver's hands touching the steering wheel? Answer with yes or no."
CANDS = {  # id -> (hub id, model class, est. need GB at batch 8 bf16)
    "qwen3vl2b": ("Qwen/Qwen3-VL-2B-Instruct", "Qwen3VLForConditionalGeneration", 6.5),
    "qwen25vl3b": ("Qwen/Qwen2.5-VL-3B-Instruct", "Qwen2_5_VLForConditionalGeneration", 9.5),
    "qwen3vl4b": ("Qwen/Qwen3-VL-4B-Instruct", "Qwen3VLForConditionalGeneration", 10.5),
}
YES = ["yes", "Yes", "YES", " yes", " Yes", " YES"]
NO = ["no", "No", "NO", " no", " No", " NO"]
EXP_W_OUT, EXP_W_IN, EXP_H_DOWN, EXP_H_UP = 0.5, 0.1, 0.25, 0.05       # x1.6 horizontally toward the wheel side, x1.3 vertically
LONG_MAX, FULL_W = 896, 896
MIN_PIXELS, MAX_PIXELS = 64 * 32 * 32, 896 * 896                       # processor limits: our inputs (<= 896 px long side) are never downsized
GATE_MARGIN_GB, GATE_RAM_GB, GATE_SLEEP_S = 4.0, 28.0, 60


def nvsmi(q):
    return subprocess.run(["nvidia-smi", f"--query-gpu={q}", "--format=csv,noheader,nounits"], capture_output=True, text=True).stdout.strip()


def mem_available_gb():
    for ln in open("/proc/meminfo"):
        if ln.startswith("MemAvailable"):
            return int(ln.split()[1]) / 1e6
    return 0.0


def gpu_gate(need_gb, log):
    while True:
        free = float(nvsmi("memory.free")) / 1024; ram = mem_available_gb()
        if free >= need_gb + GATE_MARGIN_GB and ram >= GATE_RAM_GB:
            log(f"gate OK: free VRAM {free:.1f} GB >= {need_gb}+{GATE_MARGIN_GB}; MemAvailable {ram:.1f} GB >= {GATE_RAM_GB}")
            return free, ram
        log(f"gate WAIT: free VRAM {free:.1f} GB (need {need_gb + GATE_MARGIN_GB}), MemAvailable {ram:.1f} GB (need {GATE_RAM_GB}); sleeping {GATE_SLEEP_S}s")
        time.sleep(GATE_SLEEP_S)


def crop_box(box, side, W, H):
    x0, y0, x1, y1 = box; w, h = x1 - x0, y1 - y0
    if side == "right":
        nx0, nx1 = x0 - EXP_W_IN * w, x1 + EXP_W_OUT * w
    else:
        nx0, nx1 = x0 - EXP_W_OUT * w, x1 + EXP_W_IN * w
    ny0, ny1 = y0 - EXP_H_UP * h, y1 + EXP_H_DOWN * h
    return [int(max(0, round(nx0))), int(max(0, round(ny0))), int(min(W, round(nx1))), int(min(H, round(ny1)))]


def make_input(variant, f, it):
    from PIL import Image
    if variant == "overlay":
        return Image.open(CWM3 / f["file_overlay"]).convert("RGB"), None
    im = Image.open(CWM3 / f["file_clean"]).convert("RGB")
    W, H = im.size
    if variant == "full896":
        return im.resize((FULL_W, int(round(H * FULL_W / W))), Image.BICUBIC), None
    if variant == "crop":
        if f["box_native"] is None:
            im2 = im.resize((FULL_W, int(round(H * FULL_W / W))), Image.BICUBIC); return im2, "full_fallback"
        cb = crop_box(f["box_native"], it["driver_side"], W, H)
        im2 = im.crop(cb); cw, ch = im2.size
        s = min(1.0, LONG_MAX / max(cw, ch))
        if s < 1.0:
            im2 = im2.resize((max(32, int(round(cw * s))), max(32, int(round(ch * s)))), Image.BICUBIC)
        return im2, cb
    raise ValueError(variant)


class Scorer:
    def __init__(self, cand, batch, log):
        import torch, transformers
        hub, cls, _ = CANDS[cand]
        self.hub, self.batch, self.log = hub, batch, log
        torch.set_num_threads(4)
        self.proc = transformers.AutoProcessor.from_pretrained(hub)
        self.proc.tokenizer.padding_side = "left"
        size = {"longest_edge": MAX_PIXELS, "shortest_edge": MIN_PIXELS}          # pixel-count limits of the Qwen2-VL family fast image processor (SizeDict in tf 5.x)
        self.proc.image_processor.size = size
        self.model = getattr(transformers, cls).from_pretrained(hub, dtype=torch.bfloat16, attn_implementation="sdpa", device_map="cuda").eval()
        self.text = self.proc.apply_chat_template([{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": PROMPT}]}], tokenize=False, add_generation_prompt=True)
        tok = self.proc.tokenizer
        self.yes_ids = sorted({tok.encode(v, add_special_tokens=False)[0] for v in YES if len(tok.encode(v, add_special_tokens=False)) == 1})
        self.no_ids = sorted({tok.encode(v, add_special_tokens=False)[0] for v in NO if len(tok.encode(v, add_special_tokens=False)) == 1})
        assert self.yes_ids and self.no_ids and not set(self.yes_ids) & set(self.no_ids)
        self.meta = {"model_id": hub, "class": cls, "dtype": "bfloat16", "attn": "sdpa", "batch": batch, "padding_side": "left", "prompt": PROMPT,
                     "chat_template_text": self.text, "yes_tokens": {tok.decode([i]): i for i in self.yes_ids}, "no_tokens": {tok.decode([i]): i for i in self.no_ids},
                     "image_processor_size": size, "transformers": transformers.__version__, "torch": torch.__version__,
                     "n_params_M": round(sum(p.numel() for p in self.model.parameters()) / 1e6, 1)}
        self.use_keep = True
        log(f"loaded {hub}: {self.meta['n_params_M']} M params; yes ids {self.meta['yes_tokens']} no ids {self.meta['no_tokens']}")

    @staticmethod
    def _logits_last(out, n):
        lg = out.logits
        return lg[:, -1, :] if lg.dim() == 3 else lg

    def score(self, ims):
        import torch
        inputs = self.proc(text=[self.text] * len(ims), images=ims, return_tensors="pt", padding=True).to("cuda")
        n_tok = int(inputs["attention_mask"].sum(1).max()); n_img_tok = int(inputs["image_grid_thw"].prod(-1).sum() // 4) if "image_grid_thw" in inputs else -1
        with torch.no_grad():
            if self.use_keep:
                try:
                    out = self.model(**inputs, logits_to_keep=1)
                except TypeError:
                    self.use_keep = False; out = self.model(**inputs)
            else:
                out = self.model(**inputs)
            lg = self._logits_last(out, len(ims)).float()
            full = torch.softmax(lg, -1)
            top = lg.argmax(-1)
            sub = torch.cat([lg[:, self.yes_ids], lg[:, self.no_ids]], 1)
            p = torch.softmax(sub, -1)
            p_yes = p[:, :len(self.yes_ids)].sum(1)
            mass = full[:, self.yes_ids].sum(1) + full[:, self.no_ids].sum(1)
        return p_yes.cpu().numpy(), mass.cpu().numpy(), top.cpu().numpy(), n_tok, n_img_tok


def main(argv=None) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--cand", required=True, choices=list(CANDS))
    ap.add_argument("--variants", default="crop,full896,overlay")
    ap.add_argument("--batch", type=int, default=8)
    ap.add_argument("--limit", type=int, default=0, help="debug: only the first N frames")
    a = ap.parse_args(argv)
    OUT.mkdir(parents=True, exist_ok=True)
    logf = open(OUT / f"scores_{a.cand}.log", "a")

    def log(s):
        s = f"[{time.strftime('%H:%M:%S')}] {s}"; print(s, flush=True); logf.write(s + "\n"); logf.flush()

    t_start = time.time()
    val = json.load(open(VAL))
    frames = [(it, f) for it in val["items"] for f in it["frames"]]
    if a.limit:
        frames = frames[:a.limit]
    log(f"cand {a.cand}: {len(frames)} frames, variants {a.variants}, batch {a.batch}")
    free0, ram0 = gpu_gate(CANDS[a.cand][2], log)
    import torch
    torch.cuda.reset_peak_memory_stats()
    t_load = time.time(); sc = Scorer(a.cand, a.batch, log); t_load = time.time() - t_load
    vram_after_load = torch.cuda.memory_allocated() / 1e9
    log(f"load {t_load:.1f}s; allocated after load {vram_after_load:.2f} GB")
    res = {"cand": a.cand, "built_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "meta": sc.meta, "n_frames": len(frames), "gate": {"free_vram_gb": round(free0, 2), "mem_available_gb": round(ram0, 2)},
           "load_s": round(t_load, 1), "vram_after_load_gb": round(vram_after_load, 2), "variants": {}, "frames": [{"item": it["id"], "k": f["k"], "t": f["t"], "dt": f["dt"], "label": f["label"], "regime": it["regime"]} for it, f in frames]}
    for variant in a.variants.split(","):
        torch.cuda.reset_peak_memory_stats(); torch.cuda.synchronize()
        P, M, TOP, crops = np.zeros(len(frames)), np.zeros(len(frames)), np.zeros(len(frames), np.int64), []
        n_tok_max = n_img_max = 0; t_model = 0.0; t_pre = 0.0; tv0 = time.time()
        for b0 in range(0, len(frames), a.batch):
            tp = time.time(); ims = []
            for it, f in frames[b0:b0 + a.batch]:
                im, cb = make_input(variant, f, it); ims.append(im); crops.append(cb)
            t_pre += time.time() - tp
            torch.cuda.synchronize(); tm = time.time()
            p, m, top, n_tok, n_img = sc.score(ims)
            torch.cuda.synchronize(); t_model += time.time() - tm
            P[b0:b0 + len(ims)] = p; M[b0:b0 + len(ims)] = m; TOP[b0:b0 + len(ims)] = top
            n_tok_max, n_img_max = max(n_tok_max, n_tok), max(n_img_max, n_img)
            if (b0 // a.batch) % 20 == 0:
                log(f"  {variant} {b0 + len(ims)}/{len(frames)}  seq<= {n_tok} img_tok<= {n_img}  {time.time() - tv0:.1f}s")
        wall = time.time() - tv0
        peak = torch.cuda.max_memory_allocated() / 1e9; peak_res = torch.cuda.max_memory_reserved() / 1e9
        nv_used = nvsmi("memory.used")
        ids = set(sc.yes_ids) | set(sc.no_ids)
        top_in = float(np.mean([int(t) in ids for t in TOP]))
        y = np.array([f["label"] for _, f in frames])
        acc = float(((P >= 0.5).astype(int) == y).mean())
        res["variants"][variant] = {"p_yes": [round(float(v), 5) for v in P], "yes_no_mass": [round(float(v), 4) for v in M], "top1_in_yes_no": top_in,
                                    "top1_token_ids_hist": {int(k): int(v) for k, v in zip(*np.unique(TOP, return_counts=True))},
                                    "crop_boxes": crops if variant == "crop" else None, "n_full_fallback": int(sum(c == "full_fallback" for c in crops)) if variant == "crop" else 0,
                                    "seq_len_max": n_tok_max, "image_tokens_max": n_img_max, "wall_s": round(wall, 2), "model_s": round(t_model, 2), "preproc_s": round(t_pre, 2),
                                    "frames_per_s_wall": round(len(frames) / wall, 2), "frames_per_s_model": round(len(frames) / t_model, 2),
                                    "peak_vram_alloc_gb": round(peak, 2), "peak_vram_reserved_gb": round(peak_res, 2), "nvsmi_used_total_mib_after": nv_used, "quick_acc_at_0.5": round(acc, 4)}
        log(f"{variant}: {len(frames)} frames in {wall:.1f}s wall ({len(frames)/wall:.2f} f/s; model-only {len(frames)/t_model:.2f} f/s); peak VRAM alloc {peak:.2f} GB reserved {peak_res:.2f} GB; "
            f"top-1 in yes/no {top_in:.3f}; yes+no mass median {np.median(M):.3f}; quick acc@0.5 {acc:.3f}")
    out = OUT / f"scores_{a.cand}.json"
    out.write_text(json.dumps(res, indent=0))
    write_run_json(OUT, f"score_{a.cand}", [Path(__file__)], [VAL], seed=None, t_start=t_start, extra={"cand": a.cand, "variants": {k: {kk: vv for kk, vv in v.items() if kk not in ("p_yes", "yes_no_mass", "crop_boxes", "top1_token_ids_hist")} for k, v in res["variants"].items()}}, input_listing=False)
    log(f"SCORES_DONE {out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())

Xet Storage Details

Size:: 13.4 kB
Xet hash:: 9e06449c78d6741b166ed7fb07ab8db49849072c96cbaba8c894e45834b167ae

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	#!/usr/bin/env python3
	"""tools/vlm_handson/score_vlm_handson.py — local VLM hands-on scoring of the validation frames (pre-registration docs/decisions.md
	2026-09-16T23:26Z "vision-based hands-on gold"). NEW files only; frames never leave the machine (local HF models; no API).

	Per frame the model is asked EXACTLY
	"Look at the driver. Is at least one of the driver's hands touching the steering wheel? Answer with yes or no."
	and P(yes) = softmax over the FIRST answer-token logits restricted to the single-token case variants of 'yes' vs 'no' (variants summed).
	Candidates (FIXED order, first pass is used, all evaluated ones reported): qwen3vl2b = Qwen/Qwen3-VL-2B-Instruct (cached),
	qwen25vl3b = Qwen/Qwen2.5-VL-3B-Instruct (cached weights; processor / tokenizer files fetched from the hub if absent — text metadata only),
	qwen3vl4b = Qwen/Qwen3-VL-4B-Instruct (download).
	Input variants (both reported):
	crop the causal driver box expanded x1.6 horizontally toward the wheel side (+0.5 w toward the driver's outer side, +0.1 w inward) and
	x1.3 vertically (+0.25 h down, +0.05 h up), clipped, longer side resized to <= 896 px;
	full896 the whole frame resized to 896 px wide;
	overlay the owner's review jpg (640 px wide, red box + text) as-is — reports whether the overlay changes the scores (fallback check).
	GPU gate before loading: free VRAM >= need + 4 GB and MemAvailable >= 28 GB (else wait, 60 s per loop); <= 4 CPU threads; bf16; sdpa; batch 8.
	Output: results/vlm_handson/scores_<cand>.json (per-frame P(yes) per variant, timing, VRAM, token bookkeeping) + scores_<cand>.log.
	python tools/vlm_handson/score_vlm_handson.py --cand qwen3vl2b [--variants crop,full896,overlay] [--batch 8] [--limit N]
	"""
	from __future__ import annotations
	import os
	os.environ.setdefault("OMP_NUM_THREADS", "4"); os.environ.setdefault("MKL_NUM_THREADS", "4"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
	import argparse, json, subprocess, sys, time
	from pathlib import Path
	import numpy as np

	CWM3 = Path(__file__).resolve().parents[2]
	sys.path.insert(0, str(CWM3 / "common"))
	from runmeta import write_run_json # noqa: E402

	OUT = CWM3 / "results/vlm_handson"
	VAL = OUT / "validation_frames.json"
	PROMPT = "Look at the driver. Is at least one of the driver's hands touching the steering wheel? Answer with yes or no."
	CANDS = { # id -> (hub id, model class, est. need GB at batch 8 bf16)
	"qwen3vl2b": ("Qwen/Qwen3-VL-2B-Instruct", "Qwen3VLForConditionalGeneration", 6.5),
	"qwen25vl3b": ("Qwen/Qwen2.5-VL-3B-Instruct", "Qwen2_5_VLForConditionalGeneration", 9.5),
	"qwen3vl4b": ("Qwen/Qwen3-VL-4B-Instruct", "Qwen3VLForConditionalGeneration", 10.5),
	}
	YES = ["yes", "Yes", "YES", " yes", " Yes", " YES"]
	NO = ["no", "No", "NO", " no", " No", " NO"]
	EXP_W_OUT, EXP_W_IN, EXP_H_DOWN, EXP_H_UP = 0.5, 0.1, 0.25, 0.05 # x1.6 horizontally toward the wheel side, x1.3 vertically
	LONG_MAX, FULL_W = 896, 896
	MIN_PIXELS, MAX_PIXELS = 64 * 32 * 32, 896 * 896 # processor limits: our inputs (<= 896 px long side) are never downsized
	GATE_MARGIN_GB, GATE_RAM_GB, GATE_SLEEP_S = 4.0, 28.0, 60


	def nvsmi(q):
	return subprocess.run(["nvidia-smi", f"--query-gpu={q}", "--format=csv,noheader,nounits"], capture_output=True, text=True).stdout.strip()


	def mem_available_gb():
	for ln in open("/proc/meminfo"):
	if ln.startswith("MemAvailable"):
	return int(ln.split()[1]) / 1e6
	return 0.0


	def gpu_gate(need_gb, log):
	while True:
	free = float(nvsmi("memory.free")) / 1024; ram = mem_available_gb()
	if free >= need_gb + GATE_MARGIN_GB and ram >= GATE_RAM_GB:
	log(f"gate OK: free VRAM {free:.1f} GB >= {need_gb}+{GATE_MARGIN_GB}; MemAvailable {ram:.1f} GB >= {GATE_RAM_GB}")
	return free, ram
	log(f"gate WAIT: free VRAM {free:.1f} GB (need {need_gb + GATE_MARGIN_GB}), MemAvailable {ram:.1f} GB (need {GATE_RAM_GB}); sleeping {GATE_SLEEP_S}s")
	time.sleep(GATE_SLEEP_S)


	def crop_box(box, side, W, H):
	x0, y0, x1, y1 = box; w, h = x1 - x0, y1 - y0
	if side == "right":
	nx0, nx1 = x0 - EXP_W_IN * w, x1 + EXP_W_OUT * w
	else:
	nx0, nx1 = x0 - EXP_W_OUT * w, x1 + EXP_W_IN * w
	ny0, ny1 = y0 - EXP_H_UP * h, y1 + EXP_H_DOWN * h
	return [int(max(0, round(nx0))), int(max(0, round(ny0))), int(min(W, round(nx1))), int(min(H, round(ny1)))]


	def make_input(variant, f, it):
	from PIL import Image
	if variant == "overlay":
	return Image.open(CWM3 / f["file_overlay"]).convert("RGB"), None
	im = Image.open(CWM3 / f["file_clean"]).convert("RGB")
	W, H = im.size
	if variant == "full896":
	return im.resize((FULL_W, int(round(H * FULL_W / W))), Image.BICUBIC), None
	if variant == "crop":
	if f["box_native"] is None:
	im2 = im.resize((FULL_W, int(round(H * FULL_W / W))), Image.BICUBIC); return im2, "full_fallback"
	cb = crop_box(f["box_native"], it["driver_side"], W, H)
	im2 = im.crop(cb); cw, ch = im2.size
	s = min(1.0, LONG_MAX / max(cw, ch))
	if s < 1.0:
	im2 = im2.resize((max(32, int(round(cw * s))), max(32, int(round(ch * s)))), Image.BICUBIC)
	return im2, cb
	raise ValueError(variant)


	class Scorer:
	def __init__(self, cand, batch, log):
	import torch, transformers
	hub, cls, _ = CANDS[cand]
	self.hub, self.batch, self.log = hub, batch, log
	torch.set_num_threads(4)
	self.proc = transformers.AutoProcessor.from_pretrained(hub)
	self.proc.tokenizer.padding_side = "left"
	size = {"longest_edge": MAX_PIXELS, "shortest_edge": MIN_PIXELS} # pixel-count limits of the Qwen2-VL family fast image processor (SizeDict in tf 5.x)
	self.proc.image_processor.size = size
	self.model = getattr(transformers, cls).from_pretrained(hub, dtype=torch.bfloat16, attn_implementation="sdpa", device_map="cuda").eval()
	self.text = self.proc.apply_chat_template([{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": PROMPT}]}], tokenize=False, add_generation_prompt=True)
	tok = self.proc.tokenizer
	self.yes_ids = sorted({tok.encode(v, add_special_tokens=False)[0] for v in YES if len(tok.encode(v, add_special_tokens=False)) == 1})
	self.no_ids = sorted({tok.encode(v, add_special_tokens=False)[0] for v in NO if len(tok.encode(v, add_special_tokens=False)) == 1})
	assert self.yes_ids and self.no_ids and not set(self.yes_ids) & set(self.no_ids)
	self.meta = {"model_id": hub, "class": cls, "dtype": "bfloat16", "attn": "sdpa", "batch": batch, "padding_side": "left", "prompt": PROMPT,
	"chat_template_text": self.text, "yes_tokens": {tok.decode([i]): i for i in self.yes_ids}, "no_tokens": {tok.decode([i]): i for i in self.no_ids},
	"image_processor_size": size, "transformers": transformers.__version__, "torch": torch.__version__,
	"n_params_M": round(sum(p.numel() for p in self.model.parameters()) / 1e6, 1)}
	self.use_keep = True
	log(f"loaded {hub}: {self.meta['n_params_M']} M params; yes ids {self.meta['yes_tokens']} no ids {self.meta['no_tokens']}")

	@staticmethod
	def _logits_last(out, n):
	lg = out.logits
	return lg[:, -1, :] if lg.dim() == 3 else lg

	def score(self, ims):
	import torch
	inputs = self.proc(text=[self.text] * len(ims), images=ims, return_tensors="pt", padding=True).to("cuda")
	n_tok = int(inputs["attention_mask"].sum(1).max()); n_img_tok = int(inputs["image_grid_thw"].prod(-1).sum() // 4) if "image_grid_thw" in inputs else -1
	with torch.no_grad():
	if self.use_keep:
	try:
	out = self.model(**inputs, logits_to_keep=1)
	except TypeError:
	self.use_keep = False; out = self.model(**inputs)
	else:
	out = self.model(**inputs)
	lg = self._logits_last(out, len(ims)).float()
	full = torch.softmax(lg, -1)
	top = lg.argmax(-1)
	sub = torch.cat([lg[:, self.yes_ids], lg[:, self.no_ids]], 1)
	p = torch.softmax(sub, -1)
	p_yes = p[:, :len(self.yes_ids)].sum(1)
	mass = full[:, self.yes_ids].sum(1) + full[:, self.no_ids].sum(1)
	return p_yes.cpu().numpy(), mass.cpu().numpy(), top.cpu().numpy(), n_tok, n_img_tok


	def main(argv=None) -> int:
	ap = argparse.ArgumentParser()
	ap.add_argument("--cand", required=True, choices=list(CANDS))
	ap.add_argument("--variants", default="crop,full896,overlay")
	ap.add_argument("--batch", type=int, default=8)
	ap.add_argument("--limit", type=int, default=0, help="debug: only the first N frames")
	a = ap.parse_args(argv)
	OUT.mkdir(parents=True, exist_ok=True)
	logf = open(OUT / f"scores_{a.cand}.log", "a")

	def log(s):
	s = f"[{time.strftime('%H:%M:%S')}] {s}"; print(s, flush=True); logf.write(s + "\n"); logf.flush()

	t_start = time.time()
	val = json.load(open(VAL))
	frames = [(it, f) for it in val["items"] for f in it["frames"]]
	if a.limit:
	frames = frames[:a.limit]
	log(f"cand {a.cand}: {len(frames)} frames, variants {a.variants}, batch {a.batch}")
	free0, ram0 = gpu_gate(CANDS[a.cand][2], log)
	import torch
	torch.cuda.reset_peak_memory_stats()
	t_load = time.time(); sc = Scorer(a.cand, a.batch, log); t_load = time.time() - t_load
	vram_after_load = torch.cuda.memory_allocated() / 1e9
	log(f"load {t_load:.1f}s; allocated after load {vram_after_load:.2f} GB")
	res = {"cand": a.cand, "built_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "meta": sc.meta, "n_frames": len(frames), "gate": {"free_vram_gb": round(free0, 2), "mem_available_gb": round(ram0, 2)},
	"load_s": round(t_load, 1), "vram_after_load_gb": round(vram_after_load, 2), "variants": {}, "frames": [{"item": it["id"], "k": f["k"], "t": f["t"], "dt": f["dt"], "label": f["label"], "regime": it["regime"]} for it, f in frames]}
	for variant in a.variants.split(","):
	torch.cuda.reset_peak_memory_stats(); torch.cuda.synchronize()
	P, M, TOP, crops = np.zeros(len(frames)), np.zeros(len(frames)), np.zeros(len(frames), np.int64), []
	n_tok_max = n_img_max = 0; t_model = 0.0; t_pre = 0.0; tv0 = time.time()
	for b0 in range(0, len(frames), a.batch):
	tp = time.time(); ims = []
	for it, f in frames[b0:b0 + a.batch]:
	im, cb = make_input(variant, f, it); ims.append(im); crops.append(cb)
	t_pre += time.time() - tp
	torch.cuda.synchronize(); tm = time.time()
	p, m, top, n_tok, n_img = sc.score(ims)
	torch.cuda.synchronize(); t_model += time.time() - tm
	P[b0:b0 + len(ims)] = p; M[b0:b0 + len(ims)] = m; TOP[b0:b0 + len(ims)] = top
	n_tok_max, n_img_max = max(n_tok_max, n_tok), max(n_img_max, n_img)
	if (b0 // a.batch) % 20 == 0:
	log(f" {variant} {b0 + len(ims)}/{len(frames)} seq<= {n_tok} img_tok<= {n_img} {time.time() - tv0:.1f}s")
	wall = time.time() - tv0
	peak = torch.cuda.max_memory_allocated() / 1e9; peak_res = torch.cuda.max_memory_reserved() / 1e9
	nv_used = nvsmi("memory.used")
	ids = set(sc.yes_ids) \| set(sc.no_ids)
	top_in = float(np.mean([int(t) in ids for t in TOP]))
	y = np.array([f["label"] for _, f in frames])
	acc = float(((P >= 0.5).astype(int) == y).mean())
	res["variants"][variant] = {"p_yes": [round(float(v), 5) for v in P], "yes_no_mass": [round(float(v), 4) for v in M], "top1_in_yes_no": top_in,
	"top1_token_ids_hist": {int(k): int(v) for k, v in zip(*np.unique(TOP, return_counts=True))},
	"crop_boxes": crops if variant == "crop" else None, "n_full_fallback": int(sum(c == "full_fallback" for c in crops)) if variant == "crop" else 0,
	"seq_len_max": n_tok_max, "image_tokens_max": n_img_max, "wall_s": round(wall, 2), "model_s": round(t_model, 2), "preproc_s": round(t_pre, 2),
	"frames_per_s_wall": round(len(frames) / wall, 2), "frames_per_s_model": round(len(frames) / t_model, 2),
	"peak_vram_alloc_gb": round(peak, 2), "peak_vram_reserved_gb": round(peak_res, 2), "nvsmi_used_total_mib_after": nv_used, "quick_acc_at_0.5": round(acc, 4)}
	log(f"{variant}: {len(frames)} frames in {wall:.1f}s wall ({len(frames)/wall:.2f} f/s; model-only {len(frames)/t_model:.2f} f/s); peak VRAM alloc {peak:.2f} GB reserved {peak_res:.2f} GB; "
	f"top-1 in yes/no {top_in:.3f}; yes+no mass median {np.median(M):.3f}; quick acc@0.5 {acc:.3f}")
	out = OUT / f"scores_{a.cand}.json"
	out.write_text(json.dumps(res, indent=0))
	write_run_json(OUT, f"score_{a.cand}", [Path(__file__)], [VAL], seed=None, t_start=t_start, extra={"cand": a.cand, "variants": {k: {kk: vv for kk, vv in v.items() if kk not in ("p_yes", "yes_no_mass", "crop_boxes", "top1_token_ids_hist")} for k, v in res["variants"].items()}}, input_listing=False)
	log(f"SCORES_DONE {out}")
	return 0


	if __name__ == "__main__":
	sys.exit(main())