File size: 2,786 Bytes
723126c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import json
import os
import time
from pathlib import Path
import torch
from peft import PeftModel
from transformers import AutoModelForImageTextToText, AutoProcessor
from video_inputs import build_video_item, prepare
CACHE = Path("/home/s56fasla/MOTIONX/cache")
RUNS = Path("/home/s56fasla/MOTIONX/runs")
BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen3-VL-8B-Instruct")
RUN_PREFIX = os.environ.get("RUN_PREFIX", "skivlm_Qwen3-VL-8B")
TAG = os.environ.get("TAG", "q3")
ARM = os.environ.get("ARM", "ski")
LIMIT = int(os.environ.get("LIMIT", 0))
OUT = Path("/home/s56fasla/MOTIONX/evals")
QUESTION = "Describe what the person is doing in this video."
VIDEO_FPS = 4.0
MAX_FRAMES = 32
MAX_PIXELS = 200704
MAX_NEW_TOKENS = 128
OUT.mkdir(parents=True, exist_ok=True)
device = "cuda"
run_dir = None if ARM == "base" else RUNS / f"{RUN_PREFIX}_{ARM}"
proc_src = run_dir / "processor" if run_dir else BASE_MODEL
processor = AutoProcessor.from_pretrained(proc_src)
model = AutoModelForImageTextToText.from_pretrained(
BASE_MODEL, dtype=torch.bfloat16, device_map={"": 0})
if len(processor.tokenizer) != model.get_input_embeddings().weight.shape[0]:
model.resize_token_embeddings(len(processor.tokenizer))
if run_dir:
model = PeftModel.from_pretrained(model, run_dir / "lora")
model.eval()
print("arm", ARM, "lora", bool(run_dir), flush=True)
items = json.load(open(CACHE / "manifest_val_ok.json"))
if LIMIT:
items = items[:LIMIT]
print("val clips", len(items), flush=True)
preds = []
t0 = time.time()
for k, m in enumerate(items):
messages = [{"role": "user", "content": [
build_video_item(m["video"], VIDEO_FPS, MAX_FRAMES, MAX_PIXELS),
{"type": "text", "text": QUESTION}]}]
try:
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
videos, vkw = prepare(messages, m["video"])
enc = processor(text=[text], videos=videos, return_tensors="pt", **vkw).to(device)
with torch.no_grad():
out = model.generate(**enc, max_new_tokens=MAX_NEW_TOKENS, do_sample=False)
gen = processor.tokenizer.decode(
out[0][enc["input_ids"].shape[1]:], skip_special_tokens=True).strip()
except Exception as e:
print("[skip]", m["stem"], type(e).__name__, flush=True)
continue
preds.append({"stem": m["stem"], "subset": m["subset"], "ref": m["text"], "pred": gen})
if k % 25 == 0:
el = time.time() - t0
print(f"{k}/{len(items)} {el:.0f}s eta {el/max(1,k)*(len(items)-k)/60:.0f}min", flush=True)
json.dump(preds, open(OUT / f"preds_{TAG}_{ARM}.json", "w"), indent=1)
json.dump(preds, open(OUT / f"preds_{TAG}_{ARM}.json", "w"), indent=1)
print("done", len(preds), "->", OUT / f"preds_{TAG}_{ARM}.json", flush=True)
|