| import json |
| import os |
| import time |
| from pathlib import Path |
|
|
| import torch |
| from peft import PeftModel |
| from transformers import AutoModelForImageTextToText, AutoProcessor |
| from video_inputs import build_video_item, prepare |
|
|
| CACHE = Path("/home/s56fasla/MOTIONX/cache") |
| RUNS = Path("/home/s56fasla/MOTIONX/runs") |
| BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen3-VL-8B-Instruct") |
| RUN_PREFIX = os.environ.get("RUN_PREFIX", "skivlm_Qwen3-VL-8B") |
| TAG = os.environ.get("TAG", "q3") |
| ARM = os.environ.get("ARM", "ski") |
| LIMIT = int(os.environ.get("LIMIT", 0)) |
| OUT = Path("/home/s56fasla/MOTIONX/evals") |
| QUESTION = "Describe what the person is doing in this video." |
| VIDEO_FPS = 4.0 |
| MAX_FRAMES = 32 |
| MAX_PIXELS = 200704 |
| MAX_NEW_TOKENS = 128 |
|
|
| OUT.mkdir(parents=True, exist_ok=True) |
| device = "cuda" |
| run_dir = None if ARM == "base" else RUNS / f"{RUN_PREFIX}_{ARM}" |
|
|
| proc_src = run_dir / "processor" if run_dir else BASE_MODEL |
| processor = AutoProcessor.from_pretrained(proc_src) |
| model = AutoModelForImageTextToText.from_pretrained( |
| BASE_MODEL, dtype=torch.bfloat16, device_map={"": 0}) |
| if len(processor.tokenizer) != model.get_input_embeddings().weight.shape[0]: |
| model.resize_token_embeddings(len(processor.tokenizer)) |
| if run_dir: |
| model = PeftModel.from_pretrained(model, run_dir / "lora") |
| model.eval() |
| print("arm", ARM, "lora", bool(run_dir), flush=True) |
|
|
| items = json.load(open(CACHE / "manifest_val_ok.json")) |
| if LIMIT: |
| items = items[:LIMIT] |
| print("val clips", len(items), flush=True) |
|
|
| preds = [] |
| t0 = time.time() |
| for k, m in enumerate(items): |
| messages = [{"role": "user", "content": [ |
| build_video_item(m["video"], VIDEO_FPS, MAX_FRAMES, MAX_PIXELS), |
| {"type": "text", "text": QUESTION}]}] |
| try: |
| text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) |
| videos, vkw = prepare(messages, m["video"]) |
| enc = processor(text=[text], videos=videos, return_tensors="pt", **vkw).to(device) |
| with torch.no_grad(): |
| out = model.generate(**enc, max_new_tokens=MAX_NEW_TOKENS, do_sample=False) |
| gen = processor.tokenizer.decode( |
| out[0][enc["input_ids"].shape[1]:], skip_special_tokens=True).strip() |
| except Exception as e: |
| print("[skip]", m["stem"], type(e).__name__, flush=True) |
| continue |
| preds.append({"stem": m["stem"], "subset": m["subset"], "ref": m["text"], "pred": gen}) |
| if k % 25 == 0: |
| el = time.time() - t0 |
| print(f"{k}/{len(items)} {el:.0f}s eta {el/max(1,k)*(len(items)-k)/60:.0f}min", flush=True) |
| json.dump(preds, open(OUT / f"preds_{TAG}_{ARM}.json", "w"), indent=1) |
|
|
| json.dump(preds, open(OUT / f"preds_{TAG}_{ARM}.json", "w"), indent=1) |
| print("done", len(preds), "->", OUT / f"preds_{TAG}_{ARM}.json", flush=True) |
|
|
