"""Prefix-only review of official external QA; decisions never use HFD outputs.""" import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import subprocess import sys import textwrap import numpy as np from PIL import Image, ImageDraw, ImageFont from common import ROOT, sha256, write_json from external_visual import DEFAULT sys.path[:0] = [str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] from psi.data.seed.video_overlay import decode_video from audit_hfd_live_video_windows import parse, ordered_prefetch PUBLIC = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen') def media_inventory(root): rows = json.loads((root/'candidates.json').read_text()) train = ROOT.parents[1]/'videogen/hfd_visual_dataset/videos/manifest.jsonl' hashes, parents = set(), set() with train.open() as handle: for line in handle: row=json.loads(line) if row['split']=='train': hashes.add(row['media_sha256']);parents.add(row.get('parent_video_id','')) def probe(key): path=root/'media/OVO-Bench'/key data=json.loads(subprocess.check_output(['ffprobe','-v','error','-show_entries', 'format=duration:stream=width,height,codec_type','-of','json',str(path)])) visual=next(s for s in data['streams'] if s['codec_type']=='video') digest=sha256(path);parent='youtube:'+path.stem return key,dict(video_path=str(path),video_sha256=digest, duration_seconds=float(data['format']['duration']),width=visual['width'],height=visual['height'], exact_train_overlap=digest in hashes,parent_train_overlap=parent in parents, parent_id_known=path.parent.name!='AutoEvalMetaData') with ThreadPoolExecutor(max_workers=8) as pool: media=dict(pool.map(probe,sorted({r['video_relative'] for r in rows}))) eligible=[];rejected=[] for row in rows: r={**row,**media[row['video_relative']]} reason=None if r['exact_train_overlap'] or r['parent_train_overlap']:reason='known_training_visual_overlap' elif r['question_time']+1.2>r['duration_seconds']:reason='no_live_video_after_question' elif min(r['width'],r['height'])<224:reason='source_resolution_below_camera_resolution' if reason:rejected.append(dict(id=r['id'],reason=reason)) else:eligible.append(r) write_json(root/'media_inventory.json',dict(rows=eligible,rejected=rejected, split_audit=dict(exact_sha_and_known_parent_ids=True, unknown_parent_videos=sum(not r['parent_id_known'] for r in media.values()), limitation='Re-encoded/cropped duplicates require perceptual review; unknown aliases are not declared disjoint.'))) print('MEDIA',len(eligible),'QA',len(media),'videos',Counter(r['reason'] for r in rejected),flush=True) def frames(row): t=row['question_time'] # Keep entire history plus denser recent evidence; only PTS <= question time. times=np.unique(np.r_[np.linspace(0,t,24),np.linspace(max(0,t-3),t,9)]) rgb,pts=decode_video(row['video_path'],times,end_seconds=row['duration_seconds']) return [Image.fromarray(f) for f in rgb],pts.tolist() def review(a): import torch from transformers import AutoProcessor,Qwen3VLForConditionalGeneration torch.set_num_threads(4) rows=json.loads((a.root/'media_inventory.json').read_text())['rows'] contract=dict(model=str(a.model),model_config_sha256=sha256(a.model/'config.json'), code_sha256=sha256(__file__),sampled_prefix=True,max_frames=33,side=224, independent_of_hfd_predictions=True,same_judge_blind_then_verify=True, machine_screen_not_human_ground_truth=True) cp=a.root/'review_contract.json' if cp.exists() and json.loads(cp.read_text())!=contract:raise ValueError('Review contract changed') write_json(cp,contract) pending=[r for r in rows if not (a.root/'reviews'/(r['id']+'.json')).exists()] if a.limit:pending=pending[:a.limit] if not pending:return processor=AutoProcessor.from_pretrained(a.model,local_files_only=True) processor.tokenizer.padding_side='left' model=Qwen3VLForConditionalGeneration.from_pretrained(a.model,dtype=torch.bfloat16, attn_implementation='flash_attention_2',local_files_only=True).cuda().eval().requires_grad_(False) def generate(images,prompts): texts=[processor.apply_chat_template([dict(role='user',content= [dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])], tokenize=False,add_generation_prompt=True) for ims,prompt in zip(images,prompts)] batch=processor(text=texts,images=images,return_tensors='pt',padding=True,do_resize=False).to('cuda') with torch.inference_mode():out=model.generate(**batch,max_new_tokens=128,do_sample=False) return [parse(s) for s in processor.batch_decode(out[:,batch['input_ids'].shape[1]:],skip_special_tokens=True)] def prepare(batch): return [(r,*frames(r)) for r in batch] batches=[pending[i:i+a.batch_size] for i in range(0,len(pending),a.batch_size)] for batch in ordered_prefetch(batches,prepare): images=[b[1] for b in batch] prompts=[('These frames show only the observed video prefix, in timestamp order. ' 'Answer the question from seen visual evidence. Earlier frames may be remembered; no future events, ' 'unheard speech, player biography, or external facts. Respect NOW versus earlier. ' 'OCR is allowed only when clearly legible at this resolution. If uncertain, say so. ' 'Return JSON {"grounded":bool,"answer":"brief answer","audio_required":bool,"game_or_animation":bool}.\n'+ json.dumps(dict(question=r['question'],question_time=r['question_time'],frame_times=pts))) for r,_,pts in batch] blind=generate(images,prompts) checks=[('Check the official reference against ONLY these observed frames. It must be supported at ' 'the question time, not by later video or unheard audio. Does the independently produced answer ' 'convey the same factual answer? Accept natural paraphrases; reject guesses or missing evidence. ' 'Return JSON {"supported":bool,"equivalent":bool,"reason":"short"}.\n'+ json.dumps(dict(question=r['question'],reference=r['answer'],independent=b, question_time=r['question_time'],frame_times=pts))) for (r,_,pts),b in zip(batch,blind)] verified=generate(images,checks) for (r,_,pts),b,v in zip(batch,blind,verified): passed=(b.get('grounded') is True and b.get('audio_required') is False and b.get('game_or_animation') is False and bool(b.get('answer')) and v.get('supported') is True and v.get('equivalent') is True) write_json(a.root/'reviews'/(r['id']+'.json'),dict(id=r['id'],accepted=passed, video_sha256=r['video_sha256'],question=r['question'],reference=r['answer'], question_time=r['question_time'],observed_prefix_only=True,frame_pts=pts, blind=b,verification=v,contract_sha256=sha256(cp),machine_review=True)) print('REVIEW',r['id'],passed,b,v,flush=True) def contacts(a): rows=json.loads((a.root/'media_inventory.json').read_text())['rows'] font=ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',17) dest=a.root/'contacts';dest.mkdir(exist_ok=True) if a.limit:rows=rows[:a.limit] for r in rows: path=dest/(r['id']+'.jpg') if path.exists():continue images,pts=frames(r) picks=np.unique(np.r_[np.linspace(0,len(images)-1,9).astype(int),len(images)-1]) board=Image.new('RGB',(1008,1250),'white');draw=ImageDraw.Draw(board);line=8 for text in [r['id']+' '+r['original']['task']+' '+r['video_relative'], f"Question time {r['question_time']:.2f}s / video {r['duration_seconds']:.2f}s", 'Q: '+r['question'],'A: '+r['answer']]: for part in textwrap.wrap(text,105):draw.text((8,line),part,fill='black',font=font);line+=22 for k,j in enumerate(picks): x=(k%3)*336;y=210+(k//3)*344 board.paste(images[j].resize((336,312)),(x,y));draw.text((x+4,y+313),f'PTS {pts[j]:.2f}',fill='black',font=font) board.save(path,quality=92) print('CONTACTS',dest,flush=True) if __name__=='__main__': p=argparse.ArgumentParser(description=__doc__) p.add_argument('command',choices=['probe','review','contacts']) p.add_argument('--root',type=Path,default=DEFAULT) p.add_argument('--model',type=Path,default=PUBLIC/'Qwen3-VL-32B-Instruct') p.add_argument('--batch-size',type=int,default=2) p.add_argument('--limit',type=int,default=0) a=p.parse_args() if a.command=='probe':media_inventory(a.root) elif a.command=='review':review(a) else:contacts(a)