Download benchmark/review_external.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 9.05 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/review_external.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/review_external.py
-
curl -L -o review_external.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/review_external.py
9.05 kB
| """Prefix-only review of official external QA; decisions never use HFD outputs.""" | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ThreadPoolExecutor | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import textwrap | |
| import numpy as np | |
| from PIL import Image, ImageDraw, ImageFont | |
| from common import ROOT, sha256, write_json | |
| from external_visual import DEFAULT | |
| sys.path[:0] = [str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] | |
| from psi.data.seed.video_overlay import decode_video | |
| from audit_hfd_live_video_windows import parse, ordered_prefetch | |
| PUBLIC = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen') | |
| def media_inventory(root): | |
| rows = json.loads((root/'candidates.json').read_text()) | |
| train = ROOT.parents[1]/'videogen/hfd_visual_dataset/videos/manifest.jsonl' | |
| hashes, parents = set(), set() | |
| with train.open() as handle: | |
| for line in handle: | |
| row=json.loads(line) | |
| if row['split']=='train': | |
| hashes.add(row['media_sha256']);parents.add(row.get('parent_video_id','')) | |
| def probe(key): | |
| path=root/'media/OVO-Bench'/key | |
| data=json.loads(subprocess.check_output(['ffprobe','-v','error','-show_entries', | |
| 'format=duration:stream=width,height,codec_type','-of','json',str(path)])) | |
| visual=next(s for s in data['streams'] if s['codec_type']=='video') | |
| digest=sha256(path);parent='youtube:'+path.stem | |
| return key,dict(video_path=str(path),video_sha256=digest, | |
| duration_seconds=float(data['format']['duration']),width=visual['width'],height=visual['height'], | |
| exact_train_overlap=digest in hashes,parent_train_overlap=parent in parents, | |
| parent_id_known=path.parent.name!='AutoEvalMetaData') | |
| with ThreadPoolExecutor(max_workers=8) as pool: | |
| media=dict(pool.map(probe,sorted({r['video_relative'] for r in rows}))) | |
| eligible=[];rejected=[] | |
| for row in rows: | |
| r={**row,**media[row['video_relative']]} | |
| reason=None | |
| if r['exact_train_overlap'] or r['parent_train_overlap']:reason='known_training_visual_overlap' | |
| elif r['question_time']+1.2>r['duration_seconds']:reason='no_live_video_after_question' | |
| elif min(r['width'],r['height'])<224:reason='source_resolution_below_camera_resolution' | |
| if reason:rejected.append(dict(id=r['id'],reason=reason)) | |
| else:eligible.append(r) | |
| write_json(root/'media_inventory.json',dict(rows=eligible,rejected=rejected, | |
| split_audit=dict(exact_sha_and_known_parent_ids=True, | |
| unknown_parent_videos=sum(not r['parent_id_known'] for r in media.values()), | |
| limitation='Re-encoded/cropped duplicates require perceptual review; unknown aliases are not declared disjoint.'))) | |
| print('MEDIA',len(eligible),'QA',len(media),'videos',Counter(r['reason'] for r in rejected),flush=True) | |
| def frames(row): | |
| t=row['question_time'] | |
| # Keep entire history plus denser recent evidence; only PTS <= question time. | |
| times=np.unique(np.r_[np.linspace(0,t,24),np.linspace(max(0,t-3),t,9)]) | |
| rgb,pts=decode_video(row['video_path'],times,end_seconds=row['duration_seconds']) | |
| return [Image.fromarray(f) for f in rgb],pts.tolist() | |
| def review(a): | |
| import torch | |
| from transformers import AutoProcessor,Qwen3VLForConditionalGeneration | |
| torch.set_num_threads(4) | |
| rows=json.loads((a.root/'media_inventory.json').read_text())['rows'] | |
| contract=dict(model=str(a.model),model_config_sha256=sha256(a.model/'config.json'), | |
| code_sha256=sha256(__file__),sampled_prefix=True,max_frames=33,side=224, | |
| independent_of_hfd_predictions=True,same_judge_blind_then_verify=True, | |
| machine_screen_not_human_ground_truth=True) | |
| cp=a.root/'review_contract.json' | |
| if cp.exists() and json.loads(cp.read_text())!=contract:raise ValueError('Review contract changed') | |
| write_json(cp,contract) | |
| pending=[r for r in rows if not (a.root/'reviews'/(r['id']+'.json')).exists()] | |
| if a.limit:pending=pending[:a.limit] | |
| if not pending:return | |
| processor=AutoProcessor.from_pretrained(a.model,local_files_only=True) | |
| processor.tokenizer.padding_side='left' | |
| model=Qwen3VLForConditionalGeneration.from_pretrained(a.model,dtype=torch.bfloat16, | |
| attn_implementation='flash_attention_2',local_files_only=True).cuda().eval().requires_grad_(False) | |
| def generate(images,prompts): | |
| texts=[processor.apply_chat_template([dict(role='user',content= | |
| [dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])], | |
| tokenize=False,add_generation_prompt=True) for ims,prompt in zip(images,prompts)] | |
| batch=processor(text=texts,images=images,return_tensors='pt',padding=True,do_resize=False).to('cuda') | |
| with torch.inference_mode():out=model.generate(**batch,max_new_tokens=128,do_sample=False) | |
| return [parse(s) for s in processor.batch_decode(out[:,batch['input_ids'].shape[1]:],skip_special_tokens=True)] | |
| def prepare(batch): | |
| return [(r,*frames(r)) for r in batch] | |
| batches=[pending[i:i+a.batch_size] for i in range(0,len(pending),a.batch_size)] | |
| for batch in ordered_prefetch(batches,prepare): | |
| images=[b[1] for b in batch] | |
| prompts=[('These frames show only the observed video prefix, in timestamp order. ' | |
| 'Answer the question from seen visual evidence. Earlier frames may be remembered; no future events, ' | |
| 'unheard speech, player biography, or external facts. Respect NOW versus earlier. ' | |
| 'OCR is allowed only when clearly legible at this resolution. If uncertain, say so. ' | |
| 'Return JSON {"grounded":bool,"answer":"brief answer","audio_required":bool,"game_or_animation":bool}.\n'+ | |
| json.dumps(dict(question=r['question'],question_time=r['question_time'],frame_times=pts))) for r,_,pts in batch] | |
| blind=generate(images,prompts) | |
| checks=[('Check the official reference against ONLY these observed frames. It must be supported at ' | |
| 'the question time, not by later video or unheard audio. Does the independently produced answer ' | |
| 'convey the same factual answer? Accept natural paraphrases; reject guesses or missing evidence. ' | |
| 'Return JSON {"supported":bool,"equivalent":bool,"reason":"short"}.\n'+ | |
| json.dumps(dict(question=r['question'],reference=r['answer'],independent=b, | |
| question_time=r['question_time'],frame_times=pts))) for (r,_,pts),b in zip(batch,blind)] | |
| verified=generate(images,checks) | |
| for (r,_,pts),b,v in zip(batch,blind,verified): | |
| passed=(b.get('grounded') is True and b.get('audio_required') is False | |
| and b.get('game_or_animation') is False and bool(b.get('answer')) | |
| and v.get('supported') is True and v.get('equivalent') is True) | |
| write_json(a.root/'reviews'/(r['id']+'.json'),dict(id=r['id'],accepted=passed, | |
| video_sha256=r['video_sha256'],question=r['question'],reference=r['answer'], | |
| question_time=r['question_time'],observed_prefix_only=True,frame_pts=pts, | |
| blind=b,verification=v,contract_sha256=sha256(cp),machine_review=True)) | |
| print('REVIEW',r['id'],passed,b,v,flush=True) | |
| def contacts(a): | |
| rows=json.loads((a.root/'media_inventory.json').read_text())['rows'] | |
| font=ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',17) | |
| dest=a.root/'contacts';dest.mkdir(exist_ok=True) | |
| if a.limit:rows=rows[:a.limit] | |
| for r in rows: | |
| path=dest/(r['id']+'.jpg') | |
| if path.exists():continue | |
| images,pts=frames(r) | |
| picks=np.unique(np.r_[np.linspace(0,len(images)-1,9).astype(int),len(images)-1]) | |
| board=Image.new('RGB',(1008,1250),'white');draw=ImageDraw.Draw(board);line=8 | |
| for text in [r['id']+' '+r['original']['task']+' '+r['video_relative'], | |
| f"Question time {r['question_time']:.2f}s / video {r['duration_seconds']:.2f}s", | |
| 'Q: '+r['question'],'A: '+r['answer']]: | |
| for part in textwrap.wrap(text,105):draw.text((8,line),part,fill='black',font=font);line+=22 | |
| for k,j in enumerate(picks): | |
| x=(k%3)*336;y=210+(k//3)*344 | |
| board.paste(images[j].resize((336,312)),(x,y));draw.text((x+4,y+313),f'PTS {pts[j]:.2f}',fill='black',font=font) | |
| board.save(path,quality=92) | |
| print('CONTACTS',dest,flush=True) | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('command',choices=['probe','review','contacts']) | |
| p.add_argument('--root',type=Path,default=DEFAULT) | |
| p.add_argument('--model',type=Path,default=PUBLIC/'Qwen3-VL-32B-Instruct') | |
| p.add_argument('--batch-size',type=int,default=2) | |
| p.add_argument('--limit',type=int,default=0) | |
| a=p.parse_args() | |
| if a.command=='probe':media_inventory(a.root) | |
| elif a.command=='review':review(a) | |
| else:contacts(a) | |