code-HFD / benchmark /review_external.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
9.05 kB
"""Prefix-only review of official external QA; decisions never use HFD outputs."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import subprocess
import sys
import textwrap
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from common import ROOT, sha256, write_json
from external_visual import DEFAULT
sys.path[:0] = [str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')]
from psi.data.seed.video_overlay import decode_video
from audit_hfd_live_video_windows import parse, ordered_prefetch
PUBLIC = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen')
def media_inventory(root):
rows = json.loads((root/'candidates.json').read_text())
train = ROOT.parents[1]/'videogen/hfd_visual_dataset/videos/manifest.jsonl'
hashes, parents = set(), set()
with train.open() as handle:
for line in handle:
row=json.loads(line)
if row['split']=='train':
hashes.add(row['media_sha256']);parents.add(row.get('parent_video_id',''))
def probe(key):
path=root/'media/OVO-Bench'/key
data=json.loads(subprocess.check_output(['ffprobe','-v','error','-show_entries',
'format=duration:stream=width,height,codec_type','-of','json',str(path)]))
visual=next(s for s in data['streams'] if s['codec_type']=='video')
digest=sha256(path);parent='youtube:'+path.stem
return key,dict(video_path=str(path),video_sha256=digest,
duration_seconds=float(data['format']['duration']),width=visual['width'],height=visual['height'],
exact_train_overlap=digest in hashes,parent_train_overlap=parent in parents,
parent_id_known=path.parent.name!='AutoEvalMetaData')
with ThreadPoolExecutor(max_workers=8) as pool:
media=dict(pool.map(probe,sorted({r['video_relative'] for r in rows})))
eligible=[];rejected=[]
for row in rows:
r={**row,**media[row['video_relative']]}
reason=None
if r['exact_train_overlap'] or r['parent_train_overlap']:reason='known_training_visual_overlap'
elif r['question_time']+1.2>r['duration_seconds']:reason='no_live_video_after_question'
elif min(r['width'],r['height'])<224:reason='source_resolution_below_camera_resolution'
if reason:rejected.append(dict(id=r['id'],reason=reason))
else:eligible.append(r)
write_json(root/'media_inventory.json',dict(rows=eligible,rejected=rejected,
split_audit=dict(exact_sha_and_known_parent_ids=True,
unknown_parent_videos=sum(not r['parent_id_known'] for r in media.values()),
limitation='Re-encoded/cropped duplicates require perceptual review; unknown aliases are not declared disjoint.')))
print('MEDIA',len(eligible),'QA',len(media),'videos',Counter(r['reason'] for r in rejected),flush=True)
def frames(row):
t=row['question_time']
# Keep entire history plus denser recent evidence; only PTS <= question time.
times=np.unique(np.r_[np.linspace(0,t,24),np.linspace(max(0,t-3),t,9)])
rgb,pts=decode_video(row['video_path'],times,end_seconds=row['duration_seconds'])
return [Image.fromarray(f) for f in rgb],pts.tolist()
def review(a):
import torch
from transformers import AutoProcessor,Qwen3VLForConditionalGeneration
torch.set_num_threads(4)
rows=json.loads((a.root/'media_inventory.json').read_text())['rows']
contract=dict(model=str(a.model),model_config_sha256=sha256(a.model/'config.json'),
code_sha256=sha256(__file__),sampled_prefix=True,max_frames=33,side=224,
independent_of_hfd_predictions=True,same_judge_blind_then_verify=True,
machine_screen_not_human_ground_truth=True)
cp=a.root/'review_contract.json'
if cp.exists() and json.loads(cp.read_text())!=contract:raise ValueError('Review contract changed')
write_json(cp,contract)
pending=[r for r in rows if not (a.root/'reviews'/(r['id']+'.json')).exists()]
if a.limit:pending=pending[:a.limit]
if not pending:return
processor=AutoProcessor.from_pretrained(a.model,local_files_only=True)
processor.tokenizer.padding_side='left'
model=Qwen3VLForConditionalGeneration.from_pretrained(a.model,dtype=torch.bfloat16,
attn_implementation='flash_attention_2',local_files_only=True).cuda().eval().requires_grad_(False)
def generate(images,prompts):
texts=[processor.apply_chat_template([dict(role='user',content=
[dict(type='image') for _ in ims]+[dict(type='text',text=prompt)])],
tokenize=False,add_generation_prompt=True) for ims,prompt in zip(images,prompts)]
batch=processor(text=texts,images=images,return_tensors='pt',padding=True,do_resize=False).to('cuda')
with torch.inference_mode():out=model.generate(**batch,max_new_tokens=128,do_sample=False)
return [parse(s) for s in processor.batch_decode(out[:,batch['input_ids'].shape[1]:],skip_special_tokens=True)]
def prepare(batch):
return [(r,*frames(r)) for r in batch]
batches=[pending[i:i+a.batch_size] for i in range(0,len(pending),a.batch_size)]
for batch in ordered_prefetch(batches,prepare):
images=[b[1] for b in batch]
prompts=[('These frames show only the observed video prefix, in timestamp order. '
'Answer the question from seen visual evidence. Earlier frames may be remembered; no future events, '
'unheard speech, player biography, or external facts. Respect NOW versus earlier. '
'OCR is allowed only when clearly legible at this resolution. If uncertain, say so. '
'Return JSON {"grounded":bool,"answer":"brief answer","audio_required":bool,"game_or_animation":bool}.\n'+
json.dumps(dict(question=r['question'],question_time=r['question_time'],frame_times=pts))) for r,_,pts in batch]
blind=generate(images,prompts)
checks=[('Check the official reference against ONLY these observed frames. It must be supported at '
'the question time, not by later video or unheard audio. Does the independently produced answer '
'convey the same factual answer? Accept natural paraphrases; reject guesses or missing evidence. '
'Return JSON {"supported":bool,"equivalent":bool,"reason":"short"}.\n'+
json.dumps(dict(question=r['question'],reference=r['answer'],independent=b,
question_time=r['question_time'],frame_times=pts))) for (r,_,pts),b in zip(batch,blind)]
verified=generate(images,checks)
for (r,_,pts),b,v in zip(batch,blind,verified):
passed=(b.get('grounded') is True and b.get('audio_required') is False
and b.get('game_or_animation') is False and bool(b.get('answer'))
and v.get('supported') is True and v.get('equivalent') is True)
write_json(a.root/'reviews'/(r['id']+'.json'),dict(id=r['id'],accepted=passed,
video_sha256=r['video_sha256'],question=r['question'],reference=r['answer'],
question_time=r['question_time'],observed_prefix_only=True,frame_pts=pts,
blind=b,verification=v,contract_sha256=sha256(cp),machine_review=True))
print('REVIEW',r['id'],passed,b,v,flush=True)
def contacts(a):
rows=json.loads((a.root/'media_inventory.json').read_text())['rows']
font=ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',17)
dest=a.root/'contacts';dest.mkdir(exist_ok=True)
if a.limit:rows=rows[:a.limit]
for r in rows:
path=dest/(r['id']+'.jpg')
if path.exists():continue
images,pts=frames(r)
picks=np.unique(np.r_[np.linspace(0,len(images)-1,9).astype(int),len(images)-1])
board=Image.new('RGB',(1008,1250),'white');draw=ImageDraw.Draw(board);line=8
for text in [r['id']+' '+r['original']['task']+' '+r['video_relative'],
f"Question time {r['question_time']:.2f}s / video {r['duration_seconds']:.2f}s",
'Q: '+r['question'],'A: '+r['answer']]:
for part in textwrap.wrap(text,105):draw.text((8,line),part,fill='black',font=font);line+=22
for k,j in enumerate(picks):
x=(k%3)*336;y=210+(k//3)*344
board.paste(images[j].resize((336,312)),(x,y));draw.text((x+4,y+313),f'PTS {pts[j]:.2f}',fill='black',font=font)
board.save(path,quality=92)
print('CONTACTS',dest,flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('command',choices=['probe','review','contacts'])
p.add_argument('--root',type=Path,default=DEFAULT)
p.add_argument('--model',type=Path,default=PUBLIC/'Qwen3-VL-32B-Instruct')
p.add_argument('--batch-size',type=int,default=2)
p.add_argument('--limit',type=int,default=0)
a=p.parse_args()
if a.command=='probe':media_inventory(a.root)
elif a.command=='review':review(a)
else:contacts(a)