File size: 6,027 Bytes
fa82a9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | #!/usr/bin/env python3
"""Export authentic cached evidence for illustrative cases, with hash checks.
The input is a small JSON case plan. Extraction runs in the official decoding
environment on the data host. Images are never regenerated by an image model.
"""
import argparse
import hashlib
import html
import json
from pathlib import Path
import shutil
import sys
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'src'))
from vbench_prompts_compile import annotation_jobs as J
from cache_backend_outputs import sha
from cache_matrix_transforms import frames_for_backend,frame_hash
def main(argv=None):
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--plan',required=True);p.add_argument('--out',required=True)
p.add_argument('--vbench-root',default='/root/wenbiao_zhao/VBench')
args=p.parse_args(argv);sys.path.insert(0,args.vbench_root)
import numpy as np
from PIL import Image
from vbench.utils import load_video
cases=json.loads(Path(args.plan).read_text())['cases']
out=Path(args.out);out.mkdir(parents=True,exist_ok=True);result=[]
for case in cases:
images=[];index=case.get('frame_index',8)
if case.get('annotation_image'):
source=Path(case['annotation_image']);path=out/(case['id']+'-annotation.png')
shutil.copyfile(source,path);images.append(path.name)
else:
base=case['cache'];source=Path(base['video_path'])
if sha(source)!=base['video_sha256']:raise ValueError('video changed: '+str(source))
if case['task']=='scene':
frames=load_video(str(source),num_frames=16,return_tensor=False,width=384,height=384)
elif case['task']=='action':
from vbench.human_action import Compose,Resize,CenterCrop,ClipToTensor,Normalize
transform=Compose([Resize(256,interpolation='bilinear'),CenterCrop(size=(224,224)),ClipToTensor(),
Normalize(mean=[.485,.456,.406],std=[.229,.224,.225])])
tensor=load_video(str(source),transform,num_frames=16).unsqueeze(0)
if frame_hash(tensor.numpy())!=base['model_input_sha256']:raise ValueError('UMT input differs')
values=tensor[0].permute(1,2,3,0).numpy()
frames=np.rint(np.clip(values*np.asarray([.229,.224,.225])+np.asarray([.485,.456,.406]),0,1)*255).astype('uint8')
else:frames=frames_for_backend(source,load_video)
if case['task']!='action' and [frame_hash(f) for f in frames]!=base['frame_sha256']:
raise ValueError('sampled frames changed: '+str(source))
frame=frames[index];path=out/(case['id']+'-before.png')
Image.fromarray(frame.astype('uint8')).save(path);images.append(path.name)
changed=case.get('transformed')
if changed:
kind=changed['evidence_kind']
if kind in {'evidence_mirror','video_mirror'}:
after=np.flip(frame,axis=1 if changed['axis']=='hflip' else 0).copy()
if kind=='video_mirror':
source=Path(changed['transformed_video'])
if sha(source)!=changed['transformed_video_sha256']:raise ValueError('mirror file changed')
actual=frames_for_backend(source,load_video)[index]
if frame_hash(actual)!=changed['frame_sha256'][index] or not np.array_equal(actual,after):
raise ValueError('physical mirror pixels differ')
elif kind=='occlusion':
after=frame.copy();detail=changed['frame_transform'][index]
if detail['status']!='ok':raise ValueError('selected occlusion frame incomplete')
for x0,y0,x1,y1 in detail['rectangles']:after[y0:y1,x0:x1]=128
if frame_hash(after)!=changed['frame_sha256'][index]:raise ValueError('occlusion pixels differ')
else:raise ValueError(kind)
path=out/(case['id']+'-after.png');Image.fromarray(after.astype('uint8')).save(path);images.append(path.name)
small={k:v for k,v in case.items() if k not in {'cache','transformed'}}
small.update(images=images,image_sha256={name:sha(out/name) for name in images},
cache_record_sha256=hashlib.sha256(json.dumps(case.get('cache'),sort_keys=True).encode()).hexdigest())
result.append(small)
J.atomic_json(out/'cases.json',{'plan_sha256':sha(args.plan),'exporter_sha256':sha(__file__),'cases':result,
'note':'Illustrative selections, not a frequency estimate. Vision labels are model-derived. Evidence-only mirror pictures illustrate the exact coordinate reflection.'})
document=['<!doctype html><html lang="en"><meta charset="utf-8"><title>VBench evidence cases</title>',
'<style>body{font:16px system-ui;margin:2rem;max-width:1200px}figure{display:flex;gap:1rem;margin:1rem 0}img{max-width:47%;object-fit:contain}pre{white-space:pre-wrap;background:#f4f4f4;padding:1rem}section{border-top:1px solid #bbb;margin-top:2rem}</style>',
'<h1>Illustrative evidence and interface failures</h1><p>These examples explain failure mechanisms; aggregate frequencies belong to the full matrix tables. Model annotations are not human gold.</p>']
for row in result:
document.append('<section><h2>'+html.escape(row['title'])+'</h2><p>'+html.escape(row['note'])+'</p><figure>')
document.extend('<img src="'+html.escape(name)+'" alt="'+html.escape(row['title'])+'">' for name in row['images'])
document.append('</figure><pre>'+html.escape(json.dumps({k:v for k,v in row.items() if k not in {'title','note','images','image_sha256','cache_record_sha256'}},ensure_ascii=False,indent=2))+'</pre></section>')
document.append('</html>');(out/'index.html').write_text('\n'.join(document)+'\n')
print(json.dumps({'cases':len(result),'out':str(out)}));return 0
if __name__=='__main__':raise SystemExit(main())
|