"""Reuse training-video fingerprints to screen external benchmark overlap. Finite dHash screening is a conservative suspect detector, not proof of no cropped/transcoded overlap. Suspects are held out of publication for review. """ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor import argparse import hashlib import json from pathlib import Path import sys import numpy as np from common import ROOT,write_json from external_visual import DEFAULT sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] from prepare_hfd_video_assets import dhash from psi.data.seed.video_overlay import decode_video def main(root): bundle=ROOT.parents[1]/'videogen/hfd_visual_dataset' fingerprints=ROOT.parents[1]/'videogen/hfd_video_vqa_50k_v1/admission/fingerprints' train={};train_rows={} with (bundle/'videos/manifest.jsonl').open() as handle: for line in handle: r=json.loads(line) if r['split']=='train': train[r['media_sha256']]=r.get('parent_video_id',r['id']) train_rows[r['media_sha256']]=r def load(path): r=json.loads(path.read_text()) digest=r['signature']['video_sha256'] if r.get('accepted') and digest in train: return digest, r['hashes'] catalog={} with ThreadPoolExecutor(max_workers=12) as pool: for result in pool.map(load,fingerprints.glob('*.json')): if result:catalog[result[0]]=result[1] def missing(digest): path=root/'training_fingerprints'/(digest+'.json') if path.exists():return digest,json.loads(path.read_text())['hashes'] r=train_rows[digest];times=np.linspace(.1,r['duration_seconds']-.15,8) frames,pts=decode_video(bundle/'videos'/r['media_path'],times,end_seconds=r['duration_seconds']) values=[dhash(f) for f in frames] write_json(path,dict(video_sha256=digest,hashes=values,pts=pts.tolist())) return digest,values with ThreadPoolExecutor(max_workers=8) as pool: for digest,values in pool.map(missing,sorted(train.keys()-catalog.keys())):catalog[digest]=values catalog_sha=hashlib.sha256(json.dumps(catalog,sort_keys=True).encode()).hexdigest() print('TRAIN_FINGERPRINTS',len(catalog),'/',len(train),flush=True) bands=defaultdict(set) for digest,values in catalog.items(): for i,h in enumerate(values): for band in range(4):bands[(band,(h>>(16*band))&65535)].add((digest,i)) rows=json.loads((root/'media_inventory.json').read_text())['rows'] media={r['video_sha256']:r for r in rows} def examine(item): digest,r=item p=root/'split_reviews'/(digest+'.json') if p.exists(): prior=json.loads(p.read_text()) if prior.get('training_catalog_sha256')==catalog_sha:return prior # Sample the full source and the observed prefix, not just one still. times=np.unique(np.r_[np.linspace(.05,r['duration_seconds']-.2,32), np.linspace(.05,r['question_time'],24)]) frames,pts=decode_video(r['video_path'],times,end_seconds=r['duration_seconds']) values=[dhash(f) for f in frames];matches=defaultdict(set) for h in values: suspects=set() for band in range(4):suspects.update(bands.get((band,(h>>(16*band))&65535),())) for other,i in suspects: if (h^catalog[other][i]).bit_count()<=3:matches[other].add(i) suspects=[dict(training_sha256=d,training_parent=train[d],matching_anchors=len(indices)) for d,indices in matches.items() if len(indices)>=3] out=dict(video_sha256=digest,video_path=r['video_path'],suspects=suspects,training_catalog_sha256=catalog_sha, sampled_pts=pts.tolist(),hashes=values,passed=not suspects) write_json(p,out);print('SPLIT',r['video_relative'],len(suspects),flush=True) return out with ThreadPoolExecutor(max_workers=8) as pool: results=list(pool.map(examine,media.items())) write_json(root/'split_audit.json',dict(training_videos=len(train),fingerprinted_training_videos=len(catalog), candidate_videos=len(media),suspect_videos=sum(not r['passed'] for r in results), limitation='Exact IDs/SHA plus sampled visual fingerprints; not proof against every crop/transcode or pretrained exposure.', rows=results)) if __name__=='__main__': p=argparse.ArgumentParser(description=__doc__) p.add_argument('--root',type=Path,default=DEFAULT) main(p.parse_args().root)