Download benchmark/external_split.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_split.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/external_split.py
-
curl -L -o external_split.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/external_split.py
4.6 kB
| """Reuse training-video fingerprints to screen external benchmark overlap. | |
| Finite dHash screening is a conservative suspect detector, not proof of no | |
| cropped/transcoded overlap. Suspects are held out of publication for review. | |
| """ | |
| from collections import defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import numpy as np | |
| from common import ROOT,write_json | |
| from external_visual import DEFAULT | |
| sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')] | |
| from prepare_hfd_video_assets import dhash | |
| from psi.data.seed.video_overlay import decode_video | |
| def main(root): | |
| bundle=ROOT.parents[1]/'videogen/hfd_visual_dataset' | |
| fingerprints=ROOT.parents[1]/'videogen/hfd_video_vqa_50k_v1/admission/fingerprints' | |
| train={};train_rows={} | |
| with (bundle/'videos/manifest.jsonl').open() as handle: | |
| for line in handle: | |
| r=json.loads(line) | |
| if r['split']=='train': | |
| train[r['media_sha256']]=r.get('parent_video_id',r['id']) | |
| train_rows[r['media_sha256']]=r | |
| def load(path): | |
| r=json.loads(path.read_text()) | |
| digest=r['signature']['video_sha256'] | |
| if r.get('accepted') and digest in train: | |
| return digest, r['hashes'] | |
| catalog={} | |
| with ThreadPoolExecutor(max_workers=12) as pool: | |
| for result in pool.map(load,fingerprints.glob('*.json')): | |
| if result:catalog[result[0]]=result[1] | |
| def missing(digest): | |
| path=root/'training_fingerprints'/(digest+'.json') | |
| if path.exists():return digest,json.loads(path.read_text())['hashes'] | |
| r=train_rows[digest];times=np.linspace(.1,r['duration_seconds']-.15,8) | |
| frames,pts=decode_video(bundle/'videos'/r['media_path'],times,end_seconds=r['duration_seconds']) | |
| values=[dhash(f) for f in frames] | |
| write_json(path,dict(video_sha256=digest,hashes=values,pts=pts.tolist())) | |
| return digest,values | |
| with ThreadPoolExecutor(max_workers=8) as pool: | |
| for digest,values in pool.map(missing,sorted(train.keys()-catalog.keys())):catalog[digest]=values | |
| catalog_sha=hashlib.sha256(json.dumps(catalog,sort_keys=True).encode()).hexdigest() | |
| print('TRAIN_FINGERPRINTS',len(catalog),'/',len(train),flush=True) | |
| bands=defaultdict(set) | |
| for digest,values in catalog.items(): | |
| for i,h in enumerate(values): | |
| for band in range(4):bands[(band,(h>>(16*band))&65535)].add((digest,i)) | |
| rows=json.loads((root/'media_inventory.json').read_text())['rows'] | |
| media={r['video_sha256']:r for r in rows} | |
| def examine(item): | |
| digest,r=item | |
| p=root/'split_reviews'/(digest+'.json') | |
| if p.exists(): | |
| prior=json.loads(p.read_text()) | |
| if prior.get('training_catalog_sha256')==catalog_sha:return prior | |
| # Sample the full source and the observed prefix, not just one still. | |
| times=np.unique(np.r_[np.linspace(.05,r['duration_seconds']-.2,32), | |
| np.linspace(.05,r['question_time'],24)]) | |
| frames,pts=decode_video(r['video_path'],times,end_seconds=r['duration_seconds']) | |
| values=[dhash(f) for f in frames];matches=defaultdict(set) | |
| for h in values: | |
| suspects=set() | |
| for band in range(4):suspects.update(bands.get((band,(h>>(16*band))&65535),())) | |
| for other,i in suspects: | |
| if (h^catalog[other][i]).bit_count()<=3:matches[other].add(i) | |
| suspects=[dict(training_sha256=d,training_parent=train[d],matching_anchors=len(indices)) | |
| for d,indices in matches.items() if len(indices)>=3] | |
| out=dict(video_sha256=digest,video_path=r['video_path'],suspects=suspects,training_catalog_sha256=catalog_sha, | |
| sampled_pts=pts.tolist(),hashes=values,passed=not suspects) | |
| write_json(p,out);print('SPLIT',r['video_relative'],len(suspects),flush=True) | |
| return out | |
| with ThreadPoolExecutor(max_workers=8) as pool: | |
| results=list(pool.map(examine,media.items())) | |
| write_json(root/'split_audit.json',dict(training_videos=len(train),fingerprinted_training_videos=len(catalog), | |
| candidate_videos=len(media),suspect_videos=sum(not r['passed'] for r in results), | |
| limitation='Exact IDs/SHA plus sampled visual fingerprints; not proof against every crop/transcode or pretrained exposure.', | |
| rows=results)) | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--root',type=Path,default=DEFAULT) | |
| main(p.parse_args().root) | |