File size: 4,595 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """Reuse training-video fingerprints to screen external benchmark overlap.
Finite dHash screening is a conservative suspect detector, not proof of no
cropped/transcoded overlap. Suspects are held out of publication for review.
"""
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
import argparse
import hashlib
import json
from pathlib import Path
import sys
import numpy as np
from common import ROOT,write_json
from external_visual import DEFAULT
sys.path[:0]=[str(ROOT),str(ROOT/'src'),str(ROOT/'scripts/data')]
from prepare_hfd_video_assets import dhash
from psi.data.seed.video_overlay import decode_video
def main(root):
bundle=ROOT.parents[1]/'videogen/hfd_visual_dataset'
fingerprints=ROOT.parents[1]/'videogen/hfd_video_vqa_50k_v1/admission/fingerprints'
train={};train_rows={}
with (bundle/'videos/manifest.jsonl').open() as handle:
for line in handle:
r=json.loads(line)
if r['split']=='train':
train[r['media_sha256']]=r.get('parent_video_id',r['id'])
train_rows[r['media_sha256']]=r
def load(path):
r=json.loads(path.read_text())
digest=r['signature']['video_sha256']
if r.get('accepted') and digest in train:
return digest, r['hashes']
catalog={}
with ThreadPoolExecutor(max_workers=12) as pool:
for result in pool.map(load,fingerprints.glob('*.json')):
if result:catalog[result[0]]=result[1]
def missing(digest):
path=root/'training_fingerprints'/(digest+'.json')
if path.exists():return digest,json.loads(path.read_text())['hashes']
r=train_rows[digest];times=np.linspace(.1,r['duration_seconds']-.15,8)
frames,pts=decode_video(bundle/'videos'/r['media_path'],times,end_seconds=r['duration_seconds'])
values=[dhash(f) for f in frames]
write_json(path,dict(video_sha256=digest,hashes=values,pts=pts.tolist()))
return digest,values
with ThreadPoolExecutor(max_workers=8) as pool:
for digest,values in pool.map(missing,sorted(train.keys()-catalog.keys())):catalog[digest]=values
catalog_sha=hashlib.sha256(json.dumps(catalog,sort_keys=True).encode()).hexdigest()
print('TRAIN_FINGERPRINTS',len(catalog),'/',len(train),flush=True)
bands=defaultdict(set)
for digest,values in catalog.items():
for i,h in enumerate(values):
for band in range(4):bands[(band,(h>>(16*band))&65535)].add((digest,i))
rows=json.loads((root/'media_inventory.json').read_text())['rows']
media={r['video_sha256']:r for r in rows}
def examine(item):
digest,r=item
p=root/'split_reviews'/(digest+'.json')
if p.exists():
prior=json.loads(p.read_text())
if prior.get('training_catalog_sha256')==catalog_sha:return prior
# Sample the full source and the observed prefix, not just one still.
times=np.unique(np.r_[np.linspace(.05,r['duration_seconds']-.2,32),
np.linspace(.05,r['question_time'],24)])
frames,pts=decode_video(r['video_path'],times,end_seconds=r['duration_seconds'])
values=[dhash(f) for f in frames];matches=defaultdict(set)
for h in values:
suspects=set()
for band in range(4):suspects.update(bands.get((band,(h>>(16*band))&65535),()))
for other,i in suspects:
if (h^catalog[other][i]).bit_count()<=3:matches[other].add(i)
suspects=[dict(training_sha256=d,training_parent=train[d],matching_anchors=len(indices))
for d,indices in matches.items() if len(indices)>=3]
out=dict(video_sha256=digest,video_path=r['video_path'],suspects=suspects,training_catalog_sha256=catalog_sha,
sampled_pts=pts.tolist(),hashes=values,passed=not suspects)
write_json(p,out);print('SPLIT',r['video_relative'],len(suspects),flush=True)
return out
with ThreadPoolExecutor(max_workers=8) as pool:
results=list(pool.map(examine,media.items()))
write_json(root/'split_audit.json',dict(training_videos=len(train),fingerprinted_training_videos=len(catalog),
candidate_videos=len(media),suspect_videos=sum(not r['passed'] for r in results),
limitation='Exact IDs/SHA plus sampled visual fingerprints; not proof against every crop/transcode or pretrained exposure.',
rows=results))
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--root',type=Path,default=DEFAULT)
main(p.parse_args().root)
|