#!/usr/bin/env python3 """Reuse approved static VQAv2 assets without their old motion pairing. Only an index is written. Images, independent question WAVs and frozen visual frontend features stay where they are. No old mixed microphone or language KV is reused. Dynamic-camera assets require their own temporal adapter. """ import argparse from collections import Counter, defaultdict from functools import lru_cache import hashlib import json from pathlib import Path @lru_cache(maxsize=150000) def sha(path): value = hashlib.sha256() with Path(path).open('rb') as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b''): value.update(chunk) return value.hexdigest() def export(source_plan, root): rows = json.loads(source_plan.read_text()) features = {} for directory in sorted(root.glob('components_r*/visual')): for path in directory.glob('*.pt'): features.setdefault(path.stem, path) assets = [] for row in rows: if row.get('cohort') != 'vqav2_static': continue question = row['question'] vr, ar = row['visual_review'], row['asr_review'] if not (vr.get('accepted') and vr.get('blind_to_reference') and ar.get('accepted')): raise ValueError(f"Unapproved asset in admitted source plan: {row['id']}") image = question['image_path'] audio = row['timeline']['question_voice'] image_hash = sha(image) audio_hash = sha(audio) if image_hash != question['image_sha256'] or image_hash != vr['image_sha256']: raise ValueError(f"Reviewed image changed: {row['id']}") if audio_hash != ar['wave_sha256']: raise ValueError(f"Reviewed question waveform changed: {row['id']}") feature = features.get(image_hash) if feature is None or feature.stat().st_size == 0: raise FileNotFoundError(f"Missing frozen visual feature: {row['id']}") assets.append(dict(asset_id=row['id'], split=row['split'], voice=row['voice'], question=question['question'], answer=question['answer'], image_path=image, question_audio_path=audio, image_sha256=image_hash, question_audio_sha256=audio_hash, visual_features_path=str(feature), data_domain='vqav2_static', visual_receipt=str(root/'visual_review'/(row['id']+'.json')), asr_receipt=str(root/'asr'/(Path(audio).stem+'.json')))) if len(assets) % 5000 == 0: print('EXPORTED', len(assets), flush=True) splits = defaultdict(set) for row in assets: splits[row['image_sha256']].add(row['split']) if any('train' in s and len(s) > 1 for s in splits.values()): raise ValueError('VQAv2 image overlaps train and held-out splits') if not assets: raise ValueError('No approved static VQAv2 assets') return dict(format='hfd_overlay_assets_v1', assets=assets, provenance=dict(source_plan=str(source_plan), source_plan_sha256=sha(str(source_plan)), motion_pairing_discarded=True, raw_assets_copied=False)) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--source-plan', type=Path, required=True) p.add_argument('--root', type=Path, required=True) p.add_argument('--output', type=Path, required=True) a = p.parse_args() payload = export(a.source_plan.resolve(), a.root.resolve()) a.output.parent.mkdir(parents=True, exist_ok=True) partial = a.output.with_suffix('.partial.json') partial.write_text(json.dumps(payload, ensure_ascii=False, separators=(',', ':'))) partial.replace(a.output) print(json.dumps(dict(admitted=len(payload['assets']), splits=dict(Counter(r['split'] for r in payload['assets'])), output=str(a.output))), flush=True) if __name__ == '__main__': main()