Download scripts/data/export_hfd_overlay_assets.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 3.89 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/export_hfd_overlay_assets.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/export_hfd_overlay_assets.py
-
curl -L -o export_hfd_overlay_assets.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/export_hfd_overlay_assets.py
3.89 kB
| #!/usr/bin/env python3 | |
| """Reuse approved static VQAv2 assets without their old motion pairing. | |
| Only an index is written. Images, independent question WAVs and frozen visual | |
| frontend features stay where they are. No old mixed microphone or language KV | |
| is reused. Dynamic-camera assets require their own temporal adapter. | |
| """ | |
| import argparse | |
| from collections import Counter, defaultdict | |
| from functools import lru_cache | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| def sha(path): | |
| value = hashlib.sha256() | |
| with Path(path).open('rb') as handle: | |
| for chunk in iter(lambda: handle.read(1024 * 1024), b''): | |
| value.update(chunk) | |
| return value.hexdigest() | |
| def export(source_plan, root): | |
| rows = json.loads(source_plan.read_text()) | |
| features = {} | |
| for directory in sorted(root.glob('components_r*/visual')): | |
| for path in directory.glob('*.pt'): | |
| features.setdefault(path.stem, path) | |
| assets = [] | |
| for row in rows: | |
| if row.get('cohort') != 'vqav2_static': | |
| continue | |
| question = row['question'] | |
| vr, ar = row['visual_review'], row['asr_review'] | |
| if not (vr.get('accepted') and vr.get('blind_to_reference') and ar.get('accepted')): | |
| raise ValueError(f"Unapproved asset in admitted source plan: {row['id']}") | |
| image = question['image_path'] | |
| audio = row['timeline']['question_voice'] | |
| image_hash = sha(image) | |
| audio_hash = sha(audio) | |
| if image_hash != question['image_sha256'] or image_hash != vr['image_sha256']: | |
| raise ValueError(f"Reviewed image changed: {row['id']}") | |
| if audio_hash != ar['wave_sha256']: | |
| raise ValueError(f"Reviewed question waveform changed: {row['id']}") | |
| feature = features.get(image_hash) | |
| if feature is None or feature.stat().st_size == 0: | |
| raise FileNotFoundError(f"Missing frozen visual feature: {row['id']}") | |
| assets.append(dict(asset_id=row['id'], split=row['split'], voice=row['voice'], | |
| question=question['question'], answer=question['answer'], image_path=image, | |
| question_audio_path=audio, image_sha256=image_hash, question_audio_sha256=audio_hash, | |
| visual_features_path=str(feature), data_domain='vqav2_static', | |
| visual_receipt=str(root/'visual_review'/(row['id']+'.json')), | |
| asr_receipt=str(root/'asr'/(Path(audio).stem+'.json')))) | |
| if len(assets) % 5000 == 0: | |
| print('EXPORTED', len(assets), flush=True) | |
| splits = defaultdict(set) | |
| for row in assets: | |
| splits[row['image_sha256']].add(row['split']) | |
| if any('train' in s and len(s) > 1 for s in splits.values()): | |
| raise ValueError('VQAv2 image overlaps train and held-out splits') | |
| if not assets: | |
| raise ValueError('No approved static VQAv2 assets') | |
| return dict(format='hfd_overlay_assets_v1', assets=assets, | |
| provenance=dict(source_plan=str(source_plan), source_plan_sha256=sha(str(source_plan)), | |
| motion_pairing_discarded=True, raw_assets_copied=False)) | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--source-plan', type=Path, required=True) | |
| p.add_argument('--root', type=Path, required=True) | |
| p.add_argument('--output', type=Path, required=True) | |
| a = p.parse_args() | |
| payload = export(a.source_plan.resolve(), a.root.resolve()) | |
| a.output.parent.mkdir(parents=True, exist_ok=True) | |
| partial = a.output.with_suffix('.partial.json') | |
| partial.write_text(json.dumps(payload, ensure_ascii=False, separators=(',', ':'))) | |
| partial.replace(a.output) | |
| print(json.dumps(dict(admitted=len(payload['assets']), | |
| splits=dict(Counter(r['split'] for r in payload['assets'])), output=str(a.output))), flush=True) | |
| if __name__ == '__main__': | |
| main() | |