Download scripts/data/check_hfd_overlay_manifest.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/check_hfd_overlay_manifest.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/check_hfd_overlay_manifest.py
-
curl -L -o check_hfd_overlay_manifest.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/check_hfd_overlay_manifest.py
5.7 kB
| #!/usr/bin/env python3 | |
| """Fast startup gate for independently published image/video overlay assets.""" | |
| import argparse | |
| from collections import Counter,defaultdict | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import sys | |
| sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'src')) | |
| def admitted_rows(path, base_model): | |
| if path.is_dir(): | |
| from psi.data.seed.visual_dataset import overlay_rows | |
| yield from overlay_rows(path, verify=True, base_model=base_model, lazy_paths=True) | |
| return | |
| raw=path.read_bytes();data=json.loads(raw) | |
| if data.get('format')!='hfd_overlay_assets_v1' or not data.get('assets'): | |
| raise ValueError('Invalid/empty overlay manifest') | |
| if any(r.get('video_path') for r in data['assets']): | |
| report=json.loads((path.parent/'admission/acceptance.json').read_text()) | |
| if not report.get('passed') or report['manifest_sha256']!=hashlib.sha256(raw).hexdigest(): | |
| raise ValueError('Unaccepted or modified dynamic video manifest') | |
| contract=data['provenance']['contract'] | |
| if base_model: | |
| for name,key in [('config.json','model_config_sha256'),('model.safetensors.index.json','weight_index_sha256')]: | |
| if hashlib.sha256((Path(base_model)/name).read_bytes()).hexdigest()!=contract[key]: | |
| raise ValueError('Video frontend does not match active Qwen') | |
| yield from data['assets'] | |
| def check(paths,base_model=None,*,require_eight_voices=False,require_live_video_qa=False,visual_input_mode='cached'): | |
| if visual_input_mode not in ('cached','raw'):raise ValueError('visual_input_mode must be cached or raw') | |
| identities=defaultdict(set);counts=Counter();samples={};voices=set() | |
| last_points = None; last_duration = None; points_valid = False | |
| for path in map(Path,paths): | |
| groups=defaultdict(set) | |
| for row in admitted_rows(path,base_model): | |
| groups[row['asset_id']].add(row['voice']) | |
| if require_live_video_qa and row.get('video_path'): | |
| from psi.data.seed.causal_video_qa import valid_point | |
| if row.get('causal_qa_points'): | |
| points, duration = row['causal_qa_points'], row.get('duration_seconds',0.) | |
| # Unified QA shares the same immutable point list across | |
| # eight voices; validate it once, not eight NumPy loops. | |
| if points is not last_points or duration != last_duration: | |
| points_valid = all(valid_point(p, duration) for p in points) | |
| last_points, last_duration = points, duration | |
| valid = row.get('live_qa_receipt') and points_valid | |
| else: | |
| valid = row.get('live_qa_windows') and row.get('live_qa_receipt') | |
| if not valid: | |
| raise ValueError(f'Video QA needs reviewed causal-history question times: {path}') | |
| identity=row.get('video_sha256') or row.get('image_sha256') | |
| if not identity:raise ValueError('Missing asset checksum') | |
| identities[identity].add(row['split']) | |
| if row.get('parent_video_id'):identities['parent:'+row['parent_video_id']].add(row['split']) | |
| counts[(row['data_domain'],row['split'])]+=1;voices.add(row['voice']) | |
| samples.setdefault((row['data_domain'],row['split'],row['voice']),row) | |
| if not groups: | |
| raise ValueError(f'No admitted overlay assets: {path}') | |
| if require_eight_voices: | |
| expected={f'{sex}_{n:02d}_{lang}' for sex in ('female','male') for n in (1,2) for lang in ('en','zh')} | |
| if any(value!=expected for value in groups.values()): | |
| raise ValueError(f'Image AND video QA require all eight voices per asset; expansion not complete: {path}') | |
| if any('train' in splits and len(splits)>1 for splits in identities.values()):raise ValueError('Overlay train/heldout leak') | |
| from psi.data.seed.live_multimodal_overlay import visual_features,question_wave | |
| from psi.data.seed.video_overlay import dynamic_features | |
| from psi.data.seed.visual_dataset import resolve_overlay_row | |
| for row in samples.values(): | |
| row = resolve_overlay_row(row) | |
| media=row.get('video_path') or row.get('image_path') | |
| if not media or not Path(media).is_file():raise FileNotFoundError(media) | |
| question_wave(row['question_audio_path']) | |
| if visual_input_mode=='cached': | |
| (dynamic_features if row.get('video_path') else visual_features)(row['visual_features_path']) | |
| else: | |
| from psi.data.seed.native_visual import request_frames | |
| request_frames(dict(path=media,kind='video' if row.get('video_path') else 'image', | |
| duration=row.get('duration_seconds'),index=0,count=5)) | |
| return dict(counts={':'.join(k):v for k,v in counts.items()},voices=sorted(voices), | |
| samples_checked=len(samples),note='All asset split IDs checked; expensive checks attested offline, representative file reads only at startup') | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser();p.add_argument('--manifests',type=Path,nargs='+',required=True) | |
| p.add_argument('--base-model',type=Path) | |
| p.add_argument('--require-eight-voices',action='store_true') | |
| p.add_argument('--visual-input-mode',choices=('cached','raw'),default='cached') | |
| p.add_argument('--require-live-video-qa',action='store_true');a=p.parse_args() | |
| print(json.dumps(check(a.manifests,a.base_model,require_eight_voices=a.require_eight_voices, | |
| require_live_video_qa=a.require_live_video_qa,visual_input_mode=a.visual_input_mode),ensure_ascii=False)) | |