#!/usr/bin/env python3 """Fast startup gate for independently published image/video overlay assets.""" import argparse from collections import Counter,defaultdict import hashlib import json from pathlib import Path import sys sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'src')) def admitted_rows(path, base_model): if path.is_dir(): from psi.data.seed.visual_dataset import overlay_rows yield from overlay_rows(path, verify=True, base_model=base_model, lazy_paths=True) return raw=path.read_bytes();data=json.loads(raw) if data.get('format')!='hfd_overlay_assets_v1' or not data.get('assets'): raise ValueError('Invalid/empty overlay manifest') if any(r.get('video_path') for r in data['assets']): report=json.loads((path.parent/'admission/acceptance.json').read_text()) if not report.get('passed') or report['manifest_sha256']!=hashlib.sha256(raw).hexdigest(): raise ValueError('Unaccepted or modified dynamic video manifest') contract=data['provenance']['contract'] if base_model: for name,key in [('config.json','model_config_sha256'),('model.safetensors.index.json','weight_index_sha256')]: if hashlib.sha256((Path(base_model)/name).read_bytes()).hexdigest()!=contract[key]: raise ValueError('Video frontend does not match active Qwen') yield from data['assets'] def check(paths,base_model=None,*,require_eight_voices=False,require_live_video_qa=False,visual_input_mode='cached'): if visual_input_mode not in ('cached','raw'):raise ValueError('visual_input_mode must be cached or raw') identities=defaultdict(set);counts=Counter();samples={};voices=set() last_points = None; last_duration = None; points_valid = False for path in map(Path,paths): groups=defaultdict(set) for row in admitted_rows(path,base_model): groups[row['asset_id']].add(row['voice']) if require_live_video_qa and row.get('video_path'): from psi.data.seed.causal_video_qa import valid_point if row.get('causal_qa_points'): points, duration = row['causal_qa_points'], row.get('duration_seconds',0.) # Unified QA shares the same immutable point list across # eight voices; validate it once, not eight NumPy loops. if points is not last_points or duration != last_duration: points_valid = all(valid_point(p, duration) for p in points) last_points, last_duration = points, duration valid = row.get('live_qa_receipt') and points_valid else: valid = row.get('live_qa_windows') and row.get('live_qa_receipt') if not valid: raise ValueError(f'Video QA needs reviewed causal-history question times: {path}') identity=row.get('video_sha256') or row.get('image_sha256') if not identity:raise ValueError('Missing asset checksum') identities[identity].add(row['split']) if row.get('parent_video_id'):identities['parent:'+row['parent_video_id']].add(row['split']) counts[(row['data_domain'],row['split'])]+=1;voices.add(row['voice']) samples.setdefault((row['data_domain'],row['split'],row['voice']),row) if not groups: raise ValueError(f'No admitted overlay assets: {path}') if require_eight_voices: expected={f'{sex}_{n:02d}_{lang}' for sex in ('female','male') for n in (1,2) for lang in ('en','zh')} if any(value!=expected for value in groups.values()): raise ValueError(f'Image AND video QA require all eight voices per asset; expansion not complete: {path}') if any('train' in splits and len(splits)>1 for splits in identities.values()):raise ValueError('Overlay train/heldout leak') from psi.data.seed.live_multimodal_overlay import visual_features,question_wave from psi.data.seed.video_overlay import dynamic_features from psi.data.seed.visual_dataset import resolve_overlay_row for row in samples.values(): row = resolve_overlay_row(row) media=row.get('video_path') or row.get('image_path') if not media or not Path(media).is_file():raise FileNotFoundError(media) question_wave(row['question_audio_path']) if visual_input_mode=='cached': (dynamic_features if row.get('video_path') else visual_features)(row['visual_features_path']) else: from psi.data.seed.native_visual import request_frames request_frames(dict(path=media,kind='video' if row.get('video_path') else 'image', duration=row.get('duration_seconds'),index=0,count=5)) return dict(counts={':'.join(k):v for k,v in counts.items()},voices=sorted(voices), samples_checked=len(samples),note='All asset split IDs checked; expensive checks attested offline, representative file reads only at startup') if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('--manifests',type=Path,nargs='+',required=True) p.add_argument('--base-model',type=Path) p.add_argument('--require-eight-voices',action='store_true') p.add_argument('--visual-input-mode',choices=('cached','raw'),default='cached') p.add_argument('--require-live-video-qa',action='store_true');a=p.parse_args() print(json.dumps(check(a.manifests,a.base_model,require_eight_voices=a.require_eight_voices, require_live_video_qa=a.require_live_video_qa,visual_input_mode=a.visual_input_mode),ensure_ascii=False))