code-HFD / scripts /data /check_hfd_overlay_manifest.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
5.7 kB
#!/usr/bin/env python3
"""Fast startup gate for independently published image/video overlay assets."""
import argparse
from collections import Counter,defaultdict
import hashlib
import json
from pathlib import Path
import sys
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'src'))
def admitted_rows(path, base_model):
if path.is_dir():
from psi.data.seed.visual_dataset import overlay_rows
yield from overlay_rows(path, verify=True, base_model=base_model, lazy_paths=True)
return
raw=path.read_bytes();data=json.loads(raw)
if data.get('format')!='hfd_overlay_assets_v1' or not data.get('assets'):
raise ValueError('Invalid/empty overlay manifest')
if any(r.get('video_path') for r in data['assets']):
report=json.loads((path.parent/'admission/acceptance.json').read_text())
if not report.get('passed') or report['manifest_sha256']!=hashlib.sha256(raw).hexdigest():
raise ValueError('Unaccepted or modified dynamic video manifest')
contract=data['provenance']['contract']
if base_model:
for name,key in [('config.json','model_config_sha256'),('model.safetensors.index.json','weight_index_sha256')]:
if hashlib.sha256((Path(base_model)/name).read_bytes()).hexdigest()!=contract[key]:
raise ValueError('Video frontend does not match active Qwen')
yield from data['assets']
def check(paths,base_model=None,*,require_eight_voices=False,require_live_video_qa=False,visual_input_mode='cached'):
if visual_input_mode not in ('cached','raw'):raise ValueError('visual_input_mode must be cached or raw')
identities=defaultdict(set);counts=Counter();samples={};voices=set()
last_points = None; last_duration = None; points_valid = False
for path in map(Path,paths):
groups=defaultdict(set)
for row in admitted_rows(path,base_model):
groups[row['asset_id']].add(row['voice'])
if require_live_video_qa and row.get('video_path'):
from psi.data.seed.causal_video_qa import valid_point
if row.get('causal_qa_points'):
points, duration = row['causal_qa_points'], row.get('duration_seconds',0.)
# Unified QA shares the same immutable point list across
# eight voices; validate it once, not eight NumPy loops.
if points is not last_points or duration != last_duration:
points_valid = all(valid_point(p, duration) for p in points)
last_points, last_duration = points, duration
valid = row.get('live_qa_receipt') and points_valid
else:
valid = row.get('live_qa_windows') and row.get('live_qa_receipt')
if not valid:
raise ValueError(f'Video QA needs reviewed causal-history question times: {path}')
identity=row.get('video_sha256') or row.get('image_sha256')
if not identity:raise ValueError('Missing asset checksum')
identities[identity].add(row['split'])
if row.get('parent_video_id'):identities['parent:'+row['parent_video_id']].add(row['split'])
counts[(row['data_domain'],row['split'])]+=1;voices.add(row['voice'])
samples.setdefault((row['data_domain'],row['split'],row['voice']),row)
if not groups:
raise ValueError(f'No admitted overlay assets: {path}')
if require_eight_voices:
expected={f'{sex}_{n:02d}_{lang}' for sex in ('female','male') for n in (1,2) for lang in ('en','zh')}
if any(value!=expected for value in groups.values()):
raise ValueError(f'Image AND video QA require all eight voices per asset; expansion not complete: {path}')
if any('train' in splits and len(splits)>1 for splits in identities.values()):raise ValueError('Overlay train/heldout leak')
from psi.data.seed.live_multimodal_overlay import visual_features,question_wave
from psi.data.seed.video_overlay import dynamic_features
from psi.data.seed.visual_dataset import resolve_overlay_row
for row in samples.values():
row = resolve_overlay_row(row)
media=row.get('video_path') or row.get('image_path')
if not media or not Path(media).is_file():raise FileNotFoundError(media)
question_wave(row['question_audio_path'])
if visual_input_mode=='cached':
(dynamic_features if row.get('video_path') else visual_features)(row['visual_features_path'])
else:
from psi.data.seed.native_visual import request_frames
request_frames(dict(path=media,kind='video' if row.get('video_path') else 'image',
duration=row.get('duration_seconds'),index=0,count=5))
return dict(counts={':'.join(k):v for k,v in counts.items()},voices=sorted(voices),
samples_checked=len(samples),note='All asset split IDs checked; expensive checks attested offline, representative file reads only at startup')
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--manifests',type=Path,nargs='+',required=True)
p.add_argument('--base-model',type=Path)
p.add_argument('--require-eight-voices',action='store_true')
p.add_argument('--visual-input-mode',choices=('cached','raw'),default='cached')
p.add_argument('--require-live-video-qa',action='store_true');a=p.parse_args()
print(json.dumps(check(a.manifests,a.base_model,require_eight_voices=a.require_eight_voices,
require_live_video_qa=a.require_live_video_qa,visual_input_mode=a.visual_input_mode),ensure_ascii=False))