"""Check HFD source-level split, checkpoint identity and episode coverage.""" import argparse import json import sys from pathlib import Path from common import ROOT, WORK, write_json, sha256 sys.path.insert(0,str(ROOT/'src')) import numpy as np import torch from psi.data.seed.source_split import motion_source_id, source_hash_u64 def checkpoint_from_identity(identity): """Generation may use a new run with the same frozen evaluation cohort.""" run = Path(identity['run']) if sha256(run/'run_config.json') != identity['config_sha256']: raise ValueError('Generation run configuration changed') return run/'checkpoints'/f"ckpt_{int(identity['step'])}" def main(): p=argparse.ArgumentParser() p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048') p.add_argument('--generated',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated') a=p.parse_args() manifest=json.loads((a.cohort/'manifest.json').read_text()) identity=json.loads((a.generated/'identity.json').read_text()) if identity['cohort_sha256']!=sha256(a.cohort/'manifest.json'): raise ValueError('Generated outputs do not match frozen cohort') admitted=None if manifest.get('external_assets_path'): asset_path=Path(manifest['external_assets_path']) if sha256(asset_path)!=manifest['external_assets_sha256']:raise ValueError('Admission manifest changed') from session_overlay import reviewed from revision import same_asset assets=json.loads(asset_path.read_text())['assets'] if not all(reviewed(r) for r in assets):raise ValueError('External review no longer valid') admitted={(r['asset_id'],r['voice']):r for r in assets} run=Path(manifest['run']);config=json.loads((run/'run_config.json').read_text()) data=config['data'];rows=[];names=[];verified_assets={} def verify_file(path, expected): if not expected:raise ValueError('Missing external asset hash: '+str(path)) if path not in verified_assets:verified_assets[path]=sha256(path) if verified_assets[path]!=expected:raise ValueError('External asset changed: '+str(path)) for row in manifest['rows']: result=json.loads((a.generated/(row['sample_id']+'.json')).read_text()) if any(result.get(k)!=v for k,v in identity.items()):raise ValueError('Mixed generated result identities') if admitted is not None: asset=admitted.get((row['overlay']['asset_id'],row['voice'])) if asset is None or not same_asset(row,asset):raise ValueError('Sample visual QA not admitted') for field in ('source_file','reference_file','microphone_file'): if field in row and sha256(a.cohort/row[field])!=row[field.replace('_file','_sha256')]: raise ValueError('Frozen benchmark file changed') source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True) if source['source_id']!=row['source_id']:raise ValueError('Source identity mismatch') motion_names=source['turn_motion_names'] for name in motion_names: sid=motion_source_id(data['split_source_namespace'],name) if source_hash_u64(sid,data['split_seed'])>=int(data['val_ratio']*(1<<64)): raise ValueError(f'Not held out: {sid}') names.extend(motion_names) with np.load(a.generated/(row['sample_id']+'.npz')) as z:pred=z['absolute'] with np.load(a.cohort/row['reference_file']) as z:gt=z['absolute'];labels=z['controls'] n=min(len(pred),len(gt)) if len(pred)1/16000.+1e-8: raise ValueError('External question time was shifted') if row['visual_kind']=='video' and relation['camera_duration_seconds']