File size: 6,797 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """Check HFD source-level split, checkpoint identity and episode coverage."""
import argparse
import json
import sys
from pathlib import Path
from common import ROOT, WORK, write_json, sha256
sys.path.insert(0,str(ROOT/'src'))
import numpy as np
import torch
from psi.data.seed.source_split import motion_source_id, source_hash_u64
def checkpoint_from_identity(identity):
"""Generation may use a new run with the same frozen evaluation cohort."""
run = Path(identity['run'])
if sha256(run/'run_config.json') != identity['config_sha256']:
raise ValueError('Generation run configuration changed')
return run/'checkpoints'/f"ckpt_{int(identity['step'])}"
def main():
p=argparse.ArgumentParser()
p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
p.add_argument('--generated',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated')
a=p.parse_args()
manifest=json.loads((a.cohort/'manifest.json').read_text())
identity=json.loads((a.generated/'identity.json').read_text())
if identity['cohort_sha256']!=sha256(a.cohort/'manifest.json'):
raise ValueError('Generated outputs do not match frozen cohort')
admitted=None
if manifest.get('external_assets_path'):
asset_path=Path(manifest['external_assets_path'])
if sha256(asset_path)!=manifest['external_assets_sha256']:raise ValueError('Admission manifest changed')
from session_overlay import reviewed
from revision import same_asset
assets=json.loads(asset_path.read_text())['assets']
if not all(reviewed(r) for r in assets):raise ValueError('External review no longer valid')
admitted={(r['asset_id'],r['voice']):r for r in assets}
run=Path(manifest['run']);config=json.loads((run/'run_config.json').read_text())
data=config['data'];rows=[];names=[];verified_assets={}
def verify_file(path, expected):
if not expected:raise ValueError('Missing external asset hash: '+str(path))
if path not in verified_assets:verified_assets[path]=sha256(path)
if verified_assets[path]!=expected:raise ValueError('External asset changed: '+str(path))
for row in manifest['rows']:
result=json.loads((a.generated/(row['sample_id']+'.json')).read_text())
if any(result.get(k)!=v for k,v in identity.items()):raise ValueError('Mixed generated result identities')
if admitted is not None:
asset=admitted.get((row['overlay']['asset_id'],row['voice']))
if asset is None or not same_asset(row,asset):raise ValueError('Sample visual QA not admitted')
for field in ('source_file','reference_file','microphone_file'):
if field in row and sha256(a.cohort/row[field])!=row[field.replace('_file','_sha256')]:
raise ValueError('Frozen benchmark file changed')
source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
if source['source_id']!=row['source_id']:raise ValueError('Source identity mismatch')
motion_names=source['turn_motion_names']
for name in motion_names:
sid=motion_source_id(data['split_source_namespace'],name)
if source_hash_u64(sid,data['split_seed'])>=int(data['val_ratio']*(1<<64)):
raise ValueError(f'Not held out: {sid}')
names.extend(motion_names)
with np.load(a.generated/(row['sample_id']+'.npz')) as z:pred=z['absolute']
with np.load(a.cohort/row['reference_file']) as z:gt=z['absolute'];labels=z['controls']
n=min(len(pred),len(gt))
if len(pred)<row['score_end']:raise ValueError('Incomplete motion')
if 'segments' in row:
if len(pred)<len(gt):raise ValueError('Incomplete session')
relation=row['overlay']
if relation['mode']!='qa':raise ValueError('Missing visual question')
external=relation.get('external_benchmark')
if manifest.get('external_assets_sha256'):
if not external:raise ValueError('Missing external benchmark provenance')
verify_file(relation['video_path'],external['source_video_sha256'])
verify_file(relation['visual_features_path'],external['feature_sha256'])
verify_file(external['review_receipt'],external['review_receipt_sha256'])
verify_file(relation['question_audio_path'],external['question_audio_sha256'])
if abs(relation['question_end']-external['question_time'])>1/16000.+1e-8:
raise ValueError('External question time was shifted')
if row['visual_kind']=='video' and relation['camera_duration_seconds']<relation['session_required_seconds']+.2:
raise ValueError('Video ends before session')
for segment in row['segments']:
if segment['eligible']:
s,e=segment['score_start'],segment['score_end']
if not 0<=s<e<=len(gt) or not (labels[s:e]==1).all():
raise ValueError('Invalid source slice')
rows.append(dict(sample_id=row['sample_id'],motion_names=motion_names,
source_sha256=sha256(a.cohort/row['source_file']),reference_sha256=sha256(a.cohort/row['reference_file']),
generated_sha256=sha256(a.generated/(row['sample_id']+'.npz')),
generated_frames=len(pred),reference_frames=len(gt),scored_full_frames=n,
unscored_generated_tail=len(pred)-n,unscored_reference_tail=len(gt)-n,
phase_frames={name:int((labels[:n]==value).sum()) for name,value in [('idle',2),('listen',0),('motion',1)]}))
if manifest.get('schema')!='hfd_bench_session_v3' and len(names)!=len(set(names)):
raise ValueError('Repeated motion source in legacy cohort')
if len({r['source_id'] for r in manifest['rows']})!=len(manifest['rows']):
raise ValueError('Duplicated complete session')
ident=json.loads((a.generated/'identity.json').read_text())
ckpt=checkpoint_from_identity(ident)
write_json(a.generated.parent/'audit.json',dict(source_split_verified=True,unique_motion_sources=len(set(names)),
model_sha256=sha256(ckpt/'model.safetensors'),
formal_rollout_sha256=sha256(ROOT/'src/psi/runtime/swa_rollout.py'),
fused_manifest_sha256=sha256(ckpt/'hfd_fused_delta_manifest.json'),
initialization='formal deployment source initial idle state; no GT subsequent states/RTC/duration',
scope='held-out validation, not separately reserved final test; full multimodal sessions; final-command motion slices',
source_occurrences=len(names),session_count=len(manifest['rows']),
external_assets_verified=len(verified_assets),
rows=rows))
if __name__=='__main__':main()
|