File size: 6,797 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
"""Check HFD source-level split, checkpoint identity and episode coverage."""
import argparse
import json
import sys
from pathlib import Path
from common import ROOT, WORK, write_json, sha256
sys.path.insert(0,str(ROOT/'src'))
import numpy as np
import torch
from psi.data.seed.source_split import motion_source_id, source_hash_u64

def checkpoint_from_identity(identity):
    """Generation may use a new run with the same frozen evaluation cohort."""
    run = Path(identity['run'])
    if sha256(run/'run_config.json') != identity['config_sha256']:
        raise ValueError('Generation run configuration changed')
    return run/'checkpoints'/f"ckpt_{int(identity['step'])}"

def main():
    p=argparse.ArgumentParser()
    p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048')
    p.add_argument('--generated',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated')
    a=p.parse_args()
    manifest=json.loads((a.cohort/'manifest.json').read_text())
    identity=json.loads((a.generated/'identity.json').read_text())
    if identity['cohort_sha256']!=sha256(a.cohort/'manifest.json'):
        raise ValueError('Generated outputs do not match frozen cohort')
    admitted=None
    if manifest.get('external_assets_path'):
        asset_path=Path(manifest['external_assets_path'])
        if sha256(asset_path)!=manifest['external_assets_sha256']:raise ValueError('Admission manifest changed')
        from session_overlay import reviewed
        from revision import same_asset
        assets=json.loads(asset_path.read_text())['assets']
        if not all(reviewed(r) for r in assets):raise ValueError('External review no longer valid')
        admitted={(r['asset_id'],r['voice']):r for r in assets}
    run=Path(manifest['run']);config=json.loads((run/'run_config.json').read_text())
    data=config['data'];rows=[];names=[];verified_assets={}
    def verify_file(path, expected):
        if not expected:raise ValueError('Missing external asset hash: '+str(path))
        if path not in verified_assets:verified_assets[path]=sha256(path)
        if verified_assets[path]!=expected:raise ValueError('External asset changed: '+str(path))
    for row in manifest['rows']:
        result=json.loads((a.generated/(row['sample_id']+'.json')).read_text())
        if any(result.get(k)!=v for k,v in identity.items()):raise ValueError('Mixed generated result identities')
        if admitted is not None:
            asset=admitted.get((row['overlay']['asset_id'],row['voice']))
            if asset is None or not same_asset(row,asset):raise ValueError('Sample visual QA not admitted')
        for field in ('source_file','reference_file','microphone_file'):
            if field in row and sha256(a.cohort/row[field])!=row[field.replace('_file','_sha256')]:
                raise ValueError('Frozen benchmark file changed')
        source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True)
        if source['source_id']!=row['source_id']:raise ValueError('Source identity mismatch')
        motion_names=source['turn_motion_names']
        for name in motion_names:
            sid=motion_source_id(data['split_source_namespace'],name)
            if source_hash_u64(sid,data['split_seed'])>=int(data['val_ratio']*(1<<64)):
                raise ValueError(f'Not held out: {sid}')
        names.extend(motion_names)
        with np.load(a.generated/(row['sample_id']+'.npz')) as z:pred=z['absolute']
        with np.load(a.cohort/row['reference_file']) as z:gt=z['absolute'];labels=z['controls']
        n=min(len(pred),len(gt))
        if len(pred)<row['score_end']:raise ValueError('Incomplete motion')
        if 'segments' in row:
            if len(pred)<len(gt):raise ValueError('Incomplete session')
            relation=row['overlay']
            if relation['mode']!='qa':raise ValueError('Missing visual question')
            external=relation.get('external_benchmark')
            if manifest.get('external_assets_sha256'):
                if not external:raise ValueError('Missing external benchmark provenance')
                verify_file(relation['video_path'],external['source_video_sha256'])
                verify_file(relation['visual_features_path'],external['feature_sha256'])
                verify_file(external['review_receipt'],external['review_receipt_sha256'])
                verify_file(relation['question_audio_path'],external['question_audio_sha256'])
                if abs(relation['question_end']-external['question_time'])>1/16000.+1e-8:
                    raise ValueError('External question time was shifted')
            if row['visual_kind']=='video' and relation['camera_duration_seconds']<relation['session_required_seconds']+.2:
                raise ValueError('Video ends before session')
            for segment in row['segments']:
                if segment['eligible']:
                    s,e=segment['score_start'],segment['score_end']
                    if not 0<=s<e<=len(gt) or not (labels[s:e]==1).all():
                        raise ValueError('Invalid source slice')
        rows.append(dict(sample_id=row['sample_id'],motion_names=motion_names,
            source_sha256=sha256(a.cohort/row['source_file']),reference_sha256=sha256(a.cohort/row['reference_file']),
            generated_sha256=sha256(a.generated/(row['sample_id']+'.npz')),
            generated_frames=len(pred),reference_frames=len(gt),scored_full_frames=n,
            unscored_generated_tail=len(pred)-n,unscored_reference_tail=len(gt)-n,
            phase_frames={name:int((labels[:n]==value).sum()) for name,value in [('idle',2),('listen',0),('motion',1)]}))
    if manifest.get('schema')!='hfd_bench_session_v3' and len(names)!=len(set(names)):
        raise ValueError('Repeated motion source in legacy cohort')
    if len({r['source_id'] for r in manifest['rows']})!=len(manifest['rows']):
        raise ValueError('Duplicated complete session')
    ident=json.loads((a.generated/'identity.json').read_text())
    ckpt=checkpoint_from_identity(ident)
    write_json(a.generated.parent/'audit.json',dict(source_split_verified=True,unique_motion_sources=len(set(names)),
        model_sha256=sha256(ckpt/'model.safetensors'),
        formal_rollout_sha256=sha256(ROOT/'src/psi/runtime/swa_rollout.py'),
        fused_manifest_sha256=sha256(ckpt/'hfd_fused_delta_manifest.json'),
        initialization='formal deployment source initial idle state; no GT subsequent states/RTC/duration',
        scope='held-out validation, not separately reserved final test; full multimodal sessions; final-command motion slices',
        source_occurrences=len(names),session_count=len(manifest['rows']),
        external_assets_verified=len(verified_assets),
        rows=rows))

if __name__=='__main__':main()