Download benchmark/audit.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 6.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/audit.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/audit.py
-
curl -L -o audit.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/audit.py
6.8 kB
| """Check HFD source-level split, checkpoint identity and episode coverage.""" | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| from common import ROOT, WORK, write_json, sha256 | |
| sys.path.insert(0,str(ROOT/'src')) | |
| import numpy as np | |
| import torch | |
| from psi.data.seed.source_split import motion_source_id, source_hash_u64 | |
| def checkpoint_from_identity(identity): | |
| """Generation may use a new run with the same frozen evaluation cohort.""" | |
| run = Path(identity['run']) | |
| if sha256(run/'run_config.json') != identity['config_sha256']: | |
| raise ValueError('Generation run configuration changed') | |
| return run/'checkpoints'/f"ckpt_{int(identity['step'])}" | |
| def main(): | |
| p=argparse.ArgumentParser() | |
| p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048') | |
| p.add_argument('--generated',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated') | |
| a=p.parse_args() | |
| manifest=json.loads((a.cohort/'manifest.json').read_text()) | |
| identity=json.loads((a.generated/'identity.json').read_text()) | |
| if identity['cohort_sha256']!=sha256(a.cohort/'manifest.json'): | |
| raise ValueError('Generated outputs do not match frozen cohort') | |
| admitted=None | |
| if manifest.get('external_assets_path'): | |
| asset_path=Path(manifest['external_assets_path']) | |
| if sha256(asset_path)!=manifest['external_assets_sha256']:raise ValueError('Admission manifest changed') | |
| from session_overlay import reviewed | |
| from revision import same_asset | |
| assets=json.loads(asset_path.read_text())['assets'] | |
| if not all(reviewed(r) for r in assets):raise ValueError('External review no longer valid') | |
| admitted={(r['asset_id'],r['voice']):r for r in assets} | |
| run=Path(manifest['run']);config=json.loads((run/'run_config.json').read_text()) | |
| data=config['data'];rows=[];names=[];verified_assets={} | |
| def verify_file(path, expected): | |
| if not expected:raise ValueError('Missing external asset hash: '+str(path)) | |
| if path not in verified_assets:verified_assets[path]=sha256(path) | |
| if verified_assets[path]!=expected:raise ValueError('External asset changed: '+str(path)) | |
| for row in manifest['rows']: | |
| result=json.loads((a.generated/(row['sample_id']+'.json')).read_text()) | |
| if any(result.get(k)!=v for k,v in identity.items()):raise ValueError('Mixed generated result identities') | |
| if admitted is not None: | |
| asset=admitted.get((row['overlay']['asset_id'],row['voice'])) | |
| if asset is None or not same_asset(row,asset):raise ValueError('Sample visual QA not admitted') | |
| for field in ('source_file','reference_file','microphone_file'): | |
| if field in row and sha256(a.cohort/row[field])!=row[field.replace('_file','_sha256')]: | |
| raise ValueError('Frozen benchmark file changed') | |
| source=torch.load(a.cohort/row['source_file'],map_location='cpu',weights_only=False,mmap=True) | |
| if source['source_id']!=row['source_id']:raise ValueError('Source identity mismatch') | |
| motion_names=source['turn_motion_names'] | |
| for name in motion_names: | |
| sid=motion_source_id(data['split_source_namespace'],name) | |
| if source_hash_u64(sid,data['split_seed'])>=int(data['val_ratio']*(1<<64)): | |
| raise ValueError(f'Not held out: {sid}') | |
| names.extend(motion_names) | |
| with np.load(a.generated/(row['sample_id']+'.npz')) as z:pred=z['absolute'] | |
| with np.load(a.cohort/row['reference_file']) as z:gt=z['absolute'];labels=z['controls'] | |
| n=min(len(pred),len(gt)) | |
| if len(pred)<row['score_end']:raise ValueError('Incomplete motion') | |
| if 'segments' in row: | |
| if len(pred)<len(gt):raise ValueError('Incomplete session') | |
| relation=row['overlay'] | |
| if relation['mode']!='qa':raise ValueError('Missing visual question') | |
| external=relation.get('external_benchmark') | |
| if manifest.get('external_assets_sha256'): | |
| if not external:raise ValueError('Missing external benchmark provenance') | |
| verify_file(relation['video_path'],external['source_video_sha256']) | |
| verify_file(relation['visual_features_path'],external['feature_sha256']) | |
| verify_file(external['review_receipt'],external['review_receipt_sha256']) | |
| verify_file(relation['question_audio_path'],external['question_audio_sha256']) | |
| if abs(relation['question_end']-external['question_time'])>1/16000.+1e-8: | |
| raise ValueError('External question time was shifted') | |
| if row['visual_kind']=='video' and relation['camera_duration_seconds']<relation['session_required_seconds']+.2: | |
| raise ValueError('Video ends before session') | |
| for segment in row['segments']: | |
| if segment['eligible']: | |
| s,e=segment['score_start'],segment['score_end'] | |
| if not 0<=s<e<=len(gt) or not (labels[s:e]==1).all(): | |
| raise ValueError('Invalid source slice') | |
| rows.append(dict(sample_id=row['sample_id'],motion_names=motion_names, | |
| source_sha256=sha256(a.cohort/row['source_file']),reference_sha256=sha256(a.cohort/row['reference_file']), | |
| generated_sha256=sha256(a.generated/(row['sample_id']+'.npz')), | |
| generated_frames=len(pred),reference_frames=len(gt),scored_full_frames=n, | |
| unscored_generated_tail=len(pred)-n,unscored_reference_tail=len(gt)-n, | |
| phase_frames={name:int((labels[:n]==value).sum()) for name,value in [('idle',2),('listen',0),('motion',1)]})) | |
| if manifest.get('schema')!='hfd_bench_session_v3' and len(names)!=len(set(names)): | |
| raise ValueError('Repeated motion source in legacy cohort') | |
| if len({r['source_id'] for r in manifest['rows']})!=len(manifest['rows']): | |
| raise ValueError('Duplicated complete session') | |
| ident=json.loads((a.generated/'identity.json').read_text()) | |
| ckpt=checkpoint_from_identity(ident) | |
| write_json(a.generated.parent/'audit.json',dict(source_split_verified=True,unique_motion_sources=len(set(names)), | |
| model_sha256=sha256(ckpt/'model.safetensors'), | |
| formal_rollout_sha256=sha256(ROOT/'src/psi/runtime/swa_rollout.py'), | |
| fused_manifest_sha256=sha256(ckpt/'hfd_fused_delta_manifest.json'), | |
| initialization='formal deployment source initial idle state; no GT subsequent states/RTC/duration', | |
| scope='held-out validation, not separately reserved final test; full multimodal sessions; final-command motion slices', | |
| source_occurrences=len(names),session_count=len(manifest['rows']), | |
| external_assets_verified=len(verified_assets), | |
| rows=rows)) | |
| if __name__=='__main__':main() | |