#!/usr/bin/env python3 """CPU-only donor admission and stratified current-pipeline replay. Use --headers-only for the entire file/split inventory, or a saved --run-config for current motion/RTC/overlay compatibility. No donor code is imported. Outputs go to --output (outside the repository), never into the donor package. """ import argparse from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path import wave import numpy as np from psi.data.seed.third_party_dataset import read_recipes, contained, file_hash def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--package', type=Path, required=True) p.add_argument('--output', type=Path, required=True) p.add_argument('--run-config', type=Path) p.add_argument('--per-cell', type=int, default=3) p.add_argument('--workers', type=int, default=16) p.add_argument('--headers-only', action='store_true') p.add_argument('--hash-audio', action='store_true') a = p.parse_args() root = (a.package/'production').resolve() selections = {s: a.package/'training_scale_all_commands'/n for s,n in [('train','selection.json'),('val','validation.json')]} recipes = {s: read_recipes(root,path,s) for s,path in selections.items()} primitive = {s:{t['motion_filename'] for r in rs for t in r['turns']} for s,rs in recipes.items()} audio_ids = {s:{r['audio']['sha256'] for r in rs} for s,rs in recipes.items()} assert not primitive['train'] & primitive['val'], 'Motion train/val leakage' assert not audio_ids['train'] & audio_ids['val'], 'Microphone train/val leakage' report = {} def check(row): path=contained(root,row['audio']['path']) with wave.open(str(path),'rb') as f: assert (f.getframerate(),f.getnchannels(),f.getsampwidth(),f.getnframes()) == ( 16000,1,2,row['audio']['sample_count']), str(path) if a.hash_audio: assert file_hash(path)==row['audio']['sha256'], str(path) return path.stat().st_size + contained(root,row['timeline']['path']).stat().st_size for split,rs in recipes.items(): with ThreadPoolExecutor(max_workers=a.workers) as pool: sizes=list(pool.map(check,rs)) report[split]=dict(sessions=len(rs),hours=sum(r['audio']['duration_seconds'] for r in rs)/3600, bytes=sum(sizes),profiles=dict(Counter(r['session_type'] for r in rs)), voices=dict(Counter(r['session_voice_name'] for r in rs)), turns=dict(Counter(len(r['turns']) for r in rs)), duration_quantiles=np.quantile([r['audio']['duration_seconds'] for r in rs],[0,.5,.9,.99,1]).tolist()) print('INVENTORY',split,report[split],flush=True) if not a.headers_only: from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig from psi.config.transform import SeedPosttrainRepackTransform if not a.run_config: p.error('--run-config required for replay') data=json.loads(a.run_config.read_text())['data'] config=FullDuplexAudioCommandDataConfig.model_validate(data) config.transform.repack=SeedPosttrainRepackTransform.model_validate(data['transform']['repack']) config.third_party_root=str(root) config.third_party_train_selection=str(selections['train']) config.third_party_val_selection=str(selections['val']) for split in recipes: dataset=config(split=split).raw_dataset overlay=dataset if hasattr(dataset,'compose') else None mixed=dataset.raw_dataset if overlay else dataset cells=Counter(); results=[] for i,r in enumerate(mixed.recipes): key=(r['session_type'],r['background_language']) if cells[key]>=a.per_cell:continue cells[key]+=1 source=mixed[len(mixed.raw_dataset)+i] assert all(np.isfinite(s['states']).all() and np.isfinite(s['actions']).all() for s in source['steps']) # Root40 repack on first, boundary and final windows. indices={0,len(source['steps'])-1,*[s['step_index'] for s in source['steps'] if s['control_transition']]} for j in indices: transformed=config.transform.repack(dict(source['steps'][j])) assert transformed['actions'].shape[-1]==40 assert np.isfinite(transformed['actions']).all() composed=overlay.compose(source) if overlay else source result=dict(session_id=r['session_id'],profile=key[0],background_language=key[1], voice=source['session_voice_name'],steps=len(source['steps']), overlay=composed.get('multimodal_overlay',{})) results.append(result);print('REPLAY',split,r['session_id'],key,flush=True) report[split]['replayed']=results a.output.parent.mkdir(parents=True,exist_ok=True) a.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)) print('PASSED',a.output,flush=True) if __name__=='__main__':main()