File size: 5,277 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""CPU-only donor admission and stratified current-pipeline replay.
Use --headers-only for the entire file/split inventory, or a saved --run-config
for current motion/RTC/overlay compatibility. No donor code is imported.
Outputs go to --output (outside the repository), never into the donor package.
"""
import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import wave
import numpy as np
from psi.data.seed.third_party_dataset import read_recipes, contained, file_hash
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--package', type=Path, required=True)
p.add_argument('--output', type=Path, required=True)
p.add_argument('--run-config', type=Path)
p.add_argument('--per-cell', type=int, default=3)
p.add_argument('--workers', type=int, default=16)
p.add_argument('--headers-only', action='store_true')
p.add_argument('--hash-audio', action='store_true')
a = p.parse_args()
root = (a.package/'production').resolve()
selections = {s: a.package/'training_scale_all_commands'/n for s,n in
[('train','selection.json'),('val','validation.json')]}
recipes = {s: read_recipes(root,path,s) for s,path in selections.items()}
primitive = {s:{t['motion_filename'] for r in rs for t in r['turns']} for s,rs in recipes.items()}
audio_ids = {s:{r['audio']['sha256'] for r in rs} for s,rs in recipes.items()}
assert not primitive['train'] & primitive['val'], 'Motion train/val leakage'
assert not audio_ids['train'] & audio_ids['val'], 'Microphone train/val leakage'
report = {}
def check(row):
path=contained(root,row['audio']['path'])
with wave.open(str(path),'rb') as f:
assert (f.getframerate(),f.getnchannels(),f.getsampwidth(),f.getnframes()) == (
16000,1,2,row['audio']['sample_count']), str(path)
if a.hash_audio:
assert file_hash(path)==row['audio']['sha256'], str(path)
return path.stat().st_size + contained(root,row['timeline']['path']).stat().st_size
for split,rs in recipes.items():
with ThreadPoolExecutor(max_workers=a.workers) as pool:
sizes=list(pool.map(check,rs))
report[split]=dict(sessions=len(rs),hours=sum(r['audio']['duration_seconds'] for r in rs)/3600,
bytes=sum(sizes),profiles=dict(Counter(r['session_type'] for r in rs)),
voices=dict(Counter(r['session_voice_name'] for r in rs)),
turns=dict(Counter(len(r['turns']) for r in rs)),
duration_quantiles=np.quantile([r['audio']['duration_seconds'] for r in rs],[0,.5,.9,.99,1]).tolist())
print('INVENTORY',split,report[split],flush=True)
if not a.headers_only:
from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig
from psi.config.transform import SeedPosttrainRepackTransform
if not a.run_config:
p.error('--run-config required for replay')
data=json.loads(a.run_config.read_text())['data']
config=FullDuplexAudioCommandDataConfig.model_validate(data)
config.transform.repack=SeedPosttrainRepackTransform.model_validate(data['transform']['repack'])
config.third_party_root=str(root)
config.third_party_train_selection=str(selections['train'])
config.third_party_val_selection=str(selections['val'])
for split in recipes:
dataset=config(split=split).raw_dataset
overlay=dataset if hasattr(dataset,'compose') else None
mixed=dataset.raw_dataset if overlay else dataset
cells=Counter(); results=[]
for i,r in enumerate(mixed.recipes):
key=(r['session_type'],r['background_language'])
if cells[key]>=a.per_cell:continue
cells[key]+=1
source=mixed[len(mixed.raw_dataset)+i]
assert all(np.isfinite(s['states']).all() and np.isfinite(s['actions']).all() for s in source['steps'])
# Root40 repack on first, boundary and final windows.
indices={0,len(source['steps'])-1,*[s['step_index'] for s in source['steps'] if s['control_transition']]}
for j in indices:
transformed=config.transform.repack(dict(source['steps'][j]))
assert transformed['actions'].shape[-1]==40
assert np.isfinite(transformed['actions']).all()
composed=overlay.compose(source) if overlay else source
result=dict(session_id=r['session_id'],profile=key[0],background_language=key[1],
voice=source['session_voice_name'],steps=len(source['steps']),
overlay=composed.get('multimodal_overlay',{}))
results.append(result);print('REPLAY',split,r['session_id'],key,flush=True)
report[split]['replayed']=results
a.output.parent.mkdir(parents=True,exist_ok=True)
a.output.write_text(json.dumps(report,ensure_ascii=False,indent=2))
print('PASSED',a.output,flush=True)
if __name__=='__main__':main()
|