Download scripts/data/audit_third_party.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 5.28 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_third_party.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/data/audit_third_party.py
-
curl -L -o audit_third_party.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/data/audit_third_party.py
5.28 kB
| #!/usr/bin/env python3 | |
| """CPU-only donor admission and stratified current-pipeline replay. | |
| Use --headers-only for the entire file/split inventory, or a saved --run-config | |
| for current motion/RTC/overlay compatibility. No donor code is imported. | |
| Outputs go to --output (outside the repository), never into the donor package. | |
| """ | |
| import argparse | |
| from collections import Counter, defaultdict | |
| from concurrent.futures import ThreadPoolExecutor | |
| import json | |
| from pathlib import Path | |
| import wave | |
| import numpy as np | |
| from psi.data.seed.third_party_dataset import read_recipes, contained, file_hash | |
| def main(): | |
| p = argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--package', type=Path, required=True) | |
| p.add_argument('--output', type=Path, required=True) | |
| p.add_argument('--run-config', type=Path) | |
| p.add_argument('--per-cell', type=int, default=3) | |
| p.add_argument('--workers', type=int, default=16) | |
| p.add_argument('--headers-only', action='store_true') | |
| p.add_argument('--hash-audio', action='store_true') | |
| a = p.parse_args() | |
| root = (a.package/'production').resolve() | |
| selections = {s: a.package/'training_scale_all_commands'/n for s,n in | |
| [('train','selection.json'),('val','validation.json')]} | |
| recipes = {s: read_recipes(root,path,s) for s,path in selections.items()} | |
| primitive = {s:{t['motion_filename'] for r in rs for t in r['turns']} for s,rs in recipes.items()} | |
| audio_ids = {s:{r['audio']['sha256'] for r in rs} for s,rs in recipes.items()} | |
| assert not primitive['train'] & primitive['val'], 'Motion train/val leakage' | |
| assert not audio_ids['train'] & audio_ids['val'], 'Microphone train/val leakage' | |
| report = {} | |
| def check(row): | |
| path=contained(root,row['audio']['path']) | |
| with wave.open(str(path),'rb') as f: | |
| assert (f.getframerate(),f.getnchannels(),f.getsampwidth(),f.getnframes()) == ( | |
| 16000,1,2,row['audio']['sample_count']), str(path) | |
| if a.hash_audio: | |
| assert file_hash(path)==row['audio']['sha256'], str(path) | |
| return path.stat().st_size + contained(root,row['timeline']['path']).stat().st_size | |
| for split,rs in recipes.items(): | |
| with ThreadPoolExecutor(max_workers=a.workers) as pool: | |
| sizes=list(pool.map(check,rs)) | |
| report[split]=dict(sessions=len(rs),hours=sum(r['audio']['duration_seconds'] for r in rs)/3600, | |
| bytes=sum(sizes),profiles=dict(Counter(r['session_type'] for r in rs)), | |
| voices=dict(Counter(r['session_voice_name'] for r in rs)), | |
| turns=dict(Counter(len(r['turns']) for r in rs)), | |
| duration_quantiles=np.quantile([r['audio']['duration_seconds'] for r in rs],[0,.5,.9,.99,1]).tolist()) | |
| print('INVENTORY',split,report[split],flush=True) | |
| if not a.headers_only: | |
| from psi.config.data_full_duplex_audio_command import FullDuplexAudioCommandDataConfig | |
| from psi.config.transform import SeedPosttrainRepackTransform | |
| if not a.run_config: | |
| p.error('--run-config required for replay') | |
| data=json.loads(a.run_config.read_text())['data'] | |
| config=FullDuplexAudioCommandDataConfig.model_validate(data) | |
| config.transform.repack=SeedPosttrainRepackTransform.model_validate(data['transform']['repack']) | |
| config.third_party_root=str(root) | |
| config.third_party_train_selection=str(selections['train']) | |
| config.third_party_val_selection=str(selections['val']) | |
| for split in recipes: | |
| dataset=config(split=split).raw_dataset | |
| overlay=dataset if hasattr(dataset,'compose') else None | |
| mixed=dataset.raw_dataset if overlay else dataset | |
| cells=Counter(); results=[] | |
| for i,r in enumerate(mixed.recipes): | |
| key=(r['session_type'],r['background_language']) | |
| if cells[key]>=a.per_cell:continue | |
| cells[key]+=1 | |
| source=mixed[len(mixed.raw_dataset)+i] | |
| assert all(np.isfinite(s['states']).all() and np.isfinite(s['actions']).all() for s in source['steps']) | |
| # Root40 repack on first, boundary and final windows. | |
| indices={0,len(source['steps'])-1,*[s['step_index'] for s in source['steps'] if s['control_transition']]} | |
| for j in indices: | |
| transformed=config.transform.repack(dict(source['steps'][j])) | |
| assert transformed['actions'].shape[-1]==40 | |
| assert np.isfinite(transformed['actions']).all() | |
| composed=overlay.compose(source) if overlay else source | |
| result=dict(session_id=r['session_id'],profile=key[0],background_language=key[1], | |
| voice=source['session_voice_name'],steps=len(source['steps']), | |
| overlay=composed.get('multimodal_overlay',{})) | |
| results.append(result);print('REPLAY',split,r['session_id'],key,flush=True) | |
| report[split]['replayed']=results | |
| a.output.parent.mkdir(parents=True,exist_ok=True) | |
| a.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)) | |
| print('PASSED',a.output,flush=True) | |
| if __name__=='__main__':main() | |