"""Bounded, reproducible public-synthetic TYGER V2 pilot preparation. Never downloads a complete ~2 GB tar: only explicit HTTP byte ranges. Camera, microphone and enrolled-user data are not inputs to this script. """ from __future__ import annotations import argparse from concurrent.futures import ThreadPoolExecutor import csv import hashlib import io import json import os from pathlib import Path import shutil import subprocess import tarfile import time import numpy as np import requests from scipy.signal import fftconvolve, butter, sosfilt HERE = Path(__file__).resolve().parent CACHE = Path(os.environ.get('ROBO_PILOT_CACHE', str(Path.home()/'.cache'/'robo-pilot'))) REPO = 'laion/laion-voice-profiles-annotated' REVISION = '88ea857d6a6f20839674dfde30936d8fd6b23c62' TEACHER_REVISION = '0f99f2d0ebe89ac095bcc5903c4dd8f72b367286' BASE = f'https://huggingface.co/datasets/{REPO}/resolve/{REVISION}/' SR = 16000 def guard(): for root in (CACHE, HERE): root.mkdir(parents=True, exist_ok=True) if shutil.disk_usage(root).free < 5 * 1024**3: raise RuntimeError(f'Less than 5 GiB free on {root}; stop without deleting user files') total = sum(p.stat().st_size for p in CACHE.rglob('*') if p.is_file()) if total > 1024**3: raise RuntimeError('Pilot source-cache hard budget 1 GiB reached') def get(url, **kwargs): for attempt in range(5): try: r = requests.get(url, timeout=90, **kwargs) r.raise_for_status() return r except requests.RequestException: if attempt == 4: raise time.sleep(2**attempt) def atomic_json(path, value): path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + '.tmp') temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False, allow_nan=False)) temporary.replace(path) def pcm(mp3): result = subprocess.run(['ffmpeg', '-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', '-ac', '1', '-ar', str(SR), '-f', 'f32le', 'pipe:1'], input=mp3, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) return np.frombuffer(result.stdout, ' len(content): break if member.name.endswith('.mp3'): audio[member.name[:-4]] = tf.extractfile(member).read() elif member.name.endswith('.json'): record = json.load(tf.extractfile(member)) uid = member.name[:-5] if uid not in audio: continue if not uid.startswith(row['voice'] + '__'): raise ValueError(f'Catalog shard mapping mismatch for {row["voice"]}: {uid}') meta = record.get('src_meta', {}) quality = record.get('quality', {}) if not (6 <= record.get('dur_s', 0) <= 25 and len(record.get('words', [])) >= 8): continue if float(meta.get('wer', 1)) > .3 or record.get('n_bursts', 0) > 1: continue ranking = float(quality.get('overall_quality', 0)) + float(meta.get('spk_sim', 0)) choices.append({'uid': uid, 'mp3': audio.pop(uid), 'record': record, 'source_tar': path, 'source_range_bytes': length, 'ranking': ranking, 'condition_shard': local_shard}) except (tarfile.ReadError, EOFError): pass # The bounded prefix intentionally ends before the archive does. # Two clean long enrollment clips and four different prediction clips, spanning both shards. selected = [] for shard in (1, 2): pool = sorted([x for x in choices if x['condition_shard'] == shard], key=lambda x: (-x['ranking'], -x['record']['dur_s'], x['uid'])) selected += pool[:3] if len(selected) < 4: raise RuntimeError(f'Insufficient usable clips in bounded prefixes for {row["voice"]}: {len(selected)}') clips = [] # Reserve one enrollment utterance from each condition; never use it in a soundscape. for i, x in enumerate(selected): name = hashlib.sha256(x['uid'].encode()).hexdigest()[:16] + '.mp3' (out / name).write_bytes(x['mp3']) record = x['record'] clips.append({'uid': x['uid'], 'path': str(out / name), 'role': 'teacher' if i in (0, 3) else 'prediction', 'duration_s': record['dur_s'], 'words': record['words'], 'bursts': record.get('bursts'), 'lang': record.get('lang'), 'condition': x['uid'].split('__')[2:-1], 'source_tar': x['source_tar'], 'downloaded_range_bytes': x['source_range_bytes'], 'sha256': hashlib.sha256(x['mp3']).hexdigest(), 'published_wer': record.get('src_meta', {}).get('wer'), 'published_spk_sim': record.get('src_meta', {}).get('spk_sim')}) atomic_json(manifest, clips) print(json.dumps({'source_ready': row['voice'], 'split': row['split'], 'clips': len(clips)}), flush=True) return {**row, 'clips': clips} def teachers(speakers, output_root=None): import torch from speechbrain.inference.speaker import EncoderClassifier torch.set_num_threads(6) model_root = os.environ.get('ROBO_ECAPA_ROOT', '/mnt/nvme/hf_cache/hub/models--speechbrain--spkrec-ecapa-voxceleb/snapshots/' + TEACHER_REVISION) if not Path(model_root).exists(): from huggingface_hub import snapshot_download model_root = snapshot_download('speechbrain/spkrec-ecapa-voxceleb', revision=TEACHER_REVISION, cache_dir=str(CACHE/'hub'), token=False, allow_patterns=['hyperparams.yaml', '*.ckpt', 'label_encoder.txt']) model = EncoderClassifier.from_hparams(source=model_root, overrides={'pretrained_path': model_root}, savedir=str(CACHE / 'ecapa'), run_opts={'device': 'cpu'}) output_root = Path(output_root or HERE/'pilot') output_root.mkdir(parents=True, exist_ok=True) vectors, details, probe_vectors, probe_speakers, probe_uids = [], [], [], [], [] with torch.inference_mode(): for row in speakers: v = [] for clip in row['clips']: audio = pcm(Path(clip['path']).read_bytes()) # Teacher sees complete long clean utterances, not 320-ms regression windows. e = model.encode_batch(torch.from_numpy(audio)[None]).reshape(192).float().numpy() e /= max(np.linalg.norm(e), 1e-8) if clip['role'] == 'teacher': v.append(e) else: probe_vectors.append(e) probe_speakers.append(row['speaker_idx']) probe_uids.append(clip['uid']) avg = np.mean(v, axis=0) avg /= max(np.linalg.norm(avg), 1e-8) vectors.append(avg) details.append({'speaker_idx': row['speaker_idx'], 'voice': row['voice'], 'teacher_pair_cosine': float(np.dot(v[0], v[1])) if len(v) > 1 else None, 'teacher_utterances': len(v), 'teacher_seconds': sum(c['duration_s'] for c in row['clips'] if c['role'] == 'teacher')}) print(json.dumps({'teacher_ready': row['voice'], **details[-1]}), flush=True) # Fixed public profile reference only: never user-uploaded or microphone audio. public_url = 'http://127.0.0.1:8792/api/voices/reference/emolia_c1699' reference = get(public_url).content audio = pcm(reference) e = model.encode_batch(torch.from_numpy(audio)[None]).reshape(192).float().numpy() e /= max(np.linalg.norm(e), 1e-8) artifact = HERE / 'artifacts' / 'robo_reference.npz' artifact.parent.mkdir(parents=True, exist_ok=True) np.savez_compressed(artifact, embedding=e, source_url=public_url, sha256=hashlib.sha256(reference).hexdigest(), voice='emolia_c1699', duration_s=len(audio)/SR, teacher_revision=TEACHER_REVISION) np.save(output_root / 'teacher_embeddings.npy', np.stack(vectors).astype(np.float32)) np.savez_compressed(output_root / 'probe_teacher_embeddings.npz', embeddings=np.stack(probe_vectors).astype(np.float32), speaker_idx=np.asarray(probe_speakers, np.int16), uid=np.asarray(probe_uids)) atomic_json(output_root / 'teacher_quality.json', details) def source_signal(clip): audio = pcm(Path(clip['path']).read_bytes()) active = np.zeros(len(audio), bool) for w in clip['words']: start = max(0, int((w['s'] - .035) * SR)) end = min(len(audio), int((w['e'] + .06) * SR)) active[start:end] = True return audio, active def conversation(speakers, anchor, seed, duration=24): rng = np.random.default_rng(seed) same_split = [s for s in speakers if s['split'] == anchor['split']] count = int(rng.integers(1, min(5, len(same_split)) + 1)) others = [x for x in same_split if x['speaker_idx'] != anchor['speaker_idx']] selected = [anchor] + list(rng.choice(others, size=min(count-1, len(others)), replace=False)) n = duration * SR tracks = [np.zeros(n, np.float32) for _ in selected] active_tracks = [np.zeros(n, bool) for _ in selected] events = [] # One conversation clock: mostly alternating turns, with genuine interruptions # rather than placing every speaker continuously on top of every other one. t = float(rng.uniform(.1, 1.3)) for turn in range(12): who = turn % len(selected) if turn < len(selected) else int(rng.integers(len(selected))) person = selected[who] clips = [x for x in person['clips'] if x['role'] == 'prediction'] clip = clips[int(rng.integers(len(clips)))] audio, speech = source_signal(clip) length = min(len(audio), int(rng.uniform(2.0, 4.5) * SR)) src = int(rng.integers(max(1, len(audio) - length + 1))) offset = int(t * SR) take = min(length, n - offset) if take <= 0: break gain = float(rng.uniform(.6, 1.1)) tracks[who][offset:offset+take] += audio[src:src+take] * gain active_tracks[who][offset:offset+take] |= speech[src:src+take] events.append({'speaker_idx': person['speaker_idx'], 'uid': clip['uid'], 'onset_s': offset/SR, 'offset_s': (offset+take)/SR, 'source_start_s': src/SR}) # All boundaries intentionally sample arbitrary positions, not 320-ms multiples. t += take / SR + float(rng.uniform(-.4, .9)) clean = np.sum(tracks, axis=0) scenario = ['clean', 'noise', 'reverb', 'telephone', 'echo'][seed % 5] audio = clean.copy() if scenario == 'noise': noise = rng.normal(size=n).astype(np.float32) noise = sosfilt(butter(2, 3000, fs=SR, output='sos'), noise) rms = max(float(np.sqrt(np.mean(clean**2))), .01) snr_db = float(rng.uniform(8, 25)) audio += noise / max(np.std(noise), 1e-8) * rms * 10 ** (-snr_db/20) elif scenario == 'reverb': ir = np.zeros(int(.35*SR), np.float32) ir[0] = 1 for delay, gain in [(.017,.25),(.043,.18),(.081,.12),(.14,.07)]: ir[int((delay+rng.uniform(0,.003))*SR)] = gain audio = fftconvolve(audio, ir)[:n].astype(np.float32) elif scenario == 'telephone': audio = sosfilt(butter(3, [300, 3400], fs=SR, btype='bandpass', output='sos'), audio).astype(np.float32) audio = np.round(np.clip(audio, -1, 1) * 127) / 127 playback = np.zeros(n, np.float32) if scenario == 'echo': # Public synthetic source only; distinct reference, exact known causal delay. playback = tracks[0].copy() delay = int(rng.uniform(.025, .15) * SR) audio[delay:] += playback[:-delay] * float(rng.uniform(.03, .15)) events.append({'type': 'agent_echo', 'delay_s': delay/SR, 'note': 'Reference is public synthetic audio; identity is one scenario speaker in pilot'}) peak = max(float(np.max(np.abs(audio))), 1) audio = audio / peak * .95 activity = np.stack(active_tracks) return audio.astype('= 2).mean() best = int(np.argmax(occupancy)) fraction[f] = occupancy[best] if overlap >= .2 or np.sum(occupancy > .3) >= 2: ids[f], ambiguity[f] = -2, True elif occupancy[best] >= .5: ids[f] = selected[best]['speaker_idx'] valid = (ids >= 0) & ~ambiguity np.savez_compressed(path, features=x.astype(np.float16), frame_end_s=end_s, speaker_idx=ids, valid_mask=valid, speech_fraction=fraction, ambiguity=ambiguity) record = {'path': str(path.relative_to(output_root)), 'split': anchor['split'], 'speaker_ids': [s['speaker_idx'] for s in selected], 'seed': seed, 'scenario': scenario, 'duration_s': len(audio)/SR, 'frames': len(x), 'valid_frames': int(valid.sum()), 'overlap_frames': int(ambiguity.sum()), 'silence_frames': int((ids == -1).sum()), 'feature_seconds': time.perf_counter()-start, 'events': events, 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'playback_reference_note': 'Exact procedural waveform available by replaying seed; encoder pilot uses microphone only; turn-taking unchanged'} atomic_json(metadata_path, record) records.append(record) atomic_json(output_root/'manifest.json', manifest(speakers, records)) print(json.dumps({'conversation_ready': str(path.name), 'frames': len(x), 'valid': int(valid.sum()), 'feature_seconds': record['feature_seconds']}), flush=True) atomic_json(output_root/'manifest.json', manifest(speakers, records)) def manifest(speakers, records): return {'schema_version': 1, 'dataset_repo': REPO, 'dataset_revision': REVISION, 'source_license': 'CC-BY-4.0', 'attribution': 'Christoph Schuhmann and LAION', 'teacher_repo': 'speechbrain/spkrec-ecapa-voxceleb', 'teacher_revision': TEACHER_REVISION, 'encoder_layer': 15, 'feature_dim': 1024, 'frame_ms': 80, 'decision_hop_ms': 320, 'sample_rate': SR, 'augmentation_version': 'procedural-v1', 'numpy_version': np.__version__, 'activity_rule': {'word_padding_before_s': .035, 'word_padding_after_s': .06, 'dominant_occupancy_min': .5, 'overlap_occupancy_min': .2}, 'teachers': 'teacher_embeddings.npy', 'speakers': speakers, 'conversations': records, 'limitations': ['Pilot uses two acting-condition shard prefixes per identity, not full emotion matrix', 'Teacher IDs are synthetic profile labels, not verified biological identities', 'Source forced alignments and model quality scores may be imperfect', 'Procedural impulse responses/noise only, no third-party music', 'Echo source shares one conversation identity in pilot; distinct Robo self-voice eval remains required', 'No private microphone recordings, enrollment embeddings, or real camera frames included']} def main(): p = argparse.ArgumentParser() p.add_argument('--stage', choices=['sources','teacher','features','all'], default='all') p.add_argument('--train-speakers', type=int, default=24) p.add_argument('--val-speakers', type=int, default=4) p.add_argument('--test-speakers', type=int, default=4) p.add_argument('--repeats', type=int, default=2) args = p.parse_args() started = time.time() guard() plan = profile_plan(args.train_speakers, args.val_speakers, args.test_speakers) if args.stage in ('sources','all'): with ThreadPoolExecutor(max_workers=3) as pool: speakers = list(pool.map(source_voice, plan)) atomic_json(HERE/'pilot'/'speakers.json', speakers) else: speakers = json.loads((HERE/'pilot'/'speakers.json').read_text()) if args.stage in ('teacher','all'): teachers(speakers) if args.stage in ('features','all'): extract(speakers, args.repeats) print(json.dumps({'stage_complete': args.stage, 'elapsed_s': time.time()-started}), flush=True) if __name__ == '__main__': main()