"""Streaming, restartable audio preparation for the pinned three-corpus study.""" from collections import Counter, defaultdict import gc import hashlib import json from pathlib import Path import shutil import datasets from huggingface_hub import hf_hub_download import numpy as np import pyarrow.parquet as pq from transformers import AutoFeatureExtractor from coverage_vocab import coverage_ids from full_data import encode_characters, file_hash from kathbath_data import decode_audio from multicorpus_protocol import text_hash from prototype import emit, save_json from text_utils import minimum_ctc_frames, normalize def load_mel(row): if row['mel_path'].endswith('.npz'): with np.load(row['mel_path'],allow_pickle=False) as value: return value['mel'].astype(np.float32) return np.load(row['mel_path'],allow_pickle=False).astype(np.float32,copy=False) def prepare(cfg,tokenizer,output): synthetic_path=Path(cfg['synthetic_manifest']) summary_path=synthetic_path.parent/'summary.json' if not summary_path.exists(): raise ValueError('Synthetic baseline is incomplete; do not train against a partial generation manifest') synthesis_summary=json.loads(summary_path.read_text()) synthetic_rows=[json.loads(line) for line in synthetic_path.read_text().splitlines()] if len(synthetic_rows)!=synthesis_summary['generated_clips']: raise ValueError('Synthetic manifest count differs from its completed baseline summary') prompts=json.loads(Path(cfg['prompt_manifest']).read_text()) prompt_text={f'{category}-{i+1:02d}':text for category,texts in prompts['categories'].items() for i,text in enumerate(texts)} if len({r['id'] for r in synthetic_rows})!=len(synthetic_rows) or any(prompt_text.get(r['prompt_id'])!=r['text'] for r in synthetic_rows): raise ValueError('Synthetic prompt identity mismatch') manifest_path=Path(cfg['selection_manifest']) selection=json.loads(manifest_path.read_text()) contract={'selection_sha256':file_hash(manifest_path),'model_revision':cfg['model_revision'], 'protocol':'multicorpus-fp16-compressed-v1','synthetic_manifest_sha256':file_hash(cfg['synthetic_manifest']), 'preparation_code_sha256':{name:file_hash(name) for name in ['multicorpus_data.py','text_utils.py','coverage_vocab.py','full_data.py','kathbath_data.py']}} key=hashlib.sha256(json.dumps(contract,sort_keys=True).encode()).hexdigest()[:20] cache=Path(cfg['multicorpus_cache'])/key;cache.mkdir(parents=True,exist_ok=True) save_json(output/'cache-contract.json',{**contract,'path':str(cache)}) ids=coverage_ids(tokenizer.get_vocab(),selection['training_characters']) known=set(ids) allowed={p:i for p,i in tokenizer.get_vocab().items() if i in known} extractor=AutoFeatureExtractor.from_pretrained(cfg['model_id'],revision=cfg['model_revision']) data={k:[] for k in selection['splits']} data.update(fleurs_validation=[],synthetic_diagnostic=[]) rejected=[];seen_train=set();protected_audio=set();protected_pcm=set();protected_text=set() prior=json.loads(Path('data/prior_evaluation_manifest.json').read_text()) for rows in (prior['validation'],prior['previously_inspected_test']): for row in rows: protected_audio.add(row['audio_sha256']) if row.get('pcm_sha256'):protected_pcm.add(row['pcm_sha256']) protected_text.add(text_hash(row['text'])) mel_dir=cache/'mel';mel_dir.mkdir(exist_ok=True) prepared_path=cache/'prepared.json' if prepared_path.exists(): prepared=json.loads(prepared_path.read_text()) for rows in prepared['data'].values(): for row in rows: if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']: raise ValueError('Prepared cache checksum mismatch') emit('prepared_cache_reused',path=str(prepared_path)) for name,value in [('manifest.json',prepared['data']),('data-audit.json',prepared['audit']),('rejected-recordings.json',prepared['rejected'])]:save_json(output/name,value) return prepared['data'],ids,prepared['audit'],cache def accept(source,blob,split): text=normalize(source['text']) if not text:raise ValueError('Empty selected transcript') raw=hashlib.sha256(blob).hexdigest() wav,decoder=decode_audio(blob);pcm=hashlib.sha256(wav.tobytes()).hexdigest() reason=None if split=='train': if raw in protected_audio or pcm in protected_pcm or text_hash(text) in protected_text:reason='heldout_overlap' elif raw in seen_train or pcm in seen_train:reason='duplicate_training_audio' elif not 0.1<=len(wav)/16000<=30:reason='duration_outside_0_1_30s' tokens,target=encode_characters(text,tokenizer,ids,allowed) if reason: rejected.append({'id':source['id'],'source':source['source'],'reason':reason});return path=mel_dir/(raw+'.npz') if path.exists(): with np.load(path,allow_pickle=False) as saved:mel=saved['mel'] else: batch=extractor([wav],sampling_rate=16000,return_tensors='np',truncation=False,max_length=None) mask=np.asarray(batch['input_features_mask'][0],dtype=bool);length=int(mask.sum()) if not length or not mask[:length].all() or mask[length:].any():raise ValueError('Invalid feature mask') mel=np.asarray(batch['input_features'][0,:length],dtype=np.float16) if not np.isfinite(mel).all():raise ValueError('Nonfinite mel after FP16 storage conversion') if shutil.disk_usage(cache).free < mel.nbytes+cfg['minimum_free_disk_gib']*2**30:raise RuntimeError('Feature cache would breach disk reserve') temporary=path.with_suffix('.partial.npz');np.savez_compressed(temporary,mel=mel);temporary.replace(path) frames=len(mel[::2][::2]) if split=='train' and minimum_ctc_frames(target)>frames: rejected.append({'id':source['id'],'source':source['source'],'reason':'ctc_infeasible'});return row={**source,'text':text,'seconds':len(wav)/16000,'audio_sha256':raw,'pcm_sha256':pcm, 'decoder':decoder,'gemma_ids':tokens,'target':target,'frames':frames, 'mel_path':str(path),'mel_sha256':file_hash(path)} data[split].append(row) if split=='train':seen_train.update((raw,pcm)) else: protected_audio.add(raw);protected_pcm.add(pcm);protected_text.add(text_hash(text)) if len(data[split])%1000==0:emit('audio_preparation',split=split,accepted=len(data[split]),free_gib=shutil.disk_usage(cache).free/2**30) def restore(rows,split): for row in rows: if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']:raise ValueError('Shard cache checksum mismatch') data[split].append(row) if split=='train':seen_train.update((row['audio_sha256'],row['pcm_sha256'])) else: protected_audio.add(row['audio_sha256']);protected_pcm.add(row['pcm_sha256']);protected_text.add(text_hash(row['text'])) # Protect FLEURS development and synthetic prompts/audio before any training audio is accepted. reference_cache=cache/'external-evaluation.json' if reference_cache.exists(): for split,rows in json.loads(reference_cache.read_text()).items():restore(rows,split) else: stream=datasets.load_dataset(cfg['fleurs_id'],'ml_in',revision=cfg['fleurs_revision'],split='validation',streaming=True) stream=stream.cast_column('audio',datasets.Audio(decode=False)) for item in stream: audio=item['audio'];blob=audio['bytes'] if audio['bytes'] is not None else Path(audio['path']).read_bytes() accept({'id':f"fleurs:{item['id']}:{hashlib.sha256(blob).hexdigest()[:12]}",'source':'fleurs','text':item['raw_transcription']},blob,'fleurs_validation') if len(data['fleurs_validation'])!=418:raise ValueError('Incomplete FLEURS validation') for row in synthetic_rows: path=Path(row['audio_path']);blob=path.read_bytes() if hashlib.sha256(blob).hexdigest()!=row['audio_sha256']:raise ValueError('Synthetic audio checksum mismatch') accept({'id':row['id'],'source':'synthetic','text':row['text'],'voice':row['voice'],'category':row['category'],'synthetic':True},blob,'synthetic_diagnostic') if not data['synthetic_diagnostic']:raise ValueError('Synthetic diagnostic is empty') save_json(reference_cache,{s:data[s] for s in ('fleurs_validation','synthetic_diagnostic')}) sources={s['name']:s for s in selection['sources']} # First process every real evaluation shard. Training shards cannot enter until this completes. for phase in ('evaluation','train'): if phase=='train': test_hashes={r['pcm_sha256'] for r in data['indicvoices_test']} if any(r['pcm_sha256'] in test_hashes for s in ('indicvoices_dev','kathbath_dev','fleurs_validation') for r in data[s]): raise ValueError('Exact decoded audio overlap between real test and development; inspect before optimization') grouped=defaultdict(list) for split,groups in selection['splits'].items(): if (split=='train') != (phase=='train'):continue for group in groups:grouped[(group['source'],group['shard'])].append((split,group)) for (name,shard),groups in sorted(grouped.items()): done_path=cache/(phase+'-'+name+'-'+Path(shard).stem+'.json') if done_path.exists(): done=json.loads(done_path.read_text()) for split,rows in done['data'].items():restore(rows,split) rejected.extend(done['rejected']);continue source=sources[name];downloads=cache/'downloads'/name # Files are in our owned staging directory, not the shared HF cache. path=Path(hf_hub_download(source['repo'],shard,repo_type='dataset',revision=source['revision'],local_dir=downloads)) before={split:len(data[split]) for split,_ in groups};rejected_before=len(rejected) wanted={} digests={split:hashlib.sha256() for split,_ in groups} for split,group in groups: for index in group['indices']: if index in wanted:raise ValueError('One source row assigned to multiple splits') wanted[index]=split offset=0;found=Counter() columns=['text',source['speaker_column'],source['audio_column']] emit('audio_shard',source=name,shard=shard,selected=len(wanted)) for batch in pq.ParquetFile(path).iter_batches(batch_size=8,columns=columns): for item in batch.to_pylist(): split=wanted.get(offset) if split: digests[split].update(f"{offset}:{text_hash(item['text'])}\n".encode());found[split]+=1 blob=item[source['audio_column']]['bytes'] if blob is None:raise ValueError('Expected embedded Parquet audio') accept({'id':f'{name}:{Path(shard).stem}:{offset}','source':name,'shard':shard,'row_index':offset, 'speaker_id':item[source['speaker_column']],'text':item['text']},blob,split) offset+=1 for split,group in groups: if found[split]!=len(group['indices']) or digests[split].hexdigest()!=group['selected_text_sha256']: raise ValueError('Pinned source selection mismatch') temporary=done_path.with_suffix('.tmp');save_json(temporary,{'data':{s:data[s][i:] for s,i in before.items()},'rejected':rejected[rejected_before:]});temporary.replace(done_path) path.unlink() # Only our completed staging download; never shared source data. emit('shard_prepared',source=name,shard=shard,accepted={s:len(data[s])-i for s,i in before.items()},rejected=len(rejected)-rejected_before) gc.collect() expected=selection['audit']['training_clips_by_source'];actual=Counter(r['source'] for r in data['train']) if any(actual[s]r['frames'] for r in rows)} for s,rows in data.items()} prepared={'data':data,'ids':ids,'audit':audit,'rejected':rejected} temporary=prepared_path.with_suffix('.tmp');save_json(temporary,prepared);temporary.replace(prepared_path) for name,value in [('manifest.json',data),('data-audit.json',audit),('rejected-recordings.json',rejected)]:save_json(output/name,value) save_json(output/'units.json',{'gemma_ids':ids,'blank':0,'unk':1}) emit('data_audit',splits=audit,rejections=dict(Counter(r['source']+':'+r['reason'] for r in rejected)), vocabulary_units=len(ids)+2,storage='Compressed FP16 mel, promoted to FP32 on load', test_audio_prepared_for_overlap_audit=True,test_metrics_computed=False) return data,ids,audit,cache def evaluate_existing(cfg,data,output): """Score the immutable published parent on identical selected-checkpoint audio.""" import torch from huggingface_hub import snapshot_download from kathbath_train import extract_states,mel_inputs from full_train import decode_prediction from synthetic_diagnostic import load_module,aggregate path=Path(snapshot_download(cfg['baseline_repo'],revision=cfg['baseline_revision'],cache_dir=cfg['model_cache'], allow_patterns=['*.json','*.safetensors','dulu_asr.py','reader.py'])) module=load_module('pinned_multicorpus_baseline',path/'dulu_asr.py') module.Reader=load_module('pinned_multicorpus_baseline_reader',path/'reader.py').Reader model=module.DuluASR(path,device='cuda');result={} with torch.inference_mode(): for split in cfg['baseline_splits']: predictions=[] with (output/f'predictions-parent-{split}.jsonl').open('w') as stream: for index,row in enumerate(data[split]): with torch.autocast(device_type='cuda',dtype=torch.bfloat16): states,lengths=extract_states(model.encoder,mel_inputs(row,torch.device('cuda'))) logits=model.reader(states,lengths) hypothesis=decode_prediction(logits.argmax(-1)[0].cpu().tolist(),int(lengths[0]),model.tokenizer,model.ids) record={'id':row['id'],'reference':row['text'],'hypothesis':hypothesis,'audio_sha256':row['audio_sha256']} predictions.append(record);stream.write(json.dumps(record,ensure_ascii=False)+'\n') if (index+1)%500==0:emit('parent_evaluation_progress',split=split,completed=index+1,total=len(data[split])) result[split]=aggregate(predictions);emit('parent_evaluation',split=split,**result[split]) del model;gc.collect();torch.cuda.empty_cache() return result