dulu-malayalam-ctc-778h / multicorpus_data.py
whiletruelearn's picture
Archive selected epoch 9 combined-data CTC checkpoint and evaluation
750e4eb verified
Raw History Blame Contribute Delete
15.7 kB
"""Streaming, restartable audio preparation for the pinned three-corpus study."""
from collections import Counter, defaultdict
import gc
import hashlib
import json
from pathlib import Path
import shutil
import datasets
from huggingface_hub import hf_hub_download
import numpy as np
import pyarrow.parquet as pq
from transformers import AutoFeatureExtractor
from coverage_vocab import coverage_ids
from full_data import encode_characters, file_hash
from kathbath_data import decode_audio
from multicorpus_protocol import text_hash
from prototype import emit, save_json
from text_utils import minimum_ctc_frames, normalize
def load_mel(row):
if row['mel_path'].endswith('.npz'):
with np.load(row['mel_path'],allow_pickle=False) as value:
return value['mel'].astype(np.float32)
return np.load(row['mel_path'],allow_pickle=False).astype(np.float32,copy=False)
def prepare(cfg,tokenizer,output):
synthetic_path=Path(cfg['synthetic_manifest'])
summary_path=synthetic_path.parent/'summary.json'
if not summary_path.exists():
raise ValueError('Synthetic baseline is incomplete; do not train against a partial generation manifest')
synthesis_summary=json.loads(summary_path.read_text())
synthetic_rows=[json.loads(line) for line in synthetic_path.read_text().splitlines()]
if len(synthetic_rows)!=synthesis_summary['generated_clips']:
raise ValueError('Synthetic manifest count differs from its completed baseline summary')
prompts=json.loads(Path(cfg['prompt_manifest']).read_text())
prompt_text={f'{category}-{i+1:02d}':text for category,texts in prompts['categories'].items() for i,text in enumerate(texts)}
if len({r['id'] for r in synthetic_rows})!=len(synthetic_rows) or any(prompt_text.get(r['prompt_id'])!=r['text'] for r in synthetic_rows):
raise ValueError('Synthetic prompt identity mismatch')
manifest_path=Path(cfg['selection_manifest'])
selection=json.loads(manifest_path.read_text())
contract={'selection_sha256':file_hash(manifest_path),'model_revision':cfg['model_revision'],
'protocol':'multicorpus-fp16-compressed-v1','synthetic_manifest_sha256':file_hash(cfg['synthetic_manifest']),
'preparation_code_sha256':{name:file_hash(name) for name in ['multicorpus_data.py','text_utils.py','coverage_vocab.py','full_data.py','kathbath_data.py']}}
key=hashlib.sha256(json.dumps(contract,sort_keys=True).encode()).hexdigest()[:20]
cache=Path(cfg['multicorpus_cache'])/key;cache.mkdir(parents=True,exist_ok=True)
save_json(output/'cache-contract.json',{**contract,'path':str(cache)})
ids=coverage_ids(tokenizer.get_vocab(),selection['training_characters'])
known=set(ids)
allowed={p:i for p,i in tokenizer.get_vocab().items() if i in known}
extractor=AutoFeatureExtractor.from_pretrained(cfg['model_id'],revision=cfg['model_revision'])
data={k:[] for k in selection['splits']}
data.update(fleurs_validation=[],synthetic_diagnostic=[])
rejected=[];seen_train=set();protected_audio=set();protected_pcm=set();protected_text=set()
prior=json.loads(Path('data/prior_evaluation_manifest.json').read_text())
for rows in (prior['validation'],prior['previously_inspected_test']):
for row in rows:
protected_audio.add(row['audio_sha256'])
if row.get('pcm_sha256'):protected_pcm.add(row['pcm_sha256'])
protected_text.add(text_hash(row['text']))
mel_dir=cache/'mel';mel_dir.mkdir(exist_ok=True)
prepared_path=cache/'prepared.json'
if prepared_path.exists():
prepared=json.loads(prepared_path.read_text())
for rows in prepared['data'].values():
for row in rows:
if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']:
raise ValueError('Prepared cache checksum mismatch')
emit('prepared_cache_reused',path=str(prepared_path))
for name,value in [('manifest.json',prepared['data']),('data-audit.json',prepared['audit']),('rejected-recordings.json',prepared['rejected'])]:save_json(output/name,value)
return prepared['data'],ids,prepared['audit'],cache
def accept(source,blob,split):
text=normalize(source['text'])
if not text:raise ValueError('Empty selected transcript')
raw=hashlib.sha256(blob).hexdigest()
wav,decoder=decode_audio(blob);pcm=hashlib.sha256(wav.tobytes()).hexdigest()
reason=None
if split=='train':
if raw in protected_audio or pcm in protected_pcm or text_hash(text) in protected_text:reason='heldout_overlap'
elif raw in seen_train or pcm in seen_train:reason='duplicate_training_audio'
elif not 0.1<=len(wav)/16000<=30:reason='duration_outside_0_1_30s'
tokens,target=encode_characters(text,tokenizer,ids,allowed)
if reason:
rejected.append({'id':source['id'],'source':source['source'],'reason':reason});return
path=mel_dir/(raw+'.npz')
if path.exists():
with np.load(path,allow_pickle=False) as saved:mel=saved['mel']
else:
batch=extractor([wav],sampling_rate=16000,return_tensors='np',truncation=False,max_length=None)
mask=np.asarray(batch['input_features_mask'][0],dtype=bool);length=int(mask.sum())
if not length or not mask[:length].all() or mask[length:].any():raise ValueError('Invalid feature mask')
mel=np.asarray(batch['input_features'][0,:length],dtype=np.float16)
if not np.isfinite(mel).all():raise ValueError('Nonfinite mel after FP16 storage conversion')
if shutil.disk_usage(cache).free < mel.nbytes+cfg['minimum_free_disk_gib']*2**30:raise RuntimeError('Feature cache would breach disk reserve')
temporary=path.with_suffix('.partial.npz');np.savez_compressed(temporary,mel=mel);temporary.replace(path)
frames=len(mel[::2][::2])
if split=='train' and minimum_ctc_frames(target)>frames:
rejected.append({'id':source['id'],'source':source['source'],'reason':'ctc_infeasible'});return
row={**source,'text':text,'seconds':len(wav)/16000,'audio_sha256':raw,'pcm_sha256':pcm,
'decoder':decoder,'gemma_ids':tokens,'target':target,'frames':frames,
'mel_path':str(path),'mel_sha256':file_hash(path)}
data[split].append(row)
if split=='train':seen_train.update((raw,pcm))
else:
protected_audio.add(raw);protected_pcm.add(pcm);protected_text.add(text_hash(text))
if len(data[split])%1000==0:emit('audio_preparation',split=split,accepted=len(data[split]),free_gib=shutil.disk_usage(cache).free/2**30)
def restore(rows,split):
for row in rows:
if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']:raise ValueError('Shard cache checksum mismatch')
data[split].append(row)
if split=='train':seen_train.update((row['audio_sha256'],row['pcm_sha256']))
else:
protected_audio.add(row['audio_sha256']);protected_pcm.add(row['pcm_sha256']);protected_text.add(text_hash(row['text']))
# Protect FLEURS development and synthetic prompts/audio before any training audio is accepted.
reference_cache=cache/'external-evaluation.json'
if reference_cache.exists():
for split,rows in json.loads(reference_cache.read_text()).items():restore(rows,split)
else:
stream=datasets.load_dataset(cfg['fleurs_id'],'ml_in',revision=cfg['fleurs_revision'],split='validation',streaming=True)
stream=stream.cast_column('audio',datasets.Audio(decode=False))
for item in stream:
audio=item['audio'];blob=audio['bytes'] if audio['bytes'] is not None else Path(audio['path']).read_bytes()
accept({'id':f"fleurs:{item['id']}:{hashlib.sha256(blob).hexdigest()[:12]}",'source':'fleurs','text':item['raw_transcription']},blob,'fleurs_validation')
if len(data['fleurs_validation'])!=418:raise ValueError('Incomplete FLEURS validation')
for row in synthetic_rows:
path=Path(row['audio_path']);blob=path.read_bytes()
if hashlib.sha256(blob).hexdigest()!=row['audio_sha256']:raise ValueError('Synthetic audio checksum mismatch')
accept({'id':row['id'],'source':'synthetic','text':row['text'],'voice':row['voice'],'category':row['category'],'synthetic':True},blob,'synthetic_diagnostic')
if not data['synthetic_diagnostic']:raise ValueError('Synthetic diagnostic is empty')
save_json(reference_cache,{s:data[s] for s in ('fleurs_validation','synthetic_diagnostic')})
sources={s['name']:s for s in selection['sources']}
# First process every real evaluation shard. Training shards cannot enter until this completes.
for phase in ('evaluation','train'):
if phase=='train':
test_hashes={r['pcm_sha256'] for r in data['indicvoices_test']}
if any(r['pcm_sha256'] in test_hashes for s in ('indicvoices_dev','kathbath_dev','fleurs_validation') for r in data[s]):
raise ValueError('Exact decoded audio overlap between real test and development; inspect before optimization')
grouped=defaultdict(list)
for split,groups in selection['splits'].items():
if (split=='train') != (phase=='train'):continue
for group in groups:grouped[(group['source'],group['shard'])].append((split,group))
for (name,shard),groups in sorted(grouped.items()):
done_path=cache/(phase+'-'+name+'-'+Path(shard).stem+'.json')
if done_path.exists():
done=json.loads(done_path.read_text())
for split,rows in done['data'].items():restore(rows,split)
rejected.extend(done['rejected']);continue
source=sources[name];downloads=cache/'downloads'/name
# Files are in our owned staging directory, not the shared HF cache.
path=Path(hf_hub_download(source['repo'],shard,repo_type='dataset',revision=source['revision'],local_dir=downloads))
before={split:len(data[split]) for split,_ in groups};rejected_before=len(rejected)
wanted={}
digests={split:hashlib.sha256() for split,_ in groups}
for split,group in groups:
for index in group['indices']:
if index in wanted:raise ValueError('One source row assigned to multiple splits')
wanted[index]=split
offset=0;found=Counter()
columns=['text',source['speaker_column'],source['audio_column']]
emit('audio_shard',source=name,shard=shard,selected=len(wanted))
for batch in pq.ParquetFile(path).iter_batches(batch_size=8,columns=columns):
for item in batch.to_pylist():
split=wanted.get(offset)
if split:
digests[split].update(f"{offset}:{text_hash(item['text'])}\n".encode());found[split]+=1
blob=item[source['audio_column']]['bytes']
if blob is None:raise ValueError('Expected embedded Parquet audio')
accept({'id':f'{name}:{Path(shard).stem}:{offset}','source':name,'shard':shard,'row_index':offset,
'speaker_id':item[source['speaker_column']],'text':item['text']},blob,split)
offset+=1
for split,group in groups:
if found[split]!=len(group['indices']) or digests[split].hexdigest()!=group['selected_text_sha256']:
raise ValueError('Pinned source selection mismatch')
temporary=done_path.with_suffix('.tmp');save_json(temporary,{'data':{s:data[s][i:] for s,i in before.items()},'rejected':rejected[rejected_before:]});temporary.replace(done_path)
path.unlink() # Only our completed staging download; never shared source data.
emit('shard_prepared',source=name,shard=shard,accepted={s:len(data[s])-i for s,i in before.items()},rejected=len(rejected)-rejected_before)
gc.collect()
expected=selection['audit']['training_clips_by_source'];actual=Counter(r['source'] for r in data['train'])
if any(actual[s]<expected[s]*.95 for s in expected):
save_json(output/'rejected-recordings.json',rejected)
raise ValueError('More than 5% runtime rejection in a source; inspect before optimization')
for split,rows in data.items():
if not rows:raise ValueError('Empty required split '+split)
rows.sort(key=lambda r:hashlib.sha256(f"{cfg['seed']}:{r['id']}".encode()).hexdigest())
audit={s:{'recordings':len(rows),'hours':sum(r['seconds'] for r in rows)/3600,
'unique_transcripts':len({r['text'] for r in rows}),
'by_source':dict(Counter(r['source'] for r in rows)),
'ctc_infeasible':sum(minimum_ctc_frames(r['target'])>r['frames'] for r in rows)} for s,rows in data.items()}
prepared={'data':data,'ids':ids,'audit':audit,'rejected':rejected}
temporary=prepared_path.with_suffix('.tmp');save_json(temporary,prepared);temporary.replace(prepared_path)
for name,value in [('manifest.json',data),('data-audit.json',audit),('rejected-recordings.json',rejected)]:save_json(output/name,value)
save_json(output/'units.json',{'gemma_ids':ids,'blank':0,'unk':1})
emit('data_audit',splits=audit,rejections=dict(Counter(r['source']+':'+r['reason'] for r in rejected)),
vocabulary_units=len(ids)+2,storage='Compressed FP16 mel, promoted to FP32 on load',
test_audio_prepared_for_overlap_audit=True,test_metrics_computed=False)
return data,ids,audit,cache
def evaluate_existing(cfg,data,output):
"""Score the immutable published parent on identical selected-checkpoint audio."""
import torch
from huggingface_hub import snapshot_download
from kathbath_train import extract_states,mel_inputs
from full_train import decode_prediction
from synthetic_diagnostic import load_module,aggregate
path=Path(snapshot_download(cfg['baseline_repo'],revision=cfg['baseline_revision'],cache_dir=cfg['model_cache'],
allow_patterns=['*.json','*.safetensors','dulu_asr.py','reader.py']))
module=load_module('pinned_multicorpus_baseline',path/'dulu_asr.py')
module.Reader=load_module('pinned_multicorpus_baseline_reader',path/'reader.py').Reader
model=module.DuluASR(path,device='cuda');result={}
with torch.inference_mode():
for split in cfg['baseline_splits']:
predictions=[]
with (output/f'predictions-parent-{split}.jsonl').open('w') as stream:
for index,row in enumerate(data[split]):
with torch.autocast(device_type='cuda',dtype=torch.bfloat16):
states,lengths=extract_states(model.encoder,mel_inputs(row,torch.device('cuda')))
logits=model.reader(states,lengths)
hypothesis=decode_prediction(logits.argmax(-1)[0].cpu().tolist(),int(lengths[0]),model.tokenizer,model.ids)
record={'id':row['id'],'reference':row['text'],'hypothesis':hypothesis,'audio_sha256':row['audio_sha256']}
predictions.append(record);stream.write(json.dumps(record,ensure_ascii=False)+'\n')
if (index+1)%500==0:emit('parent_evaluation_progress',split=split,completed=index+1,total=len(data[split]))
result[split]=aggregate(predictions);emit('parent_evaluation',split=split,**result[split])
del model;gc.collect();torch.cuda.empty_cache()
return result