Download multicorpus_data.py from whiletruelearn/dulu-malayalam-ctc-778h: direct link, hf CLI and curl.
- Browser
- Download file 15.7 kB
-
https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/multicorpus_data.py
- Command line
-
hf download hf://whiletruelearn/dulu-malayalam-ctc-778h/multicorpus_data.py
-
curl -L -o multicorpus_data.py https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/multicorpus_data.py
15.7 kB
| """Streaming, restartable audio preparation for the pinned three-corpus study.""" | |
| from collections import Counter, defaultdict | |
| import gc | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import shutil | |
| import datasets | |
| from huggingface_hub import hf_hub_download | |
| import numpy as np | |
| import pyarrow.parquet as pq | |
| from transformers import AutoFeatureExtractor | |
| from coverage_vocab import coverage_ids | |
| from full_data import encode_characters, file_hash | |
| from kathbath_data import decode_audio | |
| from multicorpus_protocol import text_hash | |
| from prototype import emit, save_json | |
| from text_utils import minimum_ctc_frames, normalize | |
| def load_mel(row): | |
| if row['mel_path'].endswith('.npz'): | |
| with np.load(row['mel_path'],allow_pickle=False) as value: | |
| return value['mel'].astype(np.float32) | |
| return np.load(row['mel_path'],allow_pickle=False).astype(np.float32,copy=False) | |
| def prepare(cfg,tokenizer,output): | |
| synthetic_path=Path(cfg['synthetic_manifest']) | |
| summary_path=synthetic_path.parent/'summary.json' | |
| if not summary_path.exists(): | |
| raise ValueError('Synthetic baseline is incomplete; do not train against a partial generation manifest') | |
| synthesis_summary=json.loads(summary_path.read_text()) | |
| synthetic_rows=[json.loads(line) for line in synthetic_path.read_text().splitlines()] | |
| if len(synthetic_rows)!=synthesis_summary['generated_clips']: | |
| raise ValueError('Synthetic manifest count differs from its completed baseline summary') | |
| prompts=json.loads(Path(cfg['prompt_manifest']).read_text()) | |
| prompt_text={f'{category}-{i+1:02d}':text for category,texts in prompts['categories'].items() for i,text in enumerate(texts)} | |
| if len({r['id'] for r in synthetic_rows})!=len(synthetic_rows) or any(prompt_text.get(r['prompt_id'])!=r['text'] for r in synthetic_rows): | |
| raise ValueError('Synthetic prompt identity mismatch') | |
| manifest_path=Path(cfg['selection_manifest']) | |
| selection=json.loads(manifest_path.read_text()) | |
| contract={'selection_sha256':file_hash(manifest_path),'model_revision':cfg['model_revision'], | |
| 'protocol':'multicorpus-fp16-compressed-v1','synthetic_manifest_sha256':file_hash(cfg['synthetic_manifest']), | |
| 'preparation_code_sha256':{name:file_hash(name) for name in ['multicorpus_data.py','text_utils.py','coverage_vocab.py','full_data.py','kathbath_data.py']}} | |
| key=hashlib.sha256(json.dumps(contract,sort_keys=True).encode()).hexdigest()[:20] | |
| cache=Path(cfg['multicorpus_cache'])/key;cache.mkdir(parents=True,exist_ok=True) | |
| save_json(output/'cache-contract.json',{**contract,'path':str(cache)}) | |
| ids=coverage_ids(tokenizer.get_vocab(),selection['training_characters']) | |
| known=set(ids) | |
| allowed={p:i for p,i in tokenizer.get_vocab().items() if i in known} | |
| extractor=AutoFeatureExtractor.from_pretrained(cfg['model_id'],revision=cfg['model_revision']) | |
| data={k:[] for k in selection['splits']} | |
| data.update(fleurs_validation=[],synthetic_diagnostic=[]) | |
| rejected=[];seen_train=set();protected_audio=set();protected_pcm=set();protected_text=set() | |
| prior=json.loads(Path('data/prior_evaluation_manifest.json').read_text()) | |
| for rows in (prior['validation'],prior['previously_inspected_test']): | |
| for row in rows: | |
| protected_audio.add(row['audio_sha256']) | |
| if row.get('pcm_sha256'):protected_pcm.add(row['pcm_sha256']) | |
| protected_text.add(text_hash(row['text'])) | |
| mel_dir=cache/'mel';mel_dir.mkdir(exist_ok=True) | |
| prepared_path=cache/'prepared.json' | |
| if prepared_path.exists(): | |
| prepared=json.loads(prepared_path.read_text()) | |
| for rows in prepared['data'].values(): | |
| for row in rows: | |
| if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']: | |
| raise ValueError('Prepared cache checksum mismatch') | |
| emit('prepared_cache_reused',path=str(prepared_path)) | |
| for name,value in [('manifest.json',prepared['data']),('data-audit.json',prepared['audit']),('rejected-recordings.json',prepared['rejected'])]:save_json(output/name,value) | |
| return prepared['data'],ids,prepared['audit'],cache | |
| def accept(source,blob,split): | |
| text=normalize(source['text']) | |
| if not text:raise ValueError('Empty selected transcript') | |
| raw=hashlib.sha256(blob).hexdigest() | |
| wav,decoder=decode_audio(blob);pcm=hashlib.sha256(wav.tobytes()).hexdigest() | |
| reason=None | |
| if split=='train': | |
| if raw in protected_audio or pcm in protected_pcm or text_hash(text) in protected_text:reason='heldout_overlap' | |
| elif raw in seen_train or pcm in seen_train:reason='duplicate_training_audio' | |
| elif not 0.1<=len(wav)/16000<=30:reason='duration_outside_0_1_30s' | |
| tokens,target=encode_characters(text,tokenizer,ids,allowed) | |
| if reason: | |
| rejected.append({'id':source['id'],'source':source['source'],'reason':reason});return | |
| path=mel_dir/(raw+'.npz') | |
| if path.exists(): | |
| with np.load(path,allow_pickle=False) as saved:mel=saved['mel'] | |
| else: | |
| batch=extractor([wav],sampling_rate=16000,return_tensors='np',truncation=False,max_length=None) | |
| mask=np.asarray(batch['input_features_mask'][0],dtype=bool);length=int(mask.sum()) | |
| if not length or not mask[:length].all() or mask[length:].any():raise ValueError('Invalid feature mask') | |
| mel=np.asarray(batch['input_features'][0,:length],dtype=np.float16) | |
| if not np.isfinite(mel).all():raise ValueError('Nonfinite mel after FP16 storage conversion') | |
| if shutil.disk_usage(cache).free < mel.nbytes+cfg['minimum_free_disk_gib']*2**30:raise RuntimeError('Feature cache would breach disk reserve') | |
| temporary=path.with_suffix('.partial.npz');np.savez_compressed(temporary,mel=mel);temporary.replace(path) | |
| frames=len(mel[::2][::2]) | |
| if split=='train' and minimum_ctc_frames(target)>frames: | |
| rejected.append({'id':source['id'],'source':source['source'],'reason':'ctc_infeasible'});return | |
| row={**source,'text':text,'seconds':len(wav)/16000,'audio_sha256':raw,'pcm_sha256':pcm, | |
| 'decoder':decoder,'gemma_ids':tokens,'target':target,'frames':frames, | |
| 'mel_path':str(path),'mel_sha256':file_hash(path)} | |
| data[split].append(row) | |
| if split=='train':seen_train.update((raw,pcm)) | |
| else: | |
| protected_audio.add(raw);protected_pcm.add(pcm);protected_text.add(text_hash(text)) | |
| if len(data[split])%1000==0:emit('audio_preparation',split=split,accepted=len(data[split]),free_gib=shutil.disk_usage(cache).free/2**30) | |
| def restore(rows,split): | |
| for row in rows: | |
| if not Path(row['mel_path']).exists() or file_hash(row['mel_path'])!=row['mel_sha256']:raise ValueError('Shard cache checksum mismatch') | |
| data[split].append(row) | |
| if split=='train':seen_train.update((row['audio_sha256'],row['pcm_sha256'])) | |
| else: | |
| protected_audio.add(row['audio_sha256']);protected_pcm.add(row['pcm_sha256']);protected_text.add(text_hash(row['text'])) | |
| # Protect FLEURS development and synthetic prompts/audio before any training audio is accepted. | |
| reference_cache=cache/'external-evaluation.json' | |
| if reference_cache.exists(): | |
| for split,rows in json.loads(reference_cache.read_text()).items():restore(rows,split) | |
| else: | |
| stream=datasets.load_dataset(cfg['fleurs_id'],'ml_in',revision=cfg['fleurs_revision'],split='validation',streaming=True) | |
| stream=stream.cast_column('audio',datasets.Audio(decode=False)) | |
| for item in stream: | |
| audio=item['audio'];blob=audio['bytes'] if audio['bytes'] is not None else Path(audio['path']).read_bytes() | |
| accept({'id':f"fleurs:{item['id']}:{hashlib.sha256(blob).hexdigest()[:12]}",'source':'fleurs','text':item['raw_transcription']},blob,'fleurs_validation') | |
| if len(data['fleurs_validation'])!=418:raise ValueError('Incomplete FLEURS validation') | |
| for row in synthetic_rows: | |
| path=Path(row['audio_path']);blob=path.read_bytes() | |
| if hashlib.sha256(blob).hexdigest()!=row['audio_sha256']:raise ValueError('Synthetic audio checksum mismatch') | |
| accept({'id':row['id'],'source':'synthetic','text':row['text'],'voice':row['voice'],'category':row['category'],'synthetic':True},blob,'synthetic_diagnostic') | |
| if not data['synthetic_diagnostic']:raise ValueError('Synthetic diagnostic is empty') | |
| save_json(reference_cache,{s:data[s] for s in ('fleurs_validation','synthetic_diagnostic')}) | |
| sources={s['name']:s for s in selection['sources']} | |
| # First process every real evaluation shard. Training shards cannot enter until this completes. | |
| for phase in ('evaluation','train'): | |
| if phase=='train': | |
| test_hashes={r['pcm_sha256'] for r in data['indicvoices_test']} | |
| if any(r['pcm_sha256'] in test_hashes for s in ('indicvoices_dev','kathbath_dev','fleurs_validation') for r in data[s]): | |
| raise ValueError('Exact decoded audio overlap between real test and development; inspect before optimization') | |
| grouped=defaultdict(list) | |
| for split,groups in selection['splits'].items(): | |
| if (split=='train') != (phase=='train'):continue | |
| for group in groups:grouped[(group['source'],group['shard'])].append((split,group)) | |
| for (name,shard),groups in sorted(grouped.items()): | |
| done_path=cache/(phase+'-'+name+'-'+Path(shard).stem+'.json') | |
| if done_path.exists(): | |
| done=json.loads(done_path.read_text()) | |
| for split,rows in done['data'].items():restore(rows,split) | |
| rejected.extend(done['rejected']);continue | |
| source=sources[name];downloads=cache/'downloads'/name | |
| # Files are in our owned staging directory, not the shared HF cache. | |
| path=Path(hf_hub_download(source['repo'],shard,repo_type='dataset',revision=source['revision'],local_dir=downloads)) | |
| before={split:len(data[split]) for split,_ in groups};rejected_before=len(rejected) | |
| wanted={} | |
| digests={split:hashlib.sha256() for split,_ in groups} | |
| for split,group in groups: | |
| for index in group['indices']: | |
| if index in wanted:raise ValueError('One source row assigned to multiple splits') | |
| wanted[index]=split | |
| offset=0;found=Counter() | |
| columns=['text',source['speaker_column'],source['audio_column']] | |
| emit('audio_shard',source=name,shard=shard,selected=len(wanted)) | |
| for batch in pq.ParquetFile(path).iter_batches(batch_size=8,columns=columns): | |
| for item in batch.to_pylist(): | |
| split=wanted.get(offset) | |
| if split: | |
| digests[split].update(f"{offset}:{text_hash(item['text'])}\n".encode());found[split]+=1 | |
| blob=item[source['audio_column']]['bytes'] | |
| if blob is None:raise ValueError('Expected embedded Parquet audio') | |
| accept({'id':f'{name}:{Path(shard).stem}:{offset}','source':name,'shard':shard,'row_index':offset, | |
| 'speaker_id':item[source['speaker_column']],'text':item['text']},blob,split) | |
| offset+=1 | |
| for split,group in groups: | |
| if found[split]!=len(group['indices']) or digests[split].hexdigest()!=group['selected_text_sha256']: | |
| raise ValueError('Pinned source selection mismatch') | |
| temporary=done_path.with_suffix('.tmp');save_json(temporary,{'data':{s:data[s][i:] for s,i in before.items()},'rejected':rejected[rejected_before:]});temporary.replace(done_path) | |
| path.unlink() # Only our completed staging download; never shared source data. | |
| emit('shard_prepared',source=name,shard=shard,accepted={s:len(data[s])-i for s,i in before.items()},rejected=len(rejected)-rejected_before) | |
| gc.collect() | |
| expected=selection['audit']['training_clips_by_source'];actual=Counter(r['source'] for r in data['train']) | |
| if any(actual[s]<expected[s]*.95 for s in expected): | |
| save_json(output/'rejected-recordings.json',rejected) | |
| raise ValueError('More than 5% runtime rejection in a source; inspect before optimization') | |
| for split,rows in data.items(): | |
| if not rows:raise ValueError('Empty required split '+split) | |
| rows.sort(key=lambda r:hashlib.sha256(f"{cfg['seed']}:{r['id']}".encode()).hexdigest()) | |
| audit={s:{'recordings':len(rows),'hours':sum(r['seconds'] for r in rows)/3600, | |
| 'unique_transcripts':len({r['text'] for r in rows}), | |
| 'by_source':dict(Counter(r['source'] for r in rows)), | |
| 'ctc_infeasible':sum(minimum_ctc_frames(r['target'])>r['frames'] for r in rows)} for s,rows in data.items()} | |
| prepared={'data':data,'ids':ids,'audit':audit,'rejected':rejected} | |
| temporary=prepared_path.with_suffix('.tmp');save_json(temporary,prepared);temporary.replace(prepared_path) | |
| for name,value in [('manifest.json',data),('data-audit.json',audit),('rejected-recordings.json',rejected)]:save_json(output/name,value) | |
| save_json(output/'units.json',{'gemma_ids':ids,'blank':0,'unk':1}) | |
| emit('data_audit',splits=audit,rejections=dict(Counter(r['source']+':'+r['reason'] for r in rejected)), | |
| vocabulary_units=len(ids)+2,storage='Compressed FP16 mel, promoted to FP32 on load', | |
| test_audio_prepared_for_overlap_audit=True,test_metrics_computed=False) | |
| return data,ids,audit,cache | |
| def evaluate_existing(cfg,data,output): | |
| """Score the immutable published parent on identical selected-checkpoint audio.""" | |
| import torch | |
| from huggingface_hub import snapshot_download | |
| from kathbath_train import extract_states,mel_inputs | |
| from full_train import decode_prediction | |
| from synthetic_diagnostic import load_module,aggregate | |
| path=Path(snapshot_download(cfg['baseline_repo'],revision=cfg['baseline_revision'],cache_dir=cfg['model_cache'], | |
| allow_patterns=['*.json','*.safetensors','dulu_asr.py','reader.py'])) | |
| module=load_module('pinned_multicorpus_baseline',path/'dulu_asr.py') | |
| module.Reader=load_module('pinned_multicorpus_baseline_reader',path/'reader.py').Reader | |
| model=module.DuluASR(path,device='cuda');result={} | |
| with torch.inference_mode(): | |
| for split in cfg['baseline_splits']: | |
| predictions=[] | |
| with (output/f'predictions-parent-{split}.jsonl').open('w') as stream: | |
| for index,row in enumerate(data[split]): | |
| with torch.autocast(device_type='cuda',dtype=torch.bfloat16): | |
| states,lengths=extract_states(model.encoder,mel_inputs(row,torch.device('cuda'))) | |
| logits=model.reader(states,lengths) | |
| hypothesis=decode_prediction(logits.argmax(-1)[0].cpu().tolist(),int(lengths[0]),model.tokenizer,model.ids) | |
| record={'id':row['id'],'reference':row['text'],'hypothesis':hypothesis,'audio_sha256':row['audio_sha256']} | |
| predictions.append(record);stream.write(json.dumps(record,ensure_ascii=False)+'\n') | |
| if (index+1)%500==0:emit('parent_evaluation_progress',split=split,completed=index+1,total=len(data[split])) | |
| result[split]=aggregate(predictions);emit('parent_evaluation',split=split,**result[split]) | |
| del model;gc.collect();torch.cuda.empty_cache() | |
| return result | |