voxpreference / backend /utils /split_shuffle_data.py
thebickersteth's picture
Preference Grammar pipeline: one image for frontend and backend
9b3bb37 verified
Raw History Blame Contribute Delete
1.79 kB
import random
import pandas as pd
# Load both TSVs
male = pd.read_csv('data/line_index_male.tsv', sep='\t', names=['path', 'transcript'])
female = pd.read_csv('data/line_index_female.tsv', sep='\t', names=['path', 'transcript'])
# Combine
combined = pd.concat([male, female], ignore_index=True)
# File ids look like ngf_00295_00053583157: gender prefix + speaker number + utterance number
combined['speaker'] = combined['path'].str.split('_').str[:2].str.join('_')
# Add the folder path to the audio files
combined['path'] = combined['path'].apply(lambda x: f"data/SpeechFiles/{x}.wav")
# Split by speaker so validation measures performance on unseen voices (a random utterance split puts
# every speaker, and some identical sentences, on both sides). Validation speakers are drawn per gender
# until each gender has about 10% of its utterances held out.
rng = random.Random(42)
val_speakers = set()
for prefix in ('ngm', 'ngf'):
counts = combined[combined['speaker'].str.startswith(prefix)].groupby('speaker').size()
speakers = sorted(counts.index)
rng.shuffle(speakers)
held = 0
for spk in speakers:
if held >= 0.1 * counts.sum():
break
val_speakers.add(spk)
held += counts[spk]
is_val = combined['speaker'].isin(val_speakers)
train_df, val_df = combined[~is_val], combined[is_val]
# Drop sentences from validation that also occur in training (the corpus repeats some prompts)
val_df = val_df[~val_df['transcript'].isin(set(train_df['transcript']))]
# Save as CSV
train_df.to_csv('data/train.csv', index=False)
val_df.to_csv('data/val.csv', index=False)
print(f"Train samples: {len(train_df)} ({train_df['speaker'].nunique()} speakers), "
f"Validation samples: {len(val_df)} ({val_df['speaker'].nunique()} speakers)")