Spaces:
Running
Running
Download backend/utils/split_shuffle_data.py from thebickersteth/voxpreference: direct link, hf CLI and curl.
- Browser
- Download file 1.79 kB
-
https://huggingface.co/spaces/thebickersteth/voxpreference/resolve/main/backend/utils/split_shuffle_data.py
- Command line
-
hf download hf://spaces/thebickersteth/voxpreference/backend/utils/split_shuffle_data.py
-
curl -L -o split_shuffle_data.py https://huggingface.co/spaces/thebickersteth/voxpreference/resolve/main/backend/utils/split_shuffle_data.py
1.79 kB
| import random | |
| import pandas as pd | |
| # Load both TSVs | |
| male = pd.read_csv('data/line_index_male.tsv', sep='\t', names=['path', 'transcript']) | |
| female = pd.read_csv('data/line_index_female.tsv', sep='\t', names=['path', 'transcript']) | |
| # Combine | |
| combined = pd.concat([male, female], ignore_index=True) | |
| # File ids look like ngf_00295_00053583157: gender prefix + speaker number + utterance number | |
| combined['speaker'] = combined['path'].str.split('_').str[:2].str.join('_') | |
| # Add the folder path to the audio files | |
| combined['path'] = combined['path'].apply(lambda x: f"data/SpeechFiles/{x}.wav") | |
| # Split by speaker so validation measures performance on unseen voices (a random utterance split puts | |
| # every speaker, and some identical sentences, on both sides). Validation speakers are drawn per gender | |
| # until each gender has about 10% of its utterances held out. | |
| rng = random.Random(42) | |
| val_speakers = set() | |
| for prefix in ('ngm', 'ngf'): | |
| counts = combined[combined['speaker'].str.startswith(prefix)].groupby('speaker').size() | |
| speakers = sorted(counts.index) | |
| rng.shuffle(speakers) | |
| held = 0 | |
| for spk in speakers: | |
| if held >= 0.1 * counts.sum(): | |
| break | |
| val_speakers.add(spk) | |
| held += counts[spk] | |
| is_val = combined['speaker'].isin(val_speakers) | |
| train_df, val_df = combined[~is_val], combined[is_val] | |
| # Drop sentences from validation that also occur in training (the corpus repeats some prompts) | |
| val_df = val_df[~val_df['transcript'].isin(set(train_df['transcript']))] | |
| # Save as CSV | |
| train_df.to_csv('data/train.csv', index=False) | |
| val_df.to_csv('data/val.csv', index=False) | |
| print(f"Train samples: {len(train_df)} ({train_df['speaker'].nunique()} speakers), " | |
| f"Validation samples: {len(val_df)} ({val_df['speaker'].nunique()} speakers)") | |