File size: 797 Bytes
080b493 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | #!/usr/bin/env bash
# Fetch the corpora OmniVLM trains on. Resumable: safe to re-run after a restart.
# LJSpeech-1.1 (public domain, LibriVox, 24 h, single speaker) ~2.6 GB
# CMUdict (BSD-2-Clause, ARPAbet pronunciations) ~3.5 MB
set -euo pipefail
DATA="${1:-/marimo/data}"
mkdir -p "$DATA"; cd "$DATA"
if [ ! -d LJSpeech-1.1/wavs ]; then
echo "[fetch] LJSpeech-1.1"
curl -sSL --retry 5 --retry-delay 3 -C - -o ljspeech.tar.bz2 \
https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2
tar xjf ljspeech.tar.bz2 && rm -f ljspeech.tar.bz2
fi
[ -f cmudict.dict ] || curl -sSL -o cmudict.dict \
https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict
echo "[fetch] wavs: $(ls LJSpeech-1.1/wavs | wc -l) cmudict entries: $(wc -l < cmudict.dict)"
|