VLM-all-inclusive / scripts /fetch_data.sh
hkblockchaintechnology's picture
repo assets: tests, scripts, Makefile, packaging, refreshed card
080b493 verified
Raw History Blame Contribute Delete
797 Bytes
#!/usr/bin/env bash
# Fetch the corpora OmniVLM trains on. Resumable: safe to re-run after a restart.
# LJSpeech-1.1 (public domain, LibriVox, 24 h, single speaker) ~2.6 GB
# CMUdict (BSD-2-Clause, ARPAbet pronunciations) ~3.5 MB
set -euo pipefail
DATA="${1:-/marimo/data}"
mkdir -p "$DATA"; cd "$DATA"
if [ ! -d LJSpeech-1.1/wavs ]; then
echo "[fetch] LJSpeech-1.1"
curl -sSL --retry 5 --retry-delay 3 -C - -o ljspeech.tar.bz2 \
https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2
tar xjf ljspeech.tar.bz2 && rm -f ljspeech.tar.bz2
fi
[ -f cmudict.dict ] || curl -sSL -o cmudict.dict \
https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict
echo "[fetch] wavs: $(ls LJSpeech-1.1/wavs | wc -l) cmudict entries: $(wc -l < cmudict.dict)"