#!/usr/bin/env bash # Fetch the corpora OmniVLM trains on. Resumable: safe to re-run after a restart. # LJSpeech-1.1 (public domain, LibriVox, 24 h, single speaker) ~2.6 GB # CMUdict (BSD-2-Clause, ARPAbet pronunciations) ~3.5 MB set -euo pipefail DATA="${1:-/marimo/data}" mkdir -p "$DATA"; cd "$DATA" if [ ! -d LJSpeech-1.1/wavs ]; then echo "[fetch] LJSpeech-1.1" curl -sSL --retry 5 --retry-delay 3 -C - -o ljspeech.tar.bz2 \ https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 tar xjf ljspeech.tar.bz2 && rm -f ljspeech.tar.bz2 fi [ -f cmudict.dict ] || curl -sSL -o cmudict.dict \ https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict echo "[fetch] wavs: $(ls LJSpeech-1.1/wavs | wc -l) cmudict entries: $(wc -l < cmudict.dict)"