| # | |
| # Stream a nuScenes archive straight into the dataset root, keeping only the | |
| # files we need. | |
| # | |
| # The trainval blob archives are ~30 GB each and contain six cameras, the LiDAR | |
| # and five radars. We use one camera. Downloading to disk and then unpacking | |
| # needs ~60 GB of headroom for ~450 MB of useful data; piping curl into tar and | |
| # filtering as it goes needs only the ~450 MB. | |
| # | |
| # Presigned nuScenes URLs expire within minutes, so copy the link from the | |
| # browser and run this immediately. | |
| # | |
| # Usage: | |
| # ./scripts/fetch_nuscenes.sh <url> <dest> <tar pattern> [<tar pattern> ...] | |
| # | |
| # Mini, to build and verify the pipeline: | |
| # ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'v1.0-mini/*' 'samples/CAM_FRONT/*' | |
| # | |
| # Trainval metadata (all 850 scenes), then one blob of images: | |
| # ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'v1.0-trainval/*' | |
| # ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'samples/CAM_FRONT/*' | |
| set -euo pipefail | |
| if [ "$#" -lt 3 ]; then | |
| sed -n '2,25p' "$0" | |
| exit 1 | |
| fi | |
| url=$1 | |
| dest=$2 | |
| shift 2 | |
| mkdir -p "$dest" | |
| echo "streaming into $dest, keeping: $*" | |
| # --wildcards must precede the patterns. tar reads the whole stream, so an | |
| # unmatched pattern is reported at the end rather than failing fast. | |
| curl -L --fail --progress-bar "$url" \ | |
| | tar -xz -C "$dest" --wildcards "$@" | |
| echo | |
| echo "now in $dest:" | |
| du -sh "$dest"/* 2>/dev/null || true | |
| find "$dest" -name '*.jpg' | wc -l | xargs echo "jpg files:" | |