#!/usr/bin/env bash
#
# Stream a nuScenes archive straight into the dataset root, keeping only the
# files we need.
#
# The trainval blob archives are ~30 GB each and contain six cameras, the LiDAR
# and five radars. We use one camera. Downloading to disk and then unpacking
# needs ~60 GB of headroom for ~450 MB of useful data; piping curl into tar and
# filtering as it goes needs only the ~450 MB.
#
# Presigned nuScenes URLs expire within minutes, so copy the link from the
# browser and run this immediately.
#
# Usage:
#   ./scripts/fetch_nuscenes.sh <url> <dest> <tar pattern> [<tar pattern> ...]
#
# Mini, to build and verify the pipeline:
#   ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'v1.0-mini/*' 'samples/CAM_FRONT/*'
#
# Trainval metadata (all 850 scenes), then one blob of images:
#   ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'v1.0-trainval/*'
#   ./scripts/fetch_nuscenes.sh "$URL" /data/nuscenes 'samples/CAM_FRONT/*'

set -euo pipefail

if [ "$#" -lt 3 ]; then
    sed -n '2,25p' "$0"
    exit 1
fi

url=$1
dest=$2
shift 2

mkdir -p "$dest"
echo "streaming into $dest, keeping: $*"

# --wildcards must precede the patterns. tar reads the whole stream, so an
# unmatched pattern is reported at the end rather than failing fast.
curl -L --fail --progress-bar "$url" \
    | tar -xz -C "$dest" --wildcards "$@"

echo
echo "now in $dest:"
du -sh "$dest"/* 2>/dev/null || true
find "$dest" -name '*.jpg' | wc -l | xargs echo "jpg files:"