Download run.sh from Lucasgitguy-co/Fleet-2: direct link, hf CLI and curl.
- Browser
- Download file 1.51 kB
-
https://huggingface.co/Lucasgitguy-co/Fleet-2/resolve/main/run.sh
- Command line
-
hf download hf://Lucasgitguy-co/Fleet-2/run.sh
-
curl -L -o run.sh https://huggingface.co/Lucasgitguy-co/Fleet-2/resolve/main/run.sh
1.51 kB
| # Fleet-2 pipeline. Usage: ./run.sh [all|smoke|status|prepare|teacher|filter|stage0|train|select|test|publish] | |
| set -euo pipefail | |
| cd "$(dirname "$0")" | |
| CMD=${1:-status} | |
| if [[ $CMD == smoke ]]; then export FLEET2_RUN=$PWD/runs/smoke FLEET2_SMOKE=1 FLEET2_MIN_GB=${FLEET2_MIN_GB:-2}; CMD=all; fi | |
| MIN_GB=${FLEET2_MIN_GB:-5} | |
| RUN=${FLEET2_RUN:-$PWD/runs/main}; export FLEET2_RUN=$RUN | |
| mkdir -p "$RUN/done" | |
| export HF_DATASETS_CACHE=$RUN/.hf-datasets CACTUS_TELEMETRY=0 JAX_PLATFORMS=cpu JAX_ENABLE_X64=false \ | |
| ENABLE_PJRT_COMPATIBILITY=1 HF_HUB_DISABLE_TELEMETRY=1 PYTHONUNBUFFERED=1 | |
| S=.venv-student/bin/python; T=.venv-teacher/bin/python | |
| guard() { local kb; kb=$(df -k "$PWD" | awk 'NR==2{print $4}'); if (( kb < MIN_GB*1024*1024 )); then echo "Stopped: less than $MIN_GB GB free disk." >&2; exit 1; fi; } | |
| stage() { local name=$1; shift | |
| if [[ -f $RUN/done/$name ]]; then echo "[$name] already done"; return; fi | |
| guard; echo "[$name] start $(date '+%F %T')"; "$@"; date '+%F %T' > "$RUN/done/$name"; echo "[$name] done"; } | |
| run_stage() { case $1 in | |
| teacher) stage teacher "$T" -I teacher.py "$RUN" ;; | |
| prepare|filter|stage0|train|select|test|publish) stage "$1" "$S" fleet2.py "$1" ;; | |
| *) echo "unknown stage $1" >&2; exit 2 ;; esac; } | |
| case $CMD in | |
| status) echo "Run: $RUN"; ls "$RUN/done" 2>/dev/null || true; tail -n 5 "$RUN/log.txt" 2>/dev/null || true ;; | |
| all) for s in prepare teacher filter stage0 train select test publish; do run_stage "$s"; done ;; | |
| *) run_stage "$CMD" ;; | |
| esac | |