Download script/train_vae.sh from WitneyWW/tactile-vae: direct link, hf CLI and curl.
- Browser
- Download file 7.15 kB
-
https://huggingface.co/WitneyWW/tactile-vae/resolve/b952694ddc8dcafe1ea141f00cc02d775431c337/script/train_vae.sh
- Command line
-
hf download hf://WitneyWW/tactile-vae@b952694ddc8dcafe1ea141f00cc02d775431c337/script/train_vae.sh
-
curl -L -o train_vae.sh https://huggingface.co/WitneyWW/tactile-vae/resolve/b952694ddc8dcafe1ea141f00cc02d775431c337/script/train_vae.sh
7.15 kB
| #SBATCH --job-name=tactile_vae | |
| #SBATCH --partition=ct | |
| #SBATCH --nodes=1 | |
| #SBATCH --ntasks-per-node=1 | |
| #SBATCH --gres=gpu:h100:1 | |
| #SBATCH --requeue | |
| #SBATCH --output=/group2/ct/weihanx/tactile_world_model/slurm-logs/tactile_vae.%j.log | |
| #SBATCH --error=/group2/ct/weihanx/tactile_world_model/slurm-logs/tactile_vae.%j.log | |
| # Train tactile_vae.model.TactileVAE on the fota_unlabeled parquet dataset. | |
| # | |
| # Each run lives at <RUNS_DIR>/<RUN_ID>/. Re-launching with the same RUN_ID | |
| # auto-resumes from ckpt_last.pt; wandb keeps the same run id, so metrics | |
| # append to the same dashboard. | |
| # | |
| # Usage (sbatch): sbatch tactile_vae/script/train_vae.sh <run_id> [config.yaml] | |
| # Usage (local): ./tactile_vae/script/train_vae.sh <run_id> [config.yaml] | |
| # | |
| # Diagnostics: set DEBUG=1 to enable `set -x` command tracing. | |
| # sbatch --export=ALL,DEBUG=1 tactile_vae/script/train_vae.sh <run_id> | |
| # Force unbuffered stdout/stderr so the slurm log shows progress live, not in | |
| # one giant flush at the end. (Without this, NFS-backed log files can look | |
| # completely empty for minutes while bash + python buffer output.) | |
| exec 1> >(stdbuf -oL -eL cat) 2>&1 | |
| set -euo pipefail | |
| [[ "${DEBUG:-0}" == "1" ]] && set -x | |
| # ============================================================ | |
| # Inputs (positional) | |
| # ============================================================ | |
| if [[ $# -lt 1 ]]; then | |
| echo "Usage: $0 <run_id> [config.yaml]" >&2 | |
| echo " run_id : required. Both the output subdir name and the wandb run id." >&2 | |
| echo " config : optional. Defaults to tactile_vae/config/train_vae.yaml." >&2 | |
| exit 2 | |
| fi | |
| RUN_ID="$1" | |
| CONFIG="${2:-tactile_vae/config/train_vae.yaml}" | |
| # ============================================================ | |
| # Paths | |
| # ============================================================ | |
| WORKDIR="/group2/ct/weihanx/tactile_world_model" | |
| # Keep this aligned with train_vae.yaml default `runs_root: runs`. | |
| RUNS_DIR="$WORKDIR/tactile_world_model/runs" | |
| RUN_DIR="$RUNS_DIR/$RUN_ID" | |
| DATA_DIR="$WORKDIR/tactile_vae/data" | |
| SPLITS_PATH="$WORKDIR/tactile_vae/dataset/splits.json" | |
| # Conda env with all required deps installed. Override via env var if you | |
| # prefer a different env (e.g. CONDA_ENV=samaudio311 sbatch ...). | |
| # torch torchvision timm numpy pyarrow PIL pyyaml wandb | |
| # `twm` is the project's standard env (matches tactile_jepa training). | |
| CONDA_ENV="${CONDA_ENV:-twm}" | |
| mkdir -p "$WORKDIR/slurm-logs" | |
| mkdir -p "$RUNS_DIR" | |
| umask 027 | |
| # ============================================================ | |
| # Print startup info IMMEDIATELY — before any heavy operation | |
| # (conda activate / python imports) so the slurm log is never | |
| # silent for more than a fraction of a second. | |
| # ============================================================ | |
| echo "=== Tactile VAE training ===" | |
| echo "Host: $(hostname)" | |
| echo "Job ID: ${SLURM_JOB_ID:-N/A}" | |
| echo "Start time: $(date)" | |
| echo "Run ID: $RUN_ID" | |
| echo "Workdir: $WORKDIR" | |
| echo "Config: $CONFIG" | |
| echo "Run dir: $RUN_DIR" | |
| echo "Conda env: $CONDA_ENV" | |
| echo | |
| # ============================================================ | |
| # Environment knobs | |
| # ============================================================ | |
| export OMP_NUM_THREADS=8 | |
| export MKL_NUM_THREADS=8 | |
| export TOKENIZERS_PARALLELISM="false" | |
| export PYTHONFAULTHANDLER=1 | |
| export PYTHONUNBUFFERED=1 # ensures `print()` in Python flushes per line | |
| # ============================================================ | |
| # Weights & Biases (mirrors jepa_training.sh — same account) | |
| # ============================================================ | |
| export WANDB_API_KEY="76cdc4261bf436617e661171fd41d80403e69e9b" | |
| export WANDB_ENTITY="weihanx-university-of-michigan" | |
| export WANDB_USERNAME="weihanx@umich.edu" | |
| export WANDB_PROJECT="tactile_vae" | |
| export WANDB_MODE="online" | |
| export WANDB_RUN_ID="$RUN_ID" | |
| export WANDB_NAME="$RUN_ID" | |
| export WANDB_SERVICE_WAIT=300 | |
| export WANDB_INIT_TIMEOUT=300 | |
| export WANDB_START_METHOD="thread" | |
| export WANDB_CONSOLE="wrap" | |
| # Keep wandb metadata/cache off network storage to speed init/resume. | |
| export WANDB_DIR="${WANDB_DIR:-/tmp/$USER/wandb/$RUN_ID}" | |
| export WANDB_CACHE_DIR="${WANDB_CACHE_DIR:-/tmp/$USER/wandb-cache}" | |
| export WANDB_DATA_DIR="${WANDB_DATA_DIR:-/tmp/$USER/wandb-data}" | |
| mkdir -p "$WANDB_DIR" "$WANDB_CACHE_DIR" "$WANDB_DATA_DIR" | |
| # Debug knob: disable wandb entirely to isolate startup stalls. | |
| # Default is enabled; set DISABLE_WANDB=1 to disable. | |
| DISABLE_WANDB="${DISABLE_WANDB:-0}" | |
| if [[ "$DISABLE_WANDB" == "1" ]]; then | |
| unset WANDB_PROJECT WANDB_ENTITY WANDB_API_KEY WANDB_USERNAME | |
| export WANDB_MODE="disabled" | |
| fi | |
| echo "--- Wandb ---" | |
| echo " project=${WANDB_PROJECT:-<disabled>} entity=${WANDB_ENTITY:-<disabled>}" | |
| echo " run_id=$WANDB_RUN_ID name=$WANDB_NAME mode=$WANDB_MODE" | |
| echo " dir=$WANDB_DIR" | |
| echo " cache_dir=$WANDB_CACHE_DIR" | |
| echo " data_dir=$WANDB_DATA_DIR" | |
| if [[ -n "${WANDB_API_KEY:-}" ]]; then | |
| echo " api_key=${WANDB_API_KEY:0:10}...${WANDB_API_KEY: -4}" | |
| else | |
| echo " api_key=<disabled>" | |
| fi | |
| echo | |
| # ============================================================ | |
| # Sanity checks (cheap; before conda activate) | |
| # ============================================================ | |
| if [[ ! -f "$WORKDIR/$CONFIG" ]] && [[ ! -f "$CONFIG" ]]; then | |
| echo "ERROR: config not found: $CONFIG (or $WORKDIR/$CONFIG)" >&2 | |
| exit 2 | |
| fi | |
| if [[ ! -d "$DATA_DIR" ]]; then | |
| echo "ERROR: data dir does not exist: $DATA_DIR" >&2 | |
| exit 2 | |
| fi | |
| if [[ ! -f "$SPLITS_PATH" ]]; then | |
| echo "ERROR: splits manifest not found: $SPLITS_PATH" >&2 | |
| echo " Generate it with: python tactile_vae/dataset/make_splits.py" >&2 | |
| exit 2 | |
| fi | |
| if [[ -f "$RUN_DIR/ckpt_last.pt" ]]; then | |
| echo "Resume: auto-resume from $RUN_DIR/ckpt_last.pt" | |
| else | |
| echo "Resume: fresh run (no $RUN_DIR/ckpt_last.pt)" | |
| fi | |
| echo | |
| # ============================================================ | |
| # Resolve Python interpreter | |
| # ============================================================ | |
| # Fast path: call env python directly to avoid expensive `conda activate` | |
| # startup on busy shared filesystems. Fallback to full activation if needed. | |
| PYTHON_BIN="${PYTHON_BIN:-$HOME/miniconda3/envs/$CONDA_ENV/bin/python}" | |
| if [[ -x "$PYTHON_BIN" ]]; then | |
| echo "[$(date +%H:%M:%S)] using env python directly: $PYTHON_BIN" | |
| else | |
| echo "[$(date +%H:%M:%S)] env python not found; falling back to conda activate..." | |
| source ~/miniconda3/etc/profile.d/conda.sh | |
| echo "[$(date +%H:%M:%S)] activating $CONDA_ENV..." | |
| conda activate "$CONDA_ENV" | |
| PYTHON_BIN="$(which python)" | |
| echo "[$(date +%H:%M:%S)] env activated. python = $PYTHON_BIN" | |
| fi | |
| echo "[$(date +%H:%M:%S)] GPU(s): ${CUDA_VISIBLE_DEVICES:-$(nvidia-smi -L 2>/dev/null | head -1 || echo none)}" | |
| echo | |
| # ============================================================ | |
| # Launch training (`-u` is also forced by PYTHONUNBUFFERED above) | |
| # ============================================================ | |
| cd "$WORKDIR" | |
| echo "[$(date +%H:%M:%S)] launching trainer..." | |
| "$PYTHON_BIN" -u tactile_vae/script/train_vae.py \ | |
| --config "$CONFIG" \ | |
| --run-id "$RUN_ID" | |
| echo | |
| echo "[$(date +%H:%M:%S)] Finished." | |
| echo "Run dir contents:" | |
| ls -lh "$RUN_DIR" 2>/dev/null || echo " (empty)" | |