Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 3,384 Bytes
b1b6f13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | #!/usr/bin/env bash
# Rebuild the training inputs of one task from the raw mp4/parquet files:
# raw/ (download.py --what raw) -> <OUT>/<tree>_latents/{tac,visual,wrist}/<ep>.pt
# -> <OUT>/<tree>_actions/<ep>/{sensor_actions_with_dforce_fps15.pt,
# sensor_pose7d_force_fps15.pt, view_actions_fps15.pt, ...}
# bash preprocessing/run_preprocess.sh rope [OUT] [episode ...]
# Default OUT = $ACTS_DATA_ROOT/rebuilt (the released trees in $ACTS_DATA_ROOT are not touched).
# Point training at the result with data.latent_root=... data.action_root=... overrides.
# Recipe (identical to the released data): 30 -> 15 fps (stride 2), 8+8-frame windows, stride 1,
# 256x256 scale+center-crop; tactile = Wan2.2 VAE + tactile LoRA, view_middle + wrist = base VAE.
# Encoding needs 1 GPU (~10-40 min per episode group); action building is CPU-only.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
source "${ACTS_REPO:-$HERE}/env.sh"
TASK="${1:?usage: run_preprocess.sh <umi_all|rope|pusht> [OUT] [episode ...]}"
OUT="${2:-$ACTS_DATA_ROOT/rebuilt}"; shift $(( $# >= 2 ? 2 : 1 ))
declare -A TREE=([umi_all]=mixed_mb [rope]=mixed_rope [pusht]=mixed_pushT)
T="${TREE[$TASK]}"; RAW="$ACTS_DATA_ROOT/raw"
VAE="${WAN22_VAE_PTH:-$ACTS_CKPT_ROOT/Wan2.2_VAE.pth}"; LORA="${VTM_TACTILE_LORA:-$ACTS_CKPT_ROOT/vae/lora_last.pt}"
LAT="$OUT/${T}_latents"; ACT="$OUT/${T}_actions"; MAN="$OUT/episodes_${TASK}.jsonl"
mkdir -p "$LAT" "$ACT"
echo "=== [1/3] manifest"
"$PYTHON_BIN" "$ACTS_REPO/preprocessing/make_manifest.py" --raw-root "$RAW" --task "$TASK" --out "$MAN" ${@:+--episodes "$@"}
EPS=$("$PYTHON_BIN" -c "import json,sys; print(' '.join(json.loads(l)['episode'] for l in open(sys.argv[1]) if l.strip()))" "$MAN")
echo "=== [2/3] Wan2.2 latents (GPU)"
COMMON=(--vae-pth "$VAE" --fps 15 --src-fps 30 --history 8 --prediction 8 --stride 1 --resize 256
--manifest "$MAN" --output-root "$LAT" --episodes $EPS)
cd "$VTM_WAN22_DIR"
"$PYTHON_BIN" -u encode_tactile_wan22.py --modality tactile --adapter "$LORA" "${COMMON[@]}"
"$PYTHON_BIN" -u encode_tactile_wan22.py --modality visual --adapter none "${COMMON[@]}"
"$PYTHON_BIN" -u encode_tactile_wan22.py --modality wrist --adapter none "${COMMON[@]}"
echo "=== [3/3] actions (CPU)"
export ACTS_PARQUET_ROOT="$RAW/parquet"
P="$ACTS_REPO/preprocessing"
export PYTHONPATH="$P:$P/data_preprocessing:$PYTHONPATH"
for ep in $EPS; do mkdir -p "$ACT/$ep"; done
AARGS=(--parquet-root "$RAW/parquet" --videos-root "$ACT" --latent-dir "$LAT/tac" --target-fps 15 --episodes $EPS)
# order matters: the two force twins first, pose7d cross-checks its valid mask against them
"$PYTHON_BIN" -u "$P/vm_diffusion/data/build_force_actions.py" --force-term delta "${AARGS[@]}"
"$PYTHON_BIN" -u "$P/vm_diffusion/data/build_force_actions.py" --force-term absolute "${AARGS[@]}"
"$PYTHON_BIN" -u "$P/vm_diffusion/data/build_pose7d_force.py" "${AARGS[@]}"
# camera-frame view actions (gel centre / normal / up points + projections), 15 fps
"$PYTHON_BIN" -u "$P/data_preprocessing/build_view_actions.py" --videos-root "$ACT" --episodes $EPS \
--stride 2 --out-name view_actions_fps15.pt
echo "PREPROCESS DONE -> $LAT $ACT"
echo "train on it: bash scripts/train.sh configs/nano_action_causal_${TASK}_v1.yaml data.latent_root=$LAT data.action_root=$ACT"
|