CharlesZhang-USTC's picture
Add Control v2 incremental experiment results (evidence, visuals, 42 fixed samples)
a64600b verified
Raw History Blame Contribute Delete
3.11 kB
#!/usr/bin/env bash
# Run INSIDE the existing SNF CUDA 12.8 image, with the updated suite mounted.
set -euo pipefail
SNF_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$SNF_ROOT"
export PYTHONPATH="$SNF_ROOT${PYTHONPATH:+:$PYTHONPATH}"
export MPLBACKEND=Agg
export PYTHONUNBUFFERED=1
SNF_V2_PYTHON="${SNF_V2_PYTHON:-python3}"
SNF_V2_OUTPUT="${SNF_V2_OUTPUT:-/outputs/method-v2}"
SNF_V2_TASKS="${SNF_V2_TASKS:-$SNF_ROOT/suite/tasks.method_v2.jsonl}"
SNF_V2_WEIGHTS="${SNF_V2_WEIGHTS:-/weights}"
SNF_V2_MODELS="${SNF_V2_MODELS:-meissonic,showo,momask}"
SNF_V2_GPUS="${SNF_V2_GPUS:-0,1,2,3,4,5,6,7}"
SNF_V2_PHASE="${1:-all}"
IFS=',' read -r -a SNF_V2_MODEL_ARRAY <<< "$SNF_V2_MODELS"
mkdir -p "$SNF_V2_OUTPUT"
plan_phase() {
"$SNF_V2_PYTHON" -m suite.method_v2 --phase "$1" --tasks "$SNF_V2_TASKS" \
--models "$SNF_V2_MODELS" --output "$SNF_V2_OUTPUT/$1-plan"
}
run_phase() {
local SNF_PHASE_NAME="$1"
shift
"$SNF_V2_PYTHON" -m suite.cli run --jobs "$SNF_V2_OUTPUT/$SNF_PHASE_NAME-plan/jobs.jsonl" \
--tasks "$SNF_V2_TASKS" --profile full --no-extras --weights "$SNF_V2_WEIGHTS" \
--output "$SNF_V2_OUTPUT/$SNF_PHASE_NAME" --gpus "$SNF_V2_GPUS" --resume "$@"
}
phase_null() {
"$SNF_V2_PYTHON" -m suite.null_commitment --output "$SNF_V2_OUTPUT/null/steps18" --steps 18 --tokens 256 --seeds 256
"$SNF_V2_PYTHON" -m suite.null_commitment --output "$SNF_V2_OUTPUT/null/steps64" --steps 64 --tokens 256 --seeds 256
}
phase_smoke() {
"$SNF_V2_PYTHON" -m suite.cli source-check
"$SNF_V2_PYTHON" -m unittest discover -s "$SNF_ROOT/suite/tests" -v
plan_phase smoke
run_phase smoke --render-motion
}
phase_dev() { plan_phase dev; run_phase dev; }
phase_fit() {
"$SNF_V2_PYTHON" -m suite.calibration_v2 fit --run "$SNF_V2_OUTPUT/dev" --output "$SNF_V2_OUTPUT/calibration"
"$SNF_V2_PYTHON" -m suite.calibration_v2 plan --mapping "$SNF_V2_OUTPUT/calibration/mappings.json" \
--tasks "$SNF_V2_TASKS" --output "$SNF_V2_OUTPUT/test-plan" --test-tasks-per-model 12 \
--seeds 3 4 5 --models "${SNF_V2_MODEL_ARRAY[@]}"
}
phase_mechanism() { plan_phase mechanism; run_phase mechanism; }
phase_test() { run_phase test; }
phase_test_sweep() { plan_phase test-sweep; run_phase test-sweep; }
phase_quality() {
"$SNF_V2_PYTHON" -m suite.cli evaluate --run "$SNF_V2_OUTPUT/test" --weights "$SNF_V2_WEIGHTS" --device cuda
}
phase_analyze() {
"$SNF_V2_PYTHON" -m suite.calibration_v2 analyze --run "$SNF_V2_OUTPUT/test" --output "$SNF_V2_OUTPUT/test-analysis"
"$SNF_V2_PYTHON" -m suite.visualize_v2 --run "$SNF_V2_OUTPUT/test" --output "$SNF_V2_OUTPUT/paper-visuals"
}
case "$SNF_V2_PHASE" in
all)
phase_null
phase_smoke
phase_dev
phase_fit
phase_mechanism
phase_test
phase_analyze
phase_quality
;;
null) phase_null ;;
smoke) phase_smoke ;;
dev) phase_dev ;;
fit) phase_fit ;;
mechanism) phase_mechanism ;;
test) phase_test ;;
analyze) phase_analyze ;;
quality) phase_quality ;;
test-sweep) phase_test_sweep ;;
*) printf '%s\n' 'Expected all|null|smoke|dev|fit|mechanism|test|analyze|quality|test-sweep' >&2; exit 2 ;;
esac