| # HY-WorldPlay evaluation: generation (2 shards) -> VBench -> aggregate -> Experiment.md | |
| # GPUS="5 6" hy_run_eval.sh <logdir> | |
| set -u | |
| C=/local/zoubin/cz/projects/comparison | |
| PY=/local/zoubin/cz/envs/hy_worldplay_july20260720/bin/python | |
| LOG="$1"; GPUS=(${GPUS:-5 6}) | |
| export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 | |
| export HF_HOME=/local/zoubin/cz/checkpoints/hy_worldplay/huggingface HF_HUB_CACHE=/local/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub | |
| export TRANSFORMERS_CACHE=/local/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub TORCH_HOME=/local/zoubin/cz/.cache/torch XDG_CACHE_HOME=/local/zoubin/cz/.cache | |
| cd "$C"; mkdir -p "$LOG" | |
| echo "=== generation on GPUs ${GPUS[*]} ($(date +%H:%M:%S)) ===" | |
| pids=() | |
| for i in "${!GPUS[@]}"; do | |
| # A shard may already have been started by hand on an idle GPU; never run the | |
| # same shard twice -- both copies would generate the same videos. | |
| if pgrep -f "hy_generate_eval.py --shard $i " > /dev/null 2>&1; then | |
| echo "shard $i already running elsewhere, not starting a second copy" | |
| continue | |
| fi | |
| CUDA_VISIBLE_DEVICES=${GPUS[$i]} $PY hy_generate_eval.py --shard $i --num-shards ${#GPUS[@]} --out-root eval_out_hy > "$LOG/hy_gen_$i.log" 2>&1 & | |
| pids+=($!) | |
| done | |
| for p in "${pids[@]:-}"; do wait "$p" 2>/dev/null; done | |
| # Also wait for any shard that was started outside this driver. | |
| while pgrep -f "[h]y_generate_eval.py" > /dev/null 2>&1; do sleep 60; done | |
| echo "=== generation done ($(date +%H:%M:%S)) ===" | |
| GPUS="${GPUS[*]}" bash hy_vbench.sh "$LOG" 2>&1 | tee "$LOG/hy_vbench_driver.log" | |
| echo "=== aggregating ($(date +%H:%M:%S)) ===" | |
| python3 hy_aggregate.py 2>&1 | tee "$LOG/hy_aggregate.log" | |
| python3 experiment_md.py | |
| echo "ALL HY EVAL DONE $(date +%H:%M:%S)" | |