File size: 2,951 Bytes
fb0074a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | #!/usr/bin/env bash
set -euo pipefail
BENCH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
ROOT=$(dirname "$BENCH")
PEER=$(dirname "$(dirname "$(dirname "$ROOT")")")/buwendong/user
RUN=${RUN:-$ROOT/runs/posttrain_full_duplex/hfd.gpus4.2610100010-thirdparty}
STEP=${STEP:-1800000}
COUNT=${COUNT:-2048}
GPUS=${GPUS:-0,1,2,3}
WORKERS_PER_GPU=${WORKERS_PER_GPU:-3}
COHORT=${COHORT:-$BENCH/outputs/sessions2048}
EXTERNAL_ASSETS=${EXTERNAL_ASSETS:-$BENCH/outputs/external_visual/overlay_assets.json}
OUTPUT=${OUTPUT:-$BENCH/outputs/session_ckpt_$STEP}
HFD_PYTHON=${HFD_PYTHON:-$ROOT/.venv-psi/bin/python}
if [[ -z ${EVAL_PYTHON:-} ]]; then
EVAL_PYTHON=$BENCH/.venv-eval/bin/python
[[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=$PEER/envs/OMG/bin/python
[[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=python
fi
export PYTHONDONTWRITEBYTECODE=1
export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=4
cd "$ROOT"
mkdir -p "$OUTPUT/logs"
exec 9>"$OUTPUT/.run.lock"
flock -n 9 || { echo 'This benchmark output is already running'; exit 1; }
[[ -f "$BENCH/vendor/manifest.json" ]] || { echo 'Missing evaluator bundle: run benchmark/vendor.py first'; exit 1; }
if [[ ! -f "$COHORT/manifest.json" ]]; then
[[ -f "$EXTERNAL_ASSETS" ]] || { echo "External QA is not published yet: $EXTERNAL_ASSETS"; exit 1; }
"$HFD_PYTHON" -u "$BENCH/prepare.py" --run "$RUN" --output "$COHORT" --count "$COUNT" \
--external-assets "$EXTERNAL_ASSETS" | tee "$OUTPUT/logs/prepare.log"
fi
"$HFD_PYTHON" -c 'import json,sys,hashlib; d=json.load(open(sys.argv[1])); assert len(d["rows"])==int(sys.argv[2]), "Frozen cohort count mismatch"; a=open(sys.argv[3],"rb").read(); assert d.get("external_assets_sha256")==hashlib.sha256(a).hexdigest(), "Cohort must be revised for updated QA admission"; assert json.loads(a)["provenance"].get("direct_inspection_sha256"), "Direct QA inspection required"' "$COHORT/manifest.json" "$COUNT" "$EXTERNAL_ASSETS"
IFS=',' read -r -a devices <<< "$GPUS"
pids=()
workers=$(( ${#devices[@]} * WORKERS_PER_GPU ))
for ((shard=0; shard<workers; shard++)); do
device=${devices[$((shard % ${#devices[@]}))]}
CUDA_VISIBLE_DEVICES=$device "$HFD_PYTHON" -u "$BENCH/generate.py" \
--run "$RUN" --step "$STEP" --cohort "$COHORT" --output "$OUTPUT/generated" \
--shard "$shard" --shards "$workers" >"$OUTPUT/logs/generate_$shard.log" 2>&1 &
pids+=("$!")
done
status=0
for pid in "${pids[@]}"; do wait "$pid" || status=1; done
(( status == 0 )) || exit 1
"$HFD_PYTHON" -B "$BENCH/audit.py" --cohort "$COHORT" --generated "$OUTPUT/generated"
if [[ ! -f "$OUTPUT/eval/motion/paper_metrics.json" ]]; then
CUDA_VISIBLE_DEVICES=${devices[0]} "$EVAL_PYTHON" -B -u "$BENCH/evaluate.py" \
--cohort "$COHORT" --generated "$OUTPUT/generated" --output "$OUTPUT/eval" | tee "$OUTPUT/logs/eval.log"
fi
"$EVAL_PYTHON" -B "$BENCH/evaluate.py" --cohort "$COHORT" --generated "$OUTPUT/generated" \
--output "$OUTPUT/eval" --verify-only
|