#!/usr/bin/env bash set -euo pipefail BENCH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) ROOT=$(dirname "$BENCH") PEER=$(dirname "$(dirname "$(dirname "$ROOT")")")/buwendong/user RUN=${RUN:-$ROOT/runs/posttrain_full_duplex/hfd.gpus4.2610100010-thirdparty} STEP=${STEP:-1800000} COUNT=${COUNT:-2048} GPUS=${GPUS:-0,1,2,3} WORKERS_PER_GPU=${WORKERS_PER_GPU:-3} COHORT=${COHORT:-$BENCH/outputs/sessions2048} EXTERNAL_ASSETS=${EXTERNAL_ASSETS:-$BENCH/outputs/external_visual/overlay_assets.json} OUTPUT=${OUTPUT:-$BENCH/outputs/session_ckpt_$STEP} HFD_PYTHON=${HFD_PYTHON:-$ROOT/.venv-psi/bin/python} if [[ -z ${EVAL_PYTHON:-} ]]; then EVAL_PYTHON=$BENCH/.venv-eval/bin/python [[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=$PEER/envs/OMG/bin/python [[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=python fi export PYTHONDONTWRITEBYTECODE=1 export OMP_NUM_THREADS=4 export MKL_NUM_THREADS=4 cd "$ROOT" mkdir -p "$OUTPUT/logs" exec 9>"$OUTPUT/.run.lock" flock -n 9 || { echo 'This benchmark output is already running'; exit 1; } [[ -f "$BENCH/vendor/manifest.json" ]] || { echo 'Missing evaluator bundle: run benchmark/vendor.py first'; exit 1; } if [[ ! -f "$COHORT/manifest.json" ]]; then [[ -f "$EXTERNAL_ASSETS" ]] || { echo "External QA is not published yet: $EXTERNAL_ASSETS"; exit 1; } "$HFD_PYTHON" -u "$BENCH/prepare.py" --run "$RUN" --output "$COHORT" --count "$COUNT" \ --external-assets "$EXTERNAL_ASSETS" | tee "$OUTPUT/logs/prepare.log" fi "$HFD_PYTHON" -c 'import json,sys,hashlib; d=json.load(open(sys.argv[1])); assert len(d["rows"])==int(sys.argv[2]), "Frozen cohort count mismatch"; a=open(sys.argv[3],"rb").read(); assert d.get("external_assets_sha256")==hashlib.sha256(a).hexdigest(), "Cohort must be revised for updated QA admission"; assert json.loads(a)["provenance"].get("direct_inspection_sha256"), "Direct QA inspection required"' "$COHORT/manifest.json" "$COUNT" "$EXTERNAL_ASSETS" IFS=',' read -r -a devices <<< "$GPUS" pids=() workers=$(( ${#devices[@]} * WORKERS_PER_GPU )) for ((shard=0; shard"$OUTPUT/logs/generate_$shard.log" 2>&1 & pids+=("$!") done status=0 for pid in "${pids[@]}"; do wait "$pid" || status=1; done (( status == 0 )) || exit 1 "$HFD_PYTHON" -B "$BENCH/audit.py" --cohort "$COHORT" --generated "$OUTPUT/generated" if [[ ! -f "$OUTPUT/eval/motion/paper_metrics.json" ]]; then CUDA_VISIBLE_DEVICES=${devices[0]} "$EVAL_PYTHON" -B -u "$BENCH/evaluate.py" \ --cohort "$COHORT" --generated "$OUTPUT/generated" --output "$OUTPUT/eval" | tee "$OUTPUT/logs/eval.log" fi "$EVAL_PYTHON" -B "$BENCH/evaluate.py" --cohort "$COHORT" --generated "$OUTPUT/generated" \ --output "$OUTPUT/eval" --verify-only