File size: 2,951 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#!/usr/bin/env bash
set -euo pipefail

BENCH=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
ROOT=$(dirname "$BENCH")
PEER=$(dirname "$(dirname "$(dirname "$ROOT")")")/buwendong/user
RUN=${RUN:-$ROOT/runs/posttrain_full_duplex/hfd.gpus4.2610100010-thirdparty}
STEP=${STEP:-1800000}
COUNT=${COUNT:-2048}
GPUS=${GPUS:-0,1,2,3}
WORKERS_PER_GPU=${WORKERS_PER_GPU:-3}
COHORT=${COHORT:-$BENCH/outputs/sessions2048}
EXTERNAL_ASSETS=${EXTERNAL_ASSETS:-$BENCH/outputs/external_visual/overlay_assets.json}
OUTPUT=${OUTPUT:-$BENCH/outputs/session_ckpt_$STEP}
HFD_PYTHON=${HFD_PYTHON:-$ROOT/.venv-psi/bin/python}
if [[ -z ${EVAL_PYTHON:-} ]]; then
    EVAL_PYTHON=$BENCH/.venv-eval/bin/python
    [[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=$PEER/envs/OMG/bin/python
    [[ -x "$EVAL_PYTHON" ]] || EVAL_PYTHON=python
fi

export PYTHONDONTWRITEBYTECODE=1
export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=4
cd "$ROOT"
mkdir -p "$OUTPUT/logs"
exec 9>"$OUTPUT/.run.lock"
flock -n 9 || { echo 'This benchmark output is already running'; exit 1; }
[[ -f "$BENCH/vendor/manifest.json" ]] || { echo 'Missing evaluator bundle: run benchmark/vendor.py first'; exit 1; }
if [[ ! -f "$COHORT/manifest.json" ]]; then
    [[ -f "$EXTERNAL_ASSETS" ]] || { echo "External QA is not published yet: $EXTERNAL_ASSETS"; exit 1; }
    "$HFD_PYTHON" -u "$BENCH/prepare.py" --run "$RUN" --output "$COHORT" --count "$COUNT" \
        --external-assets "$EXTERNAL_ASSETS" | tee "$OUTPUT/logs/prepare.log"
fi
"$HFD_PYTHON" -c 'import json,sys,hashlib; d=json.load(open(sys.argv[1])); assert len(d["rows"])==int(sys.argv[2]), "Frozen cohort count mismatch"; a=open(sys.argv[3],"rb").read(); assert d.get("external_assets_sha256")==hashlib.sha256(a).hexdigest(), "Cohort must be revised for updated QA admission"; assert json.loads(a)["provenance"].get("direct_inspection_sha256"), "Direct QA inspection required"' "$COHORT/manifest.json" "$COUNT" "$EXTERNAL_ASSETS"
IFS=',' read -r -a devices <<< "$GPUS"
pids=()
workers=$(( ${#devices[@]} * WORKERS_PER_GPU ))
for ((shard=0; shard<workers; shard++)); do
    device=${devices[$((shard % ${#devices[@]}))]}
    CUDA_VISIBLE_DEVICES=$device "$HFD_PYTHON" -u "$BENCH/generate.py" \
        --run "$RUN" --step "$STEP" --cohort "$COHORT" --output "$OUTPUT/generated" \
        --shard "$shard" --shards "$workers" >"$OUTPUT/logs/generate_$shard.log" 2>&1 &
    pids+=("$!")
done
status=0
for pid in "${pids[@]}"; do wait "$pid" || status=1; done
(( status == 0 )) || exit 1
"$HFD_PYTHON" -B "$BENCH/audit.py" --cohort "$COHORT" --generated "$OUTPUT/generated"
if [[ ! -f "$OUTPUT/eval/motion/paper_metrics.json" ]]; then
CUDA_VISIBLE_DEVICES=${devices[0]} "$EVAL_PYTHON" -B -u "$BENCH/evaluate.py" \
    --cohort "$COHORT" --generated "$OUTPUT/generated" --output "$OUTPUT/eval" | tee "$OUTPUT/logs/eval.log"
fi
"$EVAL_PYTHON" -B "$BENCH/evaluate.py" --cohort "$COHORT" --generated "$OUTPUT/generated" \
    --output "$OUTPUT/eval" --verify-only