code-HFD / scripts /data /complete_video_vqa_with_keeper.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
1.55 kB
#!/usr/bin/env bash
# 本地专用:取得独占运行锁、暂停占卡,完成全部准入流程后恢复占卡。
# 不用于DLC,不启动长训;机器断电后需要再次运行,成功回执自动复用。
set -Eeuo pipefail
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
DATA_ROOT=${HFD_VIDEO_DATA_ROOT:-$PROJECT_ROOT/../../videogen/hfd_video_vqa_50k_v1}
KEEPER_ROOT=${HFD_VIDEO_MONITOR_ROOT:-$PROJECT_ROOT/../monitor}
export HFD_VIDEO_GPUS=${HFD_VIDEO_GPUS:-0,1,2,3}
export MONITOR_GPU_INDICES=$HFD_VIDEO_GPUS
[[ "$HFD_VIDEO_GPUS" =~ ^[0-9]+(,[0-9]+)*$ ]] || exit 2
[[ -f "$KEEPER_ROOT/stop.sh" && -f "$KEEPER_ROOT/cmd.sh" ]] || exit 2
mkdir -p "$DATA_ROOT/admission"
exec 9>"$DATA_ROOT/admission/local_supervisor.lock"
flock -n 9 || { echo 'A local admission supervisor is already running'; exit 1; }
child=
restore_keeper() {
local status=$?
trap - EXIT INT TERM
if [[ -n "$child" ]]; then
kill -TERM "$child" 2>/dev/null || true
# Python coordinator joins all encoder/acceptance children before exiting.
wait "$child" 2>/dev/null || true
fi
echo "[admission] exit=$status; restoring idle-GPU keeper"
if ! bash "$KEEPER_ROOT/cmd.sh" 9>&-; then
echo '[admission] keeper launch failed; check monitor.log' >&2
(( status != 0 )) || status=1
fi
exit "$status"
}
trap restore_keeper EXIT
trap 'exit 143' TERM
trap 'exit 130' INT
bash "$KEEPER_ROOT/stop.sh"
bash "$PROJECT_ROOT/scripts/data/complete_video_vqa.sh" "$@" &
child=$!
set +e
wait "$child"
status=$?
set -e
child=
exit "$status"