code-HFD / scripts /train_with_gpu_monitor.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
fb0074a verified
Raw History Blame Contribute Delete
1.32 kB
#!/usr/bin/env bash
# Optional local GPU-holder lifecycle. Never enabled merely because files exist.
set -Eeuo pipefail
MANAGE_GPU_MONITOR=${HFD_MANAGE_GPU_MONITOR:-0}
GPU_MONITOR_ROOT=${HFD_GPU_MONITOR_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor}
MONITOR_CMD=$GPU_MONITOR_ROOT/cmd.sh
MONITOR_SCRIPT=$GPU_MONITOR_ROOT/monitor.sh
MONITOR_PY=$GPU_MONITOR_ROOT/monitor.py
MONITOR_GUARD_PID=
MONITOR_GUARD_SECONDS=${HFD_MONITOR_GUARD_SECONDS:-5}
stop_gpu_monitor() {
pkill -f "$MONITOR_SCRIPT" 2>/dev/null || true
pkill -f "$MONITOR_PY" 2>/dev/null || true
}
guard_gpu_monitor() {
while true; do
stop_gpu_monitor
sleep "$MONITOR_GUARD_SECONDS"
done
}
restore_gpu_monitor() {
if [[ -n "${MONITOR_GUARD_PID:-}" ]]; then
kill "$MONITOR_GUARD_PID" 2>/dev/null || true
wait "$MONITOR_GUARD_PID" 2>/dev/null || true
fi
if [[ -f "$MONITOR_CMD" ]]; then
set +e
stop_gpu_monitor
source "$MONITOR_CMD"
set -e
fi
}
# DLC does not need a GPU holder. Only an explicit local opt-in may stop or
# restore this user's monitor; shared storage containing cmd.sh is not consent.
if [[ "$MANAGE_GPU_MONITOR" == "1" ]]; then
trap restore_gpu_monitor EXIT
stop_gpu_monitor
guard_gpu_monitor &
MONITOR_GUARD_PID=$!
fi
"$@"