code-HFD / scripts /data /complete_video_eight_voices.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
2.66 kB
#!/usr/bin/env bash
# 本机八音色数据扩充;原单音色发布版不改,结束后自动恢复占卡。
set -Eeuo pipefail
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
export OMP_NUM_THREADS=2 PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false
DATA_ROOT=${HFD_VIDEO_EIGHT_ROOT:-$PROJECT_ROOT/../../videogen/hfd_video_vqa_8voice_v1}
KEEPER_ROOT="$PROJECT_ROOT/../monitor"
export MONITOR_GPU_INDICES=${HFD_VIDEO_GPUS:-0,1,2,3}
# 本机 H200 实测配置;每个进程独立分片,保留原合成与审查模型。
VOICE_BATCH=${HFD_VIDEO_VOICE_BATCH:-32}
WORKERS_PER_GPU=${HFD_VIDEO_WORKERS_PER_GPU:-1}
if [[ "$WORKERS_PER_GPU" == 2 ]]; then default_memory_fraction=0.46; else default_memory_fraction=0.92; fi
MEMORY_FRACTION=${HFD_VIDEO_MEMORY_FRACTION:-$default_memory_fraction}
mkdir -p "$DATA_ROOT/admission"
exec 9>"$DATA_ROOT/admission/local_supervisor.lock"
flock -n 9 || exit 1
child=
cleanup() {
local status=$?
trap - EXIT TERM INT
if [[ -n "$child" ]]; then
kill -TERM "$child" 2>/dev/null || true
wait "$child" 2>/dev/null || true
fi
bash "$KEEPER_ROOT/cmd.sh" 9>&- || true
exit "$status"
}
trap cleanup EXIT
trap 'exit 143' TERM
trap 'exit 130' INT
bash "$KEEPER_ROOT/stop.sh"
# 刚结束上一轮时,占卡launcher可能尚未写PID文件。按本人的精确脚本
# 路径等待其退出,避免它稍后抢占新计算进程的显存;不匹配其他项目。
python - "$KEEPER_ROOT" <<'PY'
import os, signal, sys, time
from pathlib import Path
targets={str(Path(sys.argv[1]).resolve()/name) for name in ('monitor.sh','monitor.py')}
quiet=0
for _ in range(100):
found=[]
for proc in Path('/proc').iterdir():
if not proc.name.isdigit():continue
try:
args=proc.joinpath('cmdline').read_bytes().decode().split('\0')
if targets.intersection(args):found.append(int(proc.name))
except (OSError,UnicodeError):continue
if not found:
quiet+=1
if quiet>=5:break
else:
quiet=0
for pid in found:
try:os.kill(pid,signal.SIGTERM)
except ProcessLookupError:pass
time.sleep(.1)
else:raise SystemExit('Own GPU keeper did not exit; refuse to overfill GPU')
PY
cd "$PROJECT_ROOT"
python scripts/data/expand_hfd_video_voices.py all --root "$DATA_ROOT" --gpus "$MONITOR_GPU_INDICES" \
--batch "$VOICE_BATCH" --workers-per-gpu "$WORKERS_PER_GPU" \
--memory-fraction "$MEMORY_FRACTION" --trim-cache "$@" &
child=$!
set +e
wait "$child"
status=$?
set -e
child=
exit "$status"