#!/usr/bin/env bash # 本机八音色数据扩充;原单音色发布版不改,结束后自动恢复占卡。 set -Eeuo pipefail PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) source "$PROJECT_ROOT/scripts/activate_psi_env.sh" export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" export OMP_NUM_THREADS=2 PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false DATA_ROOT=${HFD_VIDEO_EIGHT_ROOT:-$PROJECT_ROOT/../../videogen/hfd_video_vqa_8voice_v1} KEEPER_ROOT="$PROJECT_ROOT/../monitor" export MONITOR_GPU_INDICES=${HFD_VIDEO_GPUS:-0,1,2,3} # 本机 H200 实测配置;每个进程独立分片,保留原合成与审查模型。 VOICE_BATCH=${HFD_VIDEO_VOICE_BATCH:-32} WORKERS_PER_GPU=${HFD_VIDEO_WORKERS_PER_GPU:-1} if [[ "$WORKERS_PER_GPU" == 2 ]]; then default_memory_fraction=0.46; else default_memory_fraction=0.92; fi MEMORY_FRACTION=${HFD_VIDEO_MEMORY_FRACTION:-$default_memory_fraction} mkdir -p "$DATA_ROOT/admission" exec 9>"$DATA_ROOT/admission/local_supervisor.lock" flock -n 9 || exit 1 child= cleanup() { local status=$? trap - EXIT TERM INT if [[ -n "$child" ]]; then kill -TERM "$child" 2>/dev/null || true wait "$child" 2>/dev/null || true fi bash "$KEEPER_ROOT/cmd.sh" 9>&- || true exit "$status" } trap cleanup EXIT trap 'exit 143' TERM trap 'exit 130' INT bash "$KEEPER_ROOT/stop.sh" # 刚结束上一轮时,占卡launcher可能尚未写PID文件。按本人的精确脚本 # 路径等待其退出,避免它稍后抢占新计算进程的显存;不匹配其他项目。 python - "$KEEPER_ROOT" <<'PY' import os, signal, sys, time from pathlib import Path targets={str(Path(sys.argv[1]).resolve()/name) for name in ('monitor.sh','monitor.py')} quiet=0 for _ in range(100): found=[] for proc in Path('/proc').iterdir(): if not proc.name.isdigit():continue try: args=proc.joinpath('cmdline').read_bytes().decode().split('\0') if targets.intersection(args):found.append(int(proc.name)) except (OSError,UnicodeError):continue if not found: quiet+=1 if quiet>=5:break else: quiet=0 for pid in found: try:os.kill(pid,signal.SIGTERM) except ProcessLookupError:pass time.sleep(.1) else:raise SystemExit('Own GPU keeper did not exit; refuse to overfill GPU') PY cd "$PROJECT_ROOT" python scripts/data/expand_hfd_video_voices.py all --root "$DATA_ROOT" --gpus "$MONITOR_GPU_INDICES" \ --batch "$VOICE_BATCH" --workers-per-gpu "$WORKERS_PER_GPU" \ --memory-fraction "$MEMORY_FRACTION" --trim-cache "$@" & child=$! set +e wait "$child" status=$? set -e child= exit "$status"