code-HFD / scripts /data /complete_video_vqa.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
1.15 kB
#!/usr/bin/env bash
# 续跑本批审核后的流程;不启动长训、不创建 WandB 曲线。
# 新扩充使用新的 HFD_VIDEO_DATA_ROOT,先提供 candidates/审核/语音回执。
set -Eeuo pipefail
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
export OMP_NUM_THREADS=2
cd "$PROJECT_ROOT"
DATA_ROOT=${HFD_VIDEO_DATA_ROOT:-$PROJECT_ROOT/../../videogen/hfd_video_vqa_50k_v1}
BODY_RUN=${HFD_VIDEO_BODY_RUN:-$PROJECT_ROOT/runs/posttrain_full_duplex/hfd.gpus2.2609231722-overlay-pilot}
BASE_MODEL=${HFD_VIDEO_BASE_MODEL:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
HELDOUT_CATALOG=${HFD_VIDEO_HELDOUT_CATALOG:-$PROJECT_ROOT/../../hfd_experiments/overlay_260923/catalog_v1.json}
exec python scripts/data/prepare_hfd_video_assets.py all \
--root "$DATA_ROOT" --base-model "$BASE_MODEL" --body-run "$BODY_RUN" \
--external-catalog "$HELDOUT_CATALOG" --cpu-workers 12 --packet-batch 1 --workers-per-gpu 4 \
--gpus "${HFD_VIDEO_GPUS:-0,1}" "$@"