#!/usr/bin/env bash set -Eeuo pipefail # Archived data builder; keep relative dependencies rooted at the HFD project. PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$PROJECT_ROOT" source "$PROJECT_ROOT/scripts/activate_psi_env.sh" export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" DATA_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1} OLD_HIDDEN_ROOT=${HFD_OLD_AUDIO_COMMAND_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source} BACKGROUND_ROOT="$DATA_ROOT/background" SESSION_DIR="$BACKGROUND_ROOT/sessions" SESSION_MANIFEST="$SESSION_DIR/session_manifest.json" MIXED_HIDDEN_ROOT="$DATA_ROOT/mixed_hidden" BACKGROUND_HIDDEN_ROOT="$DATA_ROOT/background_hidden" MANIFEST_ROOT="$DATA_ROOT/manifests" ASSIGNMENT_PATH="$MANIFEST_ROOT/source_background_assignments.jsonl" MIXED_SOURCE_CONFIG="$MANIFEST_ROOT/mixed_source_config.json" BACKGROUND_SOURCE_CONFIG="$MANIFEST_ROOT/background_source_config.json" MODEL_PATH=${HFD_QWEN_BACKBONE:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B} DEMAND_ROOT=${HFD_DEMAND_ROOT:-$PROJECT_ROOT/data/full_duplex_background_review_260730/raw} CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3} WORKERS_PER_GPU=${HFD_CONTINUOUS_WORKERS_PER_GPU:-4} CPU_WORKERS=${HFD_CONTINUOUS_CPU_WORKERS:-32} LIMIT_SOURCES=${HFD_CONTINUOUS_LIMIT_SOURCES:-0} VERIFY_SPEECH_SOURCES=${HFD_CONTINUOUS_VERIFY_SPEECH_SOURCES:-64} VERIFY_BACKGROUND_SOURCES=${HFD_CONTINUOUS_VERIFY_BACKGROUND_SOURCES:-10} VERIFY_WINDOWS_PER_BACKGROUND=${HFD_CONTINUOUS_VERIFY_BACKGROUND_WINDOWS:-8} FORCE_WAVEFORM_BUILD=${HFD_CONTINUOUS_FORCE_WAVEFORM_BUILD:-0} OVERWRITE_HIDDEN=${HFD_CONTINUOUS_OVERWRITE_HIDDEN:-0} TORCHRUN_LOCAL_ADDR=${HFD_TORCHRUN_LOCAL_ADDR:-127.0.0.1} MONITOR_STOP=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/stop.sh MONITOR_START=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/cmd.sh LOG_DIR="$DATA_ROOT/logs" LOG_PATH="$LOG_DIR/prepare.log" mkdir -p "$LOG_DIR" exec > >(tee -a "$LOG_PATH") 2>&1 echo "================================================================================" echo "HFD 连续办公室背景数据" echo "结果根目录: $DATA_ROOT" echo "混合语音 hidden: $MIXED_HIDDEN_ROOT" echo "连续背景 hidden: $BACKGROUND_HIDDEN_ROOT" echo "GPU: $CUDA_VISIBLE_DEVICES" echo "每卡 B=1 进程: $WORKERS_PER_GPU" echo "source 限制: $LIMIT_SOURCES" echo "================================================================================" if [[ "$LIMIT_SOURCES" != "0" \ && "$DATA_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1" ]]; then echo "[error] 有限 source 测试不能写进正式目录。" >&2 exit 2 fi for required in \ "$MODEL_PATH/config.json" \ "$DEMAND_ROOT/OOFFICE_ch01.wav" \ "$DEMAND_ROOT/OHALLWAY_ch01.wav"; do if [[ ! -e "$required" ]]; then echo "[error] 缺少输入: $required" >&2 exit 2 fi done if [[ "$FORCE_WAVEFORM_BUILD" == "1" \ || ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then if [[ ! -d "$OLD_HIDDEN_ROOT" && ! -f "$ASSIGNMENT_PATH" ]]; then echo "[error] 首次构造需要旧 cache 的有效 source 清单;重建则可复用新 assignment。" >&2 echo " legacy=$OLD_HIDDEN_ROOT" >&2 echo " fallback=$ASSIGNMENT_PATH" >&2 exit 2 fi fi if [[ ! -f "$SESSION_MANIFEST" ]]; then python -u scripts/data/generate_office_background_candidates.py \ --production-sessions \ --output-dir "$SESSION_DIR" \ --duration-seconds 600 \ --seed 260730 \ --demand-root "$DEMAND_ROOT" \ --vad-margin-seconds 1.0 \ --minimum-real-session-seconds 20.0 fi WAVEFORM_ARGS=( scripts/data/build_continuous_audio_command_sources.py --output-root "$DATA_ROOT" --background-manifest "$SESSION_MANIFEST" --old-hidden-root "$OLD_HIDDEN_ROOT" --workers "$CPU_WORKERS" --limit-sources "$LIMIT_SOURCES" --initial-idle-max-steps 5 --stationary-blend-frames 20 --safety-steps 10 --seed 260730 ) if [[ "$FORCE_WAVEFORM_BUILD" == "1" \ || ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then python -u "${WAVEFORM_ARGS[@]}" --mode build fi python -u "${WAVEFORM_ARGS[@]}" --mode verify monitor_stopped=0 restore_monitor() { if [[ "$monitor_stopped" == "1" && -f "$MONITOR_START" ]]; then bash "$MONITOR_START" || true monitor_stopped=0 fi } trap restore_monitor EXIT if [[ -f "$MONITOR_STOP" ]]; then bash "$MONITOR_STOP" monitor_stopped=1 fi export CUDA_VISIBLE_DEVICES export NCCL_DEBUG=WARN export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VISIBLE_GPU_COUNT=$(python - <<'PY' import torch print(torch.cuda.device_count()) PY ) if [[ "$VISIBLE_GPU_COUNT" != "4" ]]; then echo "[error] 本次正式生成要求恰好四张可见 GPU,实际为 $VISIBLE_GPU_COUNT。" >&2 exit 2 fi if [[ ! "$WORKERS_PER_GPU" =~ ^[1-9][0-9]*$ ]]; then echo "[error] WORKERS_PER_GPU 必须是正整数。" >&2 exit 2 fi NPROC=$((VISIBLE_GPU_COUNT * WORKERS_PER_GPU)) export_hidden() { local output_root=$1 local source_name=$2 local source_config=$3 local limit=$4 local overwrite_args=() local limit_args=() if [[ "$OVERWRITE_HIDDEN" == "1" ]]; then overwrite_args+=(--overwrite) fi if [[ "$limit" != "0" ]]; then limit_args+=(--limit-sources "$limit") fi python -m torch.distributed.run \ --standalone \ --local_addr="$TORCHRUN_LOCAL_ADDR" \ --nproc_per_node="$NPROC" \ scripts/data/cache_omni_hidden_states.py \ --mode export \ --model-path "$MODEL_PATH" \ --output-root "$output_root" \ --tasks audio_command_text \ --sources "$source_name" \ --source-config-file "$source_config" \ --motion-fps 50 \ --audio-command-sample-step-frames 20 \ --audio-command-window-seconds 0.6 \ --audio-sampling-rate 16000 \ --audio-batch-size 1 \ --num-io-workers 2 \ --num-save-workers 4 \ --max-pending-save-batches 8 \ --save-dtype bf16 \ --storage-format source \ --no-audio-command-full-audio \ "${limit_args[@]}" \ "${overwrite_args[@]}" } verify_hidden() { local output_root=$1 local source_name=$2 local source_config=$3 local verify_sources=$4 local verify_windows=$5 CUDA_VISIBLE_DEVICES=0 python -u scripts/data/cache_omni_hidden_states.py \ --mode verify \ --model-path "$MODEL_PATH" \ --output-root "$output_root" \ --tasks audio_command_text \ --sources "$source_name" \ --source-config-file "$source_config" \ --motion-fps 50 \ --audio-command-sample-step-frames 20 \ --audio-command-window-seconds 0.6 \ --audio-sampling-rate 16000 \ --audio-batch-size 1 \ --save-dtype bf16 \ --storage-format source \ --verify-sources "$verify_sources" \ --verify-windows-per-source "$verify_windows" \ --no-audio-command-full-audio } echo "[GPU] 导出连续纯背景 hidden" export_hidden \ "$BACKGROUND_HIDDEN_ROOT" \ hfd_continuous_office_background \ "$BACKGROUND_SOURCE_CONFIG" \ 0 verify_hidden \ "$BACKGROUND_HIDDEN_ROOT" \ hfd_continuous_office_background \ "$BACKGROUND_SOURCE_CONFIG" \ "$VERIFY_BACKGROUND_SOURCES" \ "$VERIFY_WINDOWS_PER_BACKGROUND" echo "[GPU] 导出人声+连续背景 mixed hidden" export_hidden \ "$MIXED_HIDDEN_ROOT" \ qwen3_tts_seed_audio_command \ "$MIXED_SOURCE_CONFIG" \ "$LIMIT_SOURCES" verify_hidden \ "$MIXED_HIDDEN_ROOT" \ qwen3_tts_seed_audio_command \ "$MIXED_SOURCE_CONFIG" \ "$VERIFY_SPEECH_SOURCES" \ 0 python - "$DATA_ROOT" "$MODEL_PATH" <<'PY' import hashlib import json import sqlite3 import sys import time from pathlib import Path from psi.data.seed.full_duplex_audio_contract import ( validate_encoder_cache, write_json_atomic, ) root = Path(sys.argv[1]).resolve() model = Path(sys.argv[2]).resolve() waveform = json.loads((root / "reports" / "waveform_build.json").read_text()) def index_totals(cache_root, source_name): source_dir = ( cache_root / "audio" / source_name / "audio_command_text" / "sources" ) rows = {} for path in source_dir.glob("index.ws*.rank*.sqlite"): connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) try: values = connection.execute( "SELECT key, path, count FROM sources" ).fetchall() finally: connection.close() for key, relative_path, count in values: value = (str(relative_path), int(count)) previous = rows.get(str(key)) if previous is not None and previous != value: raise RuntimeError( f"Conflicting duplicate source index row: {key}: " f"{previous} != {value}" ) rows[str(key)] = value return len(rows), sum(count for _, count in rows.values()) mixed_root = root / "mixed_hidden" background_root = root / "background_hidden" mixed_contract = validate_encoder_cache(mixed_root, model) background_contract = validate_encoder_cache(background_root, model) mixed_sources, mixed_windows = index_totals( mixed_root, "qwen3_tts_seed_audio_command" ) background_sources, background_windows = index_totals( background_root, "hfd_continuous_office_background" ) expected_sources = int(waveform["source_count"]) expected_mixed = int(waveform["mixed_window_count"]) expected_background_sources = int(waveform["background_session_count"]) expected_background = int(waveform["background_window_count"]) actual = ( mixed_sources, mixed_windows, background_sources, background_windows, ) expected = ( expected_sources, expected_mixed, expected_background_sources, expected_background, ) if actual != expected: raise RuntimeError(f"Final cache totals mismatch: actual={actual} expected={expected}") assignment = Path(waveform["assignment_path"]) digest = hashlib.sha256(assignment.read_bytes()).hexdigest() if digest != waveform["assignment_sha256"]: raise RuntimeError("Assignment manifest changed before finalization") write_json_atomic( root / "reports" / "production_complete.json", { "format": "hfd_continuous_office_production_v1", "completed_at": time.time(), "data_root": str(root), "source_count": mixed_sources, "mixed_window_count": mixed_windows, "background_source_count": background_sources, "background_window_count": background_windows, "initial_idle_max_steps": int(waveform["initial_idle_max_steps"]), "stationary_blend_frames": int(waveform["stationary_blend_frames"]), "safety_steps": int(waveform["safety_steps"]), "assignment_seed": int(waveform["seed"]), "assignment_path": str(assignment), "assignment_sha256": digest, "online_encoder_contract": mixed_contract, "background_encoder_contract": background_contract, }, ) print( f"[complete] sources={mixed_sources:,} mixed={mixed_windows:,} " f"background={background_windows:,} root={root}" ) PY restore_monitor echo "[done] $DATA_ROOT"