Download backup/prepare_full_duplex_continuous_audio.sh from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/backup/prepare_full_duplex_continuous_audio.sh
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/backup/prepare_full_duplex_continuous_audio.sh
-
curl -L -o prepare_full_duplex_continuous_audio.sh https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/backup/prepare_full_duplex_continuous_audio.sh
11.5 kB
| set -Eeuo pipefail | |
| # Archived data builder; keep relative dependencies rooted at the HFD project. | |
| PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) | |
| cd "$PROJECT_ROOT" | |
| source "$PROJECT_ROOT/scripts/activate_psi_env.sh" | |
| export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" | |
| DATA_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1} | |
| OLD_HIDDEN_ROOT=${HFD_OLD_AUDIO_COMMAND_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source} | |
| BACKGROUND_ROOT="$DATA_ROOT/background" | |
| SESSION_DIR="$BACKGROUND_ROOT/sessions" | |
| SESSION_MANIFEST="$SESSION_DIR/session_manifest.json" | |
| MIXED_HIDDEN_ROOT="$DATA_ROOT/mixed_hidden" | |
| BACKGROUND_HIDDEN_ROOT="$DATA_ROOT/background_hidden" | |
| MANIFEST_ROOT="$DATA_ROOT/manifests" | |
| ASSIGNMENT_PATH="$MANIFEST_ROOT/source_background_assignments.jsonl" | |
| MIXED_SOURCE_CONFIG="$MANIFEST_ROOT/mixed_source_config.json" | |
| BACKGROUND_SOURCE_CONFIG="$MANIFEST_ROOT/background_source_config.json" | |
| MODEL_PATH=${HFD_QWEN_BACKBONE:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B} | |
| DEMAND_ROOT=${HFD_DEMAND_ROOT:-$PROJECT_ROOT/data/full_duplex_background_review_260730/raw} | |
| CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3} | |
| WORKERS_PER_GPU=${HFD_CONTINUOUS_WORKERS_PER_GPU:-4} | |
| CPU_WORKERS=${HFD_CONTINUOUS_CPU_WORKERS:-32} | |
| LIMIT_SOURCES=${HFD_CONTINUOUS_LIMIT_SOURCES:-0} | |
| VERIFY_SPEECH_SOURCES=${HFD_CONTINUOUS_VERIFY_SPEECH_SOURCES:-64} | |
| VERIFY_BACKGROUND_SOURCES=${HFD_CONTINUOUS_VERIFY_BACKGROUND_SOURCES:-10} | |
| VERIFY_WINDOWS_PER_BACKGROUND=${HFD_CONTINUOUS_VERIFY_BACKGROUND_WINDOWS:-8} | |
| FORCE_WAVEFORM_BUILD=${HFD_CONTINUOUS_FORCE_WAVEFORM_BUILD:-0} | |
| OVERWRITE_HIDDEN=${HFD_CONTINUOUS_OVERWRITE_HIDDEN:-0} | |
| TORCHRUN_LOCAL_ADDR=${HFD_TORCHRUN_LOCAL_ADDR:-127.0.0.1} | |
| MONITOR_STOP=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/stop.sh | |
| MONITOR_START=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/cmd.sh | |
| LOG_DIR="$DATA_ROOT/logs" | |
| LOG_PATH="$LOG_DIR/prepare.log" | |
| mkdir -p "$LOG_DIR" | |
| exec > >(tee -a "$LOG_PATH") 2>&1 | |
| echo "================================================================================" | |
| echo "HFD 连续办公室背景数据" | |
| echo "结果根目录: $DATA_ROOT" | |
| echo "混合语音 hidden: $MIXED_HIDDEN_ROOT" | |
| echo "连续背景 hidden: $BACKGROUND_HIDDEN_ROOT" | |
| echo "GPU: $CUDA_VISIBLE_DEVICES" | |
| echo "每卡 B=1 进程: $WORKERS_PER_GPU" | |
| echo "source 限制: $LIMIT_SOURCES" | |
| echo "================================================================================" | |
| if [[ "$LIMIT_SOURCES" != "0" \ | |
| && "$DATA_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1" ]]; then | |
| echo "[error] 有限 source 测试不能写进正式目录。" >&2 | |
| exit 2 | |
| fi | |
| for required in \ | |
| "$MODEL_PATH/config.json" \ | |
| "$DEMAND_ROOT/OOFFICE_ch01.wav" \ | |
| "$DEMAND_ROOT/OHALLWAY_ch01.wav"; do | |
| if [[ ! -e "$required" ]]; then | |
| echo "[error] 缺少输入: $required" >&2 | |
| exit 2 | |
| fi | |
| done | |
| if [[ "$FORCE_WAVEFORM_BUILD" == "1" \ | |
| || ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then | |
| if [[ ! -d "$OLD_HIDDEN_ROOT" && ! -f "$ASSIGNMENT_PATH" ]]; then | |
| echo "[error] 首次构造需要旧 cache 的有效 source 清单;重建则可复用新 assignment。" >&2 | |
| echo " legacy=$OLD_HIDDEN_ROOT" >&2 | |
| echo " fallback=$ASSIGNMENT_PATH" >&2 | |
| exit 2 | |
| fi | |
| fi | |
| if [[ ! -f "$SESSION_MANIFEST" ]]; then | |
| python -u scripts/data/generate_office_background_candidates.py \ | |
| --production-sessions \ | |
| --output-dir "$SESSION_DIR" \ | |
| --duration-seconds 600 \ | |
| --seed 260730 \ | |
| --demand-root "$DEMAND_ROOT" \ | |
| --vad-margin-seconds 1.0 \ | |
| --minimum-real-session-seconds 20.0 | |
| fi | |
| WAVEFORM_ARGS=( | |
| scripts/data/build_continuous_audio_command_sources.py | |
| --output-root "$DATA_ROOT" | |
| --background-manifest "$SESSION_MANIFEST" | |
| --old-hidden-root "$OLD_HIDDEN_ROOT" | |
| --workers "$CPU_WORKERS" | |
| --limit-sources "$LIMIT_SOURCES" | |
| --initial-idle-max-steps 5 | |
| --stationary-blend-frames 20 | |
| --safety-steps 10 | |
| --seed 260730 | |
| ) | |
| if [[ "$FORCE_WAVEFORM_BUILD" == "1" \ | |
| || ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then | |
| python -u "${WAVEFORM_ARGS[@]}" --mode build | |
| fi | |
| python -u "${WAVEFORM_ARGS[@]}" --mode verify | |
| monitor_stopped=0 | |
| restore_monitor() { | |
| if [[ "$monitor_stopped" == "1" && -f "$MONITOR_START" ]]; then | |
| bash "$MONITOR_START" || true | |
| monitor_stopped=0 | |
| fi | |
| } | |
| trap restore_monitor EXIT | |
| if [[ -f "$MONITOR_STOP" ]]; then | |
| bash "$MONITOR_STOP" | |
| monitor_stopped=1 | |
| fi | |
| export CUDA_VISIBLE_DEVICES | |
| export NCCL_DEBUG=WARN | |
| export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True | |
| VISIBLE_GPU_COUNT=$(python - <<'PY' | |
| import torch | |
| print(torch.cuda.device_count()) | |
| PY | |
| ) | |
| if [[ "$VISIBLE_GPU_COUNT" != "4" ]]; then | |
| echo "[error] 本次正式生成要求恰好四张可见 GPU,实际为 $VISIBLE_GPU_COUNT。" >&2 | |
| exit 2 | |
| fi | |
| if [[ ! "$WORKERS_PER_GPU" =~ ^[1-9][0-9]*$ ]]; then | |
| echo "[error] WORKERS_PER_GPU 必须是正整数。" >&2 | |
| exit 2 | |
| fi | |
| NPROC=$((VISIBLE_GPU_COUNT * WORKERS_PER_GPU)) | |
| export_hidden() { | |
| local output_root=$1 | |
| local source_name=$2 | |
| local source_config=$3 | |
| local limit=$4 | |
| local overwrite_args=() | |
| local limit_args=() | |
| if [[ "$OVERWRITE_HIDDEN" == "1" ]]; then | |
| overwrite_args+=(--overwrite) | |
| fi | |
| if [[ "$limit" != "0" ]]; then | |
| limit_args+=(--limit-sources "$limit") | |
| fi | |
| python -m torch.distributed.run \ | |
| --standalone \ | |
| --local_addr="$TORCHRUN_LOCAL_ADDR" \ | |
| --nproc_per_node="$NPROC" \ | |
| scripts/data/cache_omni_hidden_states.py \ | |
| --mode export \ | |
| --model-path "$MODEL_PATH" \ | |
| --output-root "$output_root" \ | |
| --tasks audio_command_text \ | |
| --sources "$source_name" \ | |
| --source-config-file "$source_config" \ | |
| --motion-fps 50 \ | |
| --audio-command-sample-step-frames 20 \ | |
| --audio-command-window-seconds 0.6 \ | |
| --audio-sampling-rate 16000 \ | |
| --audio-batch-size 1 \ | |
| --num-io-workers 2 \ | |
| --num-save-workers 4 \ | |
| --max-pending-save-batches 8 \ | |
| --save-dtype bf16 \ | |
| --storage-format source \ | |
| --no-audio-command-full-audio \ | |
| "${limit_args[@]}" \ | |
| "${overwrite_args[@]}" | |
| } | |
| verify_hidden() { | |
| local output_root=$1 | |
| local source_name=$2 | |
| local source_config=$3 | |
| local verify_sources=$4 | |
| local verify_windows=$5 | |
| CUDA_VISIBLE_DEVICES=0 python -u scripts/data/cache_omni_hidden_states.py \ | |
| --mode verify \ | |
| --model-path "$MODEL_PATH" \ | |
| --output-root "$output_root" \ | |
| --tasks audio_command_text \ | |
| --sources "$source_name" \ | |
| --source-config-file "$source_config" \ | |
| --motion-fps 50 \ | |
| --audio-command-sample-step-frames 20 \ | |
| --audio-command-window-seconds 0.6 \ | |
| --audio-sampling-rate 16000 \ | |
| --audio-batch-size 1 \ | |
| --save-dtype bf16 \ | |
| --storage-format source \ | |
| --verify-sources "$verify_sources" \ | |
| --verify-windows-per-source "$verify_windows" \ | |
| --no-audio-command-full-audio | |
| } | |
| echo "[GPU] 导出连续纯背景 hidden" | |
| export_hidden \ | |
| "$BACKGROUND_HIDDEN_ROOT" \ | |
| hfd_continuous_office_background \ | |
| "$BACKGROUND_SOURCE_CONFIG" \ | |
| 0 | |
| verify_hidden \ | |
| "$BACKGROUND_HIDDEN_ROOT" \ | |
| hfd_continuous_office_background \ | |
| "$BACKGROUND_SOURCE_CONFIG" \ | |
| "$VERIFY_BACKGROUND_SOURCES" \ | |
| "$VERIFY_WINDOWS_PER_BACKGROUND" | |
| echo "[GPU] 导出人声+连续背景 mixed hidden" | |
| export_hidden \ | |
| "$MIXED_HIDDEN_ROOT" \ | |
| qwen3_tts_seed_audio_command \ | |
| "$MIXED_SOURCE_CONFIG" \ | |
| "$LIMIT_SOURCES" | |
| verify_hidden \ | |
| "$MIXED_HIDDEN_ROOT" \ | |
| qwen3_tts_seed_audio_command \ | |
| "$MIXED_SOURCE_CONFIG" \ | |
| "$VERIFY_SPEECH_SOURCES" \ | |
| 0 | |
| python - "$DATA_ROOT" "$MODEL_PATH" <<'PY' | |
| import hashlib | |
| import json | |
| import sqlite3 | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from psi.data.seed.full_duplex_audio_contract import ( | |
| validate_encoder_cache, | |
| write_json_atomic, | |
| ) | |
| root = Path(sys.argv[1]).resolve() | |
| model = Path(sys.argv[2]).resolve() | |
| waveform = json.loads((root / "reports" / "waveform_build.json").read_text()) | |
| def index_totals(cache_root, source_name): | |
| source_dir = ( | |
| cache_root / "audio" / source_name / "audio_command_text" / "sources" | |
| ) | |
| rows = {} | |
| for path in source_dir.glob("index.ws*.rank*.sqlite"): | |
| connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) | |
| try: | |
| values = connection.execute( | |
| "SELECT key, path, count FROM sources" | |
| ).fetchall() | |
| finally: | |
| connection.close() | |
| for key, relative_path, count in values: | |
| value = (str(relative_path), int(count)) | |
| previous = rows.get(str(key)) | |
| if previous is not None and previous != value: | |
| raise RuntimeError( | |
| f"Conflicting duplicate source index row: {key}: " | |
| f"{previous} != {value}" | |
| ) | |
| rows[str(key)] = value | |
| return len(rows), sum(count for _, count in rows.values()) | |
| mixed_root = root / "mixed_hidden" | |
| background_root = root / "background_hidden" | |
| mixed_contract = validate_encoder_cache(mixed_root, model) | |
| background_contract = validate_encoder_cache(background_root, model) | |
| mixed_sources, mixed_windows = index_totals( | |
| mixed_root, "qwen3_tts_seed_audio_command" | |
| ) | |
| background_sources, background_windows = index_totals( | |
| background_root, "hfd_continuous_office_background" | |
| ) | |
| expected_sources = int(waveform["source_count"]) | |
| expected_mixed = int(waveform["mixed_window_count"]) | |
| expected_background_sources = int(waveform["background_session_count"]) | |
| expected_background = int(waveform["background_window_count"]) | |
| actual = ( | |
| mixed_sources, | |
| mixed_windows, | |
| background_sources, | |
| background_windows, | |
| ) | |
| expected = ( | |
| expected_sources, | |
| expected_mixed, | |
| expected_background_sources, | |
| expected_background, | |
| ) | |
| if actual != expected: | |
| raise RuntimeError(f"Final cache totals mismatch: actual={actual} expected={expected}") | |
| assignment = Path(waveform["assignment_path"]) | |
| digest = hashlib.sha256(assignment.read_bytes()).hexdigest() | |
| if digest != waveform["assignment_sha256"]: | |
| raise RuntimeError("Assignment manifest changed before finalization") | |
| write_json_atomic( | |
| root / "reports" / "production_complete.json", | |
| { | |
| "format": "hfd_continuous_office_production_v1", | |
| "completed_at": time.time(), | |
| "data_root": str(root), | |
| "source_count": mixed_sources, | |
| "mixed_window_count": mixed_windows, | |
| "background_source_count": background_sources, | |
| "background_window_count": background_windows, | |
| "initial_idle_max_steps": int(waveform["initial_idle_max_steps"]), | |
| "stationary_blend_frames": int(waveform["stationary_blend_frames"]), | |
| "safety_steps": int(waveform["safety_steps"]), | |
| "assignment_seed": int(waveform["seed"]), | |
| "assignment_path": str(assignment), | |
| "assignment_sha256": digest, | |
| "online_encoder_contract": mixed_contract, | |
| "background_encoder_contract": background_contract, | |
| }, | |
| ) | |
| print( | |
| f"[complete] sources={mixed_sources:,} mixed={mixed_windows:,} " | |
| f"background={background_windows:,} root={root}" | |
| ) | |
| PY | |
| restore_monitor | |
| echo "[done] $DATA_ROOT" | |