code-HFD / backup /prepare_full_duplex_continuous_audio.sh
Tinghong-Ye22's picture
Add files using upload-large-folder tool
5d9f970 verified
Raw History Blame Contribute Delete
11.5 kB
#!/usr/bin/env bash
set -Eeuo pipefail
# Archived data builder; keep relative dependencies rooted at the HFD project.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "$PROJECT_ROOT"
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
DATA_ROOT=${HFD_CONTINUOUS_DATA_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1}
OLD_HIDDEN_ROOT=${HFD_OLD_AUDIO_COMMAND_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_source}
BACKGROUND_ROOT="$DATA_ROOT/background"
SESSION_DIR="$BACKGROUND_ROOT/sessions"
SESSION_MANIFEST="$SESSION_DIR/session_manifest.json"
MIXED_HIDDEN_ROOT="$DATA_ROOT/mixed_hidden"
BACKGROUND_HIDDEN_ROOT="$DATA_ROOT/background_hidden"
MANIFEST_ROOT="$DATA_ROOT/manifests"
ASSIGNMENT_PATH="$MANIFEST_ROOT/source_background_assignments.jsonl"
MIXED_SOURCE_CONFIG="$MANIFEST_ROOT/mixed_source_config.json"
BACKGROUND_SOURCE_CONFIG="$MANIFEST_ROOT/background_source_config.json"
MODEL_PATH=${HFD_QWEN_BACKBONE:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B}
DEMAND_ROOT=${HFD_DEMAND_ROOT:-$PROJECT_ROOT/data/full_duplex_background_review_260730/raw}
CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
WORKERS_PER_GPU=${HFD_CONTINUOUS_WORKERS_PER_GPU:-4}
CPU_WORKERS=${HFD_CONTINUOUS_CPU_WORKERS:-32}
LIMIT_SOURCES=${HFD_CONTINUOUS_LIMIT_SOURCES:-0}
VERIFY_SPEECH_SOURCES=${HFD_CONTINUOUS_VERIFY_SPEECH_SOURCES:-64}
VERIFY_BACKGROUND_SOURCES=${HFD_CONTINUOUS_VERIFY_BACKGROUND_SOURCES:-10}
VERIFY_WINDOWS_PER_BACKGROUND=${HFD_CONTINUOUS_VERIFY_BACKGROUND_WINDOWS:-8}
FORCE_WAVEFORM_BUILD=${HFD_CONTINUOUS_FORCE_WAVEFORM_BUILD:-0}
OVERWRITE_HIDDEN=${HFD_CONTINUOUS_OVERWRITE_HIDDEN:-0}
TORCHRUN_LOCAL_ADDR=${HFD_TORCHRUN_LOCAL_ADDR:-127.0.0.1}
MONITOR_STOP=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/stop.sh
MONITOR_START=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/monitor/cmd.sh
LOG_DIR="$DATA_ROOT/logs"
LOG_PATH="$LOG_DIR/prepare.log"
mkdir -p "$LOG_DIR"
exec > >(tee -a "$LOG_PATH") 2>&1
echo "================================================================================"
echo "HFD 连续办公室背景数据"
echo "结果根目录: $DATA_ROOT"
echo "混合语音 hidden: $MIXED_HIDDEN_ROOT"
echo "连续背景 hidden: $BACKGROUND_HIDDEN_ROOT"
echo "GPU: $CUDA_VISIBLE_DEVICES"
echo "每卡 B=1 进程: $WORKERS_PER_GPU"
echo "source 限制: $LIMIT_SOURCES"
echo "================================================================================"
if [[ "$LIMIT_SOURCES" != "0" \
&& "$DATA_ROOT" == "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/hfd_audio_command_continuous_office_v1" ]]; then
echo "[error] 有限 source 测试不能写进正式目录。" >&2
exit 2
fi
for required in \
"$MODEL_PATH/config.json" \
"$DEMAND_ROOT/OOFFICE_ch01.wav" \
"$DEMAND_ROOT/OHALLWAY_ch01.wav"; do
if [[ ! -e "$required" ]]; then
echo "[error] 缺少输入: $required" >&2
exit 2
fi
done
if [[ "$FORCE_WAVEFORM_BUILD" == "1" \
|| ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then
if [[ ! -d "$OLD_HIDDEN_ROOT" && ! -f "$ASSIGNMENT_PATH" ]]; then
echo "[error] 首次构造需要旧 cache 的有效 source 清单;重建则可复用新 assignment。" >&2
echo " legacy=$OLD_HIDDEN_ROOT" >&2
echo " fallback=$ASSIGNMENT_PATH" >&2
exit 2
fi
fi
if [[ ! -f "$SESSION_MANIFEST" ]]; then
python -u scripts/data/generate_office_background_candidates.py \
--production-sessions \
--output-dir "$SESSION_DIR" \
--duration-seconds 600 \
--seed 260730 \
--demand-root "$DEMAND_ROOT" \
--vad-margin-seconds 1.0 \
--minimum-real-session-seconds 20.0
fi
WAVEFORM_ARGS=(
scripts/data/build_continuous_audio_command_sources.py
--output-root "$DATA_ROOT"
--background-manifest "$SESSION_MANIFEST"
--old-hidden-root "$OLD_HIDDEN_ROOT"
--workers "$CPU_WORKERS"
--limit-sources "$LIMIT_SOURCES"
--initial-idle-max-steps 5
--stationary-blend-frames 20
--safety-steps 10
--seed 260730
)
if [[ "$FORCE_WAVEFORM_BUILD" == "1" \
|| ! -f "$DATA_ROOT/reports/waveform_build.json" ]]; then
python -u "${WAVEFORM_ARGS[@]}" --mode build
fi
python -u "${WAVEFORM_ARGS[@]}" --mode verify
monitor_stopped=0
restore_monitor() {
if [[ "$monitor_stopped" == "1" && -f "$MONITOR_START" ]]; then
bash "$MONITOR_START" || true
monitor_stopped=0
fi
}
trap restore_monitor EXIT
if [[ -f "$MONITOR_STOP" ]]; then
bash "$MONITOR_STOP"
monitor_stopped=1
fi
export CUDA_VISIBLE_DEVICES
export NCCL_DEBUG=WARN
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
VISIBLE_GPU_COUNT=$(python - <<'PY'
import torch
print(torch.cuda.device_count())
PY
)
if [[ "$VISIBLE_GPU_COUNT" != "4" ]]; then
echo "[error] 本次正式生成要求恰好四张可见 GPU,实际为 $VISIBLE_GPU_COUNT。" >&2
exit 2
fi
if [[ ! "$WORKERS_PER_GPU" =~ ^[1-9][0-9]*$ ]]; then
echo "[error] WORKERS_PER_GPU 必须是正整数。" >&2
exit 2
fi
NPROC=$((VISIBLE_GPU_COUNT * WORKERS_PER_GPU))
export_hidden() {
local output_root=$1
local source_name=$2
local source_config=$3
local limit=$4
local overwrite_args=()
local limit_args=()
if [[ "$OVERWRITE_HIDDEN" == "1" ]]; then
overwrite_args+=(--overwrite)
fi
if [[ "$limit" != "0" ]]; then
limit_args+=(--limit-sources "$limit")
fi
python -m torch.distributed.run \
--standalone \
--local_addr="$TORCHRUN_LOCAL_ADDR" \
--nproc_per_node="$NPROC" \
scripts/data/cache_omni_hidden_states.py \
--mode export \
--model-path "$MODEL_PATH" \
--output-root "$output_root" \
--tasks audio_command_text \
--sources "$source_name" \
--source-config-file "$source_config" \
--motion-fps 50 \
--audio-command-sample-step-frames 20 \
--audio-command-window-seconds 0.6 \
--audio-sampling-rate 16000 \
--audio-batch-size 1 \
--num-io-workers 2 \
--num-save-workers 4 \
--max-pending-save-batches 8 \
--save-dtype bf16 \
--storage-format source \
--no-audio-command-full-audio \
"${limit_args[@]}" \
"${overwrite_args[@]}"
}
verify_hidden() {
local output_root=$1
local source_name=$2
local source_config=$3
local verify_sources=$4
local verify_windows=$5
CUDA_VISIBLE_DEVICES=0 python -u scripts/data/cache_omni_hidden_states.py \
--mode verify \
--model-path "$MODEL_PATH" \
--output-root "$output_root" \
--tasks audio_command_text \
--sources "$source_name" \
--source-config-file "$source_config" \
--motion-fps 50 \
--audio-command-sample-step-frames 20 \
--audio-command-window-seconds 0.6 \
--audio-sampling-rate 16000 \
--audio-batch-size 1 \
--save-dtype bf16 \
--storage-format source \
--verify-sources "$verify_sources" \
--verify-windows-per-source "$verify_windows" \
--no-audio-command-full-audio
}
echo "[GPU] 导出连续纯背景 hidden"
export_hidden \
"$BACKGROUND_HIDDEN_ROOT" \
hfd_continuous_office_background \
"$BACKGROUND_SOURCE_CONFIG" \
0
verify_hidden \
"$BACKGROUND_HIDDEN_ROOT" \
hfd_continuous_office_background \
"$BACKGROUND_SOURCE_CONFIG" \
"$VERIFY_BACKGROUND_SOURCES" \
"$VERIFY_WINDOWS_PER_BACKGROUND"
echo "[GPU] 导出人声+连续背景 mixed hidden"
export_hidden \
"$MIXED_HIDDEN_ROOT" \
qwen3_tts_seed_audio_command \
"$MIXED_SOURCE_CONFIG" \
"$LIMIT_SOURCES"
verify_hidden \
"$MIXED_HIDDEN_ROOT" \
qwen3_tts_seed_audio_command \
"$MIXED_SOURCE_CONFIG" \
"$VERIFY_SPEECH_SOURCES" \
0
python - "$DATA_ROOT" "$MODEL_PATH" <<'PY'
import hashlib
import json
import sqlite3
import sys
import time
from pathlib import Path
from psi.data.seed.full_duplex_audio_contract import (
validate_encoder_cache,
write_json_atomic,
)
root = Path(sys.argv[1]).resolve()
model = Path(sys.argv[2]).resolve()
waveform = json.loads((root / "reports" / "waveform_build.json").read_text())
def index_totals(cache_root, source_name):
source_dir = (
cache_root / "audio" / source_name / "audio_command_text" / "sources"
)
rows = {}
for path in source_dir.glob("index.ws*.rank*.sqlite"):
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
try:
values = connection.execute(
"SELECT key, path, count FROM sources"
).fetchall()
finally:
connection.close()
for key, relative_path, count in values:
value = (str(relative_path), int(count))
previous = rows.get(str(key))
if previous is not None and previous != value:
raise RuntimeError(
f"Conflicting duplicate source index row: {key}: "
f"{previous} != {value}"
)
rows[str(key)] = value
return len(rows), sum(count for _, count in rows.values())
mixed_root = root / "mixed_hidden"
background_root = root / "background_hidden"
mixed_contract = validate_encoder_cache(mixed_root, model)
background_contract = validate_encoder_cache(background_root, model)
mixed_sources, mixed_windows = index_totals(
mixed_root, "qwen3_tts_seed_audio_command"
)
background_sources, background_windows = index_totals(
background_root, "hfd_continuous_office_background"
)
expected_sources = int(waveform["source_count"])
expected_mixed = int(waveform["mixed_window_count"])
expected_background_sources = int(waveform["background_session_count"])
expected_background = int(waveform["background_window_count"])
actual = (
mixed_sources,
mixed_windows,
background_sources,
background_windows,
)
expected = (
expected_sources,
expected_mixed,
expected_background_sources,
expected_background,
)
if actual != expected:
raise RuntimeError(f"Final cache totals mismatch: actual={actual} expected={expected}")
assignment = Path(waveform["assignment_path"])
digest = hashlib.sha256(assignment.read_bytes()).hexdigest()
if digest != waveform["assignment_sha256"]:
raise RuntimeError("Assignment manifest changed before finalization")
write_json_atomic(
root / "reports" / "production_complete.json",
{
"format": "hfd_continuous_office_production_v1",
"completed_at": time.time(),
"data_root": str(root),
"source_count": mixed_sources,
"mixed_window_count": mixed_windows,
"background_source_count": background_sources,
"background_window_count": background_windows,
"initial_idle_max_steps": int(waveform["initial_idle_max_steps"]),
"stationary_blend_frames": int(waveform["stationary_blend_frames"]),
"safety_steps": int(waveform["safety_steps"]),
"assignment_seed": int(waveform["seed"]),
"assignment_path": str(assignment),
"assignment_sha256": digest,
"online_encoder_contract": mixed_contract,
"background_encoder_contract": background_contract,
},
)
print(
f"[complete] sources={mixed_sources:,} mixed={mixed_windows:,} "
f"background={background_windows:,} root={root}"
)
PY
restore_monitor
echo "[done] $DATA_ROOT"