File size: 5,196 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#!/usr/bin/env bash
set -Eeuo pipefail

PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
PYTHON_BIN=${HFD_INFER_PYTHON:-$PROJECT_ROOT/.venv-psi/bin/python}
# Plain bash: pin the newest compatible run once, then watch its checkpoints.
# To inspect an older run, edit RUN_DIR here or still pass it as argument 1.
RUN_DIR=${1:-${HFD_INFER_RUN_DIR:-auto}}
RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs}
# All tasks on the first checkpoint, then every >=20k training steps. Saving
# remains every 5k. Explicit one-shot bypasses this; 0 processes every checkpoint.
export HFD_INFER_CHECKPOINT_INTERVAL=${HFD_INFER_CHECKPOINT_INTERVAL:-20000}

cd "$PROJECT_ROOT"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
if (( $# > 1 )); then
  echo "[error] Usage: bash infer_render_auto.sh [RUN_DIR]" >&2
  exit 2
fi
RUN_DIR=$("$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" \
  "$RUN_DIR" --runs-root "$RUNS_ROOT" --resolve-only)
echo "[HFD] pinned inference run: $RUN_DIR"
# A lightweight launch check; never allocates GPUs or scans all dataset files.
if [[ "${HFD_INFER_CONFIG_ONLY:-0}" == "1" ]]; then
  echo "[HFD] inference GPUs: ${HFD_INFER_GPUS:-${CUDA_VISIBLE_DEVICES-all allocated}}"
  exit 0
fi
echo "[HFD] Inference shares its visible GPUs across all tasks. Use a separate GPU allocation to avoid slowing training."

# Audio-only cases are unchanged. SWA automatically generates native answer
# speech; the legacy ``contract`` switch below only runs an extra startup check
# (it is NOT the on/off switch for SWA speech).
SPEECH_RUNTIME_MODE=${HFD_SPEECH_RUNTIME_MODE:-off}
case "$SPEECH_RUNTIME_MODE" in
  off)
    ;;
  contract)
    QA_ARGS=()
    if [[ -n "${HFD_QA_LORA_CHECKPOINT:-}" ]]; then
      QA_ARGS+=(--qa-checkpoint "$HFD_QA_LORA_CHECKPOINT")
    fi
    "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/validate_speech_qa_runtime.py" "${QA_ARGS[@]}"
    ;;
  *)
    echo "[error] HFD_SPEECH_RUNTIME_MODE must be off or contract (optional startup validation only)." >&2
    exit 2
    ;;
esac

echo "[HFD] speech runtime contract=$SPEECH_RUNTIME_MODE solver=${HFD_SPEECH_SOLVER:-euler} steps=${HFD_SPEECH_SOLVER_STEPS:-6} codec_chunk=${HFD_SPEECH_CODEC_CHUNK_CODES:-36} first_chunk=${HFD_SPEECH_FIRST_CHUNK_CODES:-36} playback_buffer=${HFD_SPEECH_PLAYBACK_BUFFER_SECONDS:-0.20}s"

if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then
  echo "[error] A valid HFD run directory is required." >&2
  exit 2
fi
# Validate the data actually saved with this run, including legacy audio-only
# runs. A newer launcher's default must not substitute a different SWA plan.
mapfile -t SAVED_SWA < <("$PYTHON_BIN" - "$RUN_DIR/run_config.json" <<'PY'
import json, sys
with open(sys.argv[1]) as file:
    config = json.load(file)
print("enabled" if config.get("model", {}).get("swa_visual_qa_enabled", False) else "disabled")
print(config.get("data", {}).get("swa_source_plan", ""))
PY
)
SWA_DATA_MODE=${HFD_SWA_DATA_MODE:-${SAVED_SWA[0]}}
SWA_SOURCE_PLAN=${HFD_SWA_SOURCE_PLAN:-${SAVED_SWA[1]}}
SWA_FRONTEND_SAMPLE_COUNT=${HFD_SWA_FRONTEND_SAMPLE_COUNT:-2}
# Unified overlays must validate this run's actual manifests, not the retired
# standalone SWA source_plan/frontend cache still present for old checkpoints.
HAS_OVERLAY=$("$PYTHON_BIN" -c 'import json,sys; print(int(bool(json.load(open(sys.argv[1])).get("data",{}).get("multimodal_overlay_manifests"))))' "$RUN_DIR/run_config.json")
if [[ "$HAS_OVERLAY" == "1" ]]; then
  "$PYTHON_BIN" "$PROJECT_ROOT/scripts/infer/check_run_visual_data.py" "$RUN_DIR"
else
case "$SWA_DATA_MODE" in
  disabled)
    echo "[HFD] SWA visual-QA inference=disabled by explicit override"
    ;;
  contract)
    if ! [[ "$SWA_FRONTEND_SAMPLE_COUNT" =~ ^[1-9][0-9]*$ ]]; then
      echo "[error] HFD_SWA_FRONTEND_SAMPLE_COUNT must be a positive integer." >&2
      exit 2
    fi
    "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \
      --source-plan "$SWA_SOURCE_PLAN" \
      --sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" \
      --require-visual \
      --json
    echo "[ready] SWA visual-QA contract verified"
    ;;
  enabled)
    "$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/data/validate_swa_manifest.py" \
      --source-plan "$SWA_SOURCE_PLAN" --sample-frontends "$SWA_FRONTEND_SAMPLE_COUNT" --require-visual --json
    ;;
  *)
    echo "[error] HFD_SWA_DATA_MODE must be disabled, contract or enabled." >&2
    exit 2
    ;;
esac
fi

# All established interaction cases and SWA video/static train/val cases share one
# dynamic inference GPU queue (one model job per GPU). Rendering/mux use a
# separate bounded queue and never hold these model slots; default 4 workers,
# 32 outstanding videos. HFD_RENDER_WORKERS / HFD_RENDER_PENDING_LIMIT override
# these limits if needed. HFD_INCLUDE_SWA_SUITE=0 disables only SWA diagnostics.
if [[ "$SWA_DATA_MODE" == "enabled" ]]; then
  export HFD_INCLUDE_SWA_SUITE=${HFD_INCLUDE_SWA_SUITE:-1}
  echo "[HFD] SWA: model answer -> native Talker/Token2Wav -> independent speech worker -> video audio mix"
else
  export HFD_INCLUDE_SWA_SUITE=0
fi
"$PYTHON_BIN" -u "$PROJECT_ROOT/scripts/infer/run_multiturn_infer_suite.py" "$RUN_DIR"