File size: 18,350 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
#!/usr/bin/env bash
set -Eeuo pipefail

# Internal audio-case/render worker. Public entry: infer_render_auto.sh.
PROJECT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
cd "$PROJECT_ROOT"
export PYTHONPATH="$PROJECT_ROOT/src${PYTHONPATH:+:$PYTHONPATH}"
source "$PROJECT_ROOT/scripts/activate_psi_env.sh"

HFD_ENV_FILE=${HFD_ENV_FILE:-$PROJECT_ROOT/.env}
if [[ -f "$HFD_ENV_FILE" ]]; then
  set -a
  source "$HFD_ENV_FILE"
  set +a
fi

PYTHON_BIN=${HFD_INFER_PYTHON:-python}
RUNS_ROOT=${HFD_INFER_RUNS_ROOT:-$PROJECT_ROOT/runs}
RUN_DIR=${HFD_INFER_RUN_DIR:-auto}
if [[ $# -gt 0 ]]; then
  RUN_DIR=$1
fi
if [[ "$RUN_DIR" == "auto" ]]; then
  RUN_DIR=$($PYTHON_BIN - "$RUNS_ROOT" <<'PY'
import json
import sys
from pathlib import Path

candidates = []
for config_path in Path(sys.argv[1]).rglob("run_config.json"):
    try:
        config = json.loads(config_path.read_text(encoding="utf-8"))
    except Exception:
        continue
    model = config.get("model", {})
    data = config.get("data", {})
    train = config.get("train", {})
    if (
        not model.get("streaming_qwen_enabled", False)
        or not data.get("online_audio", False)
        or train.get("full_duplex_semantics_version") not in {9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}
    ):
        continue
    candidates.append((config_path.stat().st_mtime_ns, config_path.parent))
if candidates:
    print(max(candidates, key=lambda item: item[0])[1])
PY
  )
fi
if [[ -z "$RUN_DIR" || ! -f "$RUN_DIR/run_config.json" ]]; then
  echo "[error] Cannot find a full-duplex RUN_DIR. Pass it as the first argument." >&2
  exit 2
fi
RUN_DIR=$(cd "$RUN_DIR" && pwd)

if [[ -n "${HFD_INFER_GPUS:-}" ]]; then
  export CUDA_VISIBLE_DEVICES=$HFD_INFER_GPUS
elif [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
  export CUDA_VISIBLE_DEVICES=0
fi
DEVICE=${HFD_INFER_DEVICE:-cuda:0}
CHECK_INTERVAL=${HFD_INFER_CHECK_INTERVAL:-30}
INFER_ONCE=${HFD_INFER_ONCE:-0}
MOTION_FRAMES=${HFD_INFER_MOTION_FRAMES:-450}
INFERENCE_STEPS=${HFD_INFER_STEPS:-10}
PROMPT=${HFD_INFER_PROMPT:-auto}
RESPECT_CONTROL=${HFD_INFER_RESPECT_CONTROL:-1}
MAX_LISTEN_TAIL_STEPS=${HFD_INFER_MAX_LISTEN_TAIL_STEPS:-500}
INITIAL_IDLE_STEPS=${HFD_INFER_INITIAL_IDLE_STEPS:-5}
WHITE_NOISE_START_INDEX=${HFD_INFER_WHITE_NOISE_START_INDEX:-3}
OUTPUT_SUBDIR=${HFD_INFER_OUTPUT_SUBDIR:-full_duplex_auto_infer}
OUTPUT_ROOT=${HFD_INFER_OUTPUT_ROOT:-$RUN_DIR/$OUTPUT_SUBDIR}
GMR_PY=${HFD_RENDER_PYTHON:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/envs/gmr/bin/python}
GMR_XML=${HFD_RENDER_XML:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml}
SONIC_RENDER_ROOT=${HFD_RENDER_ROOT:-/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GR00T-WholeBodyControl/gear_sonic}
RENDER_FPS=${HFD_RENDER_FPS:-50}
TERMINAL_BACKGROUND_STEPS=${HFD_INFER_TERMINAL_BACKGROUND_STEPS:-0}
INTER_TURN_BACKGROUND_STEPS_BY_TURN=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS_BY_TURN:-}
MIN_CHECKPOINT_STEP=${HFD_INFER_MIN_CHECKPOINT_STEP:-0}
EXACT_CHECKPOINT_STEP=${HFD_INFER_EXACT_CHECKPOINT_STEP:-}
SAMPLE_TAG=${HFD_INFER_SAMPLE_TAG:-}
SAMPLE_PARENT=${HFD_INFER_SAMPLE_PARENT:-}
SCENARIO_FILE=${HFD_INFER_SCENARIO_FILE:-}

if [[ -n "$EXACT_CHECKPOINT_STEP" && ! "$EXACT_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
  echo "[error] HFD_INFER_EXACT_CHECKPOINT_STEP must be a non-negative integer." >&2
  exit 2
fi
if [[ -n "$SAMPLE_PARENT" && ! "$SAMPLE_PARENT" =~ ^[[:alnum:]_.-]+$ ]]; then
  echo "[error] HFD_INFER_SAMPLE_PARENT must be one safe path component." >&2
  exit 2
fi

# Default sample is one of the 50 fixed overfit sources (manifest row 48,
# "character walks forward while grabbing something"), using the exact fixed
# voice the trainer binds to that row (row_id 48 % 8 voices = female_01_en)
# and a verified neutral-idle motion as the initial pose.  Starting directly
# from the task GT first frame is outside the training construction, which
# always begins from an audited upright stationary anchor and learns a 20-frame
# transition into the task.  The previous default
# (003771, walk back-left) is not in the overfit set: a memorizing model has
# never seen a back-left walk and can only replay the forward walk it knows.
DEFAULT_AUDIO=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio/000006/female_01_en.wav
DEFAULT_INIT_MOTION=/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/code/Psi0/dataset/seed_all/g1/csv_sonic/221125/neutral_idle_loop_002__A073
AUDIO_PATHS_CSV=${HFD_INFER_AUDIO_PATHS:-$DEFAULT_AUDIO}
ADDITIONAL_AUDIO_PATHS_CSV=${HFD_INFER_ADDITIONAL_AUDIO_PATHS:-}
INTER_TURN_BACKGROUND_STEPS=${HFD_INFER_INTER_TURN_BACKGROUND_STEPS:-0}
INIT_MOTION_DIRS_CSV=${HFD_INFER_INIT_MOTION_DIRS:-$DEFAULT_INIT_MOTION}
IFS=',' read -r -a AUDIO_PATHS <<< "$AUDIO_PATHS_CSV"
ADDITIONAL_AUDIO_PATHS=()
if [[ -n "$ADDITIONAL_AUDIO_PATHS_CSV" ]]; then
  IFS=',' read -r -a ADDITIONAL_AUDIO_PATHS <<< "$ADDITIONAL_AUDIO_PATHS_CSV"
fi
IFS=',' read -r -a INIT_MOTION_DIRS <<< "$INIT_MOTION_DIRS_CSV"
if [[ ${#INIT_MOTION_DIRS[@]} -ne 1 && ${#INIT_MOTION_DIRS[@]} -ne ${#AUDIO_PATHS[@]} ]]; then
  echo "[error] HFD_INFER_INIT_MOTION_DIRS must contain one path or match HFD_INFER_AUDIO_PATHS." >&2
  exit 2
fi

checkpoint_ready() {
  local checkpoint_dir=$1
  "$PYTHON_BIN" - "$checkpoint_dir" <<'PY'
import json
import sys
from pathlib import Path

checkpoint = Path(sys.argv[1])
single = checkpoint / "model.safetensors"
manifest = checkpoint / "hfd_fused_delta_manifest.json"
boundary = checkpoint / "full_duplex_source_boundary.json"
if (
    single.is_file()
    and single.stat().st_size > 0
    and manifest.is_file()
    and manifest.stat().st_size > 0
    and boundary.is_file()
    and boundary.stat().st_size > 0
):
    payload = json.loads(boundary.read_text(encoding="utf-8"))
    if bool(payload.get("source_group_complete", False)):
        raise SystemExit(0)
raise SystemExit(1)
PY
}

checkpoint_step() {
  local checkpoint_name=$1
  if [[ "$checkpoint_name" == "best_ckpt" ]]; then
    echo best
  else
    echo "${checkpoint_name#ckpt_}"
  fi
}

checkpoint_output_name() {
  local checkpoint_dir=$1
  local captured_global_step=${2:-}
  local checkpoint_name
  checkpoint_name=$(basename "$checkpoint_dir")
  if [[ "$checkpoint_name" != "best_ckpt" ]]; then
    echo "$checkpoint_name"
    return
  fi
  if [[ -z "$captured_global_step" ]]; then
    captured_global_step=$(checkpoint_global_step "$checkpoint_dir")
  fi
  echo "best_ckpt_step_${captured_global_step}"
}

checkpoint_global_step() {
  local checkpoint_dir=$1
  "$PYTHON_BIN" - "$checkpoint_dir/full_duplex_source_boundary.json" <<'PY'
import json
import sys
from pathlib import Path

marker_path = Path(sys.argv[1])
if not marker_path.is_file():
    raise SystemExit(f"checkpoint has no source-boundary marker: {marker_path}")
payload = json.loads(marker_path.read_text(encoding="utf-8"))
step = int(payload["global_step"])
if step <= 0:
    raise SystemExit(f"checkpoint has invalid global_step={step}")
print(step)
PY
}

safe_sample_name() {
  local audio_path=$1
  local index=$2
  local stem
  stem=$(basename "$audio_path")
  stem=${stem%.*}
  stem=$(echo "$stem" | tr -cs '[:alnum:]_.-' '_')
  if [[ -n "$SAMPLE_TAG" ]]; then
    stem="${SAMPLE_TAG}_${stem}"
  fi
  if [[ ${#ADDITIONAL_AUDIO_PATHS[@]} -gt 0 ]]; then
    printf '%02d_%s_session%dturn' \
      "$index" "$stem" "$(( ${#ADDITIONAL_AUDIO_PATHS[@]} + 1 ))"
  else
    printf '%02d_%s' "$index" "$stem"
  fi
}

sample_output_dir() {
  local checkpoint_name=$1
  local sample_name=$2
  if [[ -n "$SAMPLE_PARENT" ]]; then
    printf '%s/%s/%s/%s\n' \
      "$OUTPUT_ROOT" "$checkpoint_name" "$SAMPLE_PARENT" "$sample_name"
  else
    printf '%s/%s/%s\n' "$OUTPUT_ROOT" "$checkpoint_name" "$sample_name"
  fi
}

render_sample() {
  local sample_dir=$1
  (
    cd "$SONIC_RENDER_ROOT"
    env -u PYTHONPATH \
      PYOPENGL_PLATFORM=egl \
      MUJOCO_GL=egl \
      QT_X11_NO_MITSHM=1 \
      SONIC_RENDER_HEADLESS_CAMERA=1 \
      SONIC_RENDER_LOCAL_TMP_OUTPUT=1 \
      SONIC_RENDER_TMPDIR=/tmp \
      "$GMR_PY" scripts/batch_render_reference_motion_mujoco.py \
        --motion-dir "$sample_dir" \
        --xml-path "$GMR_XML" \
        --source-fps 50 \
        --target-fps "$RENDER_FPS" \
        --dof-order isaaclab
  )
}

process_sample() {
  local checkpoint_name=$1
  local step=$2
  local sample_index=$3
  local expected_global_step=$4
  local audio_path=${AUDIO_PATHS[$sample_index]}
  local init_index=$sample_index
  if [[ ${#INIT_MOTION_DIRS[@]} -eq 1 ]]; then
    init_index=0
  fi
  local init_motion_dir=${INIT_MOTION_DIRS[$init_index]}
  local sample_name
  sample_name=$(safe_sample_name "$audio_path" "$sample_index")
  local sample_dir
  sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
  mkdir -p "$sample_dir"

  if [[ ! -s "$sample_dir/infer_meta.json" || ! -s "$sample_dir/body_pos.csv" ]]; then
    echo "[infer] checkpoint=$checkpoint_name sample=$sample_name audio=$audio_path"
    control_args=()
    additional_audio_args=()
    inter_turn_args=()
    for additional_audio_path in "${ADDITIONAL_AUDIO_PATHS[@]}"; do
      additional_audio_args+=(--additional-audio-path "$additional_audio_path")
    done
    if [[ -n "$INTER_TURN_BACKGROUND_STEPS_BY_TURN" ]]; then
      inter_turn_args+=(
        --inter-turn-background-steps-by-turn
        "$INTER_TURN_BACKGROUND_STEPS_BY_TURN"
      )
    fi
    if [[ "$RESPECT_CONTROL" != "1" && "$RESPECT_CONTROL" != "true" ]]; then
      control_args+=(--ignore-control-token)
    fi
    rm -f "$sample_dir/infer_failure.json"
    if "$PYTHON_BIN" -u scripts/infer/full_duplex_audio_to_motion.py \
        --run-dir "$RUN_DIR" \
        --ckpt-step "$step" \
        --output-dir "$sample_dir" \
        --audio-path "$audio_path" \
        "${additional_audio_args[@]}" \
        --init-motion-dir "$init_motion_dir" \
        --motion-frames "$MOTION_FRAMES" \
        --inter-turn-background-steps "$INTER_TURN_BACKGROUND_STEPS" \
        "${inter_turn_args[@]}" \
        --terminal-background-steps "$TERMINAL_BACKGROUND_STEPS" \
        --num-inference-steps "$INFERENCE_STEPS" \
        --prompt "$PROMPT" \
        --max-listen-tail-steps "$MAX_LISTEN_TAIL_STEPS" \
        --initial-idle-steps "$INITIAL_IDLE_STEPS" \
        --white-noise-start-index "$WHITE_NOISE_START_INDEX" \
        --expected-checkpoint-global-step "$expected_global_step" \
        --device "$DEVICE" \
        "${control_args[@]}" 2>&1 | tee "$sample_dir/infer.log"; then
      :
    else
      local infer_status=${PIPESTATUS[0]}
      return "$infer_status"
    fi
  else
    echo "[skip] inference exists: $sample_dir"
  fi

  if [[ "${HFD_INFER_SKIP_RENDER:-0}" == "1" ]]; then
    touch "$sample_dir/.inferred"
    echo "[inferred] queued independent rendering: $sample_dir"
    return 0
  fi
  render_and_mux_sample "$sample_dir"
}

render_and_mux_sample() {
  local sample_dir=$1

  if [[ ! -f "$sample_dir/.complete" ]]; then
    # The renderer validates the frame count of an existing intermediate MP4.
    # A cancelled render/mux must never be accepted merely because it is nonempty.
    echo "[render] sample=$sample_dir fps=$RENDER_FPS"
    if ! render_sample "$sample_dir"; then
      echo "[error] render failed: $sample_dir" >&2
      return 1
    fi
  else
    echo "[skip] render exists: $sample_dir/reference_motion.mp4"
  fi

  if [[ ! -f "$sample_dir/.complete" ]]; then
    echo "[mux] use the complete background/speech/background streaming audio timeline"
    if ! "$PYTHON_BIN" scripts/infer/mux_full_duplex_audio.py \
        --video "$sample_dir/reference_motion.mp4" \
        --audio "$sample_dir/assembled_input_audio_16k.wav" \
        --output "$sample_dir/reference_motion_with_audio.mp4" \
        --infer-meta "$sample_dir/infer_meta.json" \
        --audio-is-timeline \
        --expected-fps "$RENDER_FPS"; then
      echo "[error] audio mux failed: $sample_dir" >&2
      return 1
    fi
  else
    echo "[skip] synchronized video exists: $sample_dir/reference_motion_with_audio.mp4"
  fi
  touch "$sample_dir/.complete"
}

if [[ -n "${HFD_RENDER_ONLY_DIR:-}" ]]; then
  sample_dir=$HFD_RENDER_ONLY_DIR
  if [[ "${HFD_RENDER_TASK:-}" == "swa" ]]; then
    # Separate CPU-only preparation/composition; never reload a checkpoint.
    "$PYTHON_BIN" scripts/infer/render_swa_result.py prepare \
      --run-dir "$RUN_DIR" --output-dir "$sample_dir"
    render_sample "$sample_dir/render_motion"
    "$PYTHON_BIN" scripts/infer/render_swa_result.py compose \
      --run-dir "$RUN_DIR" --output-dir "$sample_dir"
    touch "$sample_dir/.complete"
    exit 0
  fi
  for required_file in infer_meta.json body_pos.csv body_quat.csv joint_pos.csv assembled_input_audio_16k.wav; do
    if [[ ! -s "$sample_dir/$required_file" ]]; then
      echo "[error] render-only input missing: $sample_dir/$required_file" >&2
      exit 1
    fi
  done
  render_and_mux_sample "$sample_dir"
  exit 0
fi

mkdir -p "$OUTPUT_ROOT"
if [[ -n "$SCENARIO_FILE" ]]; then
  if [[ ! -f "$SCENARIO_FILE" ]]; then
    echo "[error] HFD_INFER_SCENARIO_FILE does not exist: $SCENARIO_FILE" >&2
    exit 2
  fi
  scenario_snapshot="$OUTPUT_ROOT/scenario.snapshot.json"
  if [[ -n "$SAMPLE_PARENT" ]]; then
    scenario_snapshot="$OUTPUT_ROOT/scenario.${SAMPLE_PARENT}.snapshot.json"
  fi
  cp -- "$SCENARIO_FILE" "$scenario_snapshot"
fi
if [[ "$MIN_CHECKPOINT_STEP" == "auto" ]]; then
  MIN_CHECKPOINT_STEP=0
  if [[ -z "$EXACT_CHECKPOINT_STEP" ]]; then
    while IFS= read -r candidate_checkpoint; do
      checkpoint_ready "$candidate_checkpoint" || continue
      candidate_name=$(basename "$candidate_checkpoint")
      candidate_step=${candidate_name#ckpt_}
      if (( candidate_step > MIN_CHECKPOINT_STEP )); then
        MIN_CHECKPOINT_STEP=$candidate_step
      fi
    done < <(
      find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
        -name 'ckpt_[0-9]*' -print 2>/dev/null
    )
  fi
fi
if [[ ! "$MIN_CHECKPOINT_STEP" =~ ^[0-9]+$ ]]; then
  echo "[error] HFD_INFER_MIN_CHECKPOINT_STEP must be auto or a non-negative integer." >&2
  exit 2
fi
echo "RUN_DIR: $RUN_DIR"
echo "OUTPUT_ROOT: $OUTPUT_ROOT"
echo "TIMING: audio 16kHz, 0.6s window, 0.4s stride; action 30/20/10 at 50fps"
echo "RENDER: ${RENDER_FPS}fps; speech follows ${INITIAL_IDLE_STEPS} idle windows; motion begins at recorded frame boundary"
echo "CONTROL: respect=$RESPECT_CONTROL initial_idle_steps=$INITIAL_IDLE_STEPS max_listen_tail_steps=$MAX_LISTEN_TAIL_STEPS white_noise_start=$WHITE_NOISE_START_INDEX"
echo "SESSION: primary_samples=${#AUDIO_PATHS[@]} additional_turns=${#ADDITIONAL_AUDIO_PATHS[@]} inter_turn_background_steps=$INTER_TURN_BACKGROUND_STEPS by_turn=${INTER_TURN_BACKGROUND_STEPS_BY_TURN:-uniform} terminal_background_steps=$TERMINAL_BACKGROUND_STEPS"
echo "CHECKPOINT POLICY: immutable ckpt_<step> only; mutable best_ckpt is never watched; min_step=$MIN_CHECKPOINT_STEP exact_step=${EXACT_CHECKPOINT_STEP:-none}"

while true; do
  if [[ -n "$EXACT_CHECKPOINT_STEP" ]]; then
    CHECKPOINT_DIRS=("$RUN_DIR/checkpoints/ckpt_$EXACT_CHECKPOINT_STEP")
  else
    mapfile -t CHECKPOINT_DIRS < <(
      find "$RUN_DIR/checkpoints" -mindepth 1 -maxdepth 1 -type d \
        -name 'ckpt_[0-9]*' -print 2>/dev/null | sort -V
    )
  fi
  pending=()
  for checkpoint_dir in "${CHECKPOINT_DIRS[@]}"; do
    [[ -d "$checkpoint_dir" ]] || continue
    checkpoint_dir_name=$(basename "$checkpoint_dir")
    checkpoint_number=${checkpoint_dir_name#ckpt_}
    if (( checkpoint_number < MIN_CHECKPOINT_STEP )); then
      continue
    fi
    checkpoint_ready "$checkpoint_dir" || continue
    if ! captured_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
      echo "[wait] checkpoint identity is not ready: $checkpoint_dir" >&2
      continue
    fi
    checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$captured_global_step")
    settled=1
    for sample_index in "${!AUDIO_PATHS[@]}"; do
      sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
      sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
      if [[ ! -f "$sample_dir/.complete" && ! -f "$sample_dir/.failed" ]]; then
        settled=0
        break
      fi
    done
    [[ $settled -eq 1 ]] || pending+=("$checkpoint_dir")
  done

  if [[ ${#pending[@]} -eq 0 ]]; then
    echo "[$(date '+%F %T')] no pending checkpoint"
    if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
      break
    fi
    sleep "$CHECK_INTERVAL"
    continue
  fi

  checkpoint_dir=${pending[-1]}
  checkpoint_dir_name=$(basename "$checkpoint_dir")
  if ! expected_global_step=$(checkpoint_global_step "$checkpoint_dir"); then
    echo "[wait] selected checkpoint identity changed before inference: $checkpoint_dir" >&2
    continue
  fi
  checkpoint_name=$(checkpoint_output_name "$checkpoint_dir" "$expected_global_step")
  step=$(checkpoint_step "$checkpoint_dir_name")
  echo "[$(date '+%F %T')] process newest pending checkpoint: $checkpoint_name"
  for sample_index in "${!AUDIO_PATHS[@]}"; do
    sample_name=$(safe_sample_name "${AUDIO_PATHS[$sample_index]}" "$sample_index")
    sample_dir=$(sample_output_dir "$checkpoint_name" "$sample_name")
    if [[ -f "$sample_dir/.complete" ]]; then
      echo "[skip] completed sample: $sample_dir"
      continue
    fi
    if [[ -f "$sample_dir/.failed" ]]; then
      echo "[skip] failed sample already recorded: $sample_dir"
      continue
    fi
    if process_sample \
        "$checkpoint_name" \
        "$step" \
        "$sample_index" \
        "$expected_global_step"; then
      continue
    else
      status=$?
      printf \
        'checkpoint=%s\nsample=%s\nexit_status=%s\nfailed_at=%s\n' \
        "$checkpoint_name" \
        "$sample_name" \
        "$status" \
        "$(date --iso-8601=seconds)" \
        > "$sample_dir/.failed"
      echo "[failed] checkpoint=$checkpoint_name sample=$sample_name status=$status; details: $sample_dir/infer.log" >&2
    fi
  done
  if [[ "$INFER_ONCE" == "1" || "$INFER_ONCE" == "true" ]]; then
    break
  fi
done

echo "[done] automatic full-duplex inference/render loop exited cleanly"