#!/usr/bin/env python3 """Probe whether the production HFD control input separates four meanings. This is a read-only diagnostic. It streams production-format 0.6 s audio windows through a real fused checkpoint, keeps the native Qwen KV for the complete synthetic session, and captures ``condition.memory_token``. With state-aware serial control disabled (the semantics-v22 production policy), that tensor is exactly the vector consumed by ``control_head``. Four event types share the same prefix action and continuous office background: a new action instruction, an emergency Stop, a short backchannel, or background only. A small linear probe is trained on seven voices and evaluated on the eighth, so speaker memorisation cannot count as semantic separation. """ from __future__ import annotations import argparse import json import math import os import sys from dataclasses import dataclass from pathlib import Path from typing import Any import numpy as np import torch PROJECT_ROOT = Path(__file__).resolve().parents[2] DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data" if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) if str(DATA_SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(DATA_SCRIPT_DIR)) from psi.config.config import LaunchConfig # noqa: E402 from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402 _read_pcm16_wave_segment, _read_resampled_speech_float, ) from psi.data.seed.qwen_streaming_audio import ( # noqa: E402 prepare_qwen_streaming_audio_window, ) from psi.models.full_duplex_psi0 import FullDuplexPsi0Model # noqa: E402 from psi.utils import parse_args_to_tyro_config # noqa: E402 AUDIO_RATE = 16_000 WINDOW_SAMPLES = 9_600 STRIDE_SAMPLES = 6_400 PREFIX_SAMPLES = WINDOW_SAMPLES - STRIDE_SAMPLES VOICE_NAMES = ( "female_01_en", "female_02_en", "male_01_en", "male_02_en", "female_01_zh", "female_02_zh", "male_01_zh", "male_02_zh", ) CLASS_NAMES = ("action", "stop", "backchannel", "background") EXPECTED_CONTROL_ID = { "action": 1, "stop": 2, "backchannel": 1, "background": 1, } CONTROL_NAMES = ("listen", "motion", "idle") @dataclass(frozen=True) class Speech: path: Path scale: float text: str @dataclass(frozen=True) class ProbeRow: label: str voice: str variant: int base: Speech event: Speech | None background_windows: int = 0 def _window_count(sample_count: int) -> int: return int( math.ceil((PREFIX_SAMPLES + int(sample_count)) / STRIDE_SAMPLES) ) def _load_jsonl(path: Path) -> list[dict[str, Any]]: with path.open("r", encoding="utf-8") as handle: return [json.loads(line) for line in handle if line.strip()] def _checkpoint_name(run_dir: Path, raw: str) -> str: requested = str(raw).strip() if requested not in {"", "latest"}: return requested candidates = [] for path in (run_dir / "checkpoints").glob("ckpt_*"): try: candidates.append((int(path.name.removeprefix("ckpt_")), path.name)) except ValueError: continue if not candidates: raise FileNotFoundError(f"No immutable checkpoint under {run_dir}") return max(candidates)[1] def _speech_scale_by_path(assignment_path: Path) -> dict[str, float]: return { os.path.normpath(str(row["original_audio_path"])): float( row["speech_scale"] ) for row in _load_jsonl(assignment_path) } def _seed_speech( directory: Path, voice: str, scales: dict[str, float], ) -> Speech: path = directory / f"{voice}.wav" key = os.path.normpath(str(path)) if not path.is_file() or key not in scales: raise FileNotFoundError(f"Missing production seed assignment: {path}") return Speech(path=path, scale=scales[key], text=directory.name) def _catalog_speech( rows: list[dict[str, Any]], *, phrase_id: str, voice: str, ) -> Speech: matches = [ row for row in rows if row["phrase_id"] == phrase_id and row["voice_name"] == voice ] if len(matches) != 1: raise RuntimeError( f"Expected one {phrase_id}/{voice} row, got {len(matches)}" ) row = matches[0] path = Path(str(row["audio_path"])) if not path.is_file(): raise FileNotFoundError(path) return Speech( path=path, scale=float(row.get("speech_scale", 1.0)), text=str(row["text"]), ) def _build_probe_rows(config: LaunchConfig) -> list[ProbeRow]: seed_root = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio") assignment_path = Path(str(config.data.background_assignment_path)) scales = _speech_scale_by_path(assignment_path) control_rows = _load_jsonl(Path(str(config.data.control_command_manifest))) short_manifest = Path( "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" "hfd_backchannel_commands_v2_short/manifest.jsonl" ) backchannel_rows = _load_jsonl(short_manifest) rows: list[ProbeRow] = [] for voice in VOICE_NAMES: base = _seed_speech(seed_root / "000006", voice, scales) action_events = ( _seed_speech(seed_root / "000120", voice, scales), _seed_speech(seed_root / "000155", voice, scales), ) stop_events = tuple( _catalog_speech( control_rows, phrase_id=phrase_id, voice=voice, ) for phrase_id in ("stop_01", "stop_02") ) # One seen training phrase and one unseen, clean validation phrase. backchannel_events = tuple( _catalog_speech( backchannel_rows, phrase_id=phrase_id, voice=voice, ) for phrase_id in ("bc_cont_01", "bc_cont_11") ) for variant, event in enumerate(action_events): rows.append(ProbeRow("action", voice, variant, base, event)) for variant, event in enumerate(stop_events): rows.append(ProbeRow("stop", voice, variant, base, event)) for variant, event in enumerate(backchannel_events): rows.append(ProbeRow("backchannel", voice, variant, base, event)) for variant, count in enumerate((2, 3)): rows.append( ProbeRow( "background", voice, variant, base, None, background_windows=count, ) ) return rows def _read_quiet_background(config: LaunchConfig, sample_count: int) -> np.ndarray: root = Path(str(config.data.continuous_audio_root)) manifest_path = root / "background" / "sessions" / "session_manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) wanted = "sessions/synthetic_quiet_hvac_600s" matches = [row for row in manifest["sessions"] if row["key"] == wanted] if len(matches) != 1: raise RuntimeError(f"Missing quiet production background: {wanted}") return _read_pcm16_wave_segment( Path(str(matches[0]["path"])), # Avoid any file boundary special case while staying deterministic. start_sample=120 * STRIDE_SAMPLES, sample_count=int(sample_count), expected_rate=AUDIO_RATE, ) def _prepare_timelines( rows: list[ProbeRow], config: LaunchConfig, ) -> tuple[np.ndarray, list[int], list[dict[str, Any]]]: waveforms: dict[Path, np.ndarray] = {} def speech(path: Path) -> np.ndarray: if path not in waveforms: waveforms[path] = _read_resampled_speech_float(path, AUDIO_RATE) return waveforms[path] base_start = 2 base_counts = [_window_count(speech(row.base.path).size) for row in rows] event_starts = [base_start + count + 4 for count in base_counts] capture_steps = [] for row, event_start in zip(rows, event_starts): count = ( _window_count(speech(row.event.path).size) if row.event is not None else int(row.background_windows) ) # Read the decision only after four clean 0.4 s background updates. # Ordinary commands deliberately retain an onset/listen reaction; # Stop changes state only after the complete command is audible. # Reading the last speech-overlap window would therefore measure the # listening period rather than the semantic decision shared by all # four cases. capture_steps.append(event_start + count - 1 + 4) total_windows = max(capture_steps) + 1 flat_count = (total_windows - 1) * STRIDE_SAMPLES + WINDOW_SAMPLES background = _read_quiet_background(config, flat_count) timelines = [] metadata = [] for row, event_start, capture_step in zip( rows, event_starts, capture_steps ): flat = background.copy() base_wave = speech(row.base.path) base_offset = base_start * STRIDE_SAMPLES + PREFIX_SAMPLES flat[base_offset : base_offset + base_wave.size] += ( base_wave * np.float32(row.base.scale) ) if row.event is not None: event_wave = speech(row.event.path) event_offset = event_start * STRIDE_SAMPLES + PREFIX_SAMPLES flat[event_offset : event_offset + event_wave.size] += ( event_wave * np.float32(row.event.scale) ) peak = float(np.max(np.abs(flat), initial=0.0)) if peak >= 1.0: raise RuntimeError( f"Production-style probe mix clips: {row.voice}/{row.label} " f"variant={row.variant} peak={peak:.6f}" ) flat = ( np.rint(flat * 32768.0) .clip(-32768, 32767) .astype("" if row.event is None else row.event.text, "capture_step": capture_step, } ) return np.stack(timelines), capture_steps, metadata def _load_model( run_dir: Path, checkpoint: str, device: torch.device, ) -> tuple[FullDuplexPsi0Model, LaunchConfig]: template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore raw_config = json.loads( (run_dir / "run_config.json").read_text(encoding="utf-8") ) config = template.model_validate(raw_config) if bool(config.model.state_aware_serial_control): raise RuntimeError( "This diagnostic isolates semantic input only; the checkpoint has " "state-aware serial control enabled." ) backbone = Path(str(config.model.model_name_or_path)).expanduser() os.environ["PSI_PRETRAIN_BACKBONE_PATH"] = str(backbone) model = FullDuplexPsi0Model.from_streaming_checkpoint( run_dir, checkpoint, config, device, ) return model, config def _extract_features( model: FullDuplexPsi0Model, timelines: np.ndarray, capture_steps: list[int], ) -> tuple[np.ndarray, np.ndarray]: device = model.device batch_size, total_windows = timelines.shape[:2] source_ids = [f"semantic-probe-{index:03d}" for index in range(batch_size)] cache = model.begin_source(source_ids) captured_features: list[np.ndarray | None] = [None] * batch_size captured_probabilities: list[np.ndarray | None] = [None] * batch_size for step in range(total_windows): prepared = [ prepare_qwen_streaming_audio_window( model.vlm_processor, timelines[row, step], first_window=step == 0, sampling_rate=AUDIO_RATE, ) for row in range(batch_size) ] def stack(name: str) -> torch.Tensor: values = [item[name] for item in prepared] shapes = {tuple(value.shape) for value in values} if len(shapes) != 1: raise RuntimeError( f"Prepared {name} shapes differ inside fixed windows: {shapes}" ) return torch.stack(values).to(device=device) with torch.inference_mode(), torch.autocast( device_type=device.type, dtype=torch.bfloat16, enabled=device.type in {"cpu", "cuda"}, ): condition = model.encode_stream_input( source_ids=source_ids, kv_cache=cache, update_semantic_cache=True, vlm_hidden_states=None, attention_mask=stack("stream_attention_mask"), stream_input_ids=stack("stream_input_ids"), stream_input_features=stack("stream_input_features"), stream_feature_attention_mask=stack( "stream_feature_attention_mask" ), ) cache = condition.kv_cache memory = condition.memory_token[:, 0].float().cpu().numpy() probabilities = condition.control_probabilities.float().cpu().numpy() for row, capture_step in enumerate(capture_steps): if step == capture_step: captured_features[row] = memory[row].copy() captured_probabilities[row] = probabilities[row].copy() print( f"[probe] streamed window {step + 1}/{total_windows} " f"cache_tokens={cache.sequence_length}", flush=True, ) if any(value is None for value in captured_features): raise RuntimeError("At least one semantic feature was not captured") return ( np.stack([value for value in captured_features if value is not None]), np.stack( [value for value in captured_probabilities if value is not None] ), ) def _linear_probe( features: np.ndarray, labels: np.ndarray, voices: np.ndarray, variants: np.ndarray, *, permutation_repeats: int, seed: int, ) -> dict[str, Any]: from sklearn.linear_model import LogisticRegression from sklearn.metrics import confusion_matrix from sklearn.pipeline import make_pipeline from sklearn.preprocessing import Normalizer, StandardScaler def predict(train: np.ndarray, test: np.ndarray, target: np.ndarray) -> np.ndarray: classifier = make_pipeline( StandardScaler(with_std=False), Normalizer(norm="l2"), LogisticRegression( C=0.1, max_iter=4_000, solver="lbfgs", random_state=seed, ), ) classifier.fit(features[train], target[train]) return classifier.predict(features[test]) predictions = np.full(labels.shape, -1, dtype=np.int64) for voice in sorted(set(voices.tolist())): test = voices == voice train = ~test predictions[test] = predict(train, test, labels) accuracy = float(np.mean(predictions == labels)) phrase_predictions = np.full(labels.shape, -1, dtype=np.int64) for variant in sorted(set(variants.tolist())): test = variants == variant train = ~test phrase_predictions[test] = predict(train, test, labels) phrase_accuracy = float(np.mean(phrase_predictions == labels)) rng = np.random.default_rng(seed) null_accuracies = [] for _ in range(max(int(permutation_repeats), 0)): shuffled = labels.copy() for voice in sorted(set(voices.tolist())): indices = np.flatnonzero(voices == voice) shuffled[indices] = rng.permutation(shuffled[indices]) null_prediction = np.full(labels.shape, -1, dtype=np.int64) for voice in sorted(set(voices.tolist())): test = voices == voice train = ~test null_prediction[test] = predict(train, test, shuffled) null_accuracies.append(float(np.mean(null_prediction == shuffled))) return { "leave_one_voice_out_accuracy": accuracy, "leave_one_phrase_variant_out_accuracy": phrase_accuracy, "confusion_matrix": confusion_matrix( labels, predictions, labels=list(range(len(CLASS_NAMES))), ).tolist(), "permuted_label_accuracy_mean": ( float(np.mean(null_accuracies)) if null_accuracies else None ), "permuted_label_accuracy_p95": ( float(np.quantile(null_accuracies, 0.95)) if null_accuracies else None ), } def _control_summary( probabilities: np.ndarray, metadata: list[dict[str, Any]], ) -> dict[str, Any]: predictions = probabilities.argmax(axis=-1) result = {} for label in CLASS_NAMES: indices = np.asarray( [row["label"] == label for row in metadata], dtype=np.bool_, ) counts = { name: int(np.sum(predictions[indices] == token_id)) for token_id, name in enumerate(CONTROL_NAMES) } result[label] = { "expected_control": CONTROL_NAMES[EXPECTED_CONTROL_ID[label]], "expected_control_accuracy": float( np.mean(predictions[indices] == EXPECTED_CONTROL_ID[label]) ), "prediction_counts": counts, "mean_probabilities": { name: float(probabilities[indices, token_id].mean()) for token_id, name in enumerate(CONTROL_NAMES) }, } return result def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--ckpt-step", default="latest") parser.add_argument("--device", default="cpu") parser.add_argument("--cpu-threads", type=int, default=56) parser.add_argument("--permutation-repeats", type=int, default=20) parser.add_argument("--seed", type=int, default=42) parser.add_argument("--output-json", type=Path) return parser.parse_args() def main() -> None: args = parse_args() run_dir = args.run_dir.expanduser().resolve() checkpoint = _checkpoint_name(run_dir, args.ckpt_step) device = torch.device(args.device) if device.type == "cpu": torch.set_num_threads(max(int(args.cpu_threads), 1)) torch.manual_seed(int(args.seed)) model, config = _load_model(run_dir, checkpoint, device) rows = _build_probe_rows(config) timelines, capture_steps, metadata = _prepare_timelines(rows, config) features, probabilities = _extract_features( model, timelines, capture_steps, ) label_ids = np.asarray( [CLASS_NAMES.index(row["label"]) for row in metadata], dtype=np.int64, ) voices = np.asarray([row["voice"] for row in metadata]) variants = np.asarray([row["variant"] for row in metadata]) result = { "format": "hfd_control_semantic_probe_v1", "run_dir": str(run_dir), "checkpoint": checkpoint, "feature_contract": ( "condition.memory_token[:,0] == control_head input because " "state_aware_serial_control=false" ), "samples": int(features.shape[0]), "feature_dim": int(features.shape[1]), "classes": list(CLASS_NAMES), "voices": list(VOICE_NAMES), "chance_accuracy": 1.0 / len(CLASS_NAMES), "linear_probe": _linear_probe( features, label_ids, voices, variants, permutation_repeats=int(args.permutation_repeats), seed=int(args.seed), ), "trained_control_head": _control_summary(probabilities, metadata), "sample_metadata": metadata, } rendered = json.dumps(result, ensure_ascii=False, indent=2) print(rendered) if args.output_json is not None: output = args.output_json.expanduser().resolve() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(rendered + "\n", encoding="utf-8") if __name__ == "__main__": main()