Download scripts/diagnostics/probe_hfd_control_semantics.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 20.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/diagnostics/probe_hfd_control_semantics.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/diagnostics/probe_hfd_control_semantics.py
-
curl -L -o probe_hfd_control_semantics.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/diagnostics/probe_hfd_control_semantics.py
20.8 kB
| #!/usr/bin/env python3 | |
| """Probe whether the production HFD control input separates four meanings. | |
| This is a read-only diagnostic. It streams production-format 0.6 s audio | |
| windows through a real fused checkpoint, keeps the native Qwen KV for the | |
| complete synthetic session, and captures ``condition.memory_token``. With | |
| state-aware serial control disabled (the semantics-v22 production policy), | |
| that tensor is exactly the vector consumed by ``control_head``. | |
| Four event types share the same prefix action and continuous office | |
| background: a new action instruction, an emergency Stop, a short | |
| backchannel, or background only. A small linear probe is trained on seven | |
| voices and evaluated on the eighth, so speaker memorisation cannot count as | |
| semantic separation. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import math | |
| import os | |
| import sys | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| import numpy as np | |
| import torch | |
| PROJECT_ROOT = Path(__file__).resolve().parents[2] | |
| DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data" | |
| if str(PROJECT_ROOT) not in sys.path: | |
| sys.path.insert(0, str(PROJECT_ROOT)) | |
| if str(DATA_SCRIPT_DIR) not in sys.path: | |
| sys.path.insert(0, str(DATA_SCRIPT_DIR)) | |
| from psi.config.config import LaunchConfig # noqa: E402 | |
| from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402 | |
| _read_pcm16_wave_segment, | |
| _read_resampled_speech_float, | |
| ) | |
| from psi.data.seed.qwen_streaming_audio import ( # noqa: E402 | |
| prepare_qwen_streaming_audio_window, | |
| ) | |
| from psi.models.full_duplex_psi0 import FullDuplexPsi0Model # noqa: E402 | |
| from psi.utils import parse_args_to_tyro_config # noqa: E402 | |
| AUDIO_RATE = 16_000 | |
| WINDOW_SAMPLES = 9_600 | |
| STRIDE_SAMPLES = 6_400 | |
| PREFIX_SAMPLES = WINDOW_SAMPLES - STRIDE_SAMPLES | |
| VOICE_NAMES = ( | |
| "female_01_en", | |
| "female_02_en", | |
| "male_01_en", | |
| "male_02_en", | |
| "female_01_zh", | |
| "female_02_zh", | |
| "male_01_zh", | |
| "male_02_zh", | |
| ) | |
| CLASS_NAMES = ("action", "stop", "backchannel", "background") | |
| EXPECTED_CONTROL_ID = { | |
| "action": 1, | |
| "stop": 2, | |
| "backchannel": 1, | |
| "background": 1, | |
| } | |
| CONTROL_NAMES = ("listen", "motion", "idle") | |
| class Speech: | |
| path: Path | |
| scale: float | |
| text: str | |
| class ProbeRow: | |
| label: str | |
| voice: str | |
| variant: int | |
| base: Speech | |
| event: Speech | None | |
| background_windows: int = 0 | |
| def _window_count(sample_count: int) -> int: | |
| return int( | |
| math.ceil((PREFIX_SAMPLES + int(sample_count)) / STRIDE_SAMPLES) | |
| ) | |
| def _load_jsonl(path: Path) -> list[dict[str, Any]]: | |
| with path.open("r", encoding="utf-8") as handle: | |
| return [json.loads(line) for line in handle if line.strip()] | |
| def _checkpoint_name(run_dir: Path, raw: str) -> str: | |
| requested = str(raw).strip() | |
| if requested not in {"", "latest"}: | |
| return requested | |
| candidates = [] | |
| for path in (run_dir / "checkpoints").glob("ckpt_*"): | |
| try: | |
| candidates.append((int(path.name.removeprefix("ckpt_")), path.name)) | |
| except ValueError: | |
| continue | |
| if not candidates: | |
| raise FileNotFoundError(f"No immutable checkpoint under {run_dir}") | |
| return max(candidates)[1] | |
| def _speech_scale_by_path(assignment_path: Path) -> dict[str, float]: | |
| return { | |
| os.path.normpath(str(row["original_audio_path"])): float( | |
| row["speech_scale"] | |
| ) | |
| for row in _load_jsonl(assignment_path) | |
| } | |
| def _seed_speech( | |
| directory: Path, | |
| voice: str, | |
| scales: dict[str, float], | |
| ) -> Speech: | |
| path = directory / f"{voice}.wav" | |
| key = os.path.normpath(str(path)) | |
| if not path.is_file() or key not in scales: | |
| raise FileNotFoundError(f"Missing production seed assignment: {path}") | |
| return Speech(path=path, scale=scales[key], text=directory.name) | |
| def _catalog_speech( | |
| rows: list[dict[str, Any]], | |
| *, | |
| phrase_id: str, | |
| voice: str, | |
| ) -> Speech: | |
| matches = [ | |
| row | |
| for row in rows | |
| if row["phrase_id"] == phrase_id and row["voice_name"] == voice | |
| ] | |
| if len(matches) != 1: | |
| raise RuntimeError( | |
| f"Expected one {phrase_id}/{voice} row, got {len(matches)}" | |
| ) | |
| row = matches[0] | |
| path = Path(str(row["audio_path"])) | |
| if not path.is_file(): | |
| raise FileNotFoundError(path) | |
| return Speech( | |
| path=path, | |
| scale=float(row.get("speech_scale", 1.0)), | |
| text=str(row["text"]), | |
| ) | |
| def _build_probe_rows(config: LaunchConfig) -> list[ProbeRow]: | |
| seed_root = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio") | |
| assignment_path = Path(str(config.data.background_assignment_path)) | |
| scales = _speech_scale_by_path(assignment_path) | |
| control_rows = _load_jsonl(Path(str(config.data.control_command_manifest))) | |
| short_manifest = Path( | |
| "/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/" | |
| "hfd_backchannel_commands_v2_short/manifest.jsonl" | |
| ) | |
| backchannel_rows = _load_jsonl(short_manifest) | |
| rows: list[ProbeRow] = [] | |
| for voice in VOICE_NAMES: | |
| base = _seed_speech(seed_root / "000006", voice, scales) | |
| action_events = ( | |
| _seed_speech(seed_root / "000120", voice, scales), | |
| _seed_speech(seed_root / "000155", voice, scales), | |
| ) | |
| stop_events = tuple( | |
| _catalog_speech( | |
| control_rows, | |
| phrase_id=phrase_id, | |
| voice=voice, | |
| ) | |
| for phrase_id in ("stop_01", "stop_02") | |
| ) | |
| # One seen training phrase and one unseen, clean validation phrase. | |
| backchannel_events = tuple( | |
| _catalog_speech( | |
| backchannel_rows, | |
| phrase_id=phrase_id, | |
| voice=voice, | |
| ) | |
| for phrase_id in ("bc_cont_01", "bc_cont_11") | |
| ) | |
| for variant, event in enumerate(action_events): | |
| rows.append(ProbeRow("action", voice, variant, base, event)) | |
| for variant, event in enumerate(stop_events): | |
| rows.append(ProbeRow("stop", voice, variant, base, event)) | |
| for variant, event in enumerate(backchannel_events): | |
| rows.append(ProbeRow("backchannel", voice, variant, base, event)) | |
| for variant, count in enumerate((2, 3)): | |
| rows.append( | |
| ProbeRow( | |
| "background", | |
| voice, | |
| variant, | |
| base, | |
| None, | |
| background_windows=count, | |
| ) | |
| ) | |
| return rows | |
| def _read_quiet_background(config: LaunchConfig, sample_count: int) -> np.ndarray: | |
| root = Path(str(config.data.continuous_audio_root)) | |
| manifest_path = root / "background" / "sessions" / "session_manifest.json" | |
| manifest = json.loads(manifest_path.read_text(encoding="utf-8")) | |
| wanted = "sessions/synthetic_quiet_hvac_600s" | |
| matches = [row for row in manifest["sessions"] if row["key"] == wanted] | |
| if len(matches) != 1: | |
| raise RuntimeError(f"Missing quiet production background: {wanted}") | |
| return _read_pcm16_wave_segment( | |
| Path(str(matches[0]["path"])), | |
| # Avoid any file boundary special case while staying deterministic. | |
| start_sample=120 * STRIDE_SAMPLES, | |
| sample_count=int(sample_count), | |
| expected_rate=AUDIO_RATE, | |
| ) | |
| def _prepare_timelines( | |
| rows: list[ProbeRow], | |
| config: LaunchConfig, | |
| ) -> tuple[np.ndarray, list[int], list[dict[str, Any]]]: | |
| waveforms: dict[Path, np.ndarray] = {} | |
| def speech(path: Path) -> np.ndarray: | |
| if path not in waveforms: | |
| waveforms[path] = _read_resampled_speech_float(path, AUDIO_RATE) | |
| return waveforms[path] | |
| base_start = 2 | |
| base_counts = [_window_count(speech(row.base.path).size) for row in rows] | |
| event_starts = [base_start + count + 4 for count in base_counts] | |
| capture_steps = [] | |
| for row, event_start in zip(rows, event_starts): | |
| count = ( | |
| _window_count(speech(row.event.path).size) | |
| if row.event is not None | |
| else int(row.background_windows) | |
| ) | |
| # Read the decision only after four clean 0.4 s background updates. | |
| # Ordinary commands deliberately retain an onset/listen reaction; | |
| # Stop changes state only after the complete command is audible. | |
| # Reading the last speech-overlap window would therefore measure the | |
| # listening period rather than the semantic decision shared by all | |
| # four cases. | |
| capture_steps.append(event_start + count - 1 + 4) | |
| total_windows = max(capture_steps) + 1 | |
| flat_count = (total_windows - 1) * STRIDE_SAMPLES + WINDOW_SAMPLES | |
| background = _read_quiet_background(config, flat_count) | |
| timelines = [] | |
| metadata = [] | |
| for row, event_start, capture_step in zip( | |
| rows, event_starts, capture_steps | |
| ): | |
| flat = background.copy() | |
| base_wave = speech(row.base.path) | |
| base_offset = base_start * STRIDE_SAMPLES + PREFIX_SAMPLES | |
| flat[base_offset : base_offset + base_wave.size] += ( | |
| base_wave * np.float32(row.base.scale) | |
| ) | |
| if row.event is not None: | |
| event_wave = speech(row.event.path) | |
| event_offset = event_start * STRIDE_SAMPLES + PREFIX_SAMPLES | |
| flat[event_offset : event_offset + event_wave.size] += ( | |
| event_wave * np.float32(row.event.scale) | |
| ) | |
| peak = float(np.max(np.abs(flat), initial=0.0)) | |
| if peak >= 1.0: | |
| raise RuntimeError( | |
| f"Production-style probe mix clips: {row.voice}/{row.label} " | |
| f"variant={row.variant} peak={peak:.6f}" | |
| ) | |
| flat = ( | |
| np.rint(flat * 32768.0) | |
| .clip(-32768, 32767) | |
| .astype("<i2") | |
| .astype(np.float32) | |
| / np.float32(32768.0) | |
| ) | |
| timelines.append( | |
| np.stack( | |
| [ | |
| flat[ | |
| index * STRIDE_SAMPLES : | |
| index * STRIDE_SAMPLES + WINDOW_SAMPLES | |
| ] | |
| for index in range(total_windows) | |
| ] | |
| ) | |
| ) | |
| metadata.append( | |
| { | |
| "label": row.label, | |
| "voice": row.voice, | |
| "variant": row.variant, | |
| "event_text": "<background>" if row.event is None else row.event.text, | |
| "capture_step": capture_step, | |
| } | |
| ) | |
| return np.stack(timelines), capture_steps, metadata | |
| def _load_model( | |
| run_dir: Path, | |
| checkpoint: str, | |
| device: torch.device, | |
| ) -> tuple[FullDuplexPsi0Model, LaunchConfig]: | |
| template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore | |
| raw_config = json.loads( | |
| (run_dir / "run_config.json").read_text(encoding="utf-8") | |
| ) | |
| config = template.model_validate(raw_config) | |
| if bool(config.model.state_aware_serial_control): | |
| raise RuntimeError( | |
| "This diagnostic isolates semantic input only; the checkpoint has " | |
| "state-aware serial control enabled." | |
| ) | |
| backbone = Path(str(config.model.model_name_or_path)).expanduser() | |
| os.environ["PSI_PRETRAIN_BACKBONE_PATH"] = str(backbone) | |
| model = FullDuplexPsi0Model.from_streaming_checkpoint( | |
| run_dir, | |
| checkpoint, | |
| config, | |
| device, | |
| ) | |
| return model, config | |
| def _extract_features( | |
| model: FullDuplexPsi0Model, | |
| timelines: np.ndarray, | |
| capture_steps: list[int], | |
| ) -> tuple[np.ndarray, np.ndarray]: | |
| device = model.device | |
| batch_size, total_windows = timelines.shape[:2] | |
| source_ids = [f"semantic-probe-{index:03d}" for index in range(batch_size)] | |
| cache = model.begin_source(source_ids) | |
| captured_features: list[np.ndarray | None] = [None] * batch_size | |
| captured_probabilities: list[np.ndarray | None] = [None] * batch_size | |
| for step in range(total_windows): | |
| prepared = [ | |
| prepare_qwen_streaming_audio_window( | |
| model.vlm_processor, | |
| timelines[row, step], | |
| first_window=step == 0, | |
| sampling_rate=AUDIO_RATE, | |
| ) | |
| for row in range(batch_size) | |
| ] | |
| def stack(name: str) -> torch.Tensor: | |
| values = [item[name] for item in prepared] | |
| shapes = {tuple(value.shape) for value in values} | |
| if len(shapes) != 1: | |
| raise RuntimeError( | |
| f"Prepared {name} shapes differ inside fixed windows: {shapes}" | |
| ) | |
| return torch.stack(values).to(device=device) | |
| with torch.inference_mode(), torch.autocast( | |
| device_type=device.type, | |
| dtype=torch.bfloat16, | |
| enabled=device.type in {"cpu", "cuda"}, | |
| ): | |
| condition = model.encode_stream_input( | |
| source_ids=source_ids, | |
| kv_cache=cache, | |
| update_semantic_cache=True, | |
| vlm_hidden_states=None, | |
| attention_mask=stack("stream_attention_mask"), | |
| stream_input_ids=stack("stream_input_ids"), | |
| stream_input_features=stack("stream_input_features"), | |
| stream_feature_attention_mask=stack( | |
| "stream_feature_attention_mask" | |
| ), | |
| ) | |
| cache = condition.kv_cache | |
| memory = condition.memory_token[:, 0].float().cpu().numpy() | |
| probabilities = condition.control_probabilities.float().cpu().numpy() | |
| for row, capture_step in enumerate(capture_steps): | |
| if step == capture_step: | |
| captured_features[row] = memory[row].copy() | |
| captured_probabilities[row] = probabilities[row].copy() | |
| print( | |
| f"[probe] streamed window {step + 1}/{total_windows} " | |
| f"cache_tokens={cache.sequence_length}", | |
| flush=True, | |
| ) | |
| if any(value is None for value in captured_features): | |
| raise RuntimeError("At least one semantic feature was not captured") | |
| return ( | |
| np.stack([value for value in captured_features if value is not None]), | |
| np.stack( | |
| [value for value in captured_probabilities if value is not None] | |
| ), | |
| ) | |
| def _linear_probe( | |
| features: np.ndarray, | |
| labels: np.ndarray, | |
| voices: np.ndarray, | |
| variants: np.ndarray, | |
| *, | |
| permutation_repeats: int, | |
| seed: int, | |
| ) -> dict[str, Any]: | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.metrics import confusion_matrix | |
| from sklearn.pipeline import make_pipeline | |
| from sklearn.preprocessing import Normalizer, StandardScaler | |
| def predict(train: np.ndarray, test: np.ndarray, target: np.ndarray) -> np.ndarray: | |
| classifier = make_pipeline( | |
| StandardScaler(with_std=False), | |
| Normalizer(norm="l2"), | |
| LogisticRegression( | |
| C=0.1, | |
| max_iter=4_000, | |
| solver="lbfgs", | |
| random_state=seed, | |
| ), | |
| ) | |
| classifier.fit(features[train], target[train]) | |
| return classifier.predict(features[test]) | |
| predictions = np.full(labels.shape, -1, dtype=np.int64) | |
| for voice in sorted(set(voices.tolist())): | |
| test = voices == voice | |
| train = ~test | |
| predictions[test] = predict(train, test, labels) | |
| accuracy = float(np.mean(predictions == labels)) | |
| phrase_predictions = np.full(labels.shape, -1, dtype=np.int64) | |
| for variant in sorted(set(variants.tolist())): | |
| test = variants == variant | |
| train = ~test | |
| phrase_predictions[test] = predict(train, test, labels) | |
| phrase_accuracy = float(np.mean(phrase_predictions == labels)) | |
| rng = np.random.default_rng(seed) | |
| null_accuracies = [] | |
| for _ in range(max(int(permutation_repeats), 0)): | |
| shuffled = labels.copy() | |
| for voice in sorted(set(voices.tolist())): | |
| indices = np.flatnonzero(voices == voice) | |
| shuffled[indices] = rng.permutation(shuffled[indices]) | |
| null_prediction = np.full(labels.shape, -1, dtype=np.int64) | |
| for voice in sorted(set(voices.tolist())): | |
| test = voices == voice | |
| train = ~test | |
| null_prediction[test] = predict(train, test, shuffled) | |
| null_accuracies.append(float(np.mean(null_prediction == shuffled))) | |
| return { | |
| "leave_one_voice_out_accuracy": accuracy, | |
| "leave_one_phrase_variant_out_accuracy": phrase_accuracy, | |
| "confusion_matrix": confusion_matrix( | |
| labels, | |
| predictions, | |
| labels=list(range(len(CLASS_NAMES))), | |
| ).tolist(), | |
| "permuted_label_accuracy_mean": ( | |
| float(np.mean(null_accuracies)) if null_accuracies else None | |
| ), | |
| "permuted_label_accuracy_p95": ( | |
| float(np.quantile(null_accuracies, 0.95)) | |
| if null_accuracies | |
| else None | |
| ), | |
| } | |
| def _control_summary( | |
| probabilities: np.ndarray, | |
| metadata: list[dict[str, Any]], | |
| ) -> dict[str, Any]: | |
| predictions = probabilities.argmax(axis=-1) | |
| result = {} | |
| for label in CLASS_NAMES: | |
| indices = np.asarray( | |
| [row["label"] == label for row in metadata], | |
| dtype=np.bool_, | |
| ) | |
| counts = { | |
| name: int(np.sum(predictions[indices] == token_id)) | |
| for token_id, name in enumerate(CONTROL_NAMES) | |
| } | |
| result[label] = { | |
| "expected_control": CONTROL_NAMES[EXPECTED_CONTROL_ID[label]], | |
| "expected_control_accuracy": float( | |
| np.mean(predictions[indices] == EXPECTED_CONTROL_ID[label]) | |
| ), | |
| "prediction_counts": counts, | |
| "mean_probabilities": { | |
| name: float(probabilities[indices, token_id].mean()) | |
| for token_id, name in enumerate(CONTROL_NAMES) | |
| }, | |
| } | |
| return result | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--ckpt-step", default="latest") | |
| parser.add_argument("--device", default="cpu") | |
| parser.add_argument("--cpu-threads", type=int, default=56) | |
| parser.add_argument("--permutation-repeats", type=int, default=20) | |
| parser.add_argument("--seed", type=int, default=42) | |
| parser.add_argument("--output-json", type=Path) | |
| return parser.parse_args() | |
| def main() -> None: | |
| args = parse_args() | |
| run_dir = args.run_dir.expanduser().resolve() | |
| checkpoint = _checkpoint_name(run_dir, args.ckpt_step) | |
| device = torch.device(args.device) | |
| if device.type == "cpu": | |
| torch.set_num_threads(max(int(args.cpu_threads), 1)) | |
| torch.manual_seed(int(args.seed)) | |
| model, config = _load_model(run_dir, checkpoint, device) | |
| rows = _build_probe_rows(config) | |
| timelines, capture_steps, metadata = _prepare_timelines(rows, config) | |
| features, probabilities = _extract_features( | |
| model, | |
| timelines, | |
| capture_steps, | |
| ) | |
| label_ids = np.asarray( | |
| [CLASS_NAMES.index(row["label"]) for row in metadata], | |
| dtype=np.int64, | |
| ) | |
| voices = np.asarray([row["voice"] for row in metadata]) | |
| variants = np.asarray([row["variant"] for row in metadata]) | |
| result = { | |
| "format": "hfd_control_semantic_probe_v1", | |
| "run_dir": str(run_dir), | |
| "checkpoint": checkpoint, | |
| "feature_contract": ( | |
| "condition.memory_token[:,0] == control_head input because " | |
| "state_aware_serial_control=false" | |
| ), | |
| "samples": int(features.shape[0]), | |
| "feature_dim": int(features.shape[1]), | |
| "classes": list(CLASS_NAMES), | |
| "voices": list(VOICE_NAMES), | |
| "chance_accuracy": 1.0 / len(CLASS_NAMES), | |
| "linear_probe": _linear_probe( | |
| features, | |
| label_ids, | |
| voices, | |
| variants, | |
| permutation_repeats=int(args.permutation_repeats), | |
| seed=int(args.seed), | |
| ), | |
| "trained_control_head": _control_summary(probabilities, metadata), | |
| "sample_metadata": metadata, | |
| } | |
| rendered = json.dumps(result, ensure_ascii=False, indent=2) | |
| print(rendered) | |
| if args.output_json is not None: | |
| output = args.output_json.expanduser().resolve() | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| output.write_text(rendered + "\n", encoding="utf-8") | |
| if __name__ == "__main__": | |
| main() | |