code-HFD / scripts /diagnostics /probe_hfd_control_semantics.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
20.8 kB
#!/usr/bin/env python3
"""Probe whether the production HFD control input separates four meanings.
This is a read-only diagnostic. It streams production-format 0.6 s audio
windows through a real fused checkpoint, keeps the native Qwen KV for the
complete synthetic session, and captures ``condition.memory_token``. With
state-aware serial control disabled (the semantics-v22 production policy),
that tensor is exactly the vector consumed by ``control_head``.
Four event types share the same prefix action and continuous office
background: a new action instruction, an emergency Stop, a short
backchannel, or background only. A small linear probe is trained on seven
voices and evaluated on the eighth, so speaker memorisation cannot count as
semantic separation.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
import torch
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DATA_SCRIPT_DIR = PROJECT_ROOT / "scripts" / "data"
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
if str(DATA_SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(DATA_SCRIPT_DIR))
from psi.config.config import LaunchConfig # noqa: E402
from psi.data.seed.full_duplex_audio_command_dataset import ( # noqa: E402
_read_pcm16_wave_segment,
_read_resampled_speech_float,
)
from psi.data.seed.qwen_streaming_audio import ( # noqa: E402
prepare_qwen_streaming_audio_window,
)
from psi.models.full_duplex_psi0 import FullDuplexPsi0Model # noqa: E402
from psi.utils import parse_args_to_tyro_config # noqa: E402
AUDIO_RATE = 16_000
WINDOW_SAMPLES = 9_600
STRIDE_SAMPLES = 6_400
PREFIX_SAMPLES = WINDOW_SAMPLES - STRIDE_SAMPLES
VOICE_NAMES = (
"female_01_en",
"female_02_en",
"male_01_en",
"male_02_en",
"female_01_zh",
"female_02_zh",
"male_01_zh",
"male_02_zh",
)
CLASS_NAMES = ("action", "stop", "backchannel", "background")
EXPECTED_CONTROL_ID = {
"action": 1,
"stop": 2,
"backchannel": 1,
"background": 1,
}
CONTROL_NAMES = ("listen", "motion", "idle")
@dataclass(frozen=True)
class Speech:
path: Path
scale: float
text: str
@dataclass(frozen=True)
class ProbeRow:
label: str
voice: str
variant: int
base: Speech
event: Speech | None
background_windows: int = 0
def _window_count(sample_count: int) -> int:
return int(
math.ceil((PREFIX_SAMPLES + int(sample_count)) / STRIDE_SAMPLES)
)
def _load_jsonl(path: Path) -> list[dict[str, Any]]:
with path.open("r", encoding="utf-8") as handle:
return [json.loads(line) for line in handle if line.strip()]
def _checkpoint_name(run_dir: Path, raw: str) -> str:
requested = str(raw).strip()
if requested not in {"", "latest"}:
return requested
candidates = []
for path in (run_dir / "checkpoints").glob("ckpt_*"):
try:
candidates.append((int(path.name.removeprefix("ckpt_")), path.name))
except ValueError:
continue
if not candidates:
raise FileNotFoundError(f"No immutable checkpoint under {run_dir}")
return max(candidates)[1]
def _speech_scale_by_path(assignment_path: Path) -> dict[str, float]:
return {
os.path.normpath(str(row["original_audio_path"])): float(
row["speech_scale"]
)
for row in _load_jsonl(assignment_path)
}
def _seed_speech(
directory: Path,
voice: str,
scales: dict[str, float],
) -> Speech:
path = directory / f"{voice}.wav"
key = os.path.normpath(str(path))
if not path.is_file() or key not in scales:
raise FileNotFoundError(f"Missing production seed assignment: {path}")
return Speech(path=path, scale=scales[key], text=directory.name)
def _catalog_speech(
rows: list[dict[str, Any]],
*,
phrase_id: str,
voice: str,
) -> Speech:
matches = [
row
for row in rows
if row["phrase_id"] == phrase_id and row["voice_name"] == voice
]
if len(matches) != 1:
raise RuntimeError(
f"Expected one {phrase_id}/{voice} row, got {len(matches)}"
)
row = matches[0]
path = Path(str(row["audio_path"]))
if not path.is_file():
raise FileNotFoundError(path)
return Speech(
path=path,
scale=float(row.get("speech_scale", 1.0)),
text=str(row["text"]),
)
def _build_probe_rows(config: LaunchConfig) -> list[ProbeRow]:
seed_root = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/qwen3-tts/seed_audio")
assignment_path = Path(str(config.data.background_assignment_path))
scales = _speech_scale_by_path(assignment_path)
control_rows = _load_jsonl(Path(str(config.data.control_command_manifest)))
short_manifest = Path(
"/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/yetinghong/videogen/"
"hfd_backchannel_commands_v2_short/manifest.jsonl"
)
backchannel_rows = _load_jsonl(short_manifest)
rows: list[ProbeRow] = []
for voice in VOICE_NAMES:
base = _seed_speech(seed_root / "000006", voice, scales)
action_events = (
_seed_speech(seed_root / "000120", voice, scales),
_seed_speech(seed_root / "000155", voice, scales),
)
stop_events = tuple(
_catalog_speech(
control_rows,
phrase_id=phrase_id,
voice=voice,
)
for phrase_id in ("stop_01", "stop_02")
)
# One seen training phrase and one unseen, clean validation phrase.
backchannel_events = tuple(
_catalog_speech(
backchannel_rows,
phrase_id=phrase_id,
voice=voice,
)
for phrase_id in ("bc_cont_01", "bc_cont_11")
)
for variant, event in enumerate(action_events):
rows.append(ProbeRow("action", voice, variant, base, event))
for variant, event in enumerate(stop_events):
rows.append(ProbeRow("stop", voice, variant, base, event))
for variant, event in enumerate(backchannel_events):
rows.append(ProbeRow("backchannel", voice, variant, base, event))
for variant, count in enumerate((2, 3)):
rows.append(
ProbeRow(
"background",
voice,
variant,
base,
None,
background_windows=count,
)
)
return rows
def _read_quiet_background(config: LaunchConfig, sample_count: int) -> np.ndarray:
root = Path(str(config.data.continuous_audio_root))
manifest_path = root / "background" / "sessions" / "session_manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
wanted = "sessions/synthetic_quiet_hvac_600s"
matches = [row for row in manifest["sessions"] if row["key"] == wanted]
if len(matches) != 1:
raise RuntimeError(f"Missing quiet production background: {wanted}")
return _read_pcm16_wave_segment(
Path(str(matches[0]["path"])),
# Avoid any file boundary special case while staying deterministic.
start_sample=120 * STRIDE_SAMPLES,
sample_count=int(sample_count),
expected_rate=AUDIO_RATE,
)
def _prepare_timelines(
rows: list[ProbeRow],
config: LaunchConfig,
) -> tuple[np.ndarray, list[int], list[dict[str, Any]]]:
waveforms: dict[Path, np.ndarray] = {}
def speech(path: Path) -> np.ndarray:
if path not in waveforms:
waveforms[path] = _read_resampled_speech_float(path, AUDIO_RATE)
return waveforms[path]
base_start = 2
base_counts = [_window_count(speech(row.base.path).size) for row in rows]
event_starts = [base_start + count + 4 for count in base_counts]
capture_steps = []
for row, event_start in zip(rows, event_starts):
count = (
_window_count(speech(row.event.path).size)
if row.event is not None
else int(row.background_windows)
)
# Read the decision only after four clean 0.4 s background updates.
# Ordinary commands deliberately retain an onset/listen reaction;
# Stop changes state only after the complete command is audible.
# Reading the last speech-overlap window would therefore measure the
# listening period rather than the semantic decision shared by all
# four cases.
capture_steps.append(event_start + count - 1 + 4)
total_windows = max(capture_steps) + 1
flat_count = (total_windows - 1) * STRIDE_SAMPLES + WINDOW_SAMPLES
background = _read_quiet_background(config, flat_count)
timelines = []
metadata = []
for row, event_start, capture_step in zip(
rows, event_starts, capture_steps
):
flat = background.copy()
base_wave = speech(row.base.path)
base_offset = base_start * STRIDE_SAMPLES + PREFIX_SAMPLES
flat[base_offset : base_offset + base_wave.size] += (
base_wave * np.float32(row.base.scale)
)
if row.event is not None:
event_wave = speech(row.event.path)
event_offset = event_start * STRIDE_SAMPLES + PREFIX_SAMPLES
flat[event_offset : event_offset + event_wave.size] += (
event_wave * np.float32(row.event.scale)
)
peak = float(np.max(np.abs(flat), initial=0.0))
if peak >= 1.0:
raise RuntimeError(
f"Production-style probe mix clips: {row.voice}/{row.label} "
f"variant={row.variant} peak={peak:.6f}"
)
flat = (
np.rint(flat * 32768.0)
.clip(-32768, 32767)
.astype("<i2")
.astype(np.float32)
/ np.float32(32768.0)
)
timelines.append(
np.stack(
[
flat[
index * STRIDE_SAMPLES :
index * STRIDE_SAMPLES + WINDOW_SAMPLES
]
for index in range(total_windows)
]
)
)
metadata.append(
{
"label": row.label,
"voice": row.voice,
"variant": row.variant,
"event_text": "<background>" if row.event is None else row.event.text,
"capture_step": capture_step,
}
)
return np.stack(timelines), capture_steps, metadata
def _load_model(
run_dir: Path,
checkpoint: str,
device: torch.device,
) -> tuple[FullDuplexPsi0Model, LaunchConfig]:
template: LaunchConfig = parse_args_to_tyro_config(run_dir / "argv.txt") # type: ignore
raw_config = json.loads(
(run_dir / "run_config.json").read_text(encoding="utf-8")
)
config = template.model_validate(raw_config)
if bool(config.model.state_aware_serial_control):
raise RuntimeError(
"This diagnostic isolates semantic input only; the checkpoint has "
"state-aware serial control enabled."
)
backbone = Path(str(config.model.model_name_or_path)).expanduser()
os.environ["PSI_PRETRAIN_BACKBONE_PATH"] = str(backbone)
model = FullDuplexPsi0Model.from_streaming_checkpoint(
run_dir,
checkpoint,
config,
device,
)
return model, config
def _extract_features(
model: FullDuplexPsi0Model,
timelines: np.ndarray,
capture_steps: list[int],
) -> tuple[np.ndarray, np.ndarray]:
device = model.device
batch_size, total_windows = timelines.shape[:2]
source_ids = [f"semantic-probe-{index:03d}" for index in range(batch_size)]
cache = model.begin_source(source_ids)
captured_features: list[np.ndarray | None] = [None] * batch_size
captured_probabilities: list[np.ndarray | None] = [None] * batch_size
for step in range(total_windows):
prepared = [
prepare_qwen_streaming_audio_window(
model.vlm_processor,
timelines[row, step],
first_window=step == 0,
sampling_rate=AUDIO_RATE,
)
for row in range(batch_size)
]
def stack(name: str) -> torch.Tensor:
values = [item[name] for item in prepared]
shapes = {tuple(value.shape) for value in values}
if len(shapes) != 1:
raise RuntimeError(
f"Prepared {name} shapes differ inside fixed windows: {shapes}"
)
return torch.stack(values).to(device=device)
with torch.inference_mode(), torch.autocast(
device_type=device.type,
dtype=torch.bfloat16,
enabled=device.type in {"cpu", "cuda"},
):
condition = model.encode_stream_input(
source_ids=source_ids,
kv_cache=cache,
update_semantic_cache=True,
vlm_hidden_states=None,
attention_mask=stack("stream_attention_mask"),
stream_input_ids=stack("stream_input_ids"),
stream_input_features=stack("stream_input_features"),
stream_feature_attention_mask=stack(
"stream_feature_attention_mask"
),
)
cache = condition.kv_cache
memory = condition.memory_token[:, 0].float().cpu().numpy()
probabilities = condition.control_probabilities.float().cpu().numpy()
for row, capture_step in enumerate(capture_steps):
if step == capture_step:
captured_features[row] = memory[row].copy()
captured_probabilities[row] = probabilities[row].copy()
print(
f"[probe] streamed window {step + 1}/{total_windows} "
f"cache_tokens={cache.sequence_length}",
flush=True,
)
if any(value is None for value in captured_features):
raise RuntimeError("At least one semantic feature was not captured")
return (
np.stack([value for value in captured_features if value is not None]),
np.stack(
[value for value in captured_probabilities if value is not None]
),
)
def _linear_probe(
features: np.ndarray,
labels: np.ndarray,
voices: np.ndarray,
variants: np.ndarray,
*,
permutation_repeats: int,
seed: int,
) -> dict[str, Any]:
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer, StandardScaler
def predict(train: np.ndarray, test: np.ndarray, target: np.ndarray) -> np.ndarray:
classifier = make_pipeline(
StandardScaler(with_std=False),
Normalizer(norm="l2"),
LogisticRegression(
C=0.1,
max_iter=4_000,
solver="lbfgs",
random_state=seed,
),
)
classifier.fit(features[train], target[train])
return classifier.predict(features[test])
predictions = np.full(labels.shape, -1, dtype=np.int64)
for voice in sorted(set(voices.tolist())):
test = voices == voice
train = ~test
predictions[test] = predict(train, test, labels)
accuracy = float(np.mean(predictions == labels))
phrase_predictions = np.full(labels.shape, -1, dtype=np.int64)
for variant in sorted(set(variants.tolist())):
test = variants == variant
train = ~test
phrase_predictions[test] = predict(train, test, labels)
phrase_accuracy = float(np.mean(phrase_predictions == labels))
rng = np.random.default_rng(seed)
null_accuracies = []
for _ in range(max(int(permutation_repeats), 0)):
shuffled = labels.copy()
for voice in sorted(set(voices.tolist())):
indices = np.flatnonzero(voices == voice)
shuffled[indices] = rng.permutation(shuffled[indices])
null_prediction = np.full(labels.shape, -1, dtype=np.int64)
for voice in sorted(set(voices.tolist())):
test = voices == voice
train = ~test
null_prediction[test] = predict(train, test, shuffled)
null_accuracies.append(float(np.mean(null_prediction == shuffled)))
return {
"leave_one_voice_out_accuracy": accuracy,
"leave_one_phrase_variant_out_accuracy": phrase_accuracy,
"confusion_matrix": confusion_matrix(
labels,
predictions,
labels=list(range(len(CLASS_NAMES))),
).tolist(),
"permuted_label_accuracy_mean": (
float(np.mean(null_accuracies)) if null_accuracies else None
),
"permuted_label_accuracy_p95": (
float(np.quantile(null_accuracies, 0.95))
if null_accuracies
else None
),
}
def _control_summary(
probabilities: np.ndarray,
metadata: list[dict[str, Any]],
) -> dict[str, Any]:
predictions = probabilities.argmax(axis=-1)
result = {}
for label in CLASS_NAMES:
indices = np.asarray(
[row["label"] == label for row in metadata],
dtype=np.bool_,
)
counts = {
name: int(np.sum(predictions[indices] == token_id))
for token_id, name in enumerate(CONTROL_NAMES)
}
result[label] = {
"expected_control": CONTROL_NAMES[EXPECTED_CONTROL_ID[label]],
"expected_control_accuracy": float(
np.mean(predictions[indices] == EXPECTED_CONTROL_ID[label])
),
"prediction_counts": counts,
"mean_probabilities": {
name: float(probabilities[indices, token_id].mean())
for token_id, name in enumerate(CONTROL_NAMES)
},
}
return result
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--ckpt-step", default="latest")
parser.add_argument("--device", default="cpu")
parser.add_argument("--cpu-threads", type=int, default=56)
parser.add_argument("--permutation-repeats", type=int, default=20)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--output-json", type=Path)
return parser.parse_args()
def main() -> None:
args = parse_args()
run_dir = args.run_dir.expanduser().resolve()
checkpoint = _checkpoint_name(run_dir, args.ckpt_step)
device = torch.device(args.device)
if device.type == "cpu":
torch.set_num_threads(max(int(args.cpu_threads), 1))
torch.manual_seed(int(args.seed))
model, config = _load_model(run_dir, checkpoint, device)
rows = _build_probe_rows(config)
timelines, capture_steps, metadata = _prepare_timelines(rows, config)
features, probabilities = _extract_features(
model,
timelines,
capture_steps,
)
label_ids = np.asarray(
[CLASS_NAMES.index(row["label"]) for row in metadata],
dtype=np.int64,
)
voices = np.asarray([row["voice"] for row in metadata])
variants = np.asarray([row["variant"] for row in metadata])
result = {
"format": "hfd_control_semantic_probe_v1",
"run_dir": str(run_dir),
"checkpoint": checkpoint,
"feature_contract": (
"condition.memory_token[:,0] == control_head input because "
"state_aware_serial_control=false"
),
"samples": int(features.shape[0]),
"feature_dim": int(features.shape[1]),
"classes": list(CLASS_NAMES),
"voices": list(VOICE_NAMES),
"chance_accuracy": 1.0 / len(CLASS_NAMES),
"linear_probe": _linear_probe(
features,
label_ids,
voices,
variants,
permutation_repeats=int(args.permutation_repeats),
seed=int(args.seed),
),
"trained_control_head": _control_summary(probabilities, metadata),
"sample_metadata": metadata,
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
print(rendered)
if args.output_json is not None:
output = args.output_json.expanduser().resolve()
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(rendered + "\n", encoding="utf-8")
if __name__ == "__main__":
main()