"""Pure CPU control diagnostics; no model, file writes, or process management. ``control_diagnostics(infer_meta, control_labels, frozen_recipe)`` consumes the same saved evidence as the legacy scorer. It distinguishes an erroneous state that persists from a new motion-state transition. Neither proves that speech caused an error: causal attribution requires a paired rollout with reset caches. All times use the existing 50 Hz output alignment, not dynamic time warping. """ from __future__ import annotations from collections import Counter import hashlib import math from typing import Any, Mapping, Sequence import numpy as np PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", "continuous", "same_voice_aside") FPS, RATE, SAMPLES_PER_FRAME = 50, 16000, 320 def select_diagnostic_ids(selection: Mapping[str, Any], count: int = 24, seed: int = 26100701) -> list[str]: """Select 4/class and 2/language/class, retaining the legacy 12 and order. One high and one low background per class/language cell is preferred when possible. This is a train-only diagnostic panel, not a held-out test set. """ if count != 24 or selection.get("split", "train") != "train": raise ValueError("Diagnostic selection requires 24 train-only cases") rows = list(selection["sessions"]) by_id = {str(row["session_id"]): row for row in rows} ids = list(selection["session_ids"]) if len(by_id) != len(rows) or len(set(ids)) != len(ids) or set(ids) != set(by_id): raise ValueError("Duplicate or inconsistent training session IDs") for row in rows: if row.get("split", "train") != "train": raise ValueError("A held-out row is not eligible") legacy = list(selection["probe_session_ids"]) if len(legacy) != 12 or len(set(legacy)) != 12 or not set(legacy) <= set(by_id): raise ValueError("Expected twelve distinct legacy training probes") chosen = set(legacy) expected_cells = {(p, lang) for p in PROFILES for lang in ("zh", "en")} cell = lambda row: (row["session_type"], row["background_language"]) if set(map(cell, rows)) != expected_cells: raise ValueError("Expected the six named classes and zh/en backgrounds") counts = Counter(cell(by_id[sid]) for sid in legacy) if any(counts[c] > 2 for c in expected_cells): raise ValueError("Legacy probes cannot fit the requested balanced panel") def rank(sid: str) -> str: return hashlib.sha256(f"{seed}:control-diagnostic:{sid}".encode()).hexdigest() for key in sorted(expected_cells): available = [sid for sid, row in by_id.items() if cell(row) == key] while sum(sid in chosen for sid in available) < 2: used_levels = {bool(by_id[sid].get("background_high", False)) for sid in available if sid in chosen} candidates = [sid for sid in available if sid not in chosen] if not candidates: raise ValueError(f"Insufficient training cases for {key}") candidates.sort(key=lambda sid: ( bool(by_id[sid].get("background_high", False)) in used_levels, rank(sid))) chosen.add(candidates[0]) return legacy + sorted(chosen - set(legacy), key=rank) def _ratio(numerator: int, denominator: int) -> float | None: return numerator / denominator if denominator else None def _intervals(events: Sequence[Mapping[str, Any]]) -> list[tuple[int, int]]: result = [] for event in events: a, b = int(event["start_sample"]), int(event["end_sample"]) if not 0 <= a < b: raise ValueError("Invalid audio-event bounds") result.append((a, b)) return result def _overlaps(a: int, b: int, intervals: Sequence[tuple[int, int]]) -> bool: return any(a < end and b > start for start, end in intervals) def _first_dwell(mask: np.ndarray, frames: int) -> int | None: """First run start with at least ``frames`` consecutive true output frames.""" starts = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == 1) stops = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == -1) qualifying = starts[stops - starts >= frames] return int(qualifying[0]) if len(qualifying) else None def control_diagnostics(metadata: Mapping[str, Any], control_labels: Sequence[int], recipe: Mapping[str, Any], *, prediction_frames: int | None = None, expected_legacy: Mapping[str, Any] | None = None, dwell_seconds: float = .4) -> dict[str, Any]: """Score frozen 50 Hz controls and explicit new-event/recovery denominators. ``expected_legacy`` may be the original sample's ``metrics`` dictionary; mismatching legacy counts/rates raise rather than silently changing scores. Recovery excludes windows containing a real command; a tolerance is scored only if the complete tolerance plus required dwell is observable. ``None`` rates mean an empty denominator, never zero errors. """ gt_all = np.asarray(control_labels) if (gt_all.ndim != 1 or not len(gt_all) or not np.isin(gt_all, (0, 1, 2)).all()): raise ValueError("Expected nonempty 1D listen/motion/idle labels") if not math.isfinite(dwell_seconds) or dwell_seconds <= 0: raise ValueError("Dwell must be positive and finite") dwell = int(math.ceil(dwell_seconds * FPS)) total = int(prediction_frames if prediction_frames is not None else metadata.get("total_frames", len(gt_all))) if total <= 0: raise ValueError("Prediction must contain output frames") events = list(metadata["control_events"]) starts = [int(e["output_frame"]) for e in events] if not starts or starts[0] != 0 or any(a >= b for a, b in zip(starts, starts[1:])): raise ValueError("Control events must cover frame zero in strict order") if starts[-1] >= total: raise ValueError("Control event lies outside prediction") targets = _intervals(recipe.get("target_speech_events", [])) backgrounds = _intervals(recipe.get("interference_events", [])) predicted = np.empty(total, dtype=np.int64) input_target = np.empty(total, dtype=bool) for i, event in enumerate(events): a, b = starts[i], starts[i + 1] if i + 1 < len(events) else total value = int(event["predicted_id"]) ia, ib = int(event["input_start_sample"]), int(event["input_end_sample"]) if value not in (0, 1, 2) or not 0 <= ia < ib: raise ValueError("Invalid predicted control or consumed audio bounds") predicted[a:b] = value input_target[a:b] = _overlaps(ia, ib, targets) n = min(total, len(gt_all)) gt, pred = gt_all[:n].astype(np.int64), predicted[:n] input_target = input_target[:n] sample_starts = np.arange(n) * SAMPLES_PER_FRAME def frame_mask(intervals: Sequence[tuple[int, int]]) -> np.ndarray: mask = np.zeros(n, dtype=bool) for a, b in intervals: mask |= (sample_starts < b) & (sample_starts + SAMPLES_PER_FRAME > a) return mask target, background = frame_mask(targets), frame_mask(backgrounds) legacy_mask = background & ~target & (gt != 1) legacy_count = int(legacy_mask.sum()) legacy_errors = int(((pred == 1) & legacy_mask).sum()) legacy = {"control_scored_frames": n, "background_only_nonmotion_gt_frames": legacy_count, "background_only_false_motion_frames": legacy_errors, "background_only_false_motion_rate": _ratio(legacy_errors, legacy_count)} if expected_legacy is not None: for key in ("control_scored_frames", "background_only_nonmotion_gt_frames", "background_only_false_motion_rate"): if key not in expected_legacy: raise ValueError(f"Missing legacy validation field: {key}") old, new = expected_legacy[key], legacy[key] equal = old is None and new is None if old is not None and new is not None: equal = math.isclose(float(old), float(new), rel_tol=0, abs_tol=1e-12) if not equal: raise ValueError(f"Legacy metric mismatch: {key}: {old} != {new}") confusion = np.zeros((3, 3), dtype=np.int64) np.add.at(confusion, (gt, pred), 1) new_motion = np.r_[False, (pred[1:] == 1) & (pred[:-1] != 1)] new_mask = legacy_mask & ~input_target event_rows = [] for index, (a, b) in enumerate(backgrounds): event_mask = frame_mask([(a, b)]) first = int(np.flatnonzero(event_mask)[0]) if event_mask.any() else None eligible = bool(first is not None and first > 0 and gt[first] != 1 and pred[first - 1] != 1 and not target[first] and not input_target[first]) # Stop attribution at the next real command. Even a later non-overlap # frame could reflect that intervening command rather than the aside. future_commands = [start for start, end in targets if end > a] cutoff = min(future_commands) if future_commands else b scored = event_mask & new_mask & (sample_starts < cutoff) eligible = eligible and bool(scored.any()) event_rows.append({"event_index": index, "eligible_new_start_event": eligible, "already_motion_at_event_start": bool(first is not None and first > 0 and pred[first - 1] == 1), "new_false_motion_transitions": int((scored & new_motion).sum()), "has_new_false_motion": bool(eligible and (scored & new_motion).any())}) eligible_events = sum(row["eligible_new_start_event"] for row in event_rows) triggered_events = sum(row["has_new_false_motion"] for row in event_rows) recovery_rows = [] for boundary in (np.flatnonzero((gt[:-1] == 1) & (gt[1:] != 1)) + 1): boundary = int(boundary) # New command audio, or a new GT motion, ends the stop-recovery trial. stop_mask = (gt[boundary:] == 1) | target[boundary:] | input_target[boundary:] stops = np.flatnonzero(stop_mask) end = boundary + int(stops[0]) if len(stops) else n length = max(0, end - boundary) first_nonmotion = np.flatnonzero(pred[boundary:end] != 1) stable_nonmotion = _first_dwell(pred[boundary:end] != 1, dwell) stable_idle = _first_dwell((pred[boundary:end] == 2) & (gt[boundary:end] == 2), dwell) recovery_rows.append({"boundary_frame": boundary, "gt_after": int(gt[boundary]), "observable_frames_without_new_command": length, "prediction_motion_before_boundary": bool(pred[boundary - 1] == 1), "first_nonmotion_delay_seconds": float(first_nonmotion[0] / FPS) if len(first_nonmotion) else None, "stable_nonmotion_delay_seconds": stable_nonmotion / FPS if stable_nonmotion is not None else None, "stable_idle_delay_seconds": stable_idle / FPS if stable_idle is not None else None}) recovery_rates = {} for tolerance in (.4, .8, 1.2): observed = [row for row in recovery_rows if row["observable_frames_without_new_command"] >= int(round(tolerance * FPS)) + dwell] for name in ("nonmotion", "idle"): eligible_rows = observed if name == "nonmotion" else [r for r in observed if r["gt_after"] == 2] good = sum(row[f"stable_{name}_delay_seconds"] is not None and row[f"stable_{name}_delay_seconds"] <= tolerance for row in eligible_rows) recovery_rates[f"{name}_within_{tolerance:.1f}s"] = { "eligible_transitions": len(eligible_rows), "recovered_transitions": good, "rate": _ratio(good, len(eligible_rows))} terminal_start = None if gt[-1] == 2: nonidle = np.flatnonzero(gt != 2) terminal_start = int(nonidle[-1] + 1) if len(nonidle) else 0 terminal = pred[terminal_start:] if terminal_start is not None else np.empty(0, dtype=np.int64) not_idle = np.flatnonzero(terminal != 2) trailing = len(terminal) - int(not_idle[-1] + 1) if len(not_idle) else len(terminal) terminal_result = {"gt_terminal_idle_start_frame": terminal_start, "gt_terminal_idle_frames": len(terminal), "predicted_idle_frames": int((terminal == 2).sum()), "idle_fraction": _ratio(int((terminal == 2).sum()), len(terminal)), "trailing_predicted_idle_seconds": trailing / FPS, "trailing_idle_dwell_ge_0.4s": bool(len(terminal) >= 20 and trailing >= 20)} pre_background = {"eligible_decisions": 0, "false_motion_decisions": 0, "eligible_output_frames": []} if backgrounds: first_background = min(a for a, _ in backgrounds) for event in events: j = int(event["output_frame"]) if j >= n: continue if (gt[j] == 2 and int(event["input_end_sample"]) <= first_background and not input_target[j] and np.any(gt[:j] == 1)): pre_background["eligible_decisions"] += 1 pre_background["false_motion_decisions"] += int(pred[j] == 1) pre_background["eligible_output_frames"].append(j) pre_background["false_motion_rate"] = _ratio(pre_background["false_motion_decisions"], pre_background["eligible_decisions"]) return {"schema": 1, "session_id": recipe.get("session_id"), "fps": FPS, "legacy": legacy, "control_order": ["listen", "motion", "idle"], "control_confusion_rows_gt_columns_prediction": confusion.tolist(), "class_recall": {name: _ratio(int(confusion[i, i]), int(confusion[i].sum())) for i, name in enumerate(("listen", "motion", "idle"))}, "background_new_motion": {"eligible_events": eligible_events, "events_with_new_false_motion": triggered_events, "event_rate": _ratio(triggered_events, eligible_events), "event_denominator_definition": "Background events starting in GT nonmotion with a prior nonmotion prediction, no true-command audio in the current consumed window, and at least one eligible frame before the next command; overlapping events count separately.", "new_false_motion_transitions": int((new_motion & new_mask).sum()), "transition_definition": "Unique output-frame transitions from predicted nonmotion to motion during background-only GT nonmotion; exclude any true command in the consumed audio window.", "eligible_nonmotion_frames_no_command_window": int(new_mask.sum()), "events": event_rows}, "motion_completion_recovery": {"dwell_seconds": dwell / FPS, "transition_count": len(recovery_rows), "tolerances": recovery_rates, "denominator_definition": "GT motion-to-nonmotion boundaries with tolerance plus dwell observable before any new GT motion or real command in the consumed audio window; idle recovery additionally requires GT idle at the boundary.", "transitions": recovery_rows}, "terminal_idle": terminal_result, "pre_first_background_post_motion_idle": pre_background, "limitations": ["Train-only diagnostics; no held-out or physical-safety claim.", "Background intervals include source silence; legacy mask is intentionally unchanged.", "New state transitions are observational, not proof of causal background triggering.", "No-background frames may retain earlier background in model caches; use paired reset-cache rollouts.", "Recovery tolerances exclude censored trials and require continuous dwell; denominators differ.", "Frame zero cannot establish a new transition because no previous prediction exists."]}