code-HFD_lzk / scripts /third_party_control_diagnostics.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
15.9 kB
"""Pure CPU control diagnostics; no model, file writes, or process management.
``control_diagnostics(infer_meta, control_labels, frozen_recipe)`` consumes the
same saved evidence as the legacy scorer. It distinguishes an erroneous state
that persists from a new motion-state transition. Neither proves that speech
caused an error: causal attribution requires a paired rollout with reset caches.
All times use the existing 50 Hz output alignment, not dynamic time warping.
"""
from __future__ import annotations
from collections import Counter
import hashlib
import math
from typing import Any, Mapping, Sequence
import numpy as np
PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short",
"continuous", "same_voice_aside")
FPS, RATE, SAMPLES_PER_FRAME = 50, 16000, 320
def select_diagnostic_ids(selection: Mapping[str, Any], count: int = 24,
seed: int = 26100701) -> list[str]:
"""Select 4/class and 2/language/class, retaining the legacy 12 and order.
One high and one low background per class/language cell is preferred when
possible. This is a train-only diagnostic panel, not a held-out test set.
"""
if count != 24 or selection.get("split", "train") != "train":
raise ValueError("Diagnostic selection requires 24 train-only cases")
rows = list(selection["sessions"])
by_id = {str(row["session_id"]): row for row in rows}
ids = list(selection["session_ids"])
if len(by_id) != len(rows) or len(set(ids)) != len(ids) or set(ids) != set(by_id):
raise ValueError("Duplicate or inconsistent training session IDs")
for row in rows:
if row.get("split", "train") != "train":
raise ValueError("A held-out row is not eligible")
legacy = list(selection["probe_session_ids"])
if len(legacy) != 12 or len(set(legacy)) != 12 or not set(legacy) <= set(by_id):
raise ValueError("Expected twelve distinct legacy training probes")
chosen = set(legacy)
expected_cells = {(p, lang) for p in PROFILES for lang in ("zh", "en")}
cell = lambda row: (row["session_type"], row["background_language"])
if set(map(cell, rows)) != expected_cells:
raise ValueError("Expected the six named classes and zh/en backgrounds")
counts = Counter(cell(by_id[sid]) for sid in legacy)
if any(counts[c] > 2 for c in expected_cells):
raise ValueError("Legacy probes cannot fit the requested balanced panel")
def rank(sid: str) -> str:
return hashlib.sha256(f"{seed}:control-diagnostic:{sid}".encode()).hexdigest()
for key in sorted(expected_cells):
available = [sid for sid, row in by_id.items() if cell(row) == key]
while sum(sid in chosen for sid in available) < 2:
used_levels = {bool(by_id[sid].get("background_high", False))
for sid in available if sid in chosen}
candidates = [sid for sid in available if sid not in chosen]
if not candidates:
raise ValueError(f"Insufficient training cases for {key}")
candidates.sort(key=lambda sid: (
bool(by_id[sid].get("background_high", False)) in used_levels, rank(sid)))
chosen.add(candidates[0])
return legacy + sorted(chosen - set(legacy), key=rank)
def _ratio(numerator: int, denominator: int) -> float | None:
return numerator / denominator if denominator else None
def _intervals(events: Sequence[Mapping[str, Any]]) -> list[tuple[int, int]]:
result = []
for event in events:
a, b = int(event["start_sample"]), int(event["end_sample"])
if not 0 <= a < b:
raise ValueError("Invalid audio-event bounds")
result.append((a, b))
return result
def _overlaps(a: int, b: int, intervals: Sequence[tuple[int, int]]) -> bool:
return any(a < end and b > start for start, end in intervals)
def _first_dwell(mask: np.ndarray, frames: int) -> int | None:
"""First run start with at least ``frames`` consecutive true output frames."""
starts = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == 1)
stops = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == -1)
qualifying = starts[stops - starts >= frames]
return int(qualifying[0]) if len(qualifying) else None
def control_diagnostics(metadata: Mapping[str, Any], control_labels: Sequence[int],
recipe: Mapping[str, Any], *, prediction_frames: int | None = None,
expected_legacy: Mapping[str, Any] | None = None,
dwell_seconds: float = .4) -> dict[str, Any]:
"""Score frozen 50 Hz controls and explicit new-event/recovery denominators.
``expected_legacy`` may be the original sample's ``metrics`` dictionary;
mismatching legacy counts/rates raise rather than silently changing scores.
Recovery excludes windows containing a real command; a tolerance is scored
only if the complete tolerance plus required dwell is observable.
``None`` rates mean an empty denominator, never zero errors.
"""
gt_all = np.asarray(control_labels)
if (gt_all.ndim != 1 or not len(gt_all) or not np.isin(gt_all, (0, 1, 2)).all()):
raise ValueError("Expected nonempty 1D listen/motion/idle labels")
if not math.isfinite(dwell_seconds) or dwell_seconds <= 0:
raise ValueError("Dwell must be positive and finite")
dwell = int(math.ceil(dwell_seconds * FPS))
total = int(prediction_frames if prediction_frames is not None
else metadata.get("total_frames", len(gt_all)))
if total <= 0:
raise ValueError("Prediction must contain output frames")
events = list(metadata["control_events"])
starts = [int(e["output_frame"]) for e in events]
if not starts or starts[0] != 0 or any(a >= b for a, b in zip(starts, starts[1:])):
raise ValueError("Control events must cover frame zero in strict order")
if starts[-1] >= total:
raise ValueError("Control event lies outside prediction")
targets = _intervals(recipe.get("target_speech_events", []))
backgrounds = _intervals(recipe.get("interference_events", []))
predicted = np.empty(total, dtype=np.int64)
input_target = np.empty(total, dtype=bool)
for i, event in enumerate(events):
a, b = starts[i], starts[i + 1] if i + 1 < len(events) else total
value = int(event["predicted_id"])
ia, ib = int(event["input_start_sample"]), int(event["input_end_sample"])
if value not in (0, 1, 2) or not 0 <= ia < ib:
raise ValueError("Invalid predicted control or consumed audio bounds")
predicted[a:b] = value
input_target[a:b] = _overlaps(ia, ib, targets)
n = min(total, len(gt_all))
gt, pred = gt_all[:n].astype(np.int64), predicted[:n]
input_target = input_target[:n]
sample_starts = np.arange(n) * SAMPLES_PER_FRAME
def frame_mask(intervals: Sequence[tuple[int, int]]) -> np.ndarray:
mask = np.zeros(n, dtype=bool)
for a, b in intervals:
mask |= (sample_starts < b) & (sample_starts + SAMPLES_PER_FRAME > a)
return mask
target, background = frame_mask(targets), frame_mask(backgrounds)
legacy_mask = background & ~target & (gt != 1)
legacy_count = int(legacy_mask.sum())
legacy_errors = int(((pred == 1) & legacy_mask).sum())
legacy = {"control_scored_frames": n,
"background_only_nonmotion_gt_frames": legacy_count,
"background_only_false_motion_frames": legacy_errors,
"background_only_false_motion_rate": _ratio(legacy_errors, legacy_count)}
if expected_legacy is not None:
for key in ("control_scored_frames", "background_only_nonmotion_gt_frames",
"background_only_false_motion_rate"):
if key not in expected_legacy:
raise ValueError(f"Missing legacy validation field: {key}")
old, new = expected_legacy[key], legacy[key]
equal = old is None and new is None
if old is not None and new is not None:
equal = math.isclose(float(old), float(new), rel_tol=0, abs_tol=1e-12)
if not equal:
raise ValueError(f"Legacy metric mismatch: {key}: {old} != {new}")
confusion = np.zeros((3, 3), dtype=np.int64)
np.add.at(confusion, (gt, pred), 1)
new_motion = np.r_[False, (pred[1:] == 1) & (pred[:-1] != 1)]
new_mask = legacy_mask & ~input_target
event_rows = []
for index, (a, b) in enumerate(backgrounds):
event_mask = frame_mask([(a, b)])
first = int(np.flatnonzero(event_mask)[0]) if event_mask.any() else None
eligible = bool(first is not None and first > 0 and gt[first] != 1
and pred[first - 1] != 1 and not target[first]
and not input_target[first])
# Stop attribution at the next real command. Even a later non-overlap
# frame could reflect that intervening command rather than the aside.
future_commands = [start for start, end in targets if end > a]
cutoff = min(future_commands) if future_commands else b
scored = event_mask & new_mask & (sample_starts < cutoff)
eligible = eligible and bool(scored.any())
event_rows.append({"event_index": index, "eligible_new_start_event": eligible,
"already_motion_at_event_start": bool(first is not None and first > 0
and pred[first - 1] == 1),
"new_false_motion_transitions": int((scored & new_motion).sum()),
"has_new_false_motion": bool(eligible and (scored & new_motion).any())})
eligible_events = sum(row["eligible_new_start_event"] for row in event_rows)
triggered_events = sum(row["has_new_false_motion"] for row in event_rows)
recovery_rows = []
for boundary in (np.flatnonzero((gt[:-1] == 1) & (gt[1:] != 1)) + 1):
boundary = int(boundary)
# New command audio, or a new GT motion, ends the stop-recovery trial.
stop_mask = (gt[boundary:] == 1) | target[boundary:] | input_target[boundary:]
stops = np.flatnonzero(stop_mask)
end = boundary + int(stops[0]) if len(stops) else n
length = max(0, end - boundary)
first_nonmotion = np.flatnonzero(pred[boundary:end] != 1)
stable_nonmotion = _first_dwell(pred[boundary:end] != 1, dwell)
stable_idle = _first_dwell((pred[boundary:end] == 2) & (gt[boundary:end] == 2), dwell)
recovery_rows.append({"boundary_frame": boundary, "gt_after": int(gt[boundary]),
"observable_frames_without_new_command": length,
"prediction_motion_before_boundary": bool(pred[boundary - 1] == 1),
"first_nonmotion_delay_seconds": float(first_nonmotion[0] / FPS) if len(first_nonmotion) else None,
"stable_nonmotion_delay_seconds": stable_nonmotion / FPS if stable_nonmotion is not None else None,
"stable_idle_delay_seconds": stable_idle / FPS if stable_idle is not None else None})
recovery_rates = {}
for tolerance in (.4, .8, 1.2):
observed = [row for row in recovery_rows if row["observable_frames_without_new_command"]
>= int(round(tolerance * FPS)) + dwell]
for name in ("nonmotion", "idle"):
eligible_rows = observed if name == "nonmotion" else [r for r in observed if r["gt_after"] == 2]
good = sum(row[f"stable_{name}_delay_seconds"] is not None
and row[f"stable_{name}_delay_seconds"] <= tolerance for row in eligible_rows)
recovery_rates[f"{name}_within_{tolerance:.1f}s"] = {
"eligible_transitions": len(eligible_rows), "recovered_transitions": good,
"rate": _ratio(good, len(eligible_rows))}
terminal_start = None
if gt[-1] == 2:
nonidle = np.flatnonzero(gt != 2)
terminal_start = int(nonidle[-1] + 1) if len(nonidle) else 0
terminal = pred[terminal_start:] if terminal_start is not None else np.empty(0, dtype=np.int64)
not_idle = np.flatnonzero(terminal != 2)
trailing = len(terminal) - int(not_idle[-1] + 1) if len(not_idle) else len(terminal)
terminal_result = {"gt_terminal_idle_start_frame": terminal_start,
"gt_terminal_idle_frames": len(terminal), "predicted_idle_frames": int((terminal == 2).sum()),
"idle_fraction": _ratio(int((terminal == 2).sum()), len(terminal)),
"trailing_predicted_idle_seconds": trailing / FPS,
"trailing_idle_dwell_ge_0.4s": bool(len(terminal) >= 20 and trailing >= 20)}
pre_background = {"eligible_decisions": 0, "false_motion_decisions": 0,
"eligible_output_frames": []}
if backgrounds:
first_background = min(a for a, _ in backgrounds)
for event in events:
j = int(event["output_frame"])
if j >= n:
continue
if (gt[j] == 2 and int(event["input_end_sample"]) <= first_background
and not input_target[j] and np.any(gt[:j] == 1)):
pre_background["eligible_decisions"] += 1
pre_background["false_motion_decisions"] += int(pred[j] == 1)
pre_background["eligible_output_frames"].append(j)
pre_background["false_motion_rate"] = _ratio(pre_background["false_motion_decisions"],
pre_background["eligible_decisions"])
return {"schema": 1, "session_id": recipe.get("session_id"), "fps": FPS,
"legacy": legacy, "control_order": ["listen", "motion", "idle"],
"control_confusion_rows_gt_columns_prediction": confusion.tolist(),
"class_recall": {name: _ratio(int(confusion[i, i]), int(confusion[i].sum()))
for i, name in enumerate(("listen", "motion", "idle"))},
"background_new_motion": {"eligible_events": eligible_events,
"events_with_new_false_motion": triggered_events,
"event_rate": _ratio(triggered_events, eligible_events),
"event_denominator_definition": "Background events starting in GT nonmotion with a prior nonmotion prediction, no true-command audio in the current consumed window, and at least one eligible frame before the next command; overlapping events count separately.",
"new_false_motion_transitions": int((new_motion & new_mask).sum()),
"transition_definition": "Unique output-frame transitions from predicted nonmotion to motion during background-only GT nonmotion; exclude any true command in the consumed audio window.",
"eligible_nonmotion_frames_no_command_window": int(new_mask.sum()),
"events": event_rows},
"motion_completion_recovery": {"dwell_seconds": dwell / FPS,
"transition_count": len(recovery_rows), "tolerances": recovery_rates,
"denominator_definition": "GT motion-to-nonmotion boundaries with tolerance plus dwell observable before any new GT motion or real command in the consumed audio window; idle recovery additionally requires GT idle at the boundary.",
"transitions": recovery_rows},
"terminal_idle": terminal_result, "pre_first_background_post_motion_idle": pre_background,
"limitations": ["Train-only diagnostics; no held-out or physical-safety claim.",
"Background intervals include source silence; legacy mask is intentionally unchanged.",
"New state transitions are observational, not proof of causal background triggering.",
"No-background frames may retain earlier background in model caches; use paired reset-cache rollouts.",
"Recovery tolerances exclude censored trials and require continuous dwell; denominators differ.",
"Frame zero cannot establish a new transition because no previous prediction exists."]}