Download scripts/third_party_control_diagnostics.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 15.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_control_diagnostics.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/third_party_control_diagnostics.py
-
curl -L -o third_party_control_diagnostics.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/third_party_control_diagnostics.py
15.9 kB
| """Pure CPU control diagnostics; no model, file writes, or process management. | |
| ``control_diagnostics(infer_meta, control_labels, frozen_recipe)`` consumes the | |
| same saved evidence as the legacy scorer. It distinguishes an erroneous state | |
| that persists from a new motion-state transition. Neither proves that speech | |
| caused an error: causal attribution requires a paired rollout with reset caches. | |
| All times use the existing 50 Hz output alignment, not dynamic time warping. | |
| """ | |
| from __future__ import annotations | |
| from collections import Counter | |
| import hashlib | |
| import math | |
| from typing import Any, Mapping, Sequence | |
| import numpy as np | |
| PROFILES = ("single_short", "multi_single", "multi_babble", "mixed_short", | |
| "continuous", "same_voice_aside") | |
| FPS, RATE, SAMPLES_PER_FRAME = 50, 16000, 320 | |
| def select_diagnostic_ids(selection: Mapping[str, Any], count: int = 24, | |
| seed: int = 26100701) -> list[str]: | |
| """Select 4/class and 2/language/class, retaining the legacy 12 and order. | |
| One high and one low background per class/language cell is preferred when | |
| possible. This is a train-only diagnostic panel, not a held-out test set. | |
| """ | |
| if count != 24 or selection.get("split", "train") != "train": | |
| raise ValueError("Diagnostic selection requires 24 train-only cases") | |
| rows = list(selection["sessions"]) | |
| by_id = {str(row["session_id"]): row for row in rows} | |
| ids = list(selection["session_ids"]) | |
| if len(by_id) != len(rows) or len(set(ids)) != len(ids) or set(ids) != set(by_id): | |
| raise ValueError("Duplicate or inconsistent training session IDs") | |
| for row in rows: | |
| if row.get("split", "train") != "train": | |
| raise ValueError("A held-out row is not eligible") | |
| legacy = list(selection["probe_session_ids"]) | |
| if len(legacy) != 12 or len(set(legacy)) != 12 or not set(legacy) <= set(by_id): | |
| raise ValueError("Expected twelve distinct legacy training probes") | |
| chosen = set(legacy) | |
| expected_cells = {(p, lang) for p in PROFILES for lang in ("zh", "en")} | |
| cell = lambda row: (row["session_type"], row["background_language"]) | |
| if set(map(cell, rows)) != expected_cells: | |
| raise ValueError("Expected the six named classes and zh/en backgrounds") | |
| counts = Counter(cell(by_id[sid]) for sid in legacy) | |
| if any(counts[c] > 2 for c in expected_cells): | |
| raise ValueError("Legacy probes cannot fit the requested balanced panel") | |
| def rank(sid: str) -> str: | |
| return hashlib.sha256(f"{seed}:control-diagnostic:{sid}".encode()).hexdigest() | |
| for key in sorted(expected_cells): | |
| available = [sid for sid, row in by_id.items() if cell(row) == key] | |
| while sum(sid in chosen for sid in available) < 2: | |
| used_levels = {bool(by_id[sid].get("background_high", False)) | |
| for sid in available if sid in chosen} | |
| candidates = [sid for sid in available if sid not in chosen] | |
| if not candidates: | |
| raise ValueError(f"Insufficient training cases for {key}") | |
| candidates.sort(key=lambda sid: ( | |
| bool(by_id[sid].get("background_high", False)) in used_levels, rank(sid))) | |
| chosen.add(candidates[0]) | |
| return legacy + sorted(chosen - set(legacy), key=rank) | |
| def _ratio(numerator: int, denominator: int) -> float | None: | |
| return numerator / denominator if denominator else None | |
| def _intervals(events: Sequence[Mapping[str, Any]]) -> list[tuple[int, int]]: | |
| result = [] | |
| for event in events: | |
| a, b = int(event["start_sample"]), int(event["end_sample"]) | |
| if not 0 <= a < b: | |
| raise ValueError("Invalid audio-event bounds") | |
| result.append((a, b)) | |
| return result | |
| def _overlaps(a: int, b: int, intervals: Sequence[tuple[int, int]]) -> bool: | |
| return any(a < end and b > start for start, end in intervals) | |
| def _first_dwell(mask: np.ndarray, frames: int) -> int | None: | |
| """First run start with at least ``frames`` consecutive true output frames.""" | |
| starts = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == 1) | |
| stops = np.flatnonzero(np.diff(np.r_[False, mask, False].astype(np.int8)) == -1) | |
| qualifying = starts[stops - starts >= frames] | |
| return int(qualifying[0]) if len(qualifying) else None | |
| def control_diagnostics(metadata: Mapping[str, Any], control_labels: Sequence[int], | |
| recipe: Mapping[str, Any], *, prediction_frames: int | None = None, | |
| expected_legacy: Mapping[str, Any] | None = None, | |
| dwell_seconds: float = .4) -> dict[str, Any]: | |
| """Score frozen 50 Hz controls and explicit new-event/recovery denominators. | |
| ``expected_legacy`` may be the original sample's ``metrics`` dictionary; | |
| mismatching legacy counts/rates raise rather than silently changing scores. | |
| Recovery excludes windows containing a real command; a tolerance is scored | |
| only if the complete tolerance plus required dwell is observable. | |
| ``None`` rates mean an empty denominator, never zero errors. | |
| """ | |
| gt_all = np.asarray(control_labels) | |
| if (gt_all.ndim != 1 or not len(gt_all) or not np.isin(gt_all, (0, 1, 2)).all()): | |
| raise ValueError("Expected nonempty 1D listen/motion/idle labels") | |
| if not math.isfinite(dwell_seconds) or dwell_seconds <= 0: | |
| raise ValueError("Dwell must be positive and finite") | |
| dwell = int(math.ceil(dwell_seconds * FPS)) | |
| total = int(prediction_frames if prediction_frames is not None | |
| else metadata.get("total_frames", len(gt_all))) | |
| if total <= 0: | |
| raise ValueError("Prediction must contain output frames") | |
| events = list(metadata["control_events"]) | |
| starts = [int(e["output_frame"]) for e in events] | |
| if not starts or starts[0] != 0 or any(a >= b for a, b in zip(starts, starts[1:])): | |
| raise ValueError("Control events must cover frame zero in strict order") | |
| if starts[-1] >= total: | |
| raise ValueError("Control event lies outside prediction") | |
| targets = _intervals(recipe.get("target_speech_events", [])) | |
| backgrounds = _intervals(recipe.get("interference_events", [])) | |
| predicted = np.empty(total, dtype=np.int64) | |
| input_target = np.empty(total, dtype=bool) | |
| for i, event in enumerate(events): | |
| a, b = starts[i], starts[i + 1] if i + 1 < len(events) else total | |
| value = int(event["predicted_id"]) | |
| ia, ib = int(event["input_start_sample"]), int(event["input_end_sample"]) | |
| if value not in (0, 1, 2) or not 0 <= ia < ib: | |
| raise ValueError("Invalid predicted control or consumed audio bounds") | |
| predicted[a:b] = value | |
| input_target[a:b] = _overlaps(ia, ib, targets) | |
| n = min(total, len(gt_all)) | |
| gt, pred = gt_all[:n].astype(np.int64), predicted[:n] | |
| input_target = input_target[:n] | |
| sample_starts = np.arange(n) * SAMPLES_PER_FRAME | |
| def frame_mask(intervals: Sequence[tuple[int, int]]) -> np.ndarray: | |
| mask = np.zeros(n, dtype=bool) | |
| for a, b in intervals: | |
| mask |= (sample_starts < b) & (sample_starts + SAMPLES_PER_FRAME > a) | |
| return mask | |
| target, background = frame_mask(targets), frame_mask(backgrounds) | |
| legacy_mask = background & ~target & (gt != 1) | |
| legacy_count = int(legacy_mask.sum()) | |
| legacy_errors = int(((pred == 1) & legacy_mask).sum()) | |
| legacy = {"control_scored_frames": n, | |
| "background_only_nonmotion_gt_frames": legacy_count, | |
| "background_only_false_motion_frames": legacy_errors, | |
| "background_only_false_motion_rate": _ratio(legacy_errors, legacy_count)} | |
| if expected_legacy is not None: | |
| for key in ("control_scored_frames", "background_only_nonmotion_gt_frames", | |
| "background_only_false_motion_rate"): | |
| if key not in expected_legacy: | |
| raise ValueError(f"Missing legacy validation field: {key}") | |
| old, new = expected_legacy[key], legacy[key] | |
| equal = old is None and new is None | |
| if old is not None and new is not None: | |
| equal = math.isclose(float(old), float(new), rel_tol=0, abs_tol=1e-12) | |
| if not equal: | |
| raise ValueError(f"Legacy metric mismatch: {key}: {old} != {new}") | |
| confusion = np.zeros((3, 3), dtype=np.int64) | |
| np.add.at(confusion, (gt, pred), 1) | |
| new_motion = np.r_[False, (pred[1:] == 1) & (pred[:-1] != 1)] | |
| new_mask = legacy_mask & ~input_target | |
| event_rows = [] | |
| for index, (a, b) in enumerate(backgrounds): | |
| event_mask = frame_mask([(a, b)]) | |
| first = int(np.flatnonzero(event_mask)[0]) if event_mask.any() else None | |
| eligible = bool(first is not None and first > 0 and gt[first] != 1 | |
| and pred[first - 1] != 1 and not target[first] | |
| and not input_target[first]) | |
| # Stop attribution at the next real command. Even a later non-overlap | |
| # frame could reflect that intervening command rather than the aside. | |
| future_commands = [start for start, end in targets if end > a] | |
| cutoff = min(future_commands) if future_commands else b | |
| scored = event_mask & new_mask & (sample_starts < cutoff) | |
| eligible = eligible and bool(scored.any()) | |
| event_rows.append({"event_index": index, "eligible_new_start_event": eligible, | |
| "already_motion_at_event_start": bool(first is not None and first > 0 | |
| and pred[first - 1] == 1), | |
| "new_false_motion_transitions": int((scored & new_motion).sum()), | |
| "has_new_false_motion": bool(eligible and (scored & new_motion).any())}) | |
| eligible_events = sum(row["eligible_new_start_event"] for row in event_rows) | |
| triggered_events = sum(row["has_new_false_motion"] for row in event_rows) | |
| recovery_rows = [] | |
| for boundary in (np.flatnonzero((gt[:-1] == 1) & (gt[1:] != 1)) + 1): | |
| boundary = int(boundary) | |
| # New command audio, or a new GT motion, ends the stop-recovery trial. | |
| stop_mask = (gt[boundary:] == 1) | target[boundary:] | input_target[boundary:] | |
| stops = np.flatnonzero(stop_mask) | |
| end = boundary + int(stops[0]) if len(stops) else n | |
| length = max(0, end - boundary) | |
| first_nonmotion = np.flatnonzero(pred[boundary:end] != 1) | |
| stable_nonmotion = _first_dwell(pred[boundary:end] != 1, dwell) | |
| stable_idle = _first_dwell((pred[boundary:end] == 2) & (gt[boundary:end] == 2), dwell) | |
| recovery_rows.append({"boundary_frame": boundary, "gt_after": int(gt[boundary]), | |
| "observable_frames_without_new_command": length, | |
| "prediction_motion_before_boundary": bool(pred[boundary - 1] == 1), | |
| "first_nonmotion_delay_seconds": float(first_nonmotion[0] / FPS) if len(first_nonmotion) else None, | |
| "stable_nonmotion_delay_seconds": stable_nonmotion / FPS if stable_nonmotion is not None else None, | |
| "stable_idle_delay_seconds": stable_idle / FPS if stable_idle is not None else None}) | |
| recovery_rates = {} | |
| for tolerance in (.4, .8, 1.2): | |
| observed = [row for row in recovery_rows if row["observable_frames_without_new_command"] | |
| >= int(round(tolerance * FPS)) + dwell] | |
| for name in ("nonmotion", "idle"): | |
| eligible_rows = observed if name == "nonmotion" else [r for r in observed if r["gt_after"] == 2] | |
| good = sum(row[f"stable_{name}_delay_seconds"] is not None | |
| and row[f"stable_{name}_delay_seconds"] <= tolerance for row in eligible_rows) | |
| recovery_rates[f"{name}_within_{tolerance:.1f}s"] = { | |
| "eligible_transitions": len(eligible_rows), "recovered_transitions": good, | |
| "rate": _ratio(good, len(eligible_rows))} | |
| terminal_start = None | |
| if gt[-1] == 2: | |
| nonidle = np.flatnonzero(gt != 2) | |
| terminal_start = int(nonidle[-1] + 1) if len(nonidle) else 0 | |
| terminal = pred[terminal_start:] if terminal_start is not None else np.empty(0, dtype=np.int64) | |
| not_idle = np.flatnonzero(terminal != 2) | |
| trailing = len(terminal) - int(not_idle[-1] + 1) if len(not_idle) else len(terminal) | |
| terminal_result = {"gt_terminal_idle_start_frame": terminal_start, | |
| "gt_terminal_idle_frames": len(terminal), "predicted_idle_frames": int((terminal == 2).sum()), | |
| "idle_fraction": _ratio(int((terminal == 2).sum()), len(terminal)), | |
| "trailing_predicted_idle_seconds": trailing / FPS, | |
| "trailing_idle_dwell_ge_0.4s": bool(len(terminal) >= 20 and trailing >= 20)} | |
| pre_background = {"eligible_decisions": 0, "false_motion_decisions": 0, | |
| "eligible_output_frames": []} | |
| if backgrounds: | |
| first_background = min(a for a, _ in backgrounds) | |
| for event in events: | |
| j = int(event["output_frame"]) | |
| if j >= n: | |
| continue | |
| if (gt[j] == 2 and int(event["input_end_sample"]) <= first_background | |
| and not input_target[j] and np.any(gt[:j] == 1)): | |
| pre_background["eligible_decisions"] += 1 | |
| pre_background["false_motion_decisions"] += int(pred[j] == 1) | |
| pre_background["eligible_output_frames"].append(j) | |
| pre_background["false_motion_rate"] = _ratio(pre_background["false_motion_decisions"], | |
| pre_background["eligible_decisions"]) | |
| return {"schema": 1, "session_id": recipe.get("session_id"), "fps": FPS, | |
| "legacy": legacy, "control_order": ["listen", "motion", "idle"], | |
| "control_confusion_rows_gt_columns_prediction": confusion.tolist(), | |
| "class_recall": {name: _ratio(int(confusion[i, i]), int(confusion[i].sum())) | |
| for i, name in enumerate(("listen", "motion", "idle"))}, | |
| "background_new_motion": {"eligible_events": eligible_events, | |
| "events_with_new_false_motion": triggered_events, | |
| "event_rate": _ratio(triggered_events, eligible_events), | |
| "event_denominator_definition": "Background events starting in GT nonmotion with a prior nonmotion prediction, no true-command audio in the current consumed window, and at least one eligible frame before the next command; overlapping events count separately.", | |
| "new_false_motion_transitions": int((new_motion & new_mask).sum()), | |
| "transition_definition": "Unique output-frame transitions from predicted nonmotion to motion during background-only GT nonmotion; exclude any true command in the consumed audio window.", | |
| "eligible_nonmotion_frames_no_command_window": int(new_mask.sum()), | |
| "events": event_rows}, | |
| "motion_completion_recovery": {"dwell_seconds": dwell / FPS, | |
| "transition_count": len(recovery_rows), "tolerances": recovery_rates, | |
| "denominator_definition": "GT motion-to-nonmotion boundaries with tolerance plus dwell observable before any new GT motion or real command in the consumed audio window; idle recovery additionally requires GT idle at the boundary.", | |
| "transitions": recovery_rows}, | |
| "terminal_idle": terminal_result, "pre_first_background_post_motion_idle": pre_background, | |
| "limitations": ["Train-only diagnostics; no held-out or physical-safety claim.", | |
| "Background intervals include source silence; legacy mask is intentionally unchanged.", | |
| "New state transitions are observational, not proof of causal background triggering.", | |
| "No-background frames may retain earlier background in model caches; use paired reset-cache rollouts.", | |
| "Recovery tolerances exclude censored trials and require continuous dwell; denominators differ.", | |
| "Frame zero cannot establish a new transition because no previous prediction exists."]} | |