"""Read-only paired diagnostic: remove v5 third-party speech, not indoor noise. Production writes Q((Q(noise + target) + third_party) * global_scale), where Q is PCM16 round/clip/decode. This view returns Q(Q(noise + target) * the SAME global_scale). It neither subtracts a guessed component nor renormalizes the clean signal. It creates no files, global caches, model imports or GPU work. """ from __future__ import annotations import hashlib import io import math from pathlib import Path import wave import numpy as np from scipy.signal import resample_poly from scripts.data.hfd_third_party_common import ( SAMPLE_RATE, STRIDE_SAMPLES, audio_stats, canonical_json_sha256, quantize_pcm16, ) POLICY = "paired_remove_third_party_preserve_v5_scale_v1" def _verified_bytes(path: Path, expected: str) -> bytes: data = path.read_bytes() if hashlib.sha256(data).hexdigest() != expected: raise ValueError(f"Frozen source hash changed: {path}") return data def _payload(root: Path, relative: str) -> Path: path = (root / relative).resolve() if Path(relative).is_absolute() or not path.is_relative_to(root): raise ValueError("Frozen payload escapes production_root") return path def _decode_pcm16(data: bytes, *, expected_rate: int | None = None) -> tuple[np.ndarray, int]: with wave.open(io.BytesIO(data), "rb") as handle: rate, count = handle.getframerate(), handle.getnframes() if (handle.getnchannels(), handle.getsampwidth(), handle.getcomptype()) != (1, 2, "NONE"): raise ValueError("Expected uncompressed mono PCM16 WAV") if rate <= 0 or (expected_rate is not None and rate != expected_rate): raise ValueError("Unexpected WAV sample rate") raw = handle.readframes(count) pcm = np.frombuffer(raw, dtype=" np.ndarray: """Match original resample_poly/rounding; deliberately no source cache.""" pcm, rate = _decode_pcm16(data) speech = pcm.astype(np.float32) / np.float32(32768.) if rate != SAMPLE_RATE: divisor = math.gcd(rate, SAMPLE_RATE) speech = resample_poly(speech, SAMPLE_RATE // divisor, rate // divisor).astype(np.float32, copy=False) length = int(round(len(pcm) * SAMPLE_RATE / float(rate))) speech = speech[:length] if len(speech) >= length else np.pad(speech, (0, length - len(speech))) return np.ascontiguousarray(speech, dtype=np.float32) def load_clean_probe(production_root: Path, recipe: dict) -> tuple[np.ndarray, dict, dict]: """Return paired clean PCM-decoded audio, unchanged GT, and provenance audit. Requires a frozen v5 recipe and every original TTS/HVAC input. Missing or changed inputs are fatal; there is no fallback subtraction/denoising. All file hashes refer to the same bytes subsequently decoded. Declared third- party-free samples must exactly match the stored mixed WAV after PCM16. """ root = Path(production_root).resolve() mix = recipe["mix_metadata"] if recipe.get("profile_policy") != "multi_event_speech_v5" or mix.get("policy") != "multi_event_speech_v5": raise ValueError("Clean diagnostic requires frozen multi_event_speech_v5") if mix.get("global_scale_scope") != "entire_mixture_including_target_and_base_noise": raise ValueError("Unknown global-scale scope") scale = float(mix["global_scale"]) peak_before = float(mix["peak_before_global_scale"]) if (not math.isfinite(scale) or not 0 < scale <= 1 or not math.isfinite(peak_before) or peak_before < 0 or not math.isclose(scale, min(1., .95 / max(peak_before, 1e-12)), rel_tol=0, abs_tol=1e-12)): raise ValueError("Invalid original global peak-protection scale") info = recipe["audio"] if (info["sample_rate"], info["channels"], info["sample_width_bytes"]) != (SAMPLE_RATE, 1, 2): raise ValueError("Unexpected frozen audio metadata") original = _verified_bytes(_payload(root, info["path"]), info["sha256"]) mixed_pcm, _ = _decode_pcm16(original, expected_rate=SAMPLE_RATE) count = int(info["sample_count"]) if len(mixed_pcm) != count or hashlib.sha256(mixed_pcm.tobytes()).hexdigest() != info["pcm16_sha256"]: raise ValueError("Frozen mixed PCM geometry/hash mismatch") timeline_info = recipe["timeline"] timeline_bytes = _verified_bytes(_payload(root, timeline_info["path"]), timeline_info["file_sha256"]) with np.load(io.BytesIO(timeline_bytes), allow_pickle=False) as stored: timeline = {key: stored[key].copy() for key in ("states", "control_label")} states, labels = timeline["states"], timeline["control_label"] if (states.ndim != 2 or states.shape[1] != 36 or len(states) < 2 or not np.isfinite(states).all() or labels.shape != (len(states),) or not np.isin(labels, (0, 1, 2)).all()): raise ValueError("Invalid frozen GT states/control labels") target = np.zeros(count, dtype=np.float32) sources = [] for event in recipe["target_speech_events"]: path, expected = Path(event["original_audio_path"]), event["original_audio_sha256"] speech = _speech(_verified_bytes(path, expected)) start, end = int(event["start_sample"]), int(event["end_sample"]) gain = float(event["speech_scale"]) if (not 0 <= start < end <= count or len(speech) != int(event["sample_count"]) or end - start != len(speech) or not math.isfinite(gain) or gain <= 0): raise ValueError("Invalid target timing/gain/length") target[start:end] += speech * np.float32(gain) sources.append({"path": str(path), "sha256": expected, "start_sample": start, "end_sample": end, "speech_scale": gain}) if not sources: raise ValueError("No original target speech available") background = recipe["base_background"] noise_path = Path(background["path"]) noise_pcm, _ = _decode_pcm16(_verified_bytes(noise_path, background["sha256"]), expected_rate=SAMPLE_RATE) noise_start = int(background["start_window"]) * STRIDE_SAMPLES if noise_start < 0 or noise_start + count > len(noise_pcm): raise ValueError("Original indoor-noise interval is out of bounds") noise = noise_pcm[noise_start:noise_start + count].astype(np.float32) / np.float32(32768.) base_pcm = quantize_pcm16(noise + target) base = base_pcm.astype(np.float32) / np.float32(32768.) base_statistics = audio_stats(base) for key, expected in mix["base_audio_stats"].items(): if key not in base_statistics or not math.isclose(float(base_statistics[key]), float(expected), rel_tol=1e-10, abs_tol=1e-12): raise ValueError(f"Reconstructed base audio differs from frozen metadata: {key}") clean_pcm = quantize_pcm16(base * np.float32(scale)) interference = np.zeros(count, dtype=np.bool_) for event in recipe["interference_events"]: start, end = int(event["start_sample"]), int(event["end_sample"]) if not 0 <= start < end <= count: raise ValueError("Third-party event outside frozen session") interference[start:end] = True outside = ~interference if not np.array_equal(clean_pcm[outside], mixed_pcm[outside]): raise ValueError("Reconstructed clean PCM differs outside declared third-party events") waveform = clean_pcm.astype(np.float32) / np.float32(32768.) audit = { "policy": POLICY, "session_id": recipe["session_id"], "recipe_sha256": canonical_json_sha256(recipe), "original_mixed_wav_sha256": info["sha256"], "original_mixed_pcm16_sha256": info["pcm16_sha256"], "clean_pcm16_sha256": hashlib.sha256(clean_pcm.tobytes()).hexdigest(), "base_pcm16_sha256": hashlib.sha256(base_pcm.tobytes()).hexdigest(), "timeline_file_sha256": timeline_info["file_sha256"], "source_inputs": sources, "indoor_noise": {"path": str(noise_path), "sha256": background["sha256"], "start_sample": noise_start}, "global_scale": scale, "global_scale_recomputed": False, "target_timing_and_original_effective_gain_preserved": True, "indoor_noise_retained": True, "third_party_removed": True, "normalization": "none; original base and final PCM16 rounding/clipping retained", "outside_interference_samples": int(outside.sum()), "outside_interference_pcm_equal": True, "outside_interference_check_has_support": bool(outside.any()), "changed_pcm_samples": int(np.count_nonzero(clean_pcm != mixed_pcm)), "sample_count": count, "sample_rate": SAMPLE_RATE, "max_amplitude": float(np.max(np.abs(waveform))), "interpretation": "Paired diagnostic view only; original waveform/GT files unchanged.", } return waveform, timeline, audit