import numpy as np import ffmpeg from faster_whisper.audio import decode_audio from config import ( ASR_NO_SPEECH_CEILING, ASR_RELIABILITY_LOGPROB_FLOOR, ASR_UNRELIABLE_WEAK_FRACTION, ASR_WEAK_SEGMENT_LOGPROB, DEBUG, MAX_VIDEO_SECONDS, PRECISE_LOGPROB_GATE, RETRANSCRIBE_MAX, SAMPLE_RATE, WHISPER_COMPRESSION_RATIO_THRESHOLD, WHISPER_CONDITION_ON_PREV, WHISPER_LOGPROB_THRESHOLD, WHISPER_NO_SPEECH_THRESHOLD, ) from models import get_whisper_precise, whisper_fast from utils import fmt_time def load_audio(media_path: str) -> np.ndarray: """Decode any container to float32 mono 16 kHz waveform (PyAV, no temp file). A file with NO audio stream comes back as silence rather than an exception. PyAV raises IndexError deep inside faster_whisper when it asks for audio stream 0 of a container that has none, and that exception used to take the whole ingest down: every silent clip answered "Error reading video." in about a second, with the reason swallowed. Measured on a plain H.264 file with no audio track, 2026-09-21. Silence is the honest input here. The pipeline already handles "no speech" and falls back to visual-only synthesis, which is exactly what a screen recording without narration needs.""" try: wave = decode_audio(media_path, sampling_rate=SAMPLE_RATE) # Bounded: nothing past MAX_VIDEO_SECONDS is decoded into memory or transcribed. return wave[: MAX_VIDEO_SECONDS * SAMPLE_RATE] except (IndexError, StopIteration): print(f"[NINA] no audio stream in {str(media_path).split('/')[-1]}, " f"continuing from the picture alone", flush=True) return np.zeros(0, dtype=np.float32) def pack_audio(wave: np.ndarray) -> np.ndarray: """int16 for the cache: half the memory of float32, no audible loss for Whisper.""" if wave is None or getattr(wave, "size", 0) == 0: return np.zeros(0, dtype=np.int16) if wave.dtype == np.int16: return wave return np.clip(wave * 32767.0, -32768, 32767).astype(np.int16) def unpack_audio(wave: np.ndarray) -> np.ndarray: if wave is None or getattr(wave, "size", 0) == 0: return np.zeros(0, dtype=np.float32) if wave.dtype == np.int16: return wave.astype(np.float32) / 32767.0 return wave def media_duration(media_path: str) -> float: try: info = ffmpeg.probe(media_path) return float(info["format"]["duration"]) except Exception: return 0.0 def transcribe_fast(audio: np.ndarray) -> list[dict]: """Base-model full-video pass: greedy + Silero VAD, with hallucination guards. Stores avg_logprob, no_speech_prob and compression_ratio per segment so the pipeline can judge how trustworthy the transcript is (see transcript_reliability). """ segments_gen, _ = whisper_fast.transcribe( audio, beam_size=1, vad_filter=True, word_timestamps=False, condition_on_previous_text=WHISPER_CONDITION_ON_PREV, no_speech_threshold=WHISPER_NO_SPEECH_THRESHOLD, log_prob_threshold=WHISPER_LOGPROB_THRESHOLD, compression_ratio_threshold=WHISPER_COMPRESSION_RATIO_THRESHOLD, ) return [ { "text": seg.text.strip(), "start": round(seg.start, 2), "end": round(seg.end, 2), "logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3), "no_speech": round(float(getattr(seg, "no_speech_prob", 0.0) or 0.0), 3), "comp_ratio": round(float(getattr(seg, "compression_ratio", 0.0) or 0.0), 3), } for seg in segments_gen if seg.text.strip() ] def transcript_reliability(segments: list[dict]) -> dict: """Judge how trustworthy a base-model transcript is, so synthesis can hedge on garbled/musical audio instead of reporting Whisper's fabrications as fact. Returns {reliable: bool, score: 0..1, mean_logprob, weak_fraction, mean_no_speech}. """ if not segments: return {"reliable": False, "score": 0.0, "mean_logprob": 0.0, "weak_fraction": 1.0, "mean_no_speech": 1.0} lps = [s.get("logprob", 0.0) for s in segments] nsp = [s.get("no_speech", 0.0) for s in segments] mean_lp = sum(lps) / len(lps) mean_nsp = sum(nsp) / len(nsp) weak_frac = sum(1 for lp in lps if lp < ASR_WEAK_SEGMENT_LOGPROB) / len(lps) reliable = ( mean_lp >= ASR_RELIABILITY_LOGPROB_FLOOR and weak_frac < ASR_UNRELIABLE_WEAK_FRACTION and mean_nsp < ASR_NO_SPEECH_CEILING ) # Rough 0..1 confidence: avg_logprob of ~0 → 1.0, ~-1.0 → 0.0. score = max(0.0, min(1.0, 1.0 + mean_lp)) info = { "reliable": bool(reliable), "score": round(score, 3), "mean_logprob": round(mean_lp, 3), "weak_fraction": round(weak_frac, 3), "mean_no_speech": round(mean_nsp, 3), } if DEBUG: print(f"[DEBUG] ASR reliability: {info}") return info def retranscribe_region(audio: np.ndarray, region: dict) -> str: """Slice waveform in-memory and run precise Whisper on the slice. Always returns usable text: if the precise model is unavailable or errors out, we fall back to the region's base-model text rather than crashing the request. """ base = region.get("text", "") s = max(0, int(region["start"] * SAMPLE_RATE)) e = min(len(audio), int(region["end"] * SAMPLE_RATE)) clip = audio[s:e] if len(clip) < SAMPLE_RATE * 0.2: return base model = get_whisper_precise() if model is None: return base try: segments_gen, _ = model.transcribe(clip, beam_size=1) out = " ".join(seg.text.strip() for seg in segments_gen).strip() return out or base except Exception as exc: print(f"[NINA] Precise re-transcription failed for " f"[{fmt_time(region['start'])} → {fmt_time(region['end'])}]: {exc}") return base def fill_precise( audio: np.ndarray, regions: list[dict], all_chunks: list[dict], precise_cache: dict, budget: int, ) -> int: """ Ensure every region has `precise_text`. A sector is turbo-refined only when: • not already cached, AND • base avg_logprob is below PRECISE_LOGPROB_GATE (weak ASR), AND • still within budget. Returns remaining budget. """ reused = new = skipped = 0 for region in regions: comp = [ c for c in all_chunks if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3 ] if not comp: comp = [region] for ch in comp: key = round(ch["start"], 2) if key in precise_cache: reused += 1 continue weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE if not weak: skipped += 1 continue if budget <= 0: continue print( f"[NINA] Re-transcribing weak sector " f"[{fmt_time(ch['start'])} → {fmt_time(ch['end'])}] " f"(logprob {ch.get('logprob', 0.0):.2f})" ) precise_cache[key] = retranscribe_region(audio, ch) if DEBUG: snip = precise_cache[key] print( f"[DEBUG] Precise sector [{fmt_time(ch['start'])}→{fmt_time(ch['end'])}]: " f"{snip[:200]}{'…' if len(snip) > 200 else ''}" ) new += 1 budget -= 1 region["precise_text"] = " ".join( (precise_cache.get(round(ch["start"], 2)) or ch.get("text", "")) for ch in comp ).strip() or region.get("text", "") print(f"[NINA] Precise sectors — {new} refined, {reused} cached, {skipped} trusted (base).") return budget