Spaces:
Paused
Paused
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7 Download audio.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 7.94 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/audio.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/audio.py
-
curl -L -o audio.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/audio.py
7.94 kB
| import numpy as np | |
| import ffmpeg | |
| from faster_whisper.audio import decode_audio | |
| from config import ( | |
| ASR_NO_SPEECH_CEILING, ASR_RELIABILITY_LOGPROB_FLOOR, | |
| ASR_UNRELIABLE_WEAK_FRACTION, ASR_WEAK_SEGMENT_LOGPROB, | |
| DEBUG, MAX_VIDEO_SECONDS, PRECISE_LOGPROB_GATE, RETRANSCRIBE_MAX, SAMPLE_RATE, | |
| WHISPER_COMPRESSION_RATIO_THRESHOLD, WHISPER_CONDITION_ON_PREV, | |
| WHISPER_LOGPROB_THRESHOLD, WHISPER_NO_SPEECH_THRESHOLD, | |
| ) | |
| from models import get_whisper_precise, whisper_fast | |
| from utils import fmt_time | |
| def load_audio(media_path: str) -> np.ndarray: | |
| """Decode any container to float32 mono 16 kHz waveform (PyAV, no temp file). | |
| A file with NO audio stream comes back as silence rather than an exception. PyAV | |
| raises IndexError deep inside faster_whisper when it asks for audio stream 0 of a | |
| container that has none, and that exception used to take the whole ingest down: | |
| every silent clip answered "Error reading video." in about a second, with the | |
| reason swallowed. Measured on a plain H.264 file with no audio track, 2026-09-21. | |
| Silence is the honest input here. The pipeline already handles "no speech" and | |
| falls back to visual-only synthesis, which is exactly what a screen recording | |
| without narration needs.""" | |
| try: | |
| wave = decode_audio(media_path, sampling_rate=SAMPLE_RATE) | |
| # Bounded: nothing past MAX_VIDEO_SECONDS is decoded into memory or transcribed. | |
| return wave[: MAX_VIDEO_SECONDS * SAMPLE_RATE] | |
| except (IndexError, StopIteration): | |
| print(f"[NINA] no audio stream in {str(media_path).split('/')[-1]}, " | |
| f"continuing from the picture alone", flush=True) | |
| return np.zeros(0, dtype=np.float32) | |
| def pack_audio(wave: np.ndarray) -> np.ndarray: | |
| """int16 for the cache: half the memory of float32, no audible loss for Whisper.""" | |
| if wave is None or getattr(wave, "size", 0) == 0: | |
| return np.zeros(0, dtype=np.int16) | |
| if wave.dtype == np.int16: | |
| return wave | |
| return np.clip(wave * 32767.0, -32768, 32767).astype(np.int16) | |
| def unpack_audio(wave: np.ndarray) -> np.ndarray: | |
| if wave is None or getattr(wave, "size", 0) == 0: | |
| return np.zeros(0, dtype=np.float32) | |
| if wave.dtype == np.int16: | |
| return wave.astype(np.float32) / 32767.0 | |
| return wave | |
| def media_duration(media_path: str) -> float: | |
| try: | |
| info = ffmpeg.probe(media_path) | |
| return float(info["format"]["duration"]) | |
| except Exception: | |
| return 0.0 | |
| def transcribe_fast(audio: np.ndarray) -> list[dict]: | |
| """Base-model full-video pass: greedy + Silero VAD, with hallucination guards. | |
| Stores avg_logprob, no_speech_prob and compression_ratio per segment so the | |
| pipeline can judge how trustworthy the transcript is (see transcript_reliability). | |
| """ | |
| segments_gen, _ = whisper_fast.transcribe( | |
| audio, beam_size=1, vad_filter=True, word_timestamps=False, | |
| condition_on_previous_text=WHISPER_CONDITION_ON_PREV, | |
| no_speech_threshold=WHISPER_NO_SPEECH_THRESHOLD, | |
| log_prob_threshold=WHISPER_LOGPROB_THRESHOLD, | |
| compression_ratio_threshold=WHISPER_COMPRESSION_RATIO_THRESHOLD, | |
| ) | |
| return [ | |
| { | |
| "text": seg.text.strip(), | |
| "start": round(seg.start, 2), | |
| "end": round(seg.end, 2), | |
| "logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3), | |
| "no_speech": round(float(getattr(seg, "no_speech_prob", 0.0) or 0.0), 3), | |
| "comp_ratio": round(float(getattr(seg, "compression_ratio", 0.0) or 0.0), 3), | |
| } | |
| for seg in segments_gen | |
| if seg.text.strip() | |
| ] | |
| def transcript_reliability(segments: list[dict]) -> dict: | |
| """Judge how trustworthy a base-model transcript is, so synthesis can hedge on | |
| garbled/musical audio instead of reporting Whisper's fabrications as fact. | |
| Returns {reliable: bool, score: 0..1, mean_logprob, weak_fraction, mean_no_speech}. | |
| """ | |
| if not segments: | |
| return {"reliable": False, "score": 0.0, "mean_logprob": 0.0, | |
| "weak_fraction": 1.0, "mean_no_speech": 1.0} | |
| lps = [s.get("logprob", 0.0) for s in segments] | |
| nsp = [s.get("no_speech", 0.0) for s in segments] | |
| mean_lp = sum(lps) / len(lps) | |
| mean_nsp = sum(nsp) / len(nsp) | |
| weak_frac = sum(1 for lp in lps if lp < ASR_WEAK_SEGMENT_LOGPROB) / len(lps) | |
| reliable = ( | |
| mean_lp >= ASR_RELIABILITY_LOGPROB_FLOOR | |
| and weak_frac < ASR_UNRELIABLE_WEAK_FRACTION | |
| and mean_nsp < ASR_NO_SPEECH_CEILING | |
| ) | |
| # Rough 0..1 confidence: avg_logprob of ~0 β 1.0, ~-1.0 β 0.0. | |
| score = max(0.0, min(1.0, 1.0 + mean_lp)) | |
| info = { | |
| "reliable": bool(reliable), | |
| "score": round(score, 3), | |
| "mean_logprob": round(mean_lp, 3), | |
| "weak_fraction": round(weak_frac, 3), | |
| "mean_no_speech": round(mean_nsp, 3), | |
| } | |
| if DEBUG: | |
| print(f"[DEBUG] ASR reliability: {info}") | |
| return info | |
| def retranscribe_region(audio: np.ndarray, region: dict) -> str: | |
| """Slice waveform in-memory and run precise Whisper on the slice. | |
| Always returns usable text: if the precise model is unavailable or errors out, | |
| we fall back to the region's base-model text rather than crashing the request. | |
| """ | |
| base = region.get("text", "") | |
| s = max(0, int(region["start"] * SAMPLE_RATE)) | |
| e = min(len(audio), int(region["end"] * SAMPLE_RATE)) | |
| clip = audio[s:e] | |
| if len(clip) < SAMPLE_RATE * 0.2: | |
| return base | |
| model = get_whisper_precise() | |
| if model is None: | |
| return base | |
| try: | |
| segments_gen, _ = model.transcribe(clip, beam_size=1) | |
| out = " ".join(seg.text.strip() for seg in segments_gen).strip() | |
| return out or base | |
| except Exception as exc: | |
| print(f"[NINA] Precise re-transcription failed for " | |
| f"[{fmt_time(region['start'])} β {fmt_time(region['end'])}]: {exc}") | |
| return base | |
| def fill_precise( | |
| audio: np.ndarray, | |
| regions: list[dict], | |
| all_chunks: list[dict], | |
| precise_cache: dict, | |
| budget: int, | |
| ) -> int: | |
| """ | |
| Ensure every region has `precise_text`. A sector is turbo-refined only when: | |
| β’ not already cached, AND | |
| β’ base avg_logprob is below PRECISE_LOGPROB_GATE (weak ASR), AND | |
| β’ still within budget. | |
| Returns remaining budget. | |
| """ | |
| reused = new = skipped = 0 | |
| for region in regions: | |
| comp = [ | |
| c for c in all_chunks | |
| if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3 | |
| ] | |
| if not comp: | |
| comp = [region] | |
| for ch in comp: | |
| key = round(ch["start"], 2) | |
| if key in precise_cache: | |
| reused += 1 | |
| continue | |
| weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE | |
| if not weak: | |
| skipped += 1 | |
| continue | |
| if budget <= 0: | |
| continue | |
| print( | |
| f"[NINA] Re-transcribing weak sector " | |
| f"[{fmt_time(ch['start'])} β {fmt_time(ch['end'])}] " | |
| f"(logprob {ch.get('logprob', 0.0):.2f})" | |
| ) | |
| precise_cache[key] = retranscribe_region(audio, ch) | |
| if DEBUG: | |
| snip = precise_cache[key] | |
| print( | |
| f"[DEBUG] Precise sector [{fmt_time(ch['start'])}β{fmt_time(ch['end'])}]: " | |
| f"{snip[:200]}{'β¦' if len(snip) > 200 else ''}" | |
| ) | |
| new += 1 | |
| budget -= 1 | |
| region["precise_text"] = " ".join( | |
| (precise_cache.get(round(ch["start"], 2)) or ch.get("text", "")) | |
| for ch in comp | |
| ).strip() or region.get("text", "") | |
| print(f"[NINA] Precise sectors β {new} refined, {reused} cached, {skipped} trusted (base).") | |
| return budget | |