NINA / audio.py
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7
Raw History Blame
7.94 kB
import numpy as np
import ffmpeg
from faster_whisper.audio import decode_audio
from config import (
ASR_NO_SPEECH_CEILING, ASR_RELIABILITY_LOGPROB_FLOOR,
ASR_UNRELIABLE_WEAK_FRACTION, ASR_WEAK_SEGMENT_LOGPROB,
DEBUG, MAX_VIDEO_SECONDS, PRECISE_LOGPROB_GATE, RETRANSCRIBE_MAX, SAMPLE_RATE,
WHISPER_COMPRESSION_RATIO_THRESHOLD, WHISPER_CONDITION_ON_PREV,
WHISPER_LOGPROB_THRESHOLD, WHISPER_NO_SPEECH_THRESHOLD,
)
from models import get_whisper_precise, whisper_fast
from utils import fmt_time
def load_audio(media_path: str) -> np.ndarray:
"""Decode any container to float32 mono 16 kHz waveform (PyAV, no temp file).
A file with NO audio stream comes back as silence rather than an exception. PyAV
raises IndexError deep inside faster_whisper when it asks for audio stream 0 of a
container that has none, and that exception used to take the whole ingest down:
every silent clip answered "Error reading video." in about a second, with the
reason swallowed. Measured on a plain H.264 file with no audio track, 2026-09-21.
Silence is the honest input here. The pipeline already handles "no speech" and
falls back to visual-only synthesis, which is exactly what a screen recording
without narration needs."""
try:
wave = decode_audio(media_path, sampling_rate=SAMPLE_RATE)
# Bounded: nothing past MAX_VIDEO_SECONDS is decoded into memory or transcribed.
return wave[: MAX_VIDEO_SECONDS * SAMPLE_RATE]
except (IndexError, StopIteration):
print(f"[NINA] no audio stream in {str(media_path).split('/')[-1]}, "
f"continuing from the picture alone", flush=True)
return np.zeros(0, dtype=np.float32)
def pack_audio(wave: np.ndarray) -> np.ndarray:
"""int16 for the cache: half the memory of float32, no audible loss for Whisper."""
if wave is None or getattr(wave, "size", 0) == 0:
return np.zeros(0, dtype=np.int16)
if wave.dtype == np.int16:
return wave
return np.clip(wave * 32767.0, -32768, 32767).astype(np.int16)
def unpack_audio(wave: np.ndarray) -> np.ndarray:
if wave is None or getattr(wave, "size", 0) == 0:
return np.zeros(0, dtype=np.float32)
if wave.dtype == np.int16:
return wave.astype(np.float32) / 32767.0
return wave
def media_duration(media_path: str) -> float:
try:
info = ffmpeg.probe(media_path)
return float(info["format"]["duration"])
except Exception:
return 0.0
def transcribe_fast(audio: np.ndarray) -> list[dict]:
"""Base-model full-video pass: greedy + Silero VAD, with hallucination guards.
Stores avg_logprob, no_speech_prob and compression_ratio per segment so the
pipeline can judge how trustworthy the transcript is (see transcript_reliability).
"""
segments_gen, _ = whisper_fast.transcribe(
audio, beam_size=1, vad_filter=True, word_timestamps=False,
condition_on_previous_text=WHISPER_CONDITION_ON_PREV,
no_speech_threshold=WHISPER_NO_SPEECH_THRESHOLD,
log_prob_threshold=WHISPER_LOGPROB_THRESHOLD,
compression_ratio_threshold=WHISPER_COMPRESSION_RATIO_THRESHOLD,
)
return [
{
"text": seg.text.strip(),
"start": round(seg.start, 2),
"end": round(seg.end, 2),
"logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3),
"no_speech": round(float(getattr(seg, "no_speech_prob", 0.0) or 0.0), 3),
"comp_ratio": round(float(getattr(seg, "compression_ratio", 0.0) or 0.0), 3),
}
for seg in segments_gen
if seg.text.strip()
]
def transcript_reliability(segments: list[dict]) -> dict:
"""Judge how trustworthy a base-model transcript is, so synthesis can hedge on
garbled/musical audio instead of reporting Whisper's fabrications as fact.
Returns {reliable: bool, score: 0..1, mean_logprob, weak_fraction, mean_no_speech}.
"""
if not segments:
return {"reliable": False, "score": 0.0, "mean_logprob": 0.0,
"weak_fraction": 1.0, "mean_no_speech": 1.0}
lps = [s.get("logprob", 0.0) for s in segments]
nsp = [s.get("no_speech", 0.0) for s in segments]
mean_lp = sum(lps) / len(lps)
mean_nsp = sum(nsp) / len(nsp)
weak_frac = sum(1 for lp in lps if lp < ASR_WEAK_SEGMENT_LOGPROB) / len(lps)
reliable = (
mean_lp >= ASR_RELIABILITY_LOGPROB_FLOOR
and weak_frac < ASR_UNRELIABLE_WEAK_FRACTION
and mean_nsp < ASR_NO_SPEECH_CEILING
)
# Rough 0..1 confidence: avg_logprob of ~0 β†’ 1.0, ~-1.0 β†’ 0.0.
score = max(0.0, min(1.0, 1.0 + mean_lp))
info = {
"reliable": bool(reliable),
"score": round(score, 3),
"mean_logprob": round(mean_lp, 3),
"weak_fraction": round(weak_frac, 3),
"mean_no_speech": round(mean_nsp, 3),
}
if DEBUG:
print(f"[DEBUG] ASR reliability: {info}")
return info
def retranscribe_region(audio: np.ndarray, region: dict) -> str:
"""Slice waveform in-memory and run precise Whisper on the slice.
Always returns usable text: if the precise model is unavailable or errors out,
we fall back to the region's base-model text rather than crashing the request.
"""
base = region.get("text", "")
s = max(0, int(region["start"] * SAMPLE_RATE))
e = min(len(audio), int(region["end"] * SAMPLE_RATE))
clip = audio[s:e]
if len(clip) < SAMPLE_RATE * 0.2:
return base
model = get_whisper_precise()
if model is None:
return base
try:
segments_gen, _ = model.transcribe(clip, beam_size=1)
out = " ".join(seg.text.strip() for seg in segments_gen).strip()
return out or base
except Exception as exc:
print(f"[NINA] Precise re-transcription failed for "
f"[{fmt_time(region['start'])} β†’ {fmt_time(region['end'])}]: {exc}")
return base
def fill_precise(
audio: np.ndarray,
regions: list[dict],
all_chunks: list[dict],
precise_cache: dict,
budget: int,
) -> int:
"""
Ensure every region has `precise_text`. A sector is turbo-refined only when:
β€’ not already cached, AND
β€’ base avg_logprob is below PRECISE_LOGPROB_GATE (weak ASR), AND
β€’ still within budget.
Returns remaining budget.
"""
reused = new = skipped = 0
for region in regions:
comp = [
c for c in all_chunks
if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3
]
if not comp:
comp = [region]
for ch in comp:
key = round(ch["start"], 2)
if key in precise_cache:
reused += 1
continue
weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE
if not weak:
skipped += 1
continue
if budget <= 0:
continue
print(
f"[NINA] Re-transcribing weak sector "
f"[{fmt_time(ch['start'])} β†’ {fmt_time(ch['end'])}] "
f"(logprob {ch.get('logprob', 0.0):.2f})"
)
precise_cache[key] = retranscribe_region(audio, ch)
if DEBUG:
snip = precise_cache[key]
print(
f"[DEBUG] Precise sector [{fmt_time(ch['start'])}β†’{fmt_time(ch['end'])}]: "
f"{snip[:200]}{'…' if len(snip) > 200 else ''}"
)
new += 1
budget -= 1
region["precise_text"] = " ".join(
(precise_cache.get(round(ch["start"], 2)) or ch.get("text", ""))
for ch in comp
).strip() or region.get("text", "")
print(f"[NINA] Precise sectors β€” {new} refined, {reused} cached, {skipped} trusted (base).")
return budget