Spaces:
Paused
Paused
File size: 7,936 Bytes
05f095d 6acd603 1d697a7 6acd603 05f095d 3b71b6c 1d697a7 3b71b6c 05f095d 1d697a7 05f095d 6acd603 05f095d 6acd603 05f095d 6acd603 05f095d 6acd603 05f095d 0695bec 05f095d 0695bec 05f095d 0695bec 05f095d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | import numpy as np
import ffmpeg
from faster_whisper.audio import decode_audio
from config import (
ASR_NO_SPEECH_CEILING, ASR_RELIABILITY_LOGPROB_FLOOR,
ASR_UNRELIABLE_WEAK_FRACTION, ASR_WEAK_SEGMENT_LOGPROB,
DEBUG, MAX_VIDEO_SECONDS, PRECISE_LOGPROB_GATE, RETRANSCRIBE_MAX, SAMPLE_RATE,
WHISPER_COMPRESSION_RATIO_THRESHOLD, WHISPER_CONDITION_ON_PREV,
WHISPER_LOGPROB_THRESHOLD, WHISPER_NO_SPEECH_THRESHOLD,
)
from models import get_whisper_precise, whisper_fast
from utils import fmt_time
def load_audio(media_path: str) -> np.ndarray:
"""Decode any container to float32 mono 16 kHz waveform (PyAV, no temp file).
A file with NO audio stream comes back as silence rather than an exception. PyAV
raises IndexError deep inside faster_whisper when it asks for audio stream 0 of a
container that has none, and that exception used to take the whole ingest down:
every silent clip answered "Error reading video." in about a second, with the
reason swallowed. Measured on a plain H.264 file with no audio track, 2026-09-21.
Silence is the honest input here. The pipeline already handles "no speech" and
falls back to visual-only synthesis, which is exactly what a screen recording
without narration needs."""
try:
wave = decode_audio(media_path, sampling_rate=SAMPLE_RATE)
# Bounded: nothing past MAX_VIDEO_SECONDS is decoded into memory or transcribed.
return wave[: MAX_VIDEO_SECONDS * SAMPLE_RATE]
except (IndexError, StopIteration):
print(f"[NINA] no audio stream in {str(media_path).split('/')[-1]}, "
f"continuing from the picture alone", flush=True)
return np.zeros(0, dtype=np.float32)
def pack_audio(wave: np.ndarray) -> np.ndarray:
"""int16 for the cache: half the memory of float32, no audible loss for Whisper."""
if wave is None or getattr(wave, "size", 0) == 0:
return np.zeros(0, dtype=np.int16)
if wave.dtype == np.int16:
return wave
return np.clip(wave * 32767.0, -32768, 32767).astype(np.int16)
def unpack_audio(wave: np.ndarray) -> np.ndarray:
if wave is None or getattr(wave, "size", 0) == 0:
return np.zeros(0, dtype=np.float32)
if wave.dtype == np.int16:
return wave.astype(np.float32) / 32767.0
return wave
def media_duration(media_path: str) -> float:
try:
info = ffmpeg.probe(media_path)
return float(info["format"]["duration"])
except Exception:
return 0.0
def transcribe_fast(audio: np.ndarray) -> list[dict]:
"""Base-model full-video pass: greedy + Silero VAD, with hallucination guards.
Stores avg_logprob, no_speech_prob and compression_ratio per segment so the
pipeline can judge how trustworthy the transcript is (see transcript_reliability).
"""
segments_gen, _ = whisper_fast.transcribe(
audio, beam_size=1, vad_filter=True, word_timestamps=False,
condition_on_previous_text=WHISPER_CONDITION_ON_PREV,
no_speech_threshold=WHISPER_NO_SPEECH_THRESHOLD,
log_prob_threshold=WHISPER_LOGPROB_THRESHOLD,
compression_ratio_threshold=WHISPER_COMPRESSION_RATIO_THRESHOLD,
)
return [
{
"text": seg.text.strip(),
"start": round(seg.start, 2),
"end": round(seg.end, 2),
"logprob": round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3),
"no_speech": round(float(getattr(seg, "no_speech_prob", 0.0) or 0.0), 3),
"comp_ratio": round(float(getattr(seg, "compression_ratio", 0.0) or 0.0), 3),
}
for seg in segments_gen
if seg.text.strip()
]
def transcript_reliability(segments: list[dict]) -> dict:
"""Judge how trustworthy a base-model transcript is, so synthesis can hedge on
garbled/musical audio instead of reporting Whisper's fabrications as fact.
Returns {reliable: bool, score: 0..1, mean_logprob, weak_fraction, mean_no_speech}.
"""
if not segments:
return {"reliable": False, "score": 0.0, "mean_logprob": 0.0,
"weak_fraction": 1.0, "mean_no_speech": 1.0}
lps = [s.get("logprob", 0.0) for s in segments]
nsp = [s.get("no_speech", 0.0) for s in segments]
mean_lp = sum(lps) / len(lps)
mean_nsp = sum(nsp) / len(nsp)
weak_frac = sum(1 for lp in lps if lp < ASR_WEAK_SEGMENT_LOGPROB) / len(lps)
reliable = (
mean_lp >= ASR_RELIABILITY_LOGPROB_FLOOR
and weak_frac < ASR_UNRELIABLE_WEAK_FRACTION
and mean_nsp < ASR_NO_SPEECH_CEILING
)
# Rough 0..1 confidence: avg_logprob of ~0 β 1.0, ~-1.0 β 0.0.
score = max(0.0, min(1.0, 1.0 + mean_lp))
info = {
"reliable": bool(reliable),
"score": round(score, 3),
"mean_logprob": round(mean_lp, 3),
"weak_fraction": round(weak_frac, 3),
"mean_no_speech": round(mean_nsp, 3),
}
if DEBUG:
print(f"[DEBUG] ASR reliability: {info}")
return info
def retranscribe_region(audio: np.ndarray, region: dict) -> str:
"""Slice waveform in-memory and run precise Whisper on the slice.
Always returns usable text: if the precise model is unavailable or errors out,
we fall back to the region's base-model text rather than crashing the request.
"""
base = region.get("text", "")
s = max(0, int(region["start"] * SAMPLE_RATE))
e = min(len(audio), int(region["end"] * SAMPLE_RATE))
clip = audio[s:e]
if len(clip) < SAMPLE_RATE * 0.2:
return base
model = get_whisper_precise()
if model is None:
return base
try:
segments_gen, _ = model.transcribe(clip, beam_size=1)
out = " ".join(seg.text.strip() for seg in segments_gen).strip()
return out or base
except Exception as exc:
print(f"[NINA] Precise re-transcription failed for "
f"[{fmt_time(region['start'])} β {fmt_time(region['end'])}]: {exc}")
return base
def fill_precise(
audio: np.ndarray,
regions: list[dict],
all_chunks: list[dict],
precise_cache: dict,
budget: int,
) -> int:
"""
Ensure every region has `precise_text`. A sector is turbo-refined only when:
β’ not already cached, AND
β’ base avg_logprob is below PRECISE_LOGPROB_GATE (weak ASR), AND
β’ still within budget.
Returns remaining budget.
"""
reused = new = skipped = 0
for region in regions:
comp = [
c for c in all_chunks
if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3
]
if not comp:
comp = [region]
for ch in comp:
key = round(ch["start"], 2)
if key in precise_cache:
reused += 1
continue
weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE
if not weak:
skipped += 1
continue
if budget <= 0:
continue
print(
f"[NINA] Re-transcribing weak sector "
f"[{fmt_time(ch['start'])} β {fmt_time(ch['end'])}] "
f"(logprob {ch.get('logprob', 0.0):.2f})"
)
precise_cache[key] = retranscribe_region(audio, ch)
if DEBUG:
snip = precise_cache[key]
print(
f"[DEBUG] Precise sector [{fmt_time(ch['start'])}β{fmt_time(ch['end'])}]: "
f"{snip[:200]}{'β¦' if len(snip) > 200 else ''}"
)
new += 1
budget -= 1
region["precise_text"] = " ".join(
(precise_cache.get(round(ch["start"], 2)) or ch.get("text", ""))
for ch in comp
).strip() or region.get("text", "")
print(f"[NINA] Precise sectors β {new} refined, {reused} cached, {skipped} trusted (base).")
return budget
|