File size: 7,936 Bytes
05f095d
 
 
 
 
6acd603
 
1d697a7
6acd603
 
05f095d
 
 
 
 
 
3b71b6c
 
 
 
 
 
 
 
 
 
 
 
1d697a7
 
 
3b71b6c
 
 
 
05f095d
 
1d697a7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
05f095d
 
 
 
 
 
 
 
 
6acd603
 
 
 
 
05f095d
 
6acd603
 
 
 
05f095d
 
 
6acd603
 
 
 
 
 
05f095d
 
 
 
 
 
6acd603
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
05f095d
0695bec
 
 
 
 
 
05f095d
 
 
 
0695bec
05f095d
0695bec
 
 
 
 
 
 
 
 
 
05f095d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
import numpy as np
import ffmpeg
from faster_whisper.audio import decode_audio

from config import (
    ASR_NO_SPEECH_CEILING, ASR_RELIABILITY_LOGPROB_FLOOR,
    ASR_UNRELIABLE_WEAK_FRACTION, ASR_WEAK_SEGMENT_LOGPROB,
    DEBUG, MAX_VIDEO_SECONDS, PRECISE_LOGPROB_GATE, RETRANSCRIBE_MAX, SAMPLE_RATE,
    WHISPER_COMPRESSION_RATIO_THRESHOLD, WHISPER_CONDITION_ON_PREV,
    WHISPER_LOGPROB_THRESHOLD, WHISPER_NO_SPEECH_THRESHOLD,
)
from models import get_whisper_precise, whisper_fast
from utils import fmt_time


def load_audio(media_path: str) -> np.ndarray:
    """Decode any container to float32 mono 16 kHz waveform (PyAV, no temp file).

    A file with NO audio stream comes back as silence rather than an exception. PyAV
    raises IndexError deep inside faster_whisper when it asks for audio stream 0 of a
    container that has none, and that exception used to take the whole ingest down:
    every silent clip answered "Error reading video." in about a second, with the
    reason swallowed. Measured on a plain H.264 file with no audio track, 2026-09-21.

    Silence is the honest input here. The pipeline already handles "no speech" and
    falls back to visual-only synthesis, which is exactly what a screen recording
    without narration needs."""
    try:
        wave = decode_audio(media_path, sampling_rate=SAMPLE_RATE)
        # Bounded: nothing past MAX_VIDEO_SECONDS is decoded into memory or transcribed.
        return wave[: MAX_VIDEO_SECONDS * SAMPLE_RATE]
    except (IndexError, StopIteration):
        print(f"[NINA] no audio stream in {str(media_path).split('/')[-1]}, "
              f"continuing from the picture alone", flush=True)
        return np.zeros(0, dtype=np.float32)


def pack_audio(wave: np.ndarray) -> np.ndarray:
    """int16 for the cache: half the memory of float32, no audible loss for Whisper."""
    if wave is None or getattr(wave, "size", 0) == 0:
        return np.zeros(0, dtype=np.int16)
    if wave.dtype == np.int16:
        return wave
    return np.clip(wave * 32767.0, -32768, 32767).astype(np.int16)


def unpack_audio(wave: np.ndarray) -> np.ndarray:
    if wave is None or getattr(wave, "size", 0) == 0:
        return np.zeros(0, dtype=np.float32)
    if wave.dtype == np.int16:
        return wave.astype(np.float32) / 32767.0
    return wave


def media_duration(media_path: str) -> float:
    try:
        info = ffmpeg.probe(media_path)
        return float(info["format"]["duration"])
    except Exception:
        return 0.0


def transcribe_fast(audio: np.ndarray) -> list[dict]:
    """Base-model full-video pass: greedy + Silero VAD, with hallucination guards.

    Stores avg_logprob, no_speech_prob and compression_ratio per segment so the
    pipeline can judge how trustworthy the transcript is (see transcript_reliability).
    """
    segments_gen, _ = whisper_fast.transcribe(
        audio, beam_size=1, vad_filter=True, word_timestamps=False,
        condition_on_previous_text=WHISPER_CONDITION_ON_PREV,
        no_speech_threshold=WHISPER_NO_SPEECH_THRESHOLD,
        log_prob_threshold=WHISPER_LOGPROB_THRESHOLD,
        compression_ratio_threshold=WHISPER_COMPRESSION_RATIO_THRESHOLD,
    )
    return [
        {
            "text":        seg.text.strip(),
            "start":       round(seg.start, 2),
            "end":         round(seg.end,   2),
            "logprob":     round(float(getattr(seg, "avg_logprob", 0.0) or 0.0), 3),
            "no_speech":   round(float(getattr(seg, "no_speech_prob", 0.0) or 0.0), 3),
            "comp_ratio":  round(float(getattr(seg, "compression_ratio", 0.0) or 0.0), 3),
        }
        for seg in segments_gen
        if seg.text.strip()
    ]


def transcript_reliability(segments: list[dict]) -> dict:
    """Judge how trustworthy a base-model transcript is, so synthesis can hedge on
    garbled/musical audio instead of reporting Whisper's fabrications as fact.

    Returns {reliable: bool, score: 0..1, mean_logprob, weak_fraction, mean_no_speech}.
    """
    if not segments:
        return {"reliable": False, "score": 0.0, "mean_logprob": 0.0,
                "weak_fraction": 1.0, "mean_no_speech": 1.0}

    lps  = [s.get("logprob", 0.0)   for s in segments]
    nsp  = [s.get("no_speech", 0.0) for s in segments]
    mean_lp  = sum(lps) / len(lps)
    mean_nsp = sum(nsp) / len(nsp)
    weak_frac = sum(1 for lp in lps if lp < ASR_WEAK_SEGMENT_LOGPROB) / len(lps)

    reliable = (
        mean_lp >= ASR_RELIABILITY_LOGPROB_FLOOR
        and weak_frac < ASR_UNRELIABLE_WEAK_FRACTION
        and mean_nsp < ASR_NO_SPEECH_CEILING
    )
    # Rough 0..1 confidence: avg_logprob of ~0 β†’ 1.0, ~-1.0 β†’ 0.0.
    score = max(0.0, min(1.0, 1.0 + mean_lp))

    info = {
        "reliable":       bool(reliable),
        "score":          round(score, 3),
        "mean_logprob":   round(mean_lp, 3),
        "weak_fraction":  round(weak_frac, 3),
        "mean_no_speech": round(mean_nsp, 3),
    }
    if DEBUG:
        print(f"[DEBUG] ASR reliability: {info}")
    return info


def retranscribe_region(audio: np.ndarray, region: dict) -> str:
    """Slice waveform in-memory and run precise Whisper on the slice.

    Always returns usable text: if the precise model is unavailable or errors out,
    we fall back to the region's base-model text rather than crashing the request.
    """
    base = region.get("text", "")
    s = max(0, int(region["start"] * SAMPLE_RATE))
    e = min(len(audio), int(region["end"] * SAMPLE_RATE))
    clip = audio[s:e]
    if len(clip) < SAMPLE_RATE * 0.2:
        return base
    model = get_whisper_precise()
    if model is None:
        return base
    try:
        segments_gen, _ = model.transcribe(clip, beam_size=1)
        out = " ".join(seg.text.strip() for seg in segments_gen).strip()
        return out or base
    except Exception as exc:
        print(f"[NINA] Precise re-transcription failed for "
              f"[{fmt_time(region['start'])} β†’ {fmt_time(region['end'])}]: {exc}")
        return base


def fill_precise(
    audio: np.ndarray,
    regions: list[dict],
    all_chunks: list[dict],
    precise_cache: dict,
    budget: int,
) -> int:
    """
    Ensure every region has `precise_text`. A sector is turbo-refined only when:
      β€’ not already cached, AND
      β€’ base avg_logprob is below PRECISE_LOGPROB_GATE (weak ASR), AND
      β€’ still within budget.
    Returns remaining budget.
    """
    reused = new = skipped = 0
    for region in regions:
        comp = [
            c for c in all_chunks
            if c["start"] < region["end"] - 1e-3 and c["end"] > region["start"] + 1e-3
        ]
        if not comp:
            comp = [region]

        for ch in comp:
            key = round(ch["start"], 2)
            if key in precise_cache:
                reused += 1
                continue
            weak = ch.get("logprob", 0.0) < PRECISE_LOGPROB_GATE
            if not weak:
                skipped += 1
                continue
            if budget <= 0:
                continue
            print(
                f"[NINA] Re-transcribing weak sector "
                f"[{fmt_time(ch['start'])} β†’ {fmt_time(ch['end'])}] "
                f"(logprob {ch.get('logprob', 0.0):.2f})"
            )
            precise_cache[key] = retranscribe_region(audio, ch)
            if DEBUG:
                snip = precise_cache[key]
                print(
                    f"[DEBUG] Precise sector [{fmt_time(ch['start'])}β†’{fmt_time(ch['end'])}]: "
                    f"{snip[:200]}{'…' if len(snip) > 200 else ''}"
                )
            new += 1
            budget -= 1

        region["precise_text"] = " ".join(
            (precise_cache.get(round(ch["start"], 2)) or ch.get("text", ""))
            for ch in comp
        ).strip() or region.get("text", "")

    print(f"[NINA] Precise sectors β€” {new} refined, {reused} cached, {skipped} trusted (base).")
    return budget