NINA / pipeline.py
Malina
Claude Fable 5.1
Lazy diarization: the first answer no longer waits for pyannote; speaker labels attach to the cached transcript when ready
1d90084
Raw History Blame
25.2 kB
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from auth import verify_usage_token
from audio import (
fill_precise, load_audio, media_duration, pack_audio, transcribe_fast,
transcript_reliability, unpack_audio,
)
from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale, fingerprint
from compress import ensure_compressed, file_mb, too_large
from config import (
RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR,
MAX_UPLOAD_MB, MAX_VIDEO_SECONDS, USE_DIARIZATION, USE_SUBTITLES, USE_VLM,
USE_VOCAL_ISOLATION,
)
from diarize import (
annotate_region_speakers, attach_speakers, diarize, name_roster,
speaker_display_map, speaker_note,
)
from enhance import isolate_vocals
from frames import keyframes_for
from subtitles import extract_embedded_subs
from utils import fmt_time
from retrieval import (
add_entities_to_chunks, bi_encode_retrieve, classify_query,
merge_regions, pad_with_neighbours, rerank_and_score,
sample_chunks_for_global, semantic_chunk,
)
from synthesis import (
_fallback_summary, format_output, llm_synthesize_stream, prompt_to_words,
transcript_digest, vlm_synthesize_stream, words_to_global_chunks,
)
from visual import add_visual_tags, analyze_visual, visual_scores
# Per-video ingestion locks β€” ensure a video is ingested once even if two requests
# for it arrive together (newest-session takeover, double-tap, queue retry).
_ingest_locks: dict = {}
_ingest_guard = threading.Lock()
# Speaker diarization is the slowest branch of an ingest on CPU and the first answer
# does not need it. It runs here, one video at a time, after the answerable parts are
# done; the labels are attached to the cached transcript when it finishes.
_DIAR_POOL = ThreadPoolExecutor(max_workers=1, thread_name_prefix="nina-diar")
def _ingest_lock(key: str) -> threading.Lock:
with _ingest_guard:
lk = _ingest_locks.get(key)
if lk is None:
lk = threading.Lock()
_ingest_locks[key] = lk
return lk
def _drop_ingest_lock(key: str) -> None:
"""Forget the lock once its ingest is over; waiters hold their own reference."""
with _ingest_guard:
_ingest_locks.pop(key, None)
def _stage_log(name: str, t0: float) -> None:
print(f"[NINA ingest] stage={name} ms={int((time.time() - t0) * 1000)}", flush=True)
class _Stage:
"""Thread-safe holder for the current ingest stage. The ingest branches run in
parallel in a background thread; _run_core polls this every couple of seconds and
streams the marker so the website can show what NINA is actually doing. The base
"🎬 …" means "the parallel pass is running" (the client rotates transcribe βˆ₯
scenes βˆ₯ speakers); a worded marker is a genuine sequential step (subtitles,
vocal isolation, OCR). All markers stay 🎬-prefixed so clients keep them out of
the chat bubble."""
def __init__(self) -> None:
self._lock = threading.Lock()
self._marker = "🎬 …"
def set(self, marker: str) -> None:
with self._lock:
self._marker = marker
def get(self) -> str:
with self._lock:
return self._marker
# ── Cold-start ingestion ───────────────────────────────────────────────────────
def ingest_video(video_path: str, stage: "_Stage | None" = None) -> dict:
"""
First sight of a video: run the WHISPER (audio) and CLIP (visual) branches
fully in parallel, then return once both finish so synthesis can fuse them.
Both branches start at t=0 β€” the audio branch owns the (potentially slow) audio
decode, and the visual branch re-decodes its own frames via ffmpeg, so neither
waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release
the GIL during their heavy native compute, so the two threads genuinely overlap.
`stage` (optional) is updated as the conditional, sequential steps fire so the
caller can stream real progress to the UI.
"""
if stage is None:
stage = _Stage()
# Big uploads get an aggressive 720p re-encode first (best-effort, no-op when
# already small) so audio decode + frame sampling run on a lean file.
t_all = time.time()
video_path = ensure_compressed(video_path)
_stage_log("compress", t_all)
# Decode once up front β€” transcription AND diarization both need the waveform.
t0 = time.time()
audio = load_audio(video_path)
_stage_log("decode_audio", t0)
result = {"audio": audio}
def _audio_branch():
# WHISPER fast pass, then a fallback ladder when it looks unreliable:
# 1. a clean embedded subtitle track (best, if the file has one), else
# 2. vocal isolation + re-transcription (pull the voice out of the music).
# No waveform at all means the container had no audio stream. Whisper has
# nothing to do with an empty array, and asking it anyway is how a silent
# screen recording turns into a failed build.
t0 = time.time()
segments = transcribe_fast(audio) if getattr(audio, "size", 0) else []
_stage_log("whisper_fast", t0)
if segments:
rel = transcript_reliability(segments)
if not rel["reliable"]:
used = False
if USE_SUBTITLES:
stage.set("🎬 Reading the subtitle track…")
subs = extract_embedded_subs(video_path)
if subs:
print(f"[NINA] Transcript weak (score {rel['score']}) β€” using the "
f"embedded subtitle track instead.")
segments, used = subs, True
if not used and USE_VOCAL_ISOLATION:
print(f"[NINA] Transcript weak (score {rel['score']}) β€” trying vocal isolation…")
stage.set("🎬 Isolating the vocals…")
vocals = isolate_vocals(audio)
if vocals is not None:
seg2 = transcribe_fast(vocals)
if seg2 and transcript_reliability(seg2)["score"] > rel["score"]:
print("[NINA] Vocal isolation improved the transcript β€” using it.")
segments = seg2
result["segments"] = segments
result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else []
def _visual_branch():
# CLIP: sample frames β†’ encode.
t0 = time.time()
result["visual"] = analyze_visual(video_path, media_duration(video_path))
_stage_log("clip", t0)
def _diar_branch():
# pyannote: who-spoke-when (best-effort; [] if unavailable).
t0 = time.time()
turns = diarize(audio) if USE_DIARIZATION else []
_stage_log("diarization", t0)
return turns
print("[NINA] Ingesting β€” Whisper (audio) βˆ₯ CLIP (visual) in parallel; speakers in the background…")
with ThreadPoolExecutor(max_workers=2) as ex:
futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)]
for f in futures:
f.result() # block until BOTH finish; re-raise the first branch error
# Diarization used to be the third parallel branch and the whole ingest waited for
# it: on CPU that is minutes for a long recording, spent before the first answer.
# Now the first answer goes out without speaker labels and _merge_speakers attaches
# them to the cached transcript once this finishes.
diar = None
if USE_DIARIZATION and getattr(audio, "size", 0) and result["segments"]:
diar = _DIAR_POOL.submit(_diar_branch)
_stage_log("ingest_total", t_all)
print("[NINA] Ingest done β€” audio and visuals ready" + (", speakers pending." if diar else "."))
return {
"segments": result["segments"],
"chunks": result["chunks"],
"visual": result["visual"],
"audio": pack_audio(audio), # int16 in the cache: half the memory
"precise": {},
"turns": [] if diar is None else None,
"_diar": diar,
}
def _merge_speakers(entry: dict, fp) -> None:
"""Attach the speaker labels once the background diarization has finished. A
request that arrives earlier simply answers without them."""
fut = entry.get("_diar")
if fut is None or not fut.done():
return
entry["_diar"] = None
try:
turns = fut.result() or []
except Exception as e:
print(f"[NINA] diarization failed: {type(e).__name__}", flush=True)
turns = []
entry["turns"] = turns
if turns:
attach_speakers(entry["segments"], turns)
if fp:
try:
disk_save(fp, entry)
except Exception:
pass
print(f"[NINA] Speakers attached ({len(turns)} turns).", flush=True)
# ── Global path ────────────────────────────────────────────────────────────────
def _run_global(chunks: list[dict], visual: dict,
prompt: str, max_words: int) -> list[dict]:
"""Sampled regions carry the frames and the quotable fragments; the whole-video
digest (see transcript_digest) rides alongside so a summary covers the video and
not a few minutes of it."""
sampled = sample_chunks_for_global(chunks, words_to_global_chunks(max_words))
print(f"[NINA] Global mode β€” sampled {len(sampled)} chunks evenly (base transcript).")
for c in sampled:
c.setdefault("confidence", 0.5)
c.setdefault("bi_score", 0.0)
c.setdefault("entity_overlap", 0.0)
c.setdefault("visual_score", 0.0)
c["precise_text"] = c.get("text", "")
return add_visual_tags(sampled, visual)
# ── Visual-only path (no transcript) ───────────────────────────────────────────
def _run_visual_only(video_path: str, prompt: str, visual: dict, max_words: int):
"""No transcript available β€” synthesize from the visuals alone. One full-span
region carries the VLM's keyframes + visual tags so it can answer about what's
on screen (actions, objects, on-screen text) without any speech."""
dur = media_duration(video_path) or 0.0
if not dur:
times = visual.get("time") or []
dur = float(times[-1]) if times else 0.0
region = {
"start": 0.0, "end": float(dur), "text": "", "precise_text": "",
"confidence": 0.5, "bi_score": 0.0, "entity_overlap": 0.0, "visual_score": 0.0,
}
regions = add_visual_tags([region], visual)
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
note = "(No speech detected in this video β€” answer from what is visible on screen.)"
yield from _synthesize_streaming(regions, prompt, max_words, "global",
reliable=False, speaker_note=note, frames=frames)
# ── Streaming synthesis wrapper ────────────────────────────────────────────────
def _synthesize_streaming(regions: list[dict], prompt: str,
max_words: int, mode: str, reliable: bool = True,
speaker_note: str = "", frames: list = None, digest: str = ""):
# Primary path: the VLM actually sees the keyframes. Falls back to text-only
# synthesis inside vlm_synthesize_stream if the VLM is unavailable.
if USE_VLM and frames:
print(f"[NINA] Running VLM synthesis [{mode}] with {len(frames)} frame(s)…")
gen = vlm_synthesize_stream(regions, prompt, frames, max_words, mode,
reliable=reliable, speaker_note=speaker_note, digest=digest)
else:
print(f"[NINA] Running text synthesis [{mode}]...")
gen = llm_synthesize_stream(regions, prompt, max_words, mode,
reliable=reliable, speaker_note=speaker_note, digest=digest)
yield format_output(regions, prompt, "✍️ …", mode)
last = ""
for partial in gen:
last = partial
yield format_output(regions, prompt, partial, mode)
if not last:
yield format_output(regions, prompt, _fallback_summary(regions), mode)
# ── Main pipeline ──────────────────────────────────────────────────────────────
def run(video_path: str, prompt: str, token: str = ""):
"""Public API endpoint: verify the single-use token, then run the pipeline."""
if not verify_usage_token(token, "run"):
yield (
"β›” This request is not authorized.\n\n"
"Please use NINA at https://app.nnlabs.pl/nina (sign in first). "
"Direct API access is disabled, and each account has a usage limit."
)
return
yield from _run_core(video_path, prompt)
def transcript_text(video_path: str) -> str:
"""The cached transcript for a video as timestamped text (for chat storage)."""
entry = _cache.get(video_path)
if not entry:
return ""
return "\n".join(
f"[{fmt_time(s['start'])}] {s['text']}"
for s in (entry.get("segments") or []) if s.get("text")
)
def _run_core(video_path: str, prompt: str):
"""The pipeline without the token check β€” reused by the chat endpoint, which has
already authorized the turn (so it must not re-consume the single-use token)."""
# Every refusal starts with β›”: the website shows it as an error and does not store
# or count it as an answer.
if video_path is None:
yield "β›” Please upload a video."
return
if not prompt or not prompt.strip():
yield "β›” Please enter a prompt."
return
if too_large(video_path):
yield (
f"β›” This video is too large ({file_mb(video_path):.0f} MB).\n\n"
f"The maximum is {MAX_UPLOAD_MB} MB. Please upload a shorter clip, or compress "
f"it first (the app at https://app.nnlabs.pl/nina shrinks large videos for you)."
)
return
if video_path not in _cache:
dur = media_duration(video_path)
if dur > MAX_VIDEO_SECONDS:
yield (
f"β›” This video is {int(dur // 60)} minutes long. The limit is "
f"{MAX_VIDEO_SECONDS // 60} minutes: please trim it or upload a shorter part."
)
return
prompt = prompt.strip()
max_words = prompt_to_words(prompt)
print(f'\n[NINA] ── New request ── {len(prompt)} chars (target β‰ˆ {max_words} words)')
evict_stale()
# Stage 1–3: transcription + chunking + CLIP (cached)
fp = None
if video_path in _cache:
print("[NINA] Cache hit (memory) β€” skipping transcription + CLIP.")
entry = _cache[video_path]
else:
try:
fp = fingerprint(video_path)
except Exception:
fp = None
# One ingestion per video: a concurrent request for the SAME video waits here
# (yielding keepalives) instead of redundantly re-transcribing it, then reuses
# the cached result.
lock = _ingest_lock(video_path)
while not lock.acquire(timeout=10):
yield "🎬 …"
handed_off = False # True once the ingest thread owns the lock
try:
if video_path in _cache:
print("[NINA] Cache hit after wait β€” another request ingested it.")
entry = _cache[video_path]
else:
disk = disk_load(fp) if fp else None
if disk is not None:
disk["audio"] = pack_audio(load_audio(video_path))
entry = disk
_cache[video_path] = entry
else:
yield "🎬 …"
# Ingest in a background thread so we can stream progress while the
# parallel pass runs (keeps the SSE alive and feeds the client the
# live stage).
#
# The thread caches its own result and releases the lock itself.
# Both used to happen here, after the wait loop, which only runs if
# someone is still listening: when the website's connection dropped
# mid-ingest (2026-09-19, the Worker cancelled 30 s after the browser
# went away), Gradio closed this generator, the finished transcript
# and CLIP pass were thrown away, and the retry started from zero.
# Now the work survives the listener, and a retry for the same video
# waits on the lock while the ingest finishes, then hits the cache.
_res: dict = {}
_evt = threading.Event()
_stage = _Stage()
def _do_ingest():
try:
got = ingest_video(video_path, _stage)
# No speech is fine β€” we fall back to visual-only synthesis below.
_cache[video_path] = got
if fp:
disk_save(fp, got)
print("[NINA] Cached transcript + visuals for this video.")
_res["v"] = got
except Exception as e:
# Logged, not just stored. The caller turns this into
# "Error reading video." and returns, so without this line a
# complete failure of the ingest leaves nothing anywhere: no
# message worth reading for the user, and no trace at all for
# whoever has to find out why (2026-09-21, every analysis
# failing in one second and the logs showing only the request).
import traceback
print(f"[NINA] ingest FAILED for {str(video_path).split('/')[-1]}: "
f"{type(e).__name__}: {e}", flush=True)
traceback.print_exc()
_res["err"] = e
finally:
_evt.set()
lock.release()
_drop_ingest_lock(video_path)
handed_off = True
threading.Thread(target=_do_ingest, daemon=True).start()
while not _evt.wait(2.0):
yield _stage.get() # live stage marker; client keeps it out of the bubble
if "err" in _res:
yield "β›” NINA couldn't read this video (unsupported or damaged file). Try another file or format."
return
entry = _res["v"]
finally:
if not handed_off:
lock.release()
entry["_atime"] = time.time()
_merge_speakers(entry, fp)
segments = entry["segments"]
chunks = entry["chunks"]
visual = entry["visual"]
precise_cache = entry["precise"]
audio = unpack_audio(entry["audio"]) # float32 only for this request
if fp is None:
try:
fp = fingerprint(video_path)
except Exception:
fp = None
if not chunks:
# No usable speech anywhere β€” answer from the visuals alone. The VLM still
# sees sampled keyframes and reads any on-screen text, so questions like
# "what does the person do?" work on silent / music-only clips.
print("[NINA] No transcript β€” visual-only synthesis (VLM on keyframes).")
yield from _run_visual_only(video_path, prompt, visual, max_words)
return
# How trustworthy is the transcript? On garbled/musical audio we tell synthesis
# to flag the audio as unclear rather than report Whisper's fabrications as fact.
reliability = transcript_reliability(segments)
reliable = reliability["reliable"]
if not reliable:
print(f"[NINA] ⚠ Transcript looks UNRELIABLE {reliability} β€” synthesis will hedge.")
# Speaker labels (from diarization) β†’ friendly display names + a roster mapping a
# speaker to a name when they introduced themselves anywhere in the video. Both
# are fed to synthesis so it can attribute lines and resolve names.
spk_display = speaker_display_map(segments)
spk_roster = name_roster(segments, spk_display) if spk_display else {}
spk_note = speaker_note(spk_display, spk_roster)
if spk_note:
print(f"[NINA] {spk_note}")
# Pass 1 routing
mode = classify_query(prompt)
print(f"[NINA] Query mode (pass 1): {mode}")
if mode == "global":
regions = _run_global(chunks, visual, prompt, max_words)
annotate_region_speakers(regions, segments, spk_display)
yield "🎬 Reading the whole transcript…"
digest = transcript_digest(chunks, prompt)
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
yield from _synthesize_streaming(regions, prompt, max_words, "global",
reliable=reliable, speaker_note=spk_note, frames=frames,
digest=digest)
print(f"[NINA] Done β€” {len(regions)} region(s) [global].")
return
# SPECIFIC / HYBRID retrieval
yield "🎬 Finding the key moments…"
candidates, best_bi = bi_encode_retrieve(chunks, prompt)
# Upgrade any candidate whose precise text is already cached (improves cross-encoder accuracy)
for c in candidates:
key = round(c["start"], 2)
if key in precise_cache:
c["text"] = precise_cache[key]
_, v_norm = visual_scores(visual, prompt)
# Pass 2 routing
if best_bi < BI_SCORE_GLOBAL_FLOOR:
print(f"[NINA] bi-score {best_bi:.3f} below floor β†’ escalating to global.")
regions = _run_global(chunks, visual, prompt, max_words)
annotate_region_speakers(regions, segments, spk_display)
yield "🎬 Reading the whole transcript…"
digest = transcript_digest(chunks, prompt)
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
yield from _synthesize_streaming(regions, prompt, max_words, "global",
reliable=reliable, speaker_note=spk_note, frames=frames,
digest=digest)
print(f"[NINA] Done β€” {len(regions)} region(s) [escalated global].")
return
elif best_bi < BI_SCORE_HYBRID_FLOOR:
mode = "hybrid"
print(f"[NINA] Query mode (pass 2): {mode} bi={best_bi:.3f}")
results = rerank_and_score(candidates, prompt, visual["time"], v_norm)
if mode == "hybrid":
results = pad_with_neighbours(results, chunks)
regions = merge_regions(results)
if not regions:
print("[NINA] No regions after merge β†’ falling back to global.")
regions = _run_global(chunks, visual, prompt, max_words)
annotate_region_speakers(regions, segments, spk_display)
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
yield from _synthesize_streaming(regions, prompt, max_words, "global",
reliable=reliable, speaker_note=spk_note, frames=frames)
return
# Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual
# tagging (CLIP) are independent β€” one fills region["precise_text"], the other
# region["visual_tags"] β€” so we run them together and wait for both before the
# summary fuses audio + visual.
yield "🎬 Re-checking the details…"
before = dict(precise_cache)
def _precise_branch():
fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX)
def _tags_branch():
add_visual_tags(regions, visual) # mutates regions in place
with ThreadPoolExecutor(max_workers=2) as ex:
for f in [ex.submit(_precise_branch), ex.submit(_tags_branch)]:
f.result()
if fp and precise_cache != before:
disk_update_precise(fp, precise_cache)
# Stage 8: VLM synthesis (the model sees keyframes + reads the transcript)
annotate_region_speakers(regions, segments, spk_display)
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
yield from _synthesize_streaming(regions, prompt, max_words, mode,
reliable=reliable, speaker_note=spk_note, frames=frames)
print(f"[NINA] Done β€” {len(regions)} region(s) [{mode}].")