import threading import time from concurrent.futures import ThreadPoolExecutor from auth import verify_usage_token from audio import ( fill_precise, load_audio, media_duration, pack_audio, transcribe_fast, transcript_reliability, unpack_audio, ) from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale, fingerprint from compress import ensure_compressed, file_mb, too_large from config import ( RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR, MAX_UPLOAD_MB, MAX_VIDEO_SECONDS, USE_DIARIZATION, USE_SUBTITLES, USE_VLM, USE_VOCAL_ISOLATION, ) from diarize import ( annotate_region_speakers, attach_speakers, diarize, name_roster, speaker_display_map, speaker_note, ) from enhance import isolate_vocals from frames import keyframes_for from subtitles import extract_embedded_subs from utils import fmt_time from retrieval import ( add_entities_to_chunks, bi_encode_retrieve, classify_query, merge_regions, pad_with_neighbours, rerank_and_score, sample_chunks_for_global, semantic_chunk, ) from synthesis import ( _fallback_summary, format_output, llm_synthesize_stream, prompt_to_words, transcript_digest, vlm_synthesize_stream, words_to_global_chunks, ) from visual import add_visual_tags, analyze_visual, visual_scores # Per-video ingestion locks β€” ensure a video is ingested once even if two requests # for it arrive together (newest-session takeover, double-tap, queue retry). _ingest_locks: dict = {} _ingest_guard = threading.Lock() # Speaker diarization is the slowest branch of an ingest on CPU and the first answer # does not need it. It runs here, one video at a time, after the answerable parts are # done; the labels are attached to the cached transcript when it finishes. _DIAR_POOL = ThreadPoolExecutor(max_workers=1, thread_name_prefix="nina-diar") def _ingest_lock(key: str) -> threading.Lock: with _ingest_guard: lk = _ingest_locks.get(key) if lk is None: lk = threading.Lock() _ingest_locks[key] = lk return lk def _drop_ingest_lock(key: str) -> None: """Forget the lock once its ingest is over; waiters hold their own reference.""" with _ingest_guard: _ingest_locks.pop(key, None) def _stage_log(name: str, t0: float) -> None: print(f"[NINA ingest] stage={name} ms={int((time.time() - t0) * 1000)}", flush=True) class _Stage: """Thread-safe holder for the current ingest stage. The ingest branches run in parallel in a background thread; _run_core polls this every couple of seconds and streams the marker so the website can show what NINA is actually doing. The base "🎬 …" means "the parallel pass is running" (the client rotates transcribe βˆ₯ scenes βˆ₯ speakers); a worded marker is a genuine sequential step (subtitles, vocal isolation, OCR). All markers stay 🎬-prefixed so clients keep them out of the chat bubble.""" def __init__(self) -> None: self._lock = threading.Lock() self._marker = "🎬 …" def set(self, marker: str) -> None: with self._lock: self._marker = marker def get(self) -> str: with self._lock: return self._marker # ── Cold-start ingestion ─────────────────────────────────────────────────────── def ingest_video(video_path: str, stage: "_Stage | None" = None) -> dict: """ First sight of a video: run the WHISPER (audio) and CLIP (visual) branches fully in parallel, then return once both finish so synthesis can fuse them. Both branches start at t=0 β€” the audio branch owns the (potentially slow) audio decode, and the visual branch re-decodes its own frames via ffmpeg, so neither waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release the GIL during their heavy native compute, so the two threads genuinely overlap. `stage` (optional) is updated as the conditional, sequential steps fire so the caller can stream real progress to the UI. """ if stage is None: stage = _Stage() # Big uploads get an aggressive 720p re-encode first (best-effort, no-op when # already small) so audio decode + frame sampling run on a lean file. t_all = time.time() video_path = ensure_compressed(video_path) _stage_log("compress", t_all) # Decode once up front β€” transcription AND diarization both need the waveform. t0 = time.time() audio = load_audio(video_path) _stage_log("decode_audio", t0) result = {"audio": audio} def _audio_branch(): # WHISPER fast pass, then a fallback ladder when it looks unreliable: # 1. a clean embedded subtitle track (best, if the file has one), else # 2. vocal isolation + re-transcription (pull the voice out of the music). # No waveform at all means the container had no audio stream. Whisper has # nothing to do with an empty array, and asking it anyway is how a silent # screen recording turns into a failed build. t0 = time.time() segments = transcribe_fast(audio) if getattr(audio, "size", 0) else [] _stage_log("whisper_fast", t0) if segments: rel = transcript_reliability(segments) if not rel["reliable"]: used = False if USE_SUBTITLES: stage.set("🎬 Reading the subtitle track…") subs = extract_embedded_subs(video_path) if subs: print(f"[NINA] Transcript weak (score {rel['score']}) β€” using the " f"embedded subtitle track instead.") segments, used = subs, True if not used and USE_VOCAL_ISOLATION: print(f"[NINA] Transcript weak (score {rel['score']}) β€” trying vocal isolation…") stage.set("🎬 Isolating the vocals…") vocals = isolate_vocals(audio) if vocals is not None: seg2 = transcribe_fast(vocals) if seg2 and transcript_reliability(seg2)["score"] > rel["score"]: print("[NINA] Vocal isolation improved the transcript β€” using it.") segments = seg2 result["segments"] = segments result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else [] def _visual_branch(): # CLIP: sample frames β†’ encode. t0 = time.time() result["visual"] = analyze_visual(video_path, media_duration(video_path)) _stage_log("clip", t0) def _diar_branch(): # pyannote: who-spoke-when (best-effort; [] if unavailable). t0 = time.time() turns = diarize(audio) if USE_DIARIZATION else [] _stage_log("diarization", t0) return turns print("[NINA] Ingesting β€” Whisper (audio) βˆ₯ CLIP (visual) in parallel; speakers in the background…") with ThreadPoolExecutor(max_workers=2) as ex: futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)] for f in futures: f.result() # block until BOTH finish; re-raise the first branch error # Diarization used to be the third parallel branch and the whole ingest waited for # it: on CPU that is minutes for a long recording, spent before the first answer. # Now the first answer goes out without speaker labels and _merge_speakers attaches # them to the cached transcript once this finishes. diar = None if USE_DIARIZATION and getattr(audio, "size", 0) and result["segments"]: diar = _DIAR_POOL.submit(_diar_branch) _stage_log("ingest_total", t_all) print("[NINA] Ingest done β€” audio and visuals ready" + (", speakers pending." if diar else ".")) return { "segments": result["segments"], "chunks": result["chunks"], "visual": result["visual"], "audio": pack_audio(audio), # int16 in the cache: half the memory "precise": {}, "turns": [] if diar is None else None, "_diar": diar, } def _merge_speakers(entry: dict, fp) -> None: """Attach the speaker labels once the background diarization has finished. A request that arrives earlier simply answers without them.""" fut = entry.get("_diar") if fut is None or not fut.done(): return entry["_diar"] = None try: turns = fut.result() or [] except Exception as e: print(f"[NINA] diarization failed: {type(e).__name__}", flush=True) turns = [] entry["turns"] = turns if turns: attach_speakers(entry["segments"], turns) if fp: try: disk_save(fp, entry) except Exception: pass print(f"[NINA] Speakers attached ({len(turns)} turns).", flush=True) # ── Global path ──────────────────────────────────────────────────────────────── def _run_global(chunks: list[dict], visual: dict, prompt: str, max_words: int) -> list[dict]: """Sampled regions carry the frames and the quotable fragments; the whole-video digest (see transcript_digest) rides alongside so a summary covers the video and not a few minutes of it.""" sampled = sample_chunks_for_global(chunks, words_to_global_chunks(max_words)) print(f"[NINA] Global mode β€” sampled {len(sampled)} chunks evenly (base transcript).") for c in sampled: c.setdefault("confidence", 0.5) c.setdefault("bi_score", 0.0) c.setdefault("entity_overlap", 0.0) c.setdefault("visual_score", 0.0) c["precise_text"] = c.get("text", "") return add_visual_tags(sampled, visual) # ── Visual-only path (no transcript) ─────────────────────────────────────────── def _run_visual_only(video_path: str, prompt: str, visual: dict, max_words: int): """No transcript available β€” synthesize from the visuals alone. One full-span region carries the VLM's keyframes + visual tags so it can answer about what's on screen (actions, objects, on-screen text) without any speech.""" dur = media_duration(video_path) or 0.0 if not dur: times = visual.get("time") or [] dur = float(times[-1]) if times else 0.0 region = { "start": 0.0, "end": float(dur), "text": "", "precise_text": "", "confidence": 0.5, "bi_score": 0.0, "entity_overlap": 0.0, "visual_score": 0.0, } regions = add_visual_tags([region], visual) frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None note = "(No speech detected in this video β€” answer from what is visible on screen.)" yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=False, speaker_note=note, frames=frames) # ── Streaming synthesis wrapper ──────────────────────────────────────────────── def _synthesize_streaming(regions: list[dict], prompt: str, max_words: int, mode: str, reliable: bool = True, speaker_note: str = "", frames: list = None, digest: str = ""): # Primary path: the VLM actually sees the keyframes. Falls back to text-only # synthesis inside vlm_synthesize_stream if the VLM is unavailable. if USE_VLM and frames: print(f"[NINA] Running VLM synthesis [{mode}] with {len(frames)} frame(s)…") gen = vlm_synthesize_stream(regions, prompt, frames, max_words, mode, reliable=reliable, speaker_note=speaker_note, digest=digest) else: print(f"[NINA] Running text synthesis [{mode}]...") gen = llm_synthesize_stream(regions, prompt, max_words, mode, reliable=reliable, speaker_note=speaker_note, digest=digest) yield format_output(regions, prompt, "✍️ …", mode) last = "" for partial in gen: last = partial yield format_output(regions, prompt, partial, mode) if not last: yield format_output(regions, prompt, _fallback_summary(regions), mode) # ── Main pipeline ────────────────────────────────────────────────────────────── def run(video_path: str, prompt: str, token: str = ""): """Public API endpoint: verify the single-use token, then run the pipeline.""" if not verify_usage_token(token, "run"): yield ( "β›” This request is not authorized.\n\n" "Please use NINA at https://app.nnlabs.pl/nina (sign in first). " "Direct API access is disabled, and each account has a usage limit." ) return yield from _run_core(video_path, prompt) def transcript_text(video_path: str) -> str: """The cached transcript for a video as timestamped text (for chat storage).""" entry = _cache.get(video_path) if not entry: return "" return "\n".join( f"[{fmt_time(s['start'])}] {s['text']}" for s in (entry.get("segments") or []) if s.get("text") ) def _run_core(video_path: str, prompt: str): """The pipeline without the token check β€” reused by the chat endpoint, which has already authorized the turn (so it must not re-consume the single-use token).""" # Every refusal starts with β›”: the website shows it as an error and does not store # or count it as an answer. if video_path is None: yield "β›” Please upload a video." return if not prompt or not prompt.strip(): yield "β›” Please enter a prompt." return if too_large(video_path): yield ( f"β›” This video is too large ({file_mb(video_path):.0f} MB).\n\n" f"The maximum is {MAX_UPLOAD_MB} MB. Please upload a shorter clip, or compress " f"it first (the app at https://app.nnlabs.pl/nina shrinks large videos for you)." ) return if video_path not in _cache: dur = media_duration(video_path) if dur > MAX_VIDEO_SECONDS: yield ( f"β›” This video is {int(dur // 60)} minutes long. The limit is " f"{MAX_VIDEO_SECONDS // 60} minutes: please trim it or upload a shorter part." ) return prompt = prompt.strip() max_words = prompt_to_words(prompt) print(f'\n[NINA] ── New request ── {len(prompt)} chars (target β‰ˆ {max_words} words)') evict_stale() # Stage 1–3: transcription + chunking + CLIP (cached) fp = None if video_path in _cache: print("[NINA] Cache hit (memory) β€” skipping transcription + CLIP.") entry = _cache[video_path] else: try: fp = fingerprint(video_path) except Exception: fp = None # One ingestion per video: a concurrent request for the SAME video waits here # (yielding keepalives) instead of redundantly re-transcribing it, then reuses # the cached result. lock = _ingest_lock(video_path) while not lock.acquire(timeout=10): yield "🎬 …" handed_off = False # True once the ingest thread owns the lock try: if video_path in _cache: print("[NINA] Cache hit after wait β€” another request ingested it.") entry = _cache[video_path] else: disk = disk_load(fp) if fp else None if disk is not None: disk["audio"] = pack_audio(load_audio(video_path)) entry = disk _cache[video_path] = entry else: yield "🎬 …" # Ingest in a background thread so we can stream progress while the # parallel pass runs (keeps the SSE alive and feeds the client the # live stage). # # The thread caches its own result and releases the lock itself. # Both used to happen here, after the wait loop, which only runs if # someone is still listening: when the website's connection dropped # mid-ingest (2026-09-19, the Worker cancelled 30 s after the browser # went away), Gradio closed this generator, the finished transcript # and CLIP pass were thrown away, and the retry started from zero. # Now the work survives the listener, and a retry for the same video # waits on the lock while the ingest finishes, then hits the cache. _res: dict = {} _evt = threading.Event() _stage = _Stage() def _do_ingest(): try: got = ingest_video(video_path, _stage) # No speech is fine β€” we fall back to visual-only synthesis below. _cache[video_path] = got if fp: disk_save(fp, got) print("[NINA] Cached transcript + visuals for this video.") _res["v"] = got except Exception as e: # Logged, not just stored. The caller turns this into # "Error reading video." and returns, so without this line a # complete failure of the ingest leaves nothing anywhere: no # message worth reading for the user, and no trace at all for # whoever has to find out why (2026-09-21, every analysis # failing in one second and the logs showing only the request). import traceback print(f"[NINA] ingest FAILED for {str(video_path).split('/')[-1]}: " f"{type(e).__name__}: {e}", flush=True) traceback.print_exc() _res["err"] = e finally: _evt.set() lock.release() _drop_ingest_lock(video_path) handed_off = True threading.Thread(target=_do_ingest, daemon=True).start() while not _evt.wait(2.0): yield _stage.get() # live stage marker; client keeps it out of the bubble if "err" in _res: yield "β›” NINA couldn't read this video (unsupported or damaged file). Try another file or format." return entry = _res["v"] finally: if not handed_off: lock.release() entry["_atime"] = time.time() _merge_speakers(entry, fp) segments = entry["segments"] chunks = entry["chunks"] visual = entry["visual"] precise_cache = entry["precise"] audio = unpack_audio(entry["audio"]) # float32 only for this request if fp is None: try: fp = fingerprint(video_path) except Exception: fp = None if not chunks: # No usable speech anywhere β€” answer from the visuals alone. The VLM still # sees sampled keyframes and reads any on-screen text, so questions like # "what does the person do?" work on silent / music-only clips. print("[NINA] No transcript β€” visual-only synthesis (VLM on keyframes).") yield from _run_visual_only(video_path, prompt, visual, max_words) return # How trustworthy is the transcript? On garbled/musical audio we tell synthesis # to flag the audio as unclear rather than report Whisper's fabrications as fact. reliability = transcript_reliability(segments) reliable = reliability["reliable"] if not reliable: print(f"[NINA] ⚠ Transcript looks UNRELIABLE {reliability} β€” synthesis will hedge.") # Speaker labels (from diarization) β†’ friendly display names + a roster mapping a # speaker to a name when they introduced themselves anywhere in the video. Both # are fed to synthesis so it can attribute lines and resolve names. spk_display = speaker_display_map(segments) spk_roster = name_roster(segments, spk_display) if spk_display else {} spk_note = speaker_note(spk_display, spk_roster) if spk_note: print(f"[NINA] {spk_note}") # Pass 1 routing mode = classify_query(prompt) print(f"[NINA] Query mode (pass 1): {mode}") if mode == "global": regions = _run_global(chunks, visual, prompt, max_words) annotate_region_speakers(regions, segments, spk_display) yield "🎬 Reading the whole transcript…" digest = transcript_digest(chunks, prompt) frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable, speaker_note=spk_note, frames=frames, digest=digest) print(f"[NINA] Done β€” {len(regions)} region(s) [global].") return # SPECIFIC / HYBRID retrieval yield "🎬 Finding the key moments…" candidates, best_bi = bi_encode_retrieve(chunks, prompt) # Upgrade any candidate whose precise text is already cached (improves cross-encoder accuracy) for c in candidates: key = round(c["start"], 2) if key in precise_cache: c["text"] = precise_cache[key] _, v_norm = visual_scores(visual, prompt) # Pass 2 routing if best_bi < BI_SCORE_GLOBAL_FLOOR: print(f"[NINA] bi-score {best_bi:.3f} below floor β†’ escalating to global.") regions = _run_global(chunks, visual, prompt, max_words) annotate_region_speakers(regions, segments, spk_display) yield "🎬 Reading the whole transcript…" digest = transcript_digest(chunks, prompt) frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable, speaker_note=spk_note, frames=frames, digest=digest) print(f"[NINA] Done β€” {len(regions)} region(s) [escalated global].") return elif best_bi < BI_SCORE_HYBRID_FLOOR: mode = "hybrid" print(f"[NINA] Query mode (pass 2): {mode} bi={best_bi:.3f}") results = rerank_and_score(candidates, prompt, visual["time"], v_norm) if mode == "hybrid": results = pad_with_neighbours(results, chunks) regions = merge_regions(results) if not regions: print("[NINA] No regions after merge β†’ falling back to global.") regions = _run_global(chunks, visual, prompt, max_words) annotate_region_speakers(regions, segments, spk_display) frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None yield from _synthesize_streaming(regions, prompt, max_words, "global", reliable=reliable, speaker_note=spk_note, frames=frames) return # Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual # tagging (CLIP) are independent β€” one fills region["precise_text"], the other # region["visual_tags"] β€” so we run them together and wait for both before the # summary fuses audio + visual. yield "🎬 Re-checking the details…" before = dict(precise_cache) def _precise_branch(): fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX) def _tags_branch(): add_visual_tags(regions, visual) # mutates regions in place with ThreadPoolExecutor(max_workers=2) as ex: for f in [ex.submit(_precise_branch), ex.submit(_tags_branch)]: f.result() if fp and precise_cache != before: disk_update_precise(fp, precise_cache) # Stage 8: VLM synthesis (the model sees keyframes + reads the transcript) annotate_region_speakers(regions, segments, spk_display) frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None yield from _synthesize_streaming(regions, prompt, max_words, mode, reliable=reliable, speaker_note=spk_note, frames=frames) print(f"[NINA] Done β€” {len(regions)} region(s) [{mode}].")