Spaces:
Paused
Paused
Malina
Claude Fable 5.1
Lazy diarization: the first answer no longer waits for pyannote; speaker labels attach to the cached transcript when ready
1d90084 Download pipeline.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 25.2 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/pipeline.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/pipeline.py
-
curl -L -o pipeline.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/pipeline.py
25.2 kB
| import threading | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from auth import verify_usage_token | |
| from audio import ( | |
| fill_precise, load_audio, media_duration, pack_audio, transcribe_fast, | |
| transcript_reliability, unpack_audio, | |
| ) | |
| from cache import _cache, disk_load, disk_save, disk_update_precise, evict_stale, fingerprint | |
| from compress import ensure_compressed, file_mb, too_large | |
| from config import ( | |
| RETRANSCRIBE_MAX, BI_SCORE_GLOBAL_FLOOR, BI_SCORE_HYBRID_FLOOR, | |
| MAX_UPLOAD_MB, MAX_VIDEO_SECONDS, USE_DIARIZATION, USE_SUBTITLES, USE_VLM, | |
| USE_VOCAL_ISOLATION, | |
| ) | |
| from diarize import ( | |
| annotate_region_speakers, attach_speakers, diarize, name_roster, | |
| speaker_display_map, speaker_note, | |
| ) | |
| from enhance import isolate_vocals | |
| from frames import keyframes_for | |
| from subtitles import extract_embedded_subs | |
| from utils import fmt_time | |
| from retrieval import ( | |
| add_entities_to_chunks, bi_encode_retrieve, classify_query, | |
| merge_regions, pad_with_neighbours, rerank_and_score, | |
| sample_chunks_for_global, semantic_chunk, | |
| ) | |
| from synthesis import ( | |
| _fallback_summary, format_output, llm_synthesize_stream, prompt_to_words, | |
| transcript_digest, vlm_synthesize_stream, words_to_global_chunks, | |
| ) | |
| from visual import add_visual_tags, analyze_visual, visual_scores | |
| # Per-video ingestion locks β ensure a video is ingested once even if two requests | |
| # for it arrive together (newest-session takeover, double-tap, queue retry). | |
| _ingest_locks: dict = {} | |
| _ingest_guard = threading.Lock() | |
| # Speaker diarization is the slowest branch of an ingest on CPU and the first answer | |
| # does not need it. It runs here, one video at a time, after the answerable parts are | |
| # done; the labels are attached to the cached transcript when it finishes. | |
| _DIAR_POOL = ThreadPoolExecutor(max_workers=1, thread_name_prefix="nina-diar") | |
| def _ingest_lock(key: str) -> threading.Lock: | |
| with _ingest_guard: | |
| lk = _ingest_locks.get(key) | |
| if lk is None: | |
| lk = threading.Lock() | |
| _ingest_locks[key] = lk | |
| return lk | |
| def _drop_ingest_lock(key: str) -> None: | |
| """Forget the lock once its ingest is over; waiters hold their own reference.""" | |
| with _ingest_guard: | |
| _ingest_locks.pop(key, None) | |
| def _stage_log(name: str, t0: float) -> None: | |
| print(f"[NINA ingest] stage={name} ms={int((time.time() - t0) * 1000)}", flush=True) | |
| class _Stage: | |
| """Thread-safe holder for the current ingest stage. The ingest branches run in | |
| parallel in a background thread; _run_core polls this every couple of seconds and | |
| streams the marker so the website can show what NINA is actually doing. The base | |
| "π¬ β¦" means "the parallel pass is running" (the client rotates transcribe β₯ | |
| scenes β₯ speakers); a worded marker is a genuine sequential step (subtitles, | |
| vocal isolation, OCR). All markers stay π¬-prefixed so clients keep them out of | |
| the chat bubble.""" | |
| def __init__(self) -> None: | |
| self._lock = threading.Lock() | |
| self._marker = "π¬ β¦" | |
| def set(self, marker: str) -> None: | |
| with self._lock: | |
| self._marker = marker | |
| def get(self) -> str: | |
| with self._lock: | |
| return self._marker | |
| # ββ Cold-start ingestion βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def ingest_video(video_path: str, stage: "_Stage | None" = None) -> dict: | |
| """ | |
| First sight of a video: run the WHISPER (audio) and CLIP (visual) branches | |
| fully in parallel, then return once both finish so synthesis can fuse them. | |
| Both branches start at t=0 β the audio branch owns the (potentially slow) audio | |
| decode, and the visual branch re-decodes its own frames via ffmpeg, so neither | |
| waits on the other. faster-whisper (ctranslate2) and CLIP (torch) both release | |
| the GIL during their heavy native compute, so the two threads genuinely overlap. | |
| `stage` (optional) is updated as the conditional, sequential steps fire so the | |
| caller can stream real progress to the UI. | |
| """ | |
| if stage is None: | |
| stage = _Stage() | |
| # Big uploads get an aggressive 720p re-encode first (best-effort, no-op when | |
| # already small) so audio decode + frame sampling run on a lean file. | |
| t_all = time.time() | |
| video_path = ensure_compressed(video_path) | |
| _stage_log("compress", t_all) | |
| # Decode once up front β transcription AND diarization both need the waveform. | |
| t0 = time.time() | |
| audio = load_audio(video_path) | |
| _stage_log("decode_audio", t0) | |
| result = {"audio": audio} | |
| def _audio_branch(): | |
| # WHISPER fast pass, then a fallback ladder when it looks unreliable: | |
| # 1. a clean embedded subtitle track (best, if the file has one), else | |
| # 2. vocal isolation + re-transcription (pull the voice out of the music). | |
| # No waveform at all means the container had no audio stream. Whisper has | |
| # nothing to do with an empty array, and asking it anyway is how a silent | |
| # screen recording turns into a failed build. | |
| t0 = time.time() | |
| segments = transcribe_fast(audio) if getattr(audio, "size", 0) else [] | |
| _stage_log("whisper_fast", t0) | |
| if segments: | |
| rel = transcript_reliability(segments) | |
| if not rel["reliable"]: | |
| used = False | |
| if USE_SUBTITLES: | |
| stage.set("π¬ Reading the subtitle trackβ¦") | |
| subs = extract_embedded_subs(video_path) | |
| if subs: | |
| print(f"[NINA] Transcript weak (score {rel['score']}) β using the " | |
| f"embedded subtitle track instead.") | |
| segments, used = subs, True | |
| if not used and USE_VOCAL_ISOLATION: | |
| print(f"[NINA] Transcript weak (score {rel['score']}) β trying vocal isolationβ¦") | |
| stage.set("π¬ Isolating the vocalsβ¦") | |
| vocals = isolate_vocals(audio) | |
| if vocals is not None: | |
| seg2 = transcribe_fast(vocals) | |
| if seg2 and transcript_reliability(seg2)["score"] > rel["score"]: | |
| print("[NINA] Vocal isolation improved the transcript β using it.") | |
| segments = seg2 | |
| result["segments"] = segments | |
| result["chunks"] = add_entities_to_chunks(semantic_chunk(segments)) if segments else [] | |
| def _visual_branch(): | |
| # CLIP: sample frames β encode. | |
| t0 = time.time() | |
| result["visual"] = analyze_visual(video_path, media_duration(video_path)) | |
| _stage_log("clip", t0) | |
| def _diar_branch(): | |
| # pyannote: who-spoke-when (best-effort; [] if unavailable). | |
| t0 = time.time() | |
| turns = diarize(audio) if USE_DIARIZATION else [] | |
| _stage_log("diarization", t0) | |
| return turns | |
| print("[NINA] Ingesting β Whisper (audio) β₯ CLIP (visual) in parallel; speakers in the backgroundβ¦") | |
| with ThreadPoolExecutor(max_workers=2) as ex: | |
| futures = [ex.submit(_audio_branch), ex.submit(_visual_branch)] | |
| for f in futures: | |
| f.result() # block until BOTH finish; re-raise the first branch error | |
| # Diarization used to be the third parallel branch and the whole ingest waited for | |
| # it: on CPU that is minutes for a long recording, spent before the first answer. | |
| # Now the first answer goes out without speaker labels and _merge_speakers attaches | |
| # them to the cached transcript once this finishes. | |
| diar = None | |
| if USE_DIARIZATION and getattr(audio, "size", 0) and result["segments"]: | |
| diar = _DIAR_POOL.submit(_diar_branch) | |
| _stage_log("ingest_total", t_all) | |
| print("[NINA] Ingest done β audio and visuals ready" + (", speakers pending." if diar else ".")) | |
| return { | |
| "segments": result["segments"], | |
| "chunks": result["chunks"], | |
| "visual": result["visual"], | |
| "audio": pack_audio(audio), # int16 in the cache: half the memory | |
| "precise": {}, | |
| "turns": [] if diar is None else None, | |
| "_diar": diar, | |
| } | |
| def _merge_speakers(entry: dict, fp) -> None: | |
| """Attach the speaker labels once the background diarization has finished. A | |
| request that arrives earlier simply answers without them.""" | |
| fut = entry.get("_diar") | |
| if fut is None or not fut.done(): | |
| return | |
| entry["_diar"] = None | |
| try: | |
| turns = fut.result() or [] | |
| except Exception as e: | |
| print(f"[NINA] diarization failed: {type(e).__name__}", flush=True) | |
| turns = [] | |
| entry["turns"] = turns | |
| if turns: | |
| attach_speakers(entry["segments"], turns) | |
| if fp: | |
| try: | |
| disk_save(fp, entry) | |
| except Exception: | |
| pass | |
| print(f"[NINA] Speakers attached ({len(turns)} turns).", flush=True) | |
| # ββ Global path ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _run_global(chunks: list[dict], visual: dict, | |
| prompt: str, max_words: int) -> list[dict]: | |
| """Sampled regions carry the frames and the quotable fragments; the whole-video | |
| digest (see transcript_digest) rides alongside so a summary covers the video and | |
| not a few minutes of it.""" | |
| sampled = sample_chunks_for_global(chunks, words_to_global_chunks(max_words)) | |
| print(f"[NINA] Global mode β sampled {len(sampled)} chunks evenly (base transcript).") | |
| for c in sampled: | |
| c.setdefault("confidence", 0.5) | |
| c.setdefault("bi_score", 0.0) | |
| c.setdefault("entity_overlap", 0.0) | |
| c.setdefault("visual_score", 0.0) | |
| c["precise_text"] = c.get("text", "") | |
| return add_visual_tags(sampled, visual) | |
| # ββ Visual-only path (no transcript) βββββββββββββββββββββββββββββββββββββββββββ | |
| def _run_visual_only(video_path: str, prompt: str, visual: dict, max_words: int): | |
| """No transcript available β synthesize from the visuals alone. One full-span | |
| region carries the VLM's keyframes + visual tags so it can answer about what's | |
| on screen (actions, objects, on-screen text) without any speech.""" | |
| dur = media_duration(video_path) or 0.0 | |
| if not dur: | |
| times = visual.get("time") or [] | |
| dur = float(times[-1]) if times else 0.0 | |
| region = { | |
| "start": 0.0, "end": float(dur), "text": "", "precise_text": "", | |
| "confidence": 0.5, "bi_score": 0.0, "entity_overlap": 0.0, "visual_score": 0.0, | |
| } | |
| regions = add_visual_tags([region], visual) | |
| frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None | |
| note = "(No speech detected in this video β answer from what is visible on screen.)" | |
| yield from _synthesize_streaming(regions, prompt, max_words, "global", | |
| reliable=False, speaker_note=note, frames=frames) | |
| # ββ Streaming synthesis wrapper ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _synthesize_streaming(regions: list[dict], prompt: str, | |
| max_words: int, mode: str, reliable: bool = True, | |
| speaker_note: str = "", frames: list = None, digest: str = ""): | |
| # Primary path: the VLM actually sees the keyframes. Falls back to text-only | |
| # synthesis inside vlm_synthesize_stream if the VLM is unavailable. | |
| if USE_VLM and frames: | |
| print(f"[NINA] Running VLM synthesis [{mode}] with {len(frames)} frame(s)β¦") | |
| gen = vlm_synthesize_stream(regions, prompt, frames, max_words, mode, | |
| reliable=reliable, speaker_note=speaker_note, digest=digest) | |
| else: | |
| print(f"[NINA] Running text synthesis [{mode}]...") | |
| gen = llm_synthesize_stream(regions, prompt, max_words, mode, | |
| reliable=reliable, speaker_note=speaker_note, digest=digest) | |
| yield format_output(regions, prompt, "βοΈ β¦", mode) | |
| last = "" | |
| for partial in gen: | |
| last = partial | |
| yield format_output(regions, prompt, partial, mode) | |
| if not last: | |
| yield format_output(regions, prompt, _fallback_summary(regions), mode) | |
| # ββ Main pipeline ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def run(video_path: str, prompt: str, token: str = ""): | |
| """Public API endpoint: verify the single-use token, then run the pipeline.""" | |
| if not verify_usage_token(token, "run"): | |
| yield ( | |
| "β This request is not authorized.\n\n" | |
| "Please use NINA at https://app.nnlabs.pl/nina (sign in first). " | |
| "Direct API access is disabled, and each account has a usage limit." | |
| ) | |
| return | |
| yield from _run_core(video_path, prompt) | |
| def transcript_text(video_path: str) -> str: | |
| """The cached transcript for a video as timestamped text (for chat storage).""" | |
| entry = _cache.get(video_path) | |
| if not entry: | |
| return "" | |
| return "\n".join( | |
| f"[{fmt_time(s['start'])}] {s['text']}" | |
| for s in (entry.get("segments") or []) if s.get("text") | |
| ) | |
| def _run_core(video_path: str, prompt: str): | |
| """The pipeline without the token check β reused by the chat endpoint, which has | |
| already authorized the turn (so it must not re-consume the single-use token).""" | |
| # Every refusal starts with β: the website shows it as an error and does not store | |
| # or count it as an answer. | |
| if video_path is None: | |
| yield "β Please upload a video." | |
| return | |
| if not prompt or not prompt.strip(): | |
| yield "β Please enter a prompt." | |
| return | |
| if too_large(video_path): | |
| yield ( | |
| f"β This video is too large ({file_mb(video_path):.0f} MB).\n\n" | |
| f"The maximum is {MAX_UPLOAD_MB} MB. Please upload a shorter clip, or compress " | |
| f"it first (the app at https://app.nnlabs.pl/nina shrinks large videos for you)." | |
| ) | |
| return | |
| if video_path not in _cache: | |
| dur = media_duration(video_path) | |
| if dur > MAX_VIDEO_SECONDS: | |
| yield ( | |
| f"β This video is {int(dur // 60)} minutes long. The limit is " | |
| f"{MAX_VIDEO_SECONDS // 60} minutes: please trim it or upload a shorter part." | |
| ) | |
| return | |
| prompt = prompt.strip() | |
| max_words = prompt_to_words(prompt) | |
| print(f'\n[NINA] ββ New request ββ {len(prompt)} chars (target β {max_words} words)') | |
| evict_stale() | |
| # Stage 1β3: transcription + chunking + CLIP (cached) | |
| fp = None | |
| if video_path in _cache: | |
| print("[NINA] Cache hit (memory) β skipping transcription + CLIP.") | |
| entry = _cache[video_path] | |
| else: | |
| try: | |
| fp = fingerprint(video_path) | |
| except Exception: | |
| fp = None | |
| # One ingestion per video: a concurrent request for the SAME video waits here | |
| # (yielding keepalives) instead of redundantly re-transcribing it, then reuses | |
| # the cached result. | |
| lock = _ingest_lock(video_path) | |
| while not lock.acquire(timeout=10): | |
| yield "π¬ β¦" | |
| handed_off = False # True once the ingest thread owns the lock | |
| try: | |
| if video_path in _cache: | |
| print("[NINA] Cache hit after wait β another request ingested it.") | |
| entry = _cache[video_path] | |
| else: | |
| disk = disk_load(fp) if fp else None | |
| if disk is not None: | |
| disk["audio"] = pack_audio(load_audio(video_path)) | |
| entry = disk | |
| _cache[video_path] = entry | |
| else: | |
| yield "π¬ β¦" | |
| # Ingest in a background thread so we can stream progress while the | |
| # parallel pass runs (keeps the SSE alive and feeds the client the | |
| # live stage). | |
| # | |
| # The thread caches its own result and releases the lock itself. | |
| # Both used to happen here, after the wait loop, which only runs if | |
| # someone is still listening: when the website's connection dropped | |
| # mid-ingest (2026-09-19, the Worker cancelled 30 s after the browser | |
| # went away), Gradio closed this generator, the finished transcript | |
| # and CLIP pass were thrown away, and the retry started from zero. | |
| # Now the work survives the listener, and a retry for the same video | |
| # waits on the lock while the ingest finishes, then hits the cache. | |
| _res: dict = {} | |
| _evt = threading.Event() | |
| _stage = _Stage() | |
| def _do_ingest(): | |
| try: | |
| got = ingest_video(video_path, _stage) | |
| # No speech is fine β we fall back to visual-only synthesis below. | |
| _cache[video_path] = got | |
| if fp: | |
| disk_save(fp, got) | |
| print("[NINA] Cached transcript + visuals for this video.") | |
| _res["v"] = got | |
| except Exception as e: | |
| # Logged, not just stored. The caller turns this into | |
| # "Error reading video." and returns, so without this line a | |
| # complete failure of the ingest leaves nothing anywhere: no | |
| # message worth reading for the user, and no trace at all for | |
| # whoever has to find out why (2026-09-21, every analysis | |
| # failing in one second and the logs showing only the request). | |
| import traceback | |
| print(f"[NINA] ingest FAILED for {str(video_path).split('/')[-1]}: " | |
| f"{type(e).__name__}: {e}", flush=True) | |
| traceback.print_exc() | |
| _res["err"] = e | |
| finally: | |
| _evt.set() | |
| lock.release() | |
| _drop_ingest_lock(video_path) | |
| handed_off = True | |
| threading.Thread(target=_do_ingest, daemon=True).start() | |
| while not _evt.wait(2.0): | |
| yield _stage.get() # live stage marker; client keeps it out of the bubble | |
| if "err" in _res: | |
| yield "β NINA couldn't read this video (unsupported or damaged file). Try another file or format." | |
| return | |
| entry = _res["v"] | |
| finally: | |
| if not handed_off: | |
| lock.release() | |
| entry["_atime"] = time.time() | |
| _merge_speakers(entry, fp) | |
| segments = entry["segments"] | |
| chunks = entry["chunks"] | |
| visual = entry["visual"] | |
| precise_cache = entry["precise"] | |
| audio = unpack_audio(entry["audio"]) # float32 only for this request | |
| if fp is None: | |
| try: | |
| fp = fingerprint(video_path) | |
| except Exception: | |
| fp = None | |
| if not chunks: | |
| # No usable speech anywhere β answer from the visuals alone. The VLM still | |
| # sees sampled keyframes and reads any on-screen text, so questions like | |
| # "what does the person do?" work on silent / music-only clips. | |
| print("[NINA] No transcript β visual-only synthesis (VLM on keyframes).") | |
| yield from _run_visual_only(video_path, prompt, visual, max_words) | |
| return | |
| # How trustworthy is the transcript? On garbled/musical audio we tell synthesis | |
| # to flag the audio as unclear rather than report Whisper's fabrications as fact. | |
| reliability = transcript_reliability(segments) | |
| reliable = reliability["reliable"] | |
| if not reliable: | |
| print(f"[NINA] β Transcript looks UNRELIABLE {reliability} β synthesis will hedge.") | |
| # Speaker labels (from diarization) β friendly display names + a roster mapping a | |
| # speaker to a name when they introduced themselves anywhere in the video. Both | |
| # are fed to synthesis so it can attribute lines and resolve names. | |
| spk_display = speaker_display_map(segments) | |
| spk_roster = name_roster(segments, spk_display) if spk_display else {} | |
| spk_note = speaker_note(spk_display, spk_roster) | |
| if spk_note: | |
| print(f"[NINA] {spk_note}") | |
| # Pass 1 routing | |
| mode = classify_query(prompt) | |
| print(f"[NINA] Query mode (pass 1): {mode}") | |
| if mode == "global": | |
| regions = _run_global(chunks, visual, prompt, max_words) | |
| annotate_region_speakers(regions, segments, spk_display) | |
| yield "π¬ Reading the whole transcriptβ¦" | |
| digest = transcript_digest(chunks, prompt) | |
| frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None | |
| yield from _synthesize_streaming(regions, prompt, max_words, "global", | |
| reliable=reliable, speaker_note=spk_note, frames=frames, | |
| digest=digest) | |
| print(f"[NINA] Done β {len(regions)} region(s) [global].") | |
| return | |
| # SPECIFIC / HYBRID retrieval | |
| yield "π¬ Finding the key momentsβ¦" | |
| candidates, best_bi = bi_encode_retrieve(chunks, prompt) | |
| # Upgrade any candidate whose precise text is already cached (improves cross-encoder accuracy) | |
| for c in candidates: | |
| key = round(c["start"], 2) | |
| if key in precise_cache: | |
| c["text"] = precise_cache[key] | |
| _, v_norm = visual_scores(visual, prompt) | |
| # Pass 2 routing | |
| if best_bi < BI_SCORE_GLOBAL_FLOOR: | |
| print(f"[NINA] bi-score {best_bi:.3f} below floor β escalating to global.") | |
| regions = _run_global(chunks, visual, prompt, max_words) | |
| annotate_region_speakers(regions, segments, spk_display) | |
| yield "π¬ Reading the whole transcriptβ¦" | |
| digest = transcript_digest(chunks, prompt) | |
| frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None | |
| yield from _synthesize_streaming(regions, prompt, max_words, "global", | |
| reliable=reliable, speaker_note=spk_note, frames=frames, | |
| digest=digest) | |
| print(f"[NINA] Done β {len(regions)} region(s) [escalated global].") | |
| return | |
| elif best_bi < BI_SCORE_HYBRID_FLOOR: | |
| mode = "hybrid" | |
| print(f"[NINA] Query mode (pass 2): {mode} bi={best_bi:.3f}") | |
| results = rerank_and_score(candidates, prompt, visual["time"], v_norm) | |
| if mode == "hybrid": | |
| results = pad_with_neighbours(results, chunks) | |
| regions = merge_regions(results) | |
| if not regions: | |
| print("[NINA] No regions after merge β falling back to global.") | |
| regions = _run_global(chunks, visual, prompt, max_words) | |
| annotate_region_speakers(regions, segments, spk_display) | |
| frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None | |
| yield from _synthesize_streaming(regions, prompt, max_words, "global", | |
| reliable=reliable, speaker_note=spk_note, frames=frames) | |
| return | |
| # Stage 7 + 7b in parallel: precise re-transcription (WHISPER) and visual | |
| # tagging (CLIP) are independent β one fills region["precise_text"], the other | |
| # region["visual_tags"] β so we run them together and wait for both before the | |
| # summary fuses audio + visual. | |
| yield "π¬ Re-checking the detailsβ¦" | |
| before = dict(precise_cache) | |
| def _precise_branch(): | |
| fill_precise(audio, regions, chunks, precise_cache, RETRANSCRIBE_MAX) | |
| def _tags_branch(): | |
| add_visual_tags(regions, visual) # mutates regions in place | |
| with ThreadPoolExecutor(max_workers=2) as ex: | |
| for f in [ex.submit(_precise_branch), ex.submit(_tags_branch)]: | |
| f.result() | |
| if fp and precise_cache != before: | |
| disk_update_precise(fp, precise_cache) | |
| # Stage 8: VLM synthesis (the model sees keyframes + reads the transcript) | |
| annotate_region_speakers(regions, segments, spk_display) | |
| frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None | |
| yield from _synthesize_streaming(regions, prompt, max_words, mode, | |
| reliable=reliable, speaker_note=spk_note, frames=frames) | |
| print(f"[NINA] Done β {len(regions)} region(s) [{mode}].") | |