import re from concurrent.futures import ThreadPoolExecutor from persona import PERSONA_TEXT, PERSONA_VISION from config import ( DEBUG, GLOBAL_MAX_CHUNKS, LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE, VLM_MODEL, ) from lang import detect_language as _detect_language from models import llm_client from utils import conf_bar, fmt_time # ── Output size inference ────────────────────────────────────────────────────── _BRIEF_CUES = ( "brief", "briefly", "short", "shortly", "quick", "quickly", "concise", "in a sentence", "one sentence", "one-liner", "tl;dr", "tldr", "krótko", "w skrócie", "zwięźle", "jednym zdaniem", "w jednym zdaniu", ) _DETAIL_CUES = ( "in detail", "detailed", "in-depth", "in depth", "thorough", "thoroughly", "comprehensive", "exhaustive", "everything", "full summary", "elaborate", "step by step", "step-by-step", "deep dive", "as much as possible", "szczegółowo", "dokładnie", "wyczerpująco", "wszystko", "krok po kroku", "dogłębnie", "obszernie", ) _FACTUAL_START = re.compile( r'^\s*(what|who|whom|whose|when|where|which|how many|how much|how long|' r'is|are|was|were|does|do|did|can|could|will|would|' r'czy|kiedy|kto|kogo|gdzie|ile|jaki|jaka|jakie|który|która|które)\b', re.IGNORECASE, ) def _summary_tier(max_words: int) -> str: if max_words <= 60: return "brief / one-liner" if max_words <= 110: return "short" if max_words <= 250: return "standard" if max_words <= 400: return "detailed" return "exhaustive" def prompt_to_words(prompt: str) -> int: """Infer answer word budget from prompt cues. Clamped to [30, 500].""" p = prompt.lower() if any(c in p for c in _DETAIL_CUES): base, reason = 450, "explicit detail cue" elif any(c in p for c in _BRIEF_CUES): base, reason = 40, "explicit brevity cue" elif _FACTUAL_START.match(p): base, reason = 80, "factual question" else: n_words = len(re.findall(r'\w+', p)) base, reason = 120 + min(180, n_words * 8), f"length-scaled ({n_words} prompt words)" out = int(max(30, min(500, base))) if DEBUG: print(f"[DEBUG] Summary sizing: {reason} → {out} words [{_summary_tier(out)}]") return out def words_to_global_chunks(max_words: int) -> int: return max(3, min(GLOBAL_MAX_CHUNKS, round(max_words / 40))) # ── Whole-video digest (global mode) ────────────────────────────────────────── # A "summarize the video" used to be written from 3 to 12 sampled chunks of at most 90 # seconds each: a 45-minute recording was summarised from a few minutes of it. The # digest reads the WHOLE transcript in windows with a cheap text call each (in # parallel) and hands the notes to the synthesis call alongside the sampled regions. DIGEST_WINDOW_CHARS = 12000 DIGEST_MAX_WINDOWS = 12 DIGEST_MAX_TOKENS = 400 DIGEST_PARALLEL = 6 def _digest_window(text: str, i: int, n: int, language: str) -> str: try: r = llm_client.chat_completion( model=VLM_MODEL, messages=[ {"role": "system", "content": f"You take notes on part {i} of {n} of a video transcript. Write 4 to 8 " f"terse bullet points in {language}: who says what, decisions, numbers, names, " f"topics, in order. Keep the [m:ss] timestamp of each point from the " f"transcript lines. No preamble, no commentary, nothing not in the text."}, {"role": "user", "content": text}, ], max_tokens=DIGEST_MAX_TOKENS, temperature=0.0, stream=False, ) return (r.choices[0].message.content or "").strip() except Exception as e: print(f"[NINA] digest window {i}/{n} failed: {type(e).__name__}", flush=True) return "" def transcript_digest(chunks: list[dict], prompt: str) -> str: """Notes over the entire transcript, or "" when it fits in the sampled regions anyway.""" lines = [f"[{fmt_time(c['start'])}] {c.get('text', '')}" for c in (chunks or []) if c.get("text")] full = "\n".join(lines) if len(full) <= DIGEST_WINDOW_CHARS: return "" windows = [full[i:i + DIGEST_WINDOW_CHARS] for i in range(0, len(full), DIGEST_WINDOW_CHARS)] windows = windows[:DIGEST_MAX_WINDOWS] language = _detect_language(prompt) n = len(windows) print(f"[NINA] Global digest: {len(full)} chars in {n} window(s).", flush=True) with ThreadPoolExecutor(max_workers=min(DIGEST_PARALLEL, n)) as ex: notes = list(ex.map(lambda a: _digest_window(a[1], a[0] + 1, n, language), enumerate(windows))) notes = [x for x in notes if x] if not notes: return "" head = ("Notes over the WHOLE transcript, in order" + (f" (first {DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS} characters)" if len(full) > DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS else "") + ":") return head + "\n" + "\n".join(notes) # ── Synthesis ────────────────────────────────────────────────────────────────── def _fallback_summary(regions: list[dict]) -> str: snippets = [ f"[{fmt_time(r['start'])}] {r.get('precise_text', r['text'])[:200]}" for r in regions[:2] ] return "(LLM unavailable — raw excerpts:)\n" + "\n".join(snippets) # Adaptive answer formatting — let NINA shape the answer to the question (Markdown, # which the chat renders) instead of always emitting one prose block. Shared by the # text and vision synthesis prompts. _FORMAT_GUIDE = ( "CRITICAL OUTPUT FORMAT — your response must be visually structured. Assume the user " "is skimming, not reading word-for-word; optimize for scannability.\n" "Required (use whichever fit the answer): Markdown headings, bullet lists, numbered " "steps, and tables (Markdown `| col | col |`) when comparing items.\n" "Forbidden: large blocks of uninterrupted text; any answer longer than ~5 lines without " "formatting.\n" "(A trivial one-line factual answer may stay plain — don't pad it into a table.)\n" "Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]." ) def _make_system_msg(language: str, max_words: int, tier: str, reliable: bool = True, speaker_note: str = "") -> str: speakers = "" if not speaker_note else ( "\n\nSPEAKERS — " + speaker_note + " Fragments are tagged with their speaker. " "Attribute statements to the correct speaker. If a speaker stated their own name " "(a self-introduction), use that NAME for EVERY line by that speaker — including " "lines where the name isn't repeated. When the question asks who did or said " "something, answer with the specific speaker/name (e.g. who won, who claimed what)." ) audio_quality = "" if reliable else ( "\n\nAUDIO QUALITY (important) — This transcript was auto-extracted from " "low-quality, noisy or distorted audio and is partly UNRELIABLE. Some fragments " "may be garbled or nonsensical (speech-recognition errors), NOT real statements. " "Do not treat incoherent or nonsensical fragments as things anyone actually said, " "and do not invent a meaning for them. If the words don't form coherent sense, say " "plainly (in the user's language) that the audio is too unclear to make out — e.g. " "that you cannot reliably tell who is speaking or what is said. You may still rely on " "any clearly coherent speech and on the on-screen cues." ) return ( PERSONA_TEXT + "\n\n" "THIS ANSWER uses ONLY the transcript fragments and on-screen cues you are given.\n\n" f"LANGUAGE — Write your ENTIRE answer in the same language as the user's question " f"(it looks like {language}). Never switch languages, even if the transcript is in " f"another language: report its content in the user's language. Keep names, numbers, " f"brands, titles and direct quotes exactly as they appear — do not translate, " f"re-spell or reinterpret those.\n\n" f"LENGTH — Fit the answer to the question. Aim for about {max_words} words at most " f"({tier}); be shorter when the video covers little. Never pad. Answer the question " f"directly, with no preamble like \"The video says…\".\n\n" "GROUNDING — Use only what the fragments state. Never invent, infer, guess, or " "complete partial words, names or numbers. If the fragments don't address the " "question, say so plainly (in the user's language) instead of speculating. Don't " "fix grammar inside quoted speech." f"{speakers}" f"{audio_quality}\n\n" f"{_FORMAT_GUIDE}" ) _STRONG_CONF = 0.45 def _render_region(r: dict) -> str: spoken = (r.get("precise_text", r["text"]) or "").strip() tags = r.get("visual_tags", []) speakers = r.get("speakers", []) header = f"[{fmt_time(r['start'])} → {fmt_time(r['end'])}]" if speakers: header += f" ({' / '.join(speakers)})" block = f"{header}\n Spoken: {spoken or '(no speech)'}" if tags: block += f"\n On screen: {', '.join(tags)}" return block def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode: str) -> str: strong = [r for r in regions if r.get("confidence", 1.0) >= _STRONG_CONF] context = [r for r in regions if r.get("confidence", 1.0) < _STRONG_CONF] if not strong: # all regions are weak — treat all as strong strong, context = regions, [] fragments = "\n\n".join(_render_region(r) for r in strong) context_section = "" if context: ctx_blocks = "\n\n".join(_render_region(r) for r in context) context_section = ( "\n\n--- Supporting context (lower relevance) ---\n\n" + ctx_blocks ) if mode == "global": task_line = ( "Task: Give a natural, factual overview of what this video covers, " "drawing only from the fragments below." ) else: task_line = ( f'Task: Answer the user question "{prompt}" using ONLY what the fragments state.' ) return ( f'User question: "{prompt}"\n\n' f"Video fragments (each headed by its [m:ss → m:ss] timestamp):\n\n" f"{fragments}{context_section}\n\n" f"{task_line}\n\n" f"How to write the answer:\n" f"- Reply in the SAME language as the user's question above.\n" f"- Format for readability (per the system instructions): short paragraphs, with a " f"bullet list or a brief heading when the answer is long or has parts. Don't prefix " f"lines with timestamps.\n" f"- Ground every statement only in the fragments; never merge two fragments into a " f"single claim, and don't derive new claims from the supporting-context blocks alone.\n" f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n" f"- If nothing above is relevant, say that plainly in the user's language.\n" f"- End with one line: Sources: followed by the [m:ss] start-times you actually used." ) def llm_synthesize_stream( regions: list[dict], prompt: str, max_words: int = 150, mode: str = "specific", reliable: bool = True, speaker_note: str = "", digest: str = "", ): """Generator: yields the answer incrementally as the LLM streams tokens. `reliable=False` (poor ASR audio) makes the system prompt tell the model to flag unclear audio instead of treating garbled fragments as real speech. `speaker_note` (when multiple speakers were diarized) drives speaker/name attribution. """ if not regions: yield "(No relevant regions to summarize.)" return synth_prompt = _build_synth_prompt(regions, prompt, max_words, mode) if digest: synth_prompt = digest + "\n\n" + synth_prompt if DEBUG: print( f"[DEBUG] Synthesis [{mode}] — {len(regions)} region(s), " f"max_words={max_words} [{_summary_tier(max_words)}]." ) try: acc = "" language = _detect_language(prompt) if DEBUG: print(f"[DEBUG] Detected prompt language: {language}") stream = llm_client.chat_completion( model=LLM_MODEL, messages=[ {"role": "system", "content": _make_system_msg(language, max_words, _summary_tier(max_words), reliable=reliable, speaker_note=speaker_note)}, {"role": "user", "content": synth_prompt}, ], max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True, ) for ev in stream: if not ev.choices: continue delta = ev.choices[0].delta.content or "" if delta: acc += delta yield acc.strip() if not acc.strip(): yield _fallback_summary(regions) return except Exception as e: print(f"[NINA] Synthesis error: {e}") yield _fallback_summary(regions) # ── Vision-Language synthesis (primary) ────────────────────────────────────────── def _transcript_block(regions: list[dict]) -> str: """Plain speech transcript for the VLM (timestamps + speakers, no CLIP tags — the model reads the frames itself).""" lines = [] for r in regions: spoken = (r.get("precise_text", r["text"]) or "").strip() head = f"[{fmt_time(r['start'])} → {fmt_time(r['end'])}]" if r.get("speakers"): head += f" ({' / '.join(r['speakers'])})" lines.append(f"{head} {spoken or '(no clear speech)'}") return "\n".join(lines) if lines else "(no usable speech transcribed)" def _make_vlm_system_msg(language: str, max_words: int, tier: str, reliable: bool, speaker_note: str) -> str: speakers = "" if not speaker_note else ( "\n\nSPEAKERS — " + speaker_note + " Attribute speech to the right speaker; if " "someone states their own name, use it for all of their lines." ) audio = "" if reliable else ( "\n\nAUDIO — The speech transcript is from poor/noisy audio and is partly " "unreliable; don't trust garbled fragments. Lean on what you can SEE in the frames, " "and if neither the audio nor the frames answer the question, say so plainly." ) return ( PERSONA_VISION + "\n\n" "THIS ANSWER uses ONLY (a) the keyframes you are shown and (b) the speech transcript " "provided, nothing else.\n\n" "VISION — Actually look at the frames. Read any on-screen text (titles, captions, " "lower-thirds, names, UI, slides) verbatim, identify who/what is shown and the " "setting, and notice actions/changes across frames. Fuse this with the transcript.\n\n" f"LANGUAGE — Write the ENTIRE answer in the same language as the question (it looks " f"like {language}). Keep names, numbers, brands and on-screen text exactly as they " f"appear.\n\n" f"LENGTH — About {max_words} words at most ({tier}); shorter if there's little to say. " f"Answer directly, no preamble.\n\n" "GROUNDING — State only what is visible in the frames or present in the transcript. " "Never invent names, numbers or facts you cannot see or read.\n" "IDENTITY — Do NOT identify or name a person from their appearance or resemblance to " "someone famous. You have no way to know who someone is unless their name is WRITTEN " "on screen or SPOKEN in the transcript. If it isn't, refer to them generically (e.g. " "\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity " "can't be determined from the video. Never guess a real person's name." f"{speakers}{audio}\n\n" f"{_FORMAT_GUIDE}" ) def vlm_synthesize_stream( regions: list[dict], prompt: str, frames: list[tuple], max_words: int = 150, mode: str = "specific", reliable: bool = True, speaker_note: str = "", digest: str = "", ): """Generator: stream the answer from the vision-language model, given keyframes (list of (timestamp, data_url)) plus the transcript. Falls back to the text-only LLM if the VLM call fails before producing anything.""" language = _detect_language(prompt) transcript = _transcript_block(regions) user_text = ( f'Question: "{prompt}"\n\n' + (f"{digest}\n\n" if digest else "") + f"Speech transcript (may be imperfect):\n{transcript}\n\n" f"Below are {len(frames)} keyframes sampled from the video, each labelled with its " f"timestamp. Use them and the transcript to answer the question." ) content = [{"type": "text", "text": user_text}] for ts, url in frames: content.append({"type": "text", "text": f"Frame at {fmt_time(ts)}:"}) content.append({"type": "image_url", "image_url": {"url": url}}) if DEBUG: print(f"[DEBUG] VLM synthesis — {len(frames)} frame(s), {len(regions)} region(s), " f"lang={language}, max_words={max_words}.") acc = "" try: stream = llm_client.chat_completion( model=VLM_MODEL, messages=[ {"role": "system", "content": _make_vlm_system_msg(language, max_words, _summary_tier(max_words), reliable, speaker_note)}, {"role": "user", "content": content}, ], max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True, ) for ev in stream: if not ev.choices: continue delta = ev.choices[0].delta.content or "" if delta: acc += delta yield acc.strip() if acc.strip(): return # Empty VLM response → fall through to the text model. print("[NINA] VLM returned nothing — falling back to text synthesis.") except Exception as e: if acc.strip(): return # already produced an answer print(f"[NINA] VLM unavailable ({e}) — falling back to text synthesis.") yield from llm_synthesize_stream(regions, prompt, max_words, mode, reliable=reliable, speaker_note=speaker_note, digest=digest) # ── Output formatter ─────────────────────────────────────────────────────────── def format_output(regions: list[dict], prompt: str, summary: str, mode: str = "specific") -> str: if not regions: return ( f' Topic not found in this video.\n\n' f' Prompt : "{prompt}"\n\n' f' No relevant segments found.\n' f' Try a broader or differently worded prompt.' ) mode_tag = { "global": "🌐 global", "hybrid": "⚡ hybrid", "specific": "🎯 specific" }.get(mode, "") lines = [] if DEBUG: lines.append(f' {len(regions)} region(s) found for: "{prompt}" [{mode_tag} · audio+video]\n') for i, r in enumerate(regions, 1): dur = round(r["end"] - r["start"], 1) text = r.get("precise_text", r["text"]) ent = r.get("entity_overlap", 0) bi = r.get("bi_score", 0) vis = r.get("visual_score", 0) tags = ", ".join(r.get("visual_tags", [])) lines += [ f'\n [{i}] {fmt_time(r["start"])} → {fmt_time(r["end"])} ({dur}s)', f' {conf_bar(r["confidence"])} {r["confidence"]:.0%} ' f'(embed {bi:.0%} · entity {ent:.0%} · visual {vis:.0%})', f' 👁 {tags}' if tags else ' 👁 (no visual tags)', f' "{text[:220]}{"…" if len(text) > 220 else ""}"', ] lines.append(summary) if DEBUG: lines.append( '\n Sources: ' + ', '.join( f'[{fmt_time(r["start"])} → {fmt_time(r["end"])}]' for r in regions ) ) return "\n".join(lines)