NINA / synthesis.py
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7
Raw History Blame
20.9 kB
import re
from concurrent.futures import ThreadPoolExecutor
from persona import PERSONA_TEXT, PERSONA_VISION
from config import (
DEBUG, GLOBAL_MAX_CHUNKS, LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE, VLM_MODEL,
)
from lang import detect_language as _detect_language
from models import llm_client
from utils import conf_bar, fmt_time
# ── Output size inference ──────────────────────────────────────────────────────
_BRIEF_CUES = (
"brief", "briefly", "short", "shortly", "quick", "quickly", "concise",
"in a sentence", "one sentence", "one-liner", "tl;dr", "tldr",
"krΓ³tko", "w skrΓ³cie", "zwiΔ™ΕΊle", "jednym zdaniem", "w jednym zdaniu",
)
_DETAIL_CUES = (
"in detail", "detailed", "in-depth", "in depth", "thorough", "thoroughly",
"comprehensive", "exhaustive", "everything", "full summary", "elaborate",
"step by step", "step-by-step", "deep dive", "as much as possible",
"szczegΓ³Ε‚owo", "dokΕ‚adnie", "wyczerpujΔ…co", "wszystko", "krok po kroku",
"dogΕ‚Δ™bnie", "obszernie",
)
_FACTUAL_START = re.compile(
r'^\s*(what|who|whom|whose|when|where|which|how many|how much|how long|'
r'is|are|was|were|does|do|did|can|could|will|would|'
r'czy|kiedy|kto|kogo|gdzie|ile|jaki|jaka|jakie|ktΓ³ry|ktΓ³ra|ktΓ³re)\b',
re.IGNORECASE,
)
def _summary_tier(max_words: int) -> str:
if max_words <= 60: return "brief / one-liner"
if max_words <= 110: return "short"
if max_words <= 250: return "standard"
if max_words <= 400: return "detailed"
return "exhaustive"
def prompt_to_words(prompt: str) -> int:
"""Infer answer word budget from prompt cues. Clamped to [30, 500]."""
p = prompt.lower()
if any(c in p for c in _DETAIL_CUES):
base, reason = 450, "explicit detail cue"
elif any(c in p for c in _BRIEF_CUES):
base, reason = 40, "explicit brevity cue"
elif _FACTUAL_START.match(p):
base, reason = 80, "factual question"
else:
n_words = len(re.findall(r'\w+', p))
base, reason = 120 + min(180, n_words * 8), f"length-scaled ({n_words} prompt words)"
out = int(max(30, min(500, base)))
if DEBUG:
print(f"[DEBUG] Summary sizing: {reason} β†’ {out} words [{_summary_tier(out)}]")
return out
def words_to_global_chunks(max_words: int) -> int:
return max(3, min(GLOBAL_MAX_CHUNKS, round(max_words / 40)))
# ── Whole-video digest (global mode) ──────────────────────────────────────────
# A "summarize the video" used to be written from 3 to 12 sampled chunks of at most 90
# seconds each: a 45-minute recording was summarised from a few minutes of it. The
# digest reads the WHOLE transcript in windows with a cheap text call each (in
# parallel) and hands the notes to the synthesis call alongside the sampled regions.
DIGEST_WINDOW_CHARS = 12000
DIGEST_MAX_WINDOWS = 12
DIGEST_MAX_TOKENS = 400
DIGEST_PARALLEL = 6
def _digest_window(text: str, i: int, n: int, language: str) -> str:
try:
r = llm_client.chat_completion(
model=VLM_MODEL,
messages=[
{"role": "system", "content":
f"You take notes on part {i} of {n} of a video transcript. Write 4 to 8 "
f"terse bullet points in {language}: who says what, decisions, numbers, names, "
f"topics, in order. Keep the [m:ss] timestamp of each point from the "
f"transcript lines. No preamble, no commentary, nothing not in the text."},
{"role": "user", "content": text},
],
max_tokens=DIGEST_MAX_TOKENS, temperature=0.0, stream=False,
)
return (r.choices[0].message.content or "").strip()
except Exception as e:
print(f"[NINA] digest window {i}/{n} failed: {type(e).__name__}", flush=True)
return ""
def transcript_digest(chunks: list[dict], prompt: str) -> str:
"""Notes over the entire transcript, or "" when it fits in the sampled regions anyway."""
lines = [f"[{fmt_time(c['start'])}] {c.get('text', '')}" for c in (chunks or []) if c.get("text")]
full = "\n".join(lines)
if len(full) <= DIGEST_WINDOW_CHARS:
return ""
windows = [full[i:i + DIGEST_WINDOW_CHARS] for i in range(0, len(full), DIGEST_WINDOW_CHARS)]
windows = windows[:DIGEST_MAX_WINDOWS]
language = _detect_language(prompt)
n = len(windows)
print(f"[NINA] Global digest: {len(full)} chars in {n} window(s).", flush=True)
with ThreadPoolExecutor(max_workers=min(DIGEST_PARALLEL, n)) as ex:
notes = list(ex.map(lambda a: _digest_window(a[1], a[0] + 1, n, language), enumerate(windows)))
notes = [x for x in notes if x]
if not notes:
return ""
head = ("Notes over the WHOLE transcript, in order" +
(f" (first {DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS} characters)" if len(full) > DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS else "") + ":")
return head + "\n" + "\n".join(notes)
# ── Synthesis ──────────────────────────────────────────────────────────────────
def _fallback_summary(regions: list[dict]) -> str:
snippets = [
f"[{fmt_time(r['start'])}] {r.get('precise_text', r['text'])[:200]}"
for r in regions[:2]
]
return "(LLM unavailable β€” raw excerpts:)\n" + "\n".join(snippets)
# Adaptive answer formatting β€” let NINA shape the answer to the question (Markdown,
# which the chat renders) instead of always emitting one prose block. Shared by the
# text and vision synthesis prompts.
_FORMAT_GUIDE = (
"CRITICAL OUTPUT FORMAT β€” your response must be visually structured. Assume the user "
"is skimming, not reading word-for-word; optimize for scannability.\n"
"Required (use whichever fit the answer): Markdown headings, bullet lists, numbered "
"steps, and tables (Markdown `| col | col |`) when comparing items.\n"
"Forbidden: large blocks of uninterrupted text; any answer longer than ~5 lines without "
"formatting.\n"
"(A trivial one-line factual answer may stay plain β€” don't pad it into a table.)\n"
"Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]."
)
def _make_system_msg(language: str, max_words: int, tier: str,
reliable: bool = True, speaker_note: str = "") -> str:
speakers = "" if not speaker_note else (
"\n\nSPEAKERS β€” " + speaker_note + " Fragments are tagged with their speaker. "
"Attribute statements to the correct speaker. If a speaker stated their own name "
"(a self-introduction), use that NAME for EVERY line by that speaker β€” including "
"lines where the name isn't repeated. When the question asks who did or said "
"something, answer with the specific speaker/name (e.g. who won, who claimed what)."
)
audio_quality = "" if reliable else (
"\n\nAUDIO QUALITY (important) β€” This transcript was auto-extracted from "
"low-quality, noisy or distorted audio and is partly UNRELIABLE. Some fragments "
"may be garbled or nonsensical (speech-recognition errors), NOT real statements. "
"Do not treat incoherent or nonsensical fragments as things anyone actually said, "
"and do not invent a meaning for them. If the words don't form coherent sense, say "
"plainly (in the user's language) that the audio is too unclear to make out β€” e.g. "
"that you cannot reliably tell who is speaking or what is said. You may still rely on "
"any clearly coherent speech and on the on-screen cues."
)
return (
PERSONA_TEXT + "\n\n"
"THIS ANSWER uses ONLY the transcript fragments and on-screen cues you are given.\n\n"
f"LANGUAGE β€” Write your ENTIRE answer in the same language as the user's question "
f"(it looks like {language}). Never switch languages, even if the transcript is in "
f"another language: report its content in the user's language. Keep names, numbers, "
f"brands, titles and direct quotes exactly as they appear β€” do not translate, "
f"re-spell or reinterpret those.\n\n"
f"LENGTH β€” Fit the answer to the question. Aim for about {max_words} words at most "
f"({tier}); be shorter when the video covers little. Never pad. Answer the question "
f"directly, with no preamble like \"The video says…\".\n\n"
"GROUNDING β€” Use only what the fragments state. Never invent, infer, guess, or "
"complete partial words, names or numbers. If the fragments don't address the "
"question, say so plainly (in the user's language) instead of speculating. Don't "
"fix grammar inside quoted speech."
f"{speakers}"
f"{audio_quality}\n\n"
f"{_FORMAT_GUIDE}"
)
_STRONG_CONF = 0.45
def _render_region(r: dict) -> str:
spoken = (r.get("precise_text", r["text"]) or "").strip()
tags = r.get("visual_tags", [])
speakers = r.get("speakers", [])
header = f"[{fmt_time(r['start'])} β†’ {fmt_time(r['end'])}]"
if speakers:
header += f" ({' / '.join(speakers)})"
block = f"{header}\n Spoken: {spoken or '(no speech)'}"
if tags:
block += f"\n On screen: {', '.join(tags)}"
return block
def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode: str) -> str:
strong = [r for r in regions if r.get("confidence", 1.0) >= _STRONG_CONF]
context = [r for r in regions if r.get("confidence", 1.0) < _STRONG_CONF]
if not strong: # all regions are weak β€” treat all as strong
strong, context = regions, []
fragments = "\n\n".join(_render_region(r) for r in strong)
context_section = ""
if context:
ctx_blocks = "\n\n".join(_render_region(r) for r in context)
context_section = (
"\n\n--- Supporting context (lower relevance) ---\n\n" + ctx_blocks
)
if mode == "global":
task_line = (
"Task: Give a natural, factual overview of what this video covers, "
"drawing only from the fragments below."
)
else:
task_line = (
f'Task: Answer the user question "{prompt}" using ONLY what the fragments state.'
)
return (
f'User question: "{prompt}"\n\n'
f"Video fragments (each headed by its [m:ss β†’ m:ss] timestamp):\n\n"
f"{fragments}{context_section}\n\n"
f"{task_line}\n\n"
f"How to write the answer:\n"
f"- Reply in the SAME language as the user's question above.\n"
f"- Format for readability (per the system instructions): short paragraphs, with a "
f"bullet list or a brief heading when the answer is long or has parts. Don't prefix "
f"lines with timestamps.\n"
f"- Ground every statement only in the fragments; never merge two fragments into a "
f"single claim, and don't derive new claims from the supporting-context blocks alone.\n"
f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n"
f"- If nothing above is relevant, say that plainly in the user's language.\n"
f"- End with one line: Sources: followed by the [m:ss] start-times you actually used."
)
def llm_synthesize_stream(
regions: list[dict], prompt: str, max_words: int = 150, mode: str = "specific",
reliable: bool = True, speaker_note: str = "", digest: str = "",
):
"""Generator: yields the answer incrementally as the LLM streams tokens.
`reliable=False` (poor ASR audio) makes the system prompt tell the model to flag
unclear audio instead of treating garbled fragments as real speech. `speaker_note`
(when multiple speakers were diarized) drives speaker/name attribution.
"""
if not regions:
yield "(No relevant regions to summarize.)"
return
synth_prompt = _build_synth_prompt(regions, prompt, max_words, mode)
if digest:
synth_prompt = digest + "\n\n" + synth_prompt
if DEBUG:
print(
f"[DEBUG] Synthesis [{mode}] β€” {len(regions)} region(s), "
f"max_words={max_words} [{_summary_tier(max_words)}]."
)
try:
acc = ""
language = _detect_language(prompt)
if DEBUG:
print(f"[DEBUG] Detected prompt language: {language}")
stream = llm_client.chat_completion(
model=LLM_MODEL,
messages=[
{"role": "system",
"content": _make_system_msg(language, max_words, _summary_tier(max_words),
reliable=reliable, speaker_note=speaker_note)},
{"role": "user", "content": synth_prompt},
],
max_tokens=LLM_MAX_TOKENS,
temperature=LLM_TEMPERATURE,
stream=True,
)
for ev in stream:
if not ev.choices:
continue
delta = ev.choices[0].delta.content or ""
if delta:
acc += delta
yield acc.strip()
if not acc.strip():
yield _fallback_summary(regions)
return
except Exception as e:
print(f"[NINA] Synthesis error: {e}")
yield _fallback_summary(regions)
# ── Vision-Language synthesis (primary) ──────────────────────────────────────────
def _transcript_block(regions: list[dict]) -> str:
"""Plain speech transcript for the VLM (timestamps + speakers, no CLIP tags β€”
the model reads the frames itself)."""
lines = []
for r in regions:
spoken = (r.get("precise_text", r["text"]) or "").strip()
head = f"[{fmt_time(r['start'])} β†’ {fmt_time(r['end'])}]"
if r.get("speakers"):
head += f" ({' / '.join(r['speakers'])})"
lines.append(f"{head} {spoken or '(no clear speech)'}")
return "\n".join(lines) if lines else "(no usable speech transcribed)"
def _make_vlm_system_msg(language: str, max_words: int, tier: str,
reliable: bool, speaker_note: str) -> str:
speakers = "" if not speaker_note else (
"\n\nSPEAKERS β€” " + speaker_note + " Attribute speech to the right speaker; if "
"someone states their own name, use it for all of their lines."
)
audio = "" if reliable else (
"\n\nAUDIO β€” The speech transcript is from poor/noisy audio and is partly "
"unreliable; don't trust garbled fragments. Lean on what you can SEE in the frames, "
"and if neither the audio nor the frames answer the question, say so plainly."
)
return (
PERSONA_VISION + "\n\n"
"THIS ANSWER uses ONLY (a) the keyframes you are shown and (b) the speech transcript "
"provided, nothing else.\n\n"
"VISION β€” Actually look at the frames. Read any on-screen text (titles, captions, "
"lower-thirds, names, UI, slides) verbatim, identify who/what is shown and the "
"setting, and notice actions/changes across frames. Fuse this with the transcript.\n\n"
f"LANGUAGE β€” Write the ENTIRE answer in the same language as the question (it looks "
f"like {language}). Keep names, numbers, brands and on-screen text exactly as they "
f"appear.\n\n"
f"LENGTH β€” About {max_words} words at most ({tier}); shorter if there's little to say. "
f"Answer directly, no preamble.\n\n"
"GROUNDING β€” State only what is visible in the frames or present in the transcript. "
"Never invent names, numbers or facts you cannot see or read.\n"
"IDENTITY β€” Do NOT identify or name a person from their appearance or resemblance to "
"someone famous. You have no way to know who someone is unless their name is WRITTEN "
"on screen or SPOKEN in the transcript. If it isn't, refer to them generically (e.g. "
"\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity "
"can't be determined from the video. Never guess a real person's name."
f"{speakers}{audio}\n\n"
f"{_FORMAT_GUIDE}"
)
def vlm_synthesize_stream(
regions: list[dict], prompt: str, frames: list[tuple],
max_words: int = 150, mode: str = "specific",
reliable: bool = True, speaker_note: str = "", digest: str = "",
):
"""Generator: stream the answer from the vision-language model, given keyframes
(list of (timestamp, data_url)) plus the transcript. Falls back to the text-only
LLM if the VLM call fails before producing anything."""
language = _detect_language(prompt)
transcript = _transcript_block(regions)
user_text = (
f'Question: "{prompt}"\n\n'
+ (f"{digest}\n\n" if digest else "")
+ f"Speech transcript (may be imperfect):\n{transcript}\n\n"
f"Below are {len(frames)} keyframes sampled from the video, each labelled with its "
f"timestamp. Use them and the transcript to answer the question."
)
content = [{"type": "text", "text": user_text}]
for ts, url in frames:
content.append({"type": "text", "text": f"Frame at {fmt_time(ts)}:"})
content.append({"type": "image_url", "image_url": {"url": url}})
if DEBUG:
print(f"[DEBUG] VLM synthesis β€” {len(frames)} frame(s), {len(regions)} region(s), "
f"lang={language}, max_words={max_words}.")
acc = ""
try:
stream = llm_client.chat_completion(
model=VLM_MODEL,
messages=[
{"role": "system",
"content": _make_vlm_system_msg(language, max_words, _summary_tier(max_words),
reliable, speaker_note)},
{"role": "user", "content": content},
],
max_tokens=LLM_MAX_TOKENS,
temperature=LLM_TEMPERATURE,
stream=True,
)
for ev in stream:
if not ev.choices:
continue
delta = ev.choices[0].delta.content or ""
if delta:
acc += delta
yield acc.strip()
if acc.strip():
return
# Empty VLM response β†’ fall through to the text model.
print("[NINA] VLM returned nothing β€” falling back to text synthesis.")
except Exception as e:
if acc.strip():
return # already produced an answer
print(f"[NINA] VLM unavailable ({e}) β€” falling back to text synthesis.")
yield from llm_synthesize_stream(regions, prompt, max_words, mode,
reliable=reliable, speaker_note=speaker_note, digest=digest)
# ── Output formatter ───────────────────────────────────────────────────────────
def format_output(regions: list[dict], prompt: str, summary: str,
mode: str = "specific") -> str:
if not regions:
return (
f' Topic not found in this video.\n\n'
f' Prompt : "{prompt}"\n\n'
f' No relevant segments found.\n'
f' Try a broader or differently worded prompt.'
)
mode_tag = {
"global": "🌐 global", "hybrid": "⚑ hybrid", "specific": "🎯 specific"
}.get(mode, "")
lines = []
if DEBUG:
lines.append(f' {len(regions)} region(s) found for: "{prompt}" [{mode_tag} Β· audio+video]\n')
for i, r in enumerate(regions, 1):
dur = round(r["end"] - r["start"], 1)
text = r.get("precise_text", r["text"])
ent = r.get("entity_overlap", 0)
bi = r.get("bi_score", 0)
vis = r.get("visual_score", 0)
tags = ", ".join(r.get("visual_tags", []))
lines += [
f'\n [{i}] {fmt_time(r["start"])} β†’ {fmt_time(r["end"])} ({dur}s)',
f' {conf_bar(r["confidence"])} {r["confidence"]:.0%} '
f'(embed {bi:.0%} Β· entity {ent:.0%} Β· visual {vis:.0%})',
f' πŸ‘ {tags}' if tags else ' πŸ‘ (no visual tags)',
f' "{text[:220]}{"…" if len(text) > 220 else ""}"',
]
lines.append(summary)
if DEBUG:
lines.append(
'\n Sources: ' + ', '.join(
f'[{fmt_time(r["start"])} β†’ {fmt_time(r["end"])}]' for r in regions
)
)
return "\n".join(lines)