Spaces:
Paused
Paused
Malina
Claude Fable 5.1
v0.90: scoped single-use tokens, hardened link fetch, 45-minute cap, whole-video digest, one classification call, no OCR
1d697a7 Download synthesis.py from NextNeural/NINA: direct link, hf CLI and curl.
- Browser
- Download file 20.9 kB
-
https://huggingface.co/spaces/NextNeural/NINA/resolve/main/synthesis.py
- Command line
-
hf download hf://spaces/NextNeural/NINA/synthesis.py
-
curl -L -o synthesis.py https://huggingface.co/spaces/NextNeural/NINA/resolve/main/synthesis.py
20.9 kB
| import re | |
| from concurrent.futures import ThreadPoolExecutor | |
| from persona import PERSONA_TEXT, PERSONA_VISION | |
| from config import ( | |
| DEBUG, GLOBAL_MAX_CHUNKS, LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE, VLM_MODEL, | |
| ) | |
| from lang import detect_language as _detect_language | |
| from models import llm_client | |
| from utils import conf_bar, fmt_time | |
| # ββ Output size inference ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _BRIEF_CUES = ( | |
| "brief", "briefly", "short", "shortly", "quick", "quickly", "concise", | |
| "in a sentence", "one sentence", "one-liner", "tl;dr", "tldr", | |
| "krΓ³tko", "w skrΓ³cie", "zwiΔΕΊle", "jednym zdaniem", "w jednym zdaniu", | |
| ) | |
| _DETAIL_CUES = ( | |
| "in detail", "detailed", "in-depth", "in depth", "thorough", "thoroughly", | |
| "comprehensive", "exhaustive", "everything", "full summary", "elaborate", | |
| "step by step", "step-by-step", "deep dive", "as much as possible", | |
| "szczegΓ³Εowo", "dokΕadnie", "wyczerpujΔ co", "wszystko", "krok po kroku", | |
| "dogΕΔbnie", "obszernie", | |
| ) | |
| _FACTUAL_START = re.compile( | |
| r'^\s*(what|who|whom|whose|when|where|which|how many|how much|how long|' | |
| r'is|are|was|were|does|do|did|can|could|will|would|' | |
| r'czy|kiedy|kto|kogo|gdzie|ile|jaki|jaka|jakie|ktΓ³ry|ktΓ³ra|ktΓ³re)\b', | |
| re.IGNORECASE, | |
| ) | |
| def _summary_tier(max_words: int) -> str: | |
| if max_words <= 60: return "brief / one-liner" | |
| if max_words <= 110: return "short" | |
| if max_words <= 250: return "standard" | |
| if max_words <= 400: return "detailed" | |
| return "exhaustive" | |
| def prompt_to_words(prompt: str) -> int: | |
| """Infer answer word budget from prompt cues. Clamped to [30, 500].""" | |
| p = prompt.lower() | |
| if any(c in p for c in _DETAIL_CUES): | |
| base, reason = 450, "explicit detail cue" | |
| elif any(c in p for c in _BRIEF_CUES): | |
| base, reason = 40, "explicit brevity cue" | |
| elif _FACTUAL_START.match(p): | |
| base, reason = 80, "factual question" | |
| else: | |
| n_words = len(re.findall(r'\w+', p)) | |
| base, reason = 120 + min(180, n_words * 8), f"length-scaled ({n_words} prompt words)" | |
| out = int(max(30, min(500, base))) | |
| if DEBUG: | |
| print(f"[DEBUG] Summary sizing: {reason} β {out} words [{_summary_tier(out)}]") | |
| return out | |
| def words_to_global_chunks(max_words: int) -> int: | |
| return max(3, min(GLOBAL_MAX_CHUNKS, round(max_words / 40))) | |
| # ββ Whole-video digest (global mode) ββββββββββββββββββββββββββββββββββββββββββ | |
| # A "summarize the video" used to be written from 3 to 12 sampled chunks of at most 90 | |
| # seconds each: a 45-minute recording was summarised from a few minutes of it. The | |
| # digest reads the WHOLE transcript in windows with a cheap text call each (in | |
| # parallel) and hands the notes to the synthesis call alongside the sampled regions. | |
| DIGEST_WINDOW_CHARS = 12000 | |
| DIGEST_MAX_WINDOWS = 12 | |
| DIGEST_MAX_TOKENS = 400 | |
| DIGEST_PARALLEL = 6 | |
| def _digest_window(text: str, i: int, n: int, language: str) -> str: | |
| try: | |
| r = llm_client.chat_completion( | |
| model=VLM_MODEL, | |
| messages=[ | |
| {"role": "system", "content": | |
| f"You take notes on part {i} of {n} of a video transcript. Write 4 to 8 " | |
| f"terse bullet points in {language}: who says what, decisions, numbers, names, " | |
| f"topics, in order. Keep the [m:ss] timestamp of each point from the " | |
| f"transcript lines. No preamble, no commentary, nothing not in the text."}, | |
| {"role": "user", "content": text}, | |
| ], | |
| max_tokens=DIGEST_MAX_TOKENS, temperature=0.0, stream=False, | |
| ) | |
| return (r.choices[0].message.content or "").strip() | |
| except Exception as e: | |
| print(f"[NINA] digest window {i}/{n} failed: {type(e).__name__}", flush=True) | |
| return "" | |
| def transcript_digest(chunks: list[dict], prompt: str) -> str: | |
| """Notes over the entire transcript, or "" when it fits in the sampled regions anyway.""" | |
| lines = [f"[{fmt_time(c['start'])}] {c.get('text', '')}" for c in (chunks or []) if c.get("text")] | |
| full = "\n".join(lines) | |
| if len(full) <= DIGEST_WINDOW_CHARS: | |
| return "" | |
| windows = [full[i:i + DIGEST_WINDOW_CHARS] for i in range(0, len(full), DIGEST_WINDOW_CHARS)] | |
| windows = windows[:DIGEST_MAX_WINDOWS] | |
| language = _detect_language(prompt) | |
| n = len(windows) | |
| print(f"[NINA] Global digest: {len(full)} chars in {n} window(s).", flush=True) | |
| with ThreadPoolExecutor(max_workers=min(DIGEST_PARALLEL, n)) as ex: | |
| notes = list(ex.map(lambda a: _digest_window(a[1], a[0] + 1, n, language), enumerate(windows))) | |
| notes = [x for x in notes if x] | |
| if not notes: | |
| return "" | |
| head = ("Notes over the WHOLE transcript, in order" + | |
| (f" (first {DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS} characters)" if len(full) > DIGEST_MAX_WINDOWS * DIGEST_WINDOW_CHARS else "") + ":") | |
| return head + "\n" + "\n".join(notes) | |
| # ββ Synthesis ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _fallback_summary(regions: list[dict]) -> str: | |
| snippets = [ | |
| f"[{fmt_time(r['start'])}] {r.get('precise_text', r['text'])[:200]}" | |
| for r in regions[:2] | |
| ] | |
| return "(LLM unavailable β raw excerpts:)\n" + "\n".join(snippets) | |
| # Adaptive answer formatting β let NINA shape the answer to the question (Markdown, | |
| # which the chat renders) instead of always emitting one prose block. Shared by the | |
| # text and vision synthesis prompts. | |
| _FORMAT_GUIDE = ( | |
| "CRITICAL OUTPUT FORMAT β your response must be visually structured. Assume the user " | |
| "is skimming, not reading word-for-word; optimize for scannability.\n" | |
| "Required (use whichever fit the answer): Markdown headings, bullet lists, numbered " | |
| "steps, and tables (Markdown `| col | col |`) when comparing items.\n" | |
| "Forbidden: large blocks of uninterrupted text; any answer longer than ~5 lines without " | |
| "formatting.\n" | |
| "(A trivial one-line factual answer may stay plain β don't pad it into a table.)\n" | |
| "Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]." | |
| ) | |
| def _make_system_msg(language: str, max_words: int, tier: str, | |
| reliable: bool = True, speaker_note: str = "") -> str: | |
| speakers = "" if not speaker_note else ( | |
| "\n\nSPEAKERS β " + speaker_note + " Fragments are tagged with their speaker. " | |
| "Attribute statements to the correct speaker. If a speaker stated their own name " | |
| "(a self-introduction), use that NAME for EVERY line by that speaker β including " | |
| "lines where the name isn't repeated. When the question asks who did or said " | |
| "something, answer with the specific speaker/name (e.g. who won, who claimed what)." | |
| ) | |
| audio_quality = "" if reliable else ( | |
| "\n\nAUDIO QUALITY (important) β This transcript was auto-extracted from " | |
| "low-quality, noisy or distorted audio and is partly UNRELIABLE. Some fragments " | |
| "may be garbled or nonsensical (speech-recognition errors), NOT real statements. " | |
| "Do not treat incoherent or nonsensical fragments as things anyone actually said, " | |
| "and do not invent a meaning for them. If the words don't form coherent sense, say " | |
| "plainly (in the user's language) that the audio is too unclear to make out β e.g. " | |
| "that you cannot reliably tell who is speaking or what is said. You may still rely on " | |
| "any clearly coherent speech and on the on-screen cues." | |
| ) | |
| return ( | |
| PERSONA_TEXT + "\n\n" | |
| "THIS ANSWER uses ONLY the transcript fragments and on-screen cues you are given.\n\n" | |
| f"LANGUAGE β Write your ENTIRE answer in the same language as the user's question " | |
| f"(it looks like {language}). Never switch languages, even if the transcript is in " | |
| f"another language: report its content in the user's language. Keep names, numbers, " | |
| f"brands, titles and direct quotes exactly as they appear β do not translate, " | |
| f"re-spell or reinterpret those.\n\n" | |
| f"LENGTH β Fit the answer to the question. Aim for about {max_words} words at most " | |
| f"({tier}); be shorter when the video covers little. Never pad. Answer the question " | |
| f"directly, with no preamble like \"The video saysβ¦\".\n\n" | |
| "GROUNDING β Use only what the fragments state. Never invent, infer, guess, or " | |
| "complete partial words, names or numbers. If the fragments don't address the " | |
| "question, say so plainly (in the user's language) instead of speculating. Don't " | |
| "fix grammar inside quoted speech." | |
| f"{speakers}" | |
| f"{audio_quality}\n\n" | |
| f"{_FORMAT_GUIDE}" | |
| ) | |
| _STRONG_CONF = 0.45 | |
| def _render_region(r: dict) -> str: | |
| spoken = (r.get("precise_text", r["text"]) or "").strip() | |
| tags = r.get("visual_tags", []) | |
| speakers = r.get("speakers", []) | |
| header = f"[{fmt_time(r['start'])} β {fmt_time(r['end'])}]" | |
| if speakers: | |
| header += f" ({' / '.join(speakers)})" | |
| block = f"{header}\n Spoken: {spoken or '(no speech)'}" | |
| if tags: | |
| block += f"\n On screen: {', '.join(tags)}" | |
| return block | |
| def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode: str) -> str: | |
| strong = [r for r in regions if r.get("confidence", 1.0) >= _STRONG_CONF] | |
| context = [r for r in regions if r.get("confidence", 1.0) < _STRONG_CONF] | |
| if not strong: # all regions are weak β treat all as strong | |
| strong, context = regions, [] | |
| fragments = "\n\n".join(_render_region(r) for r in strong) | |
| context_section = "" | |
| if context: | |
| ctx_blocks = "\n\n".join(_render_region(r) for r in context) | |
| context_section = ( | |
| "\n\n--- Supporting context (lower relevance) ---\n\n" + ctx_blocks | |
| ) | |
| if mode == "global": | |
| task_line = ( | |
| "Task: Give a natural, factual overview of what this video covers, " | |
| "drawing only from the fragments below." | |
| ) | |
| else: | |
| task_line = ( | |
| f'Task: Answer the user question "{prompt}" using ONLY what the fragments state.' | |
| ) | |
| return ( | |
| f'User question: "{prompt}"\n\n' | |
| f"Video fragments (each headed by its [m:ss β m:ss] timestamp):\n\n" | |
| f"{fragments}{context_section}\n\n" | |
| f"{task_line}\n\n" | |
| f"How to write the answer:\n" | |
| f"- Reply in the SAME language as the user's question above.\n" | |
| f"- Format for readability (per the system instructions): short paragraphs, with a " | |
| f"bullet list or a brief heading when the answer is long or has parts. Don't prefix " | |
| f"lines with timestamps.\n" | |
| f"- Ground every statement only in the fragments; never merge two fragments into a " | |
| f"single claim, and don't derive new claims from the supporting-context blocks alone.\n" | |
| f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n" | |
| f"- If nothing above is relevant, say that plainly in the user's language.\n" | |
| f"- End with one line: Sources: followed by the [m:ss] start-times you actually used." | |
| ) | |
| def llm_synthesize_stream( | |
| regions: list[dict], prompt: str, max_words: int = 150, mode: str = "specific", | |
| reliable: bool = True, speaker_note: str = "", digest: str = "", | |
| ): | |
| """Generator: yields the answer incrementally as the LLM streams tokens. | |
| `reliable=False` (poor ASR audio) makes the system prompt tell the model to flag | |
| unclear audio instead of treating garbled fragments as real speech. `speaker_note` | |
| (when multiple speakers were diarized) drives speaker/name attribution. | |
| """ | |
| if not regions: | |
| yield "(No relevant regions to summarize.)" | |
| return | |
| synth_prompt = _build_synth_prompt(regions, prompt, max_words, mode) | |
| if digest: | |
| synth_prompt = digest + "\n\n" + synth_prompt | |
| if DEBUG: | |
| print( | |
| f"[DEBUG] Synthesis [{mode}] β {len(regions)} region(s), " | |
| f"max_words={max_words} [{_summary_tier(max_words)}]." | |
| ) | |
| try: | |
| acc = "" | |
| language = _detect_language(prompt) | |
| if DEBUG: | |
| print(f"[DEBUG] Detected prompt language: {language}") | |
| stream = llm_client.chat_completion( | |
| model=LLM_MODEL, | |
| messages=[ | |
| {"role": "system", | |
| "content": _make_system_msg(language, max_words, _summary_tier(max_words), | |
| reliable=reliable, speaker_note=speaker_note)}, | |
| {"role": "user", "content": synth_prompt}, | |
| ], | |
| max_tokens=LLM_MAX_TOKENS, | |
| temperature=LLM_TEMPERATURE, | |
| stream=True, | |
| ) | |
| for ev in stream: | |
| if not ev.choices: | |
| continue | |
| delta = ev.choices[0].delta.content or "" | |
| if delta: | |
| acc += delta | |
| yield acc.strip() | |
| if not acc.strip(): | |
| yield _fallback_summary(regions) | |
| return | |
| except Exception as e: | |
| print(f"[NINA] Synthesis error: {e}") | |
| yield _fallback_summary(regions) | |
| # ββ Vision-Language synthesis (primary) ββββββββββββββββββββββββββββββββββββββββββ | |
| def _transcript_block(regions: list[dict]) -> str: | |
| """Plain speech transcript for the VLM (timestamps + speakers, no CLIP tags β | |
| the model reads the frames itself).""" | |
| lines = [] | |
| for r in regions: | |
| spoken = (r.get("precise_text", r["text"]) or "").strip() | |
| head = f"[{fmt_time(r['start'])} β {fmt_time(r['end'])}]" | |
| if r.get("speakers"): | |
| head += f" ({' / '.join(r['speakers'])})" | |
| lines.append(f"{head} {spoken or '(no clear speech)'}") | |
| return "\n".join(lines) if lines else "(no usable speech transcribed)" | |
| def _make_vlm_system_msg(language: str, max_words: int, tier: str, | |
| reliable: bool, speaker_note: str) -> str: | |
| speakers = "" if not speaker_note else ( | |
| "\n\nSPEAKERS β " + speaker_note + " Attribute speech to the right speaker; if " | |
| "someone states their own name, use it for all of their lines." | |
| ) | |
| audio = "" if reliable else ( | |
| "\n\nAUDIO β The speech transcript is from poor/noisy audio and is partly " | |
| "unreliable; don't trust garbled fragments. Lean on what you can SEE in the frames, " | |
| "and if neither the audio nor the frames answer the question, say so plainly." | |
| ) | |
| return ( | |
| PERSONA_VISION + "\n\n" | |
| "THIS ANSWER uses ONLY (a) the keyframes you are shown and (b) the speech transcript " | |
| "provided, nothing else.\n\n" | |
| "VISION β Actually look at the frames. Read any on-screen text (titles, captions, " | |
| "lower-thirds, names, UI, slides) verbatim, identify who/what is shown and the " | |
| "setting, and notice actions/changes across frames. Fuse this with the transcript.\n\n" | |
| f"LANGUAGE β Write the ENTIRE answer in the same language as the question (it looks " | |
| f"like {language}). Keep names, numbers, brands and on-screen text exactly as they " | |
| f"appear.\n\n" | |
| f"LENGTH β About {max_words} words at most ({tier}); shorter if there's little to say. " | |
| f"Answer directly, no preamble.\n\n" | |
| "GROUNDING β State only what is visible in the frames or present in the transcript. " | |
| "Never invent names, numbers or facts you cannot see or read.\n" | |
| "IDENTITY β Do NOT identify or name a person from their appearance or resemblance to " | |
| "someone famous. You have no way to know who someone is unless their name is WRITTEN " | |
| "on screen or SPOKEN in the transcript. If it isn't, refer to them generically (e.g. " | |
| "\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity " | |
| "can't be determined from the video. Never guess a real person's name." | |
| f"{speakers}{audio}\n\n" | |
| f"{_FORMAT_GUIDE}" | |
| ) | |
| def vlm_synthesize_stream( | |
| regions: list[dict], prompt: str, frames: list[tuple], | |
| max_words: int = 150, mode: str = "specific", | |
| reliable: bool = True, speaker_note: str = "", digest: str = "", | |
| ): | |
| """Generator: stream the answer from the vision-language model, given keyframes | |
| (list of (timestamp, data_url)) plus the transcript. Falls back to the text-only | |
| LLM if the VLM call fails before producing anything.""" | |
| language = _detect_language(prompt) | |
| transcript = _transcript_block(regions) | |
| user_text = ( | |
| f'Question: "{prompt}"\n\n' | |
| + (f"{digest}\n\n" if digest else "") | |
| + f"Speech transcript (may be imperfect):\n{transcript}\n\n" | |
| f"Below are {len(frames)} keyframes sampled from the video, each labelled with its " | |
| f"timestamp. Use them and the transcript to answer the question." | |
| ) | |
| content = [{"type": "text", "text": user_text}] | |
| for ts, url in frames: | |
| content.append({"type": "text", "text": f"Frame at {fmt_time(ts)}:"}) | |
| content.append({"type": "image_url", "image_url": {"url": url}}) | |
| if DEBUG: | |
| print(f"[DEBUG] VLM synthesis β {len(frames)} frame(s), {len(regions)} region(s), " | |
| f"lang={language}, max_words={max_words}.") | |
| acc = "" | |
| try: | |
| stream = llm_client.chat_completion( | |
| model=VLM_MODEL, | |
| messages=[ | |
| {"role": "system", | |
| "content": _make_vlm_system_msg(language, max_words, _summary_tier(max_words), | |
| reliable, speaker_note)}, | |
| {"role": "user", "content": content}, | |
| ], | |
| max_tokens=LLM_MAX_TOKENS, | |
| temperature=LLM_TEMPERATURE, | |
| stream=True, | |
| ) | |
| for ev in stream: | |
| if not ev.choices: | |
| continue | |
| delta = ev.choices[0].delta.content or "" | |
| if delta: | |
| acc += delta | |
| yield acc.strip() | |
| if acc.strip(): | |
| return | |
| # Empty VLM response β fall through to the text model. | |
| print("[NINA] VLM returned nothing β falling back to text synthesis.") | |
| except Exception as e: | |
| if acc.strip(): | |
| return # already produced an answer | |
| print(f"[NINA] VLM unavailable ({e}) β falling back to text synthesis.") | |
| yield from llm_synthesize_stream(regions, prompt, max_words, mode, | |
| reliable=reliable, speaker_note=speaker_note, digest=digest) | |
| # ββ Output formatter βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def format_output(regions: list[dict], prompt: str, summary: str, | |
| mode: str = "specific") -> str: | |
| if not regions: | |
| return ( | |
| f' Topic not found in this video.\n\n' | |
| f' Prompt : "{prompt}"\n\n' | |
| f' No relevant segments found.\n' | |
| f' Try a broader or differently worded prompt.' | |
| ) | |
| mode_tag = { | |
| "global": "π global", "hybrid": "β‘ hybrid", "specific": "π― specific" | |
| }.get(mode, "") | |
| lines = [] | |
| if DEBUG: | |
| lines.append(f' {len(regions)} region(s) found for: "{prompt}" [{mode_tag} Β· audio+video]\n') | |
| for i, r in enumerate(regions, 1): | |
| dur = round(r["end"] - r["start"], 1) | |
| text = r.get("precise_text", r["text"]) | |
| ent = r.get("entity_overlap", 0) | |
| bi = r.get("bi_score", 0) | |
| vis = r.get("visual_score", 0) | |
| tags = ", ".join(r.get("visual_tags", [])) | |
| lines += [ | |
| f'\n [{i}] {fmt_time(r["start"])} β {fmt_time(r["end"])} ({dur}s)', | |
| f' {conf_bar(r["confidence"])} {r["confidence"]:.0%} ' | |
| f'(embed {bi:.0%} Β· entity {ent:.0%} Β· visual {vis:.0%})', | |
| f' π {tags}' if tags else ' π (no visual tags)', | |
| f' "{text[:220]}{"β¦" if len(text) > 220 else ""}"', | |
| ] | |
| lines.append(summary) | |
| if DEBUG: | |
| lines.append( | |
| '\n Sources: ' + ', '.join( | |
| f'[{fmt_time(r["start"])} β {fmt_time(r["end"])}]' for r in regions | |
| ) | |
| ) | |
| return "\n".join(lines) | |