Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit Β·
9a43844
1
Parent(s): 710a0dd
Visual-only answers, coreference + on-topic guard, 15-min cache
Browse files- pipeline: drop the "No speech detected" dead-end β when there's no usable
transcript, synthesize from the visuals alone (VLM on keyframes), so
"what does the person do?" works on silent/music clips.
- chat: resolve references to earlier turns ("explain more"); ask to clarify
when a reference is ambiguous; gate out off-topic requests and steer back to
the uploaded video; route follow-ups to context instead of re-analysis.
- app: keep uploaded media 15 min (delete_cache 900) so follow-up turns reuse
the already-uploaded video without re-uploading.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- app.py +4 -2
- chat.py +56 -7
- pipeline.py +27 -4
app.py
CHANGED
|
@@ -31,8 +31,10 @@ for _name in ("uvicorn.error", "uvicorn"):
|
|
| 31 |
logging.getLogger(_name).addFilter(_DropClientDisconnect())
|
| 32 |
|
| 33 |
# The custom front-end at app.nnlabs.pl drives these two API endpoints directly
|
| 34 |
-
# (the Gradio UI itself is unused). Privacy: purge uploaded files every
|
| 35 |
-
|
|
|
|
|
|
|
| 36 |
gr.Markdown(
|
| 37 |
"## NINA β Neural Information & Narrative Abstraction\n"
|
| 38 |
"API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
|
|
|
|
| 31 |
logging.getLogger(_name).addFilter(_DropClientDisconnect())
|
| 32 |
|
| 33 |
# The custom front-end at app.nnlabs.pl drives these two API endpoints directly
|
| 34 |
+
# (the Gradio UI itself is unused). Privacy: purge uploaded files every 15 min β
|
| 35 |
+
# long enough that follow-up turns in a chat re-use the already-uploaded video
|
| 36 |
+
# (no re-upload), short enough to honor "media is never stored".
|
| 37 |
+
with gr.Blocks(title="NINA β Multimodal Video Chat", delete_cache=(900, 900)) as demo:
|
| 38 |
gr.Markdown(
|
| 39 |
"## NINA β Neural Information & Narrative Abstraction\n"
|
| 40 |
"API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
|
chat.py
CHANGED
|
@@ -48,9 +48,13 @@ def _route(messages: list, has_video: bool) -> str:
|
|
| 48 |
model=LLM_MODEL,
|
| 49 |
messages=[
|
| 50 |
{"role": "system", "content":
|
| 51 |
-
"
|
| 52 |
-
"video for visual/audio detail
|
| 53 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
{"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
|
| 55 |
],
|
| 56 |
max_tokens=3, temperature=0.0, stream=False,
|
|
@@ -61,16 +65,51 @@ def _route(messages: list, has_video: bool) -> str:
|
|
| 61 |
return "answer"
|
| 62 |
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
def _answer_from_context(messages: list, transcript: str):
|
| 65 |
"""Stream a conversational answer grounded in the history + stored transcript."""
|
| 66 |
last = _last_user(messages)
|
| 67 |
prompt = (last.get("content") or "").strip()
|
| 68 |
language = _detect_language(prompt)
|
| 69 |
system = (
|
| 70 |
-
f"You are NINA, a video-understanding assistant continuing a conversation
|
| 71 |
-
f"Write your answer in {language}.
|
| 72 |
-
f"
|
| 73 |
-
f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
)
|
| 75 |
user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
|
| 76 |
if last.get("quote"):
|
|
@@ -120,6 +159,16 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
|
|
| 120 |
yield ["Please type a message.", False, transcript]
|
| 121 |
return
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
# Decide: analyze (re-watch) vs answer (from context).
|
| 124 |
decision = "analyze" if force else _route(messages, video_path is not None or has_video)
|
| 125 |
|
|
|
|
| 48 |
model=LLM_MODEL,
|
| 49 |
messages=[
|
| 50 |
{"role": "system", "content":
|
| 51 |
+
"You route turns for a video-chat assistant. Decide whether answering the "
|
| 52 |
+
"user's latest message needs RE-WATCHING the video for visual/audio detail "
|
| 53 |
+
"not covered yet, or can be answered from the conversation so far. "
|
| 54 |
+
"A short follow-up that refers back to the discussion ('explain more', "
|
| 55 |
+
"'why?', 'the second one', 'and then?') is almost always ANSWER β only pick "
|
| 56 |
+
"ANALYZE when genuinely new footage detail is required. "
|
| 57 |
+
"Reply with exactly one word: ANALYZE or ANSWER."},
|
| 58 |
{"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
|
| 59 |
],
|
| 60 |
max_tokens=3, temperature=0.0, stream=False,
|
|
|
|
| 65 |
return "answer"
|
| 66 |
|
| 67 |
|
| 68 |
+
def _on_topic(messages: list) -> bool:
|
| 69 |
+
"""Gate out off-topic requests ('what can I cook for dinner?'). Only checked once
|
| 70 |
+
a conversation is under way β the first message about an uploaded video is always
|
| 71 |
+
on-topic. Conservative: any error / uncertainty defaults to on-topic."""
|
| 72 |
+
if not any(m.get("role") == "assistant" for m in messages):
|
| 73 |
+
return True
|
| 74 |
+
last = _last_user(messages)
|
| 75 |
+
if not (last.get("content") or "").strip():
|
| 76 |
+
return True
|
| 77 |
+
try:
|
| 78 |
+
r = llm_client.chat_completion(
|
| 79 |
+
model=LLM_MODEL,
|
| 80 |
+
messages=[
|
| 81 |
+
{"role": "system", "content":
|
| 82 |
+
"You guard a video-understanding assistant. The user uploaded a video and is "
|
| 83 |
+
"chatting about it. Decide if their latest message is RELATED to that video or "
|
| 84 |
+
"the ongoing discussion of it β including vague follow-ups like 'explain more', "
|
| 85 |
+
"'why?', 'and then?' β or is OFF-TOPIC: a general request unrelated to the "
|
| 86 |
+
"video (recipes, coding help, world trivia, chit-chat). "
|
| 87 |
+
"Reply with exactly one word: ON or OFF."},
|
| 88 |
+
{"role": "user", "content": _render_history(messages) + "\n\nReply ON or OFF."},
|
| 89 |
+
],
|
| 90 |
+
max_tokens=2, temperature=0.0, stream=False,
|
| 91 |
+
)
|
| 92 |
+
return not (r.choices[0].message.content or "").strip().upper().startswith("OFF")
|
| 93 |
+
except Exception:
|
| 94 |
+
return True
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def _answer_from_context(messages: list, transcript: str):
|
| 98 |
"""Stream a conversational answer grounded in the history + stored transcript."""
|
| 99 |
last = _last_user(messages)
|
| 100 |
prompt = (last.get("content") or "").strip()
|
| 101 |
language = _detect_language(prompt)
|
| 102 |
system = (
|
| 103 |
+
f"You are NINA, a video-understanding assistant continuing a conversation about a video "
|
| 104 |
+
f"the user uploaded. Write your answer in {language}. "
|
| 105 |
+
f"RESOLVE REFERENCES to earlier turns: a short follow-up like 'explain more', 'why?', "
|
| 106 |
+
f"'the second one', 'and that part' refers to your previous answer or a specific earlier "
|
| 107 |
+
f"point β use the conversation to work out exactly what they mean and answer that. "
|
| 108 |
+
f"If such a reference is genuinely AMBIGUOUS (you can't tell which message or which part "
|
| 109 |
+
f"they mean), do NOT guess β reply with one short clarifying question naming the options. "
|
| 110 |
+
f"Ground everything in the conversation and the stored transcript below; do not invent "
|
| 111 |
+
f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
|
| 112 |
+
f"If the answer needs visual detail you don't have, say so and suggest re-analyzing."
|
| 113 |
)
|
| 114 |
user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
|
| 115 |
if last.get("quote"):
|
|
|
|
| 159 |
yield ["Please type a message.", False, transcript]
|
| 160 |
return
|
| 161 |
|
| 162 |
+
# Keep NINA on the uploaded video β politely refuse off-topic detours.
|
| 163 |
+
if not force and not _on_topic(messages):
|
| 164 |
+
lang = _detect_language(prompt)
|
| 165 |
+
if lang.lower().startswith("pol"):
|
| 166 |
+
msg = "Trzymajmy siΔ filmu, ktΓ³ry wgraΕeΕ β zapytaj mnie o coΕ z niego, a chΔtnie pomogΔ."
|
| 167 |
+
else:
|
| 168 |
+
msg = "Let's stick to the video you uploaded β ask me something about it and I'll dig in."
|
| 169 |
+
yield [msg, False, transcript]
|
| 170 |
+
return
|
| 171 |
+
|
| 172 |
# Decide: analyze (re-watch) vs answer (from context).
|
| 173 |
decision = "analyze" if force else _route(messages, video_path is not None or has_video)
|
| 174 |
|
pipeline.py
CHANGED
|
@@ -182,6 +182,27 @@ def _run_global(chunks: list[dict], visual: dict,
|
|
| 182 |
return add_visual_tags(sampled, visual)
|
| 183 |
|
| 184 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 185 |
# ββ Streaming synthesis wrapper ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 186 |
|
| 187 |
def _synthesize_streaming(regions: list[dict], prompt: str,
|
|
@@ -301,9 +322,7 @@ def _run_core(video_path: str, prompt: str):
|
|
| 301 |
yield "Error reading video."
|
| 302 |
return
|
| 303 |
entry = _res["v"]
|
| 304 |
-
|
| 305 |
-
yield "No speech detected."
|
| 306 |
-
return
|
| 307 |
_cache[video_path] = entry
|
| 308 |
if fp:
|
| 309 |
disk_save(fp, entry)
|
|
@@ -325,7 +344,11 @@ def _run_core(video_path: str, prompt: str):
|
|
| 325 |
fp = None
|
| 326 |
|
| 327 |
if not chunks:
|
| 328 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
return
|
| 330 |
|
| 331 |
# How trustworthy is the transcript? On garbled/musical audio we tell synthesis
|
|
|
|
| 182 |
return add_visual_tags(sampled, visual)
|
| 183 |
|
| 184 |
|
| 185 |
+
# ββ Visual-only path (no transcript) βββββββββββββββββββββββββββββββββββββββββββ
|
| 186 |
+
|
| 187 |
+
def _run_visual_only(video_path: str, prompt: str, visual: dict, max_words: int):
|
| 188 |
+
"""No transcript available β synthesize from the visuals alone. One full-span
|
| 189 |
+
region carries the VLM's keyframes + visual tags so it can answer about what's
|
| 190 |
+
on screen (actions, objects, on-screen text) without any speech."""
|
| 191 |
+
dur = media_duration(video_path) or 0.0
|
| 192 |
+
if not dur:
|
| 193 |
+
times = visual.get("time") or []
|
| 194 |
+
dur = float(times[-1]) if times else 0.0
|
| 195 |
+
region = {
|
| 196 |
+
"start": 0.0, "end": float(dur), "text": "", "precise_text": "",
|
| 197 |
+
"confidence": 0.5, "bi_score": 0.0, "entity_overlap": 0.0, "visual_score": 0.0,
|
| 198 |
+
}
|
| 199 |
+
regions = add_visual_tags([region], visual)
|
| 200 |
+
frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
|
| 201 |
+
note = "(No speech detected in this video β answer from what is visible on screen.)"
|
| 202 |
+
yield from _synthesize_streaming(regions, prompt, max_words, "global",
|
| 203 |
+
reliable=False, speaker_note=note, frames=frames)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
# ββ Streaming synthesis wrapper ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 207 |
|
| 208 |
def _synthesize_streaming(regions: list[dict], prompt: str,
|
|
|
|
| 322 |
yield "Error reading video."
|
| 323 |
return
|
| 324 |
entry = _res["v"]
|
| 325 |
+
# No speech is fine β we fall back to visual-only synthesis below.
|
|
|
|
|
|
|
| 326 |
_cache[video_path] = entry
|
| 327 |
if fp:
|
| 328 |
disk_save(fp, entry)
|
|
|
|
| 344 |
fp = None
|
| 345 |
|
| 346 |
if not chunks:
|
| 347 |
+
# No usable speech anywhere β answer from the visuals alone. The VLM still
|
| 348 |
+
# sees sampled keyframes and reads any on-screen text, so questions like
|
| 349 |
+
# "what does the person do?" work on silent / music-only clips.
|
| 350 |
+
print("[NINA] No transcript β visual-only synthesis (VLM on keyframes).")
|
| 351 |
+
yield from _run_visual_only(video_path, prompt, visual, max_words)
|
| 352 |
return
|
| 353 |
|
| 354 |
# How trustworthy is the transcript? On garbled/musical audio we tell synthesis
|