Malina Claude Opus 4.8 commited on
Commit
9a43844
Β·
1 Parent(s): 710a0dd

Visual-only answers, coreference + on-topic guard, 15-min cache

Browse files

- pipeline: drop the "No speech detected" dead-end β€” when there's no usable
transcript, synthesize from the visuals alone (VLM on keyframes), so
"what does the person do?" works on silent/music clips.
- chat: resolve references to earlier turns ("explain more"); ask to clarify
when a reference is ambiguous; gate out off-topic requests and steer back to
the uploaded video; route follow-ups to context instead of re-analysis.
- app: keep uploaded media 15 min (delete_cache 900) so follow-up turns reuse
the already-uploaded video without re-uploading.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (3) hide show
  1. app.py +4 -2
  2. chat.py +56 -7
  3. pipeline.py +27 -4
app.py CHANGED
@@ -31,8 +31,10 @@ for _name in ("uvicorn.error", "uvicorn"):
31
  logging.getLogger(_name).addFilter(_DropClientDisconnect())
32
 
33
  # The custom front-end at app.nnlabs.pl drives these two API endpoints directly
34
- # (the Gradio UI itself is unused). Privacy: purge uploaded files every 5 min.
35
- with gr.Blocks(title="NINA β€” Multimodal Video Chat", delete_cache=(300, 300)) as demo:
 
 
36
  gr.Markdown(
37
  "## NINA β€” Neural Information & Narrative Abstraction\n"
38
  "API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
 
31
  logging.getLogger(_name).addFilter(_DropClientDisconnect())
32
 
33
  # The custom front-end at app.nnlabs.pl drives these two API endpoints directly
34
+ # (the Gradio UI itself is unused). Privacy: purge uploaded files every 15 min β€”
35
+ # long enough that follow-up turns in a chat re-use the already-uploaded video
36
+ # (no re-upload), short enough to honor "media is never stored".
37
+ with gr.Blocks(title="NINA β€” Multimodal Video Chat", delete_cache=(900, 900)) as demo:
38
  gr.Markdown(
39
  "## NINA β€” Neural Information & Narrative Abstraction\n"
40
  "API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
chat.py CHANGED
@@ -48,9 +48,13 @@ def _route(messages: list, has_video: bool) -> str:
48
  model=LLM_MODEL,
49
  messages=[
50
  {"role": "system", "content":
51
- "Decide whether answering the user's latest message needs RE-WATCHING the "
52
- "video for visual/audio detail not yet covered, or can be answered from the "
53
- "conversation so far. Reply with exactly one word: ANALYZE or ANSWER."},
 
 
 
 
54
  {"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
55
  ],
56
  max_tokens=3, temperature=0.0, stream=False,
@@ -61,16 +65,51 @@ def _route(messages: list, has_video: bool) -> str:
61
  return "answer"
62
 
63
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  def _answer_from_context(messages: list, transcript: str):
65
  """Stream a conversational answer grounded in the history + stored transcript."""
66
  last = _last_user(messages)
67
  prompt = (last.get("content") or "").strip()
68
  language = _detect_language(prompt)
69
  system = (
70
- f"You are NINA, a video-understanding assistant continuing a conversation. "
71
- f"Write your answer in {language}. Ground it in the conversation and the stored video "
72
- f"transcript below β€” do not invent facts that aren't there. If the answer needs visual "
73
- f"detail you don't have, say so and suggest re-analyzing the video."
 
 
 
 
 
 
74
  )
75
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
76
  if last.get("quote"):
@@ -120,6 +159,16 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
120
  yield ["Please type a message.", False, transcript]
121
  return
122
 
 
 
 
 
 
 
 
 
 
 
123
  # Decide: analyze (re-watch) vs answer (from context).
124
  decision = "analyze" if force else _route(messages, video_path is not None or has_video)
125
 
 
48
  model=LLM_MODEL,
49
  messages=[
50
  {"role": "system", "content":
51
+ "You route turns for a video-chat assistant. Decide whether answering the "
52
+ "user's latest message needs RE-WATCHING the video for visual/audio detail "
53
+ "not covered yet, or can be answered from the conversation so far. "
54
+ "A short follow-up that refers back to the discussion ('explain more', "
55
+ "'why?', 'the second one', 'and then?') is almost always ANSWER β€” only pick "
56
+ "ANALYZE when genuinely new footage detail is required. "
57
+ "Reply with exactly one word: ANALYZE or ANSWER."},
58
  {"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
59
  ],
60
  max_tokens=3, temperature=0.0, stream=False,
 
65
  return "answer"
66
 
67
 
68
+ def _on_topic(messages: list) -> bool:
69
+ """Gate out off-topic requests ('what can I cook for dinner?'). Only checked once
70
+ a conversation is under way β€” the first message about an uploaded video is always
71
+ on-topic. Conservative: any error / uncertainty defaults to on-topic."""
72
+ if not any(m.get("role") == "assistant" for m in messages):
73
+ return True
74
+ last = _last_user(messages)
75
+ if not (last.get("content") or "").strip():
76
+ return True
77
+ try:
78
+ r = llm_client.chat_completion(
79
+ model=LLM_MODEL,
80
+ messages=[
81
+ {"role": "system", "content":
82
+ "You guard a video-understanding assistant. The user uploaded a video and is "
83
+ "chatting about it. Decide if their latest message is RELATED to that video or "
84
+ "the ongoing discussion of it β€” including vague follow-ups like 'explain more', "
85
+ "'why?', 'and then?' β€” or is OFF-TOPIC: a general request unrelated to the "
86
+ "video (recipes, coding help, world trivia, chit-chat). "
87
+ "Reply with exactly one word: ON or OFF."},
88
+ {"role": "user", "content": _render_history(messages) + "\n\nReply ON or OFF."},
89
+ ],
90
+ max_tokens=2, temperature=0.0, stream=False,
91
+ )
92
+ return not (r.choices[0].message.content or "").strip().upper().startswith("OFF")
93
+ except Exception:
94
+ return True
95
+
96
+
97
  def _answer_from_context(messages: list, transcript: str):
98
  """Stream a conversational answer grounded in the history + stored transcript."""
99
  last = _last_user(messages)
100
  prompt = (last.get("content") or "").strip()
101
  language = _detect_language(prompt)
102
  system = (
103
+ f"You are NINA, a video-understanding assistant continuing a conversation about a video "
104
+ f"the user uploaded. Write your answer in {language}. "
105
+ f"RESOLVE REFERENCES to earlier turns: a short follow-up like 'explain more', 'why?', "
106
+ f"'the second one', 'and that part' refers to your previous answer or a specific earlier "
107
+ f"point β€” use the conversation to work out exactly what they mean and answer that. "
108
+ f"If such a reference is genuinely AMBIGUOUS (you can't tell which message or which part "
109
+ f"they mean), do NOT guess β€” reply with one short clarifying question naming the options. "
110
+ f"Ground everything in the conversation and the stored transcript below; do not invent "
111
+ f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
112
+ f"If the answer needs visual detail you don't have, say so and suggest re-analyzing."
113
  )
114
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
115
  if last.get("quote"):
 
159
  yield ["Please type a message.", False, transcript]
160
  return
161
 
162
+ # Keep NINA on the uploaded video β€” politely refuse off-topic detours.
163
+ if not force and not _on_topic(messages):
164
+ lang = _detect_language(prompt)
165
+ if lang.lower().startswith("pol"):
166
+ msg = "Trzymajmy siΔ™ filmu, ktΓ³ry wgraΕ‚eΕ› β€” zapytaj mnie o coΕ› z niego, a chΔ™tnie pomogΔ™."
167
+ else:
168
+ msg = "Let's stick to the video you uploaded β€” ask me something about it and I'll dig in."
169
+ yield [msg, False, transcript]
170
+ return
171
+
172
  # Decide: analyze (re-watch) vs answer (from context).
173
  decision = "analyze" if force else _route(messages, video_path is not None or has_video)
174
 
pipeline.py CHANGED
@@ -182,6 +182,27 @@ def _run_global(chunks: list[dict], visual: dict,
182
  return add_visual_tags(sampled, visual)
183
 
184
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
185
  # ── Streaming synthesis wrapper ────────────────────────────────────────────────
186
 
187
  def _synthesize_streaming(regions: list[dict], prompt: str,
@@ -301,9 +322,7 @@ def _run_core(video_path: str, prompt: str):
301
  yield "Error reading video."
302
  return
303
  entry = _res["v"]
304
- if not entry["segments"]:
305
- yield "No speech detected."
306
- return
307
  _cache[video_path] = entry
308
  if fp:
309
  disk_save(fp, entry)
@@ -325,7 +344,11 @@ def _run_core(video_path: str, prompt: str):
325
  fp = None
326
 
327
  if not chunks:
328
- yield "No speech detected."
 
 
 
 
329
  return
330
 
331
  # How trustworthy is the transcript? On garbled/musical audio we tell synthesis
 
182
  return add_visual_tags(sampled, visual)
183
 
184
 
185
+ # ── Visual-only path (no transcript) ───────────────────────────────────────────
186
+
187
+ def _run_visual_only(video_path: str, prompt: str, visual: dict, max_words: int):
188
+ """No transcript available β€” synthesize from the visuals alone. One full-span
189
+ region carries the VLM's keyframes + visual tags so it can answer about what's
190
+ on screen (actions, objects, on-screen text) without any speech."""
191
+ dur = media_duration(video_path) or 0.0
192
+ if not dur:
193
+ times = visual.get("time") or []
194
+ dur = float(times[-1]) if times else 0.0
195
+ region = {
196
+ "start": 0.0, "end": float(dur), "text": "", "precise_text": "",
197
+ "confidence": 0.5, "bi_score": 0.0, "entity_overlap": 0.0, "visual_score": 0.0,
198
+ }
199
+ regions = add_visual_tags([region], visual)
200
+ frames = keyframes_for(video_path, regions, visual.get("time")) if USE_VLM else None
201
+ note = "(No speech detected in this video β€” answer from what is visible on screen.)"
202
+ yield from _synthesize_streaming(regions, prompt, max_words, "global",
203
+ reliable=False, speaker_note=note, frames=frames)
204
+
205
+
206
  # ── Streaming synthesis wrapper ────────────────────────────────────────────────
207
 
208
  def _synthesize_streaming(regions: list[dict], prompt: str,
 
322
  yield "Error reading video."
323
  return
324
  entry = _res["v"]
325
+ # No speech is fine β€” we fall back to visual-only synthesis below.
 
 
326
  _cache[video_path] = entry
327
  if fp:
328
  disk_save(fp, entry)
 
344
  fp = None
345
 
346
  if not chunks:
347
+ # No usable speech anywhere β€” answer from the visuals alone. The VLM still
348
+ # sees sampled keyframes and reads any on-screen text, so questions like
349
+ # "what does the person do?" work on silent / music-only clips.
350
+ print("[NINA] No transcript β€” visual-only synthesis (VLM on keyframes).")
351
+ yield from _run_visual_only(video_path, prompt, visual, max_words)
352
  return
353
 
354
  # How trustworthy is the transcript? On garbled/musical audio we tell synthesis