Malina Claude Opus 4.8 commited on
Commit
7c88737
Β·
1 Parent(s): 9a43844

Re-analysis confirm route + adaptive answer formatting

Browse files

- chat/app: add a 'route' endpoint (route_decision) so the website can ask the
user before re-watching the video ("NINA wants to analyze the video again");
add force_answer ("Don't" β†’ answer from context) alongside force_analyze.
- synthesis/chat: answers now adapt their shape to the question (Markdown) β€”
short paragraphs, bullet lists, bold, optional headings for long/multi-part
answers β€” instead of one prose block. Shared _FORMAT_GUIDE.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (3) hide show
  1. app.py +11 -1
  2. chat.py +37 -4
  3. synthesis.py +20 -7
app.py CHANGED
@@ -8,7 +8,7 @@ import os
8
 
9
  import gradio as gr
10
 
11
- from chat import chat_stream
12
  from pipeline import run
13
 
14
 
@@ -68,6 +68,16 @@ with gr.Blocks(title="NINA β€” Multimodal Video Chat", delete_cache=(900, 900))
68
  api_name="chat",
69
  )
70
 
 
 
 
 
 
 
 
 
 
 
71
  demo.queue(
72
  default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
73
  max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
 
8
 
9
  import gradio as gr
10
 
11
+ from chat import chat_stream, route_decision
12
  from pipeline import run
13
 
14
 
 
68
  api_name="chat",
69
  )
70
 
71
+ # Pre-flight: does this turn want a video re-analysis? Drives the website's
72
+ # "NINA wants to analyze the video again" popup. Returns one short string.
73
+ chat_route = gr.Textbox(label="route decision")
74
+ gr.Button("Route").click(
75
+ route_decision,
76
+ inputs=[chat_payload, chat_token],
77
+ outputs=chat_route,
78
+ api_name="route",
79
+ )
80
+
81
  demo.queue(
82
  default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
83
  max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
chat.py CHANGED
@@ -94,6 +94,28 @@ def _on_topic(messages: list) -> bool:
94
  return True
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  def _answer_from_context(messages: list, transcript: str):
98
  """Stream a conversational answer grounded in the history + stored transcript."""
99
  last = _last_user(messages)
@@ -109,7 +131,11 @@ def _answer_from_context(messages: list, transcript: str):
109
  f"they mean), do NOT guess β€” reply with one short clarifying question naming the options. "
110
  f"Ground everything in the conversation and the stored transcript below; do not invent "
111
  f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
112
- f"If the answer needs visual detail you don't have, say so and suggest re-analyzing."
 
 
 
 
113
  )
114
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
115
  if last.get("quote"):
@@ -149,7 +175,8 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
149
  messages = payload.get("messages") or []
150
  transcript = payload.get("transcript") or ""
151
  allow_analyze = bool(payload.get("allow_analyze", True))
152
- force = bool(payload.get("force_analyze", False))
 
153
  has_video = bool(payload.get("has_video", False)) # client holds the chat's video
154
 
155
  last = _last_user(messages)
@@ -169,8 +196,14 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
169
  yield [msg, False, transcript]
170
  return
171
 
172
- # Decide: analyze (re-watch) vs answer (from context).
173
- decision = "analyze" if force else _route(messages, video_path is not None or has_video)
 
 
 
 
 
 
174
 
175
  if decision == "analyze" and allow_analyze:
176
  if video_path is not None:
 
94
  return True
95
 
96
 
97
+ def route_decision(payload_json: str, token: str = "") -> str:
98
+ """Pre-flight for the website: should this turn RE-ANALYZE the video? Returns one
99
+ of: 'analyze_first' (first turn β€” analyze, no prompt), 'confirm' (a later turn
100
+ needs footage detail not covered yet AND a video is available β†’ ask the user
101
+ before spending the analysis), or 'answer' (answerable from context, or no video
102
+ to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
103
+ video again" popup."""
104
+ if not verify_usage_token(token):
105
+ return "answer"
106
+ try:
107
+ payload = json.loads(payload_json or "{}")
108
+ except Exception:
109
+ payload = {}
110
+ messages = payload.get("messages") or []
111
+ has_video = bool(payload.get("has_video", False))
112
+ if not any(m.get("role") == "assistant" for m in messages):
113
+ return "analyze_first"
114
+ if has_video and _route(messages, has_video) == "analyze":
115
+ return "confirm"
116
+ return "answer"
117
+
118
+
119
  def _answer_from_context(messages: list, transcript: str):
120
  """Stream a conversational answer grounded in the history + stored transcript."""
121
  last = _last_user(messages)
 
131
  f"they mean), do NOT guess β€” reply with one short clarifying question naming the options. "
132
  f"Ground everything in the conversation and the stored transcript below; do not invent "
133
  f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
134
+ f"If the answer needs visual detail you don't have, say so and suggest re-analyzing. "
135
+ f"FORMAT: match the answer's shape to the question (Markdown) β€” a short question gets a "
136
+ f"sentence or two; a long or multi-part answer uses short paragraphs, '- ' bullet lists "
137
+ f"for sets of items/steps, and **bold** for key terms. Never reply with one big block of "
138
+ f"text, and never over-format a small answer."
139
  )
140
  user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
141
  if last.get("quote"):
 
175
  messages = payload.get("messages") or []
176
  transcript = payload.get("transcript") or ""
177
  allow_analyze = bool(payload.get("allow_analyze", True))
178
+ force = bool(payload.get("force_analyze", False)) # user chose "Let her"
179
+ force_answer = bool(payload.get("force_answer", False)) # user chose "Don't"
180
  has_video = bool(payload.get("has_video", False)) # client holds the chat's video
181
 
182
  last = _last_user(messages)
 
196
  yield [msg, False, transcript]
197
  return
198
 
199
+ # Decide: analyze (re-watch) vs answer (from context). The website settles this
200
+ # up front via route_decision + the "analyze again?" popup, then sends force_*.
201
+ if force_answer:
202
+ decision = "answer" # "Don't" β€” answer from context / cache only
203
+ elif force:
204
+ decision = "analyze" # "Let her" / first turn β€” re-watch
205
+ else:
206
+ decision = _route(messages, video_path is not None or has_video)
207
 
208
  if decision == "analyze" and allow_analyze:
209
  if video_path is not None:
synthesis.py CHANGED
@@ -120,6 +120,21 @@ def _detect_language(text: str) -> str:
120
  return "English"
121
 
122
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
  def _make_system_msg(language: str, max_words: int, tier: str,
124
  reliable: bool = True, speaker_note: str = "") -> str:
125
  speakers = "" if not speaker_note else (
@@ -156,10 +171,7 @@ def _make_system_msg(language: str, max_words: int, tier: str,
156
  "fix grammar inside quoted speech."
157
  f"{speakers}"
158
  f"{audio_quality}\n\n"
159
- "FORMAT β€” Write a natural, flowing answer in prose (sentences or short paragraphs). "
160
- "Do NOT begin lines with timestamps. After the answer, add exactly one final line "
161
- "listing the moments you drew from, in this form:\n"
162
- "Sources: [m:ss], [m:ss]"
163
  )
164
 
165
 
@@ -211,7 +223,9 @@ def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode:
211
  f"{task_line}\n\n"
212
  f"How to write the answer:\n"
213
  f"- Reply in the SAME language as the user's question above.\n"
214
- f"- Write naturally, in prose β€” do NOT prefix lines with timestamps.\n"
 
 
215
  f"- Ground every statement only in the fragments; never merge two fragments into a "
216
  f"single claim, and don't derive new claims from the supporting-context blocks alone.\n"
217
  f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n"
@@ -329,8 +343,7 @@ def _make_vlm_system_msg(language: str, max_words: int, tier: str,
329
  "\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity "
330
  "can't be determined from the video. Never guess a real person's name."
331
  f"{speakers}{audio}\n\n"
332
- "FORMAT β€” Natural prose, no per-line timestamps. End with one line: "
333
- "Sources: [m:ss], [m:ss] (the frame/transcript moments you used)."
334
  )
335
 
336
 
 
120
  return "English"
121
 
122
 
123
+ # Adaptive answer formatting β€” let NINA shape the answer to the question (Markdown,
124
+ # which the chat renders) instead of always emitting one prose block. Shared by the
125
+ # text and vision synthesis prompts.
126
+ _FORMAT_GUIDE = (
127
+ "FORMAT β€” Match the shape of the answer to the question, using Markdown:\n"
128
+ "- A simple or short question β†’ one or two short sentences. No headings or lists.\n"
129
+ "- A longer or multi-part answer β†’ do NOT write one big block of text. Break it into "
130
+ "short paragraphs (1–3 sentences each), use a '- ' bulleted list for any set of items, "
131
+ "steps or examples, and **bold** the key terms. Add a short '## heading' only when there "
132
+ "are genuinely distinct sections.\n"
133
+ "- Never over-format a small answer or add empty headings; keep it natural and scannable.\n"
134
+ "Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]."
135
+ )
136
+
137
+
138
  def _make_system_msg(language: str, max_words: int, tier: str,
139
  reliable: bool = True, speaker_note: str = "") -> str:
140
  speakers = "" if not speaker_note else (
 
171
  "fix grammar inside quoted speech."
172
  f"{speakers}"
173
  f"{audio_quality}\n\n"
174
+ f"{_FORMAT_GUIDE}"
 
 
 
175
  )
176
 
177
 
 
223
  f"{task_line}\n\n"
224
  f"How to write the answer:\n"
225
  f"- Reply in the SAME language as the user's question above.\n"
226
+ f"- Format for readability (per the system instructions): short paragraphs, with a "
227
+ f"bullet list or a brief heading when the answer is long or has parts. Don't prefix "
228
+ f"lines with timestamps.\n"
229
  f"- Ground every statement only in the fragments; never merge two fragments into a "
230
  f"single claim, and don't derive new claims from the supporting-context blocks alone.\n"
231
  f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n"
 
343
  "\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity "
344
  "can't be determined from the video. Never guess a real person's name."
345
  f"{speakers}{audio}\n\n"
346
+ f"{_FORMAT_GUIDE}"
 
347
  )
348
 
349