Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit Β·
7c88737
1
Parent(s): 9a43844
Re-analysis confirm route + adaptive answer formatting
Browse files- chat/app: add a 'route' endpoint (route_decision) so the website can ask the
user before re-watching the video ("NINA wants to analyze the video again");
add force_answer ("Don't" β answer from context) alongside force_analyze.
- synthesis/chat: answers now adapt their shape to the question (Markdown) β
short paragraphs, bullet lists, bold, optional headings for long/multi-part
answers β instead of one prose block. Shared _FORMAT_GUIDE.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- app.py +11 -1
- chat.py +37 -4
- synthesis.py +20 -7
app.py
CHANGED
|
@@ -8,7 +8,7 @@ import os
|
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
|
| 11 |
-
from chat import chat_stream
|
| 12 |
from pipeline import run
|
| 13 |
|
| 14 |
|
|
@@ -68,6 +68,16 @@ with gr.Blocks(title="NINA β Multimodal Video Chat", delete_cache=(900, 900))
|
|
| 68 |
api_name="chat",
|
| 69 |
)
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
demo.queue(
|
| 72 |
default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
|
| 73 |
max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
|
|
|
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
|
| 11 |
+
from chat import chat_stream, route_decision
|
| 12 |
from pipeline import run
|
| 13 |
|
| 14 |
|
|
|
|
| 68 |
api_name="chat",
|
| 69 |
)
|
| 70 |
|
| 71 |
+
# Pre-flight: does this turn want a video re-analysis? Drives the website's
|
| 72 |
+
# "NINA wants to analyze the video again" popup. Returns one short string.
|
| 73 |
+
chat_route = gr.Textbox(label="route decision")
|
| 74 |
+
gr.Button("Route").click(
|
| 75 |
+
route_decision,
|
| 76 |
+
inputs=[chat_payload, chat_token],
|
| 77 |
+
outputs=chat_route,
|
| 78 |
+
api_name="route",
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
demo.queue(
|
| 82 |
default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
|
| 83 |
max_size=int(os.getenv("NINA_QUEUE_MAX", "40")),
|
chat.py
CHANGED
|
@@ -94,6 +94,28 @@ def _on_topic(messages: list) -> bool:
|
|
| 94 |
return True
|
| 95 |
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
def _answer_from_context(messages: list, transcript: str):
|
| 98 |
"""Stream a conversational answer grounded in the history + stored transcript."""
|
| 99 |
last = _last_user(messages)
|
|
@@ -109,7 +131,11 @@ def _answer_from_context(messages: list, transcript: str):
|
|
| 109 |
f"they mean), do NOT guess β reply with one short clarifying question naming the options. "
|
| 110 |
f"Ground everything in the conversation and the stored transcript below; do not invent "
|
| 111 |
f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
|
| 112 |
-
f"If the answer needs visual detail you don't have, say so and suggest re-analyzing."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
)
|
| 114 |
user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
|
| 115 |
if last.get("quote"):
|
|
@@ -149,7 +175,8 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
|
|
| 149 |
messages = payload.get("messages") or []
|
| 150 |
transcript = payload.get("transcript") or ""
|
| 151 |
allow_analyze = bool(payload.get("allow_analyze", True))
|
| 152 |
-
force = bool(payload.get("force_analyze", False))
|
|
|
|
| 153 |
has_video = bool(payload.get("has_video", False)) # client holds the chat's video
|
| 154 |
|
| 155 |
last = _last_user(messages)
|
|
@@ -169,8 +196,14 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
|
|
| 169 |
yield [msg, False, transcript]
|
| 170 |
return
|
| 171 |
|
| 172 |
-
# Decide: analyze (re-watch) vs answer (from context).
|
| 173 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
|
| 175 |
if decision == "analyze" and allow_analyze:
|
| 176 |
if video_path is not None:
|
|
|
|
| 94 |
return True
|
| 95 |
|
| 96 |
|
| 97 |
+
def route_decision(payload_json: str, token: str = "") -> str:
|
| 98 |
+
"""Pre-flight for the website: should this turn RE-ANALYZE the video? Returns one
|
| 99 |
+
of: 'analyze_first' (first turn β analyze, no prompt), 'confirm' (a later turn
|
| 100 |
+
needs footage detail not covered yet AND a video is available β ask the user
|
| 101 |
+
before spending the analysis), or 'answer' (answerable from context, or no video
|
| 102 |
+
to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
|
| 103 |
+
video again" popup."""
|
| 104 |
+
if not verify_usage_token(token):
|
| 105 |
+
return "answer"
|
| 106 |
+
try:
|
| 107 |
+
payload = json.loads(payload_json or "{}")
|
| 108 |
+
except Exception:
|
| 109 |
+
payload = {}
|
| 110 |
+
messages = payload.get("messages") or []
|
| 111 |
+
has_video = bool(payload.get("has_video", False))
|
| 112 |
+
if not any(m.get("role") == "assistant" for m in messages):
|
| 113 |
+
return "analyze_first"
|
| 114 |
+
if has_video and _route(messages, has_video) == "analyze":
|
| 115 |
+
return "confirm"
|
| 116 |
+
return "answer"
|
| 117 |
+
|
| 118 |
+
|
| 119 |
def _answer_from_context(messages: list, transcript: str):
|
| 120 |
"""Stream a conversational answer grounded in the history + stored transcript."""
|
| 121 |
last = _last_user(messages)
|
|
|
|
| 131 |
f"they mean), do NOT guess β reply with one short clarifying question naming the options. "
|
| 132 |
f"Ground everything in the conversation and the stored transcript below; do not invent "
|
| 133 |
f"facts. Stay on the video: if asked something unrelated to it, briefly steer back. "
|
| 134 |
+
f"If the answer needs visual detail you don't have, say so and suggest re-analyzing. "
|
| 135 |
+
f"FORMAT: match the answer's shape to the question (Markdown) β a short question gets a "
|
| 136 |
+
f"sentence or two; a long or multi-part answer uses short paragraphs, '- ' bullet lists "
|
| 137 |
+
f"for sets of items/steps, and **bold** for key terms. Never reply with one big block of "
|
| 138 |
+
f"text, and never over-format a small answer."
|
| 139 |
)
|
| 140 |
user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
|
| 141 |
if last.get("quote"):
|
|
|
|
| 175 |
messages = payload.get("messages") or []
|
| 176 |
transcript = payload.get("transcript") or ""
|
| 177 |
allow_analyze = bool(payload.get("allow_analyze", True))
|
| 178 |
+
force = bool(payload.get("force_analyze", False)) # user chose "Let her"
|
| 179 |
+
force_answer = bool(payload.get("force_answer", False)) # user chose "Don't"
|
| 180 |
has_video = bool(payload.get("has_video", False)) # client holds the chat's video
|
| 181 |
|
| 182 |
last = _last_user(messages)
|
|
|
|
| 196 |
yield [msg, False, transcript]
|
| 197 |
return
|
| 198 |
|
| 199 |
+
# Decide: analyze (re-watch) vs answer (from context). The website settles this
|
| 200 |
+
# up front via route_decision + the "analyze again?" popup, then sends force_*.
|
| 201 |
+
if force_answer:
|
| 202 |
+
decision = "answer" # "Don't" β answer from context / cache only
|
| 203 |
+
elif force:
|
| 204 |
+
decision = "analyze" # "Let her" / first turn β re-watch
|
| 205 |
+
else:
|
| 206 |
+
decision = _route(messages, video_path is not None or has_video)
|
| 207 |
|
| 208 |
if decision == "analyze" and allow_analyze:
|
| 209 |
if video_path is not None:
|
synthesis.py
CHANGED
|
@@ -120,6 +120,21 @@ def _detect_language(text: str) -> str:
|
|
| 120 |
return "English"
|
| 121 |
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
def _make_system_msg(language: str, max_words: int, tier: str,
|
| 124 |
reliable: bool = True, speaker_note: str = "") -> str:
|
| 125 |
speakers = "" if not speaker_note else (
|
|
@@ -156,10 +171,7 @@ def _make_system_msg(language: str, max_words: int, tier: str,
|
|
| 156 |
"fix grammar inside quoted speech."
|
| 157 |
f"{speakers}"
|
| 158 |
f"{audio_quality}\n\n"
|
| 159 |
-
"
|
| 160 |
-
"Do NOT begin lines with timestamps. After the answer, add exactly one final line "
|
| 161 |
-
"listing the moments you drew from, in this form:\n"
|
| 162 |
-
"Sources: [m:ss], [m:ss]"
|
| 163 |
)
|
| 164 |
|
| 165 |
|
|
@@ -211,7 +223,9 @@ def _build_synth_prompt(regions: list[dict], prompt: str, max_words: int, mode:
|
|
| 211 |
f"{task_line}\n\n"
|
| 212 |
f"How to write the answer:\n"
|
| 213 |
f"- Reply in the SAME language as the user's question above.\n"
|
| 214 |
-
f"-
|
|
|
|
|
|
|
| 215 |
f"- Ground every statement only in the fragments; never merge two fragments into a "
|
| 216 |
f"single claim, and don't derive new claims from the supporting-context blocks alone.\n"
|
| 217 |
f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n"
|
|
@@ -329,8 +343,7 @@ def _make_vlm_system_msg(language: str, max_words: int, tier: str,
|
|
| 329 |
"\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity "
|
| 330 |
"can't be determined from the video. Never guess a real person's name."
|
| 331 |
f"{speakers}{audio}\n\n"
|
| 332 |
-
"
|
| 333 |
-
"Sources: [m:ss], [m:ss] (the frame/transcript moments you used)."
|
| 334 |
)
|
| 335 |
|
| 336 |
|
|
|
|
| 120 |
return "English"
|
| 121 |
|
| 122 |
|
| 123 |
+
# Adaptive answer formatting β let NINA shape the answer to the question (Markdown,
|
| 124 |
+
# which the chat renders) instead of always emitting one prose block. Shared by the
|
| 125 |
+
# text and vision synthesis prompts.
|
| 126 |
+
_FORMAT_GUIDE = (
|
| 127 |
+
"FORMAT β Match the shape of the answer to the question, using Markdown:\n"
|
| 128 |
+
"- A simple or short question β one or two short sentences. No headings or lists.\n"
|
| 129 |
+
"- A longer or multi-part answer β do NOT write one big block of text. Break it into "
|
| 130 |
+
"short paragraphs (1β3 sentences each), use a '- ' bulleted list for any set of items, "
|
| 131 |
+
"steps or examples, and **bold** the key terms. Add a short '## heading' only when there "
|
| 132 |
+
"are genuinely distinct sections.\n"
|
| 133 |
+
"- Never over-format a small answer or add empty headings; keep it natural and scannable.\n"
|
| 134 |
+
"Do NOT begin lines with timestamps. End with exactly one line: Sources: [m:ss], [m:ss]."
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
def _make_system_msg(language: str, max_words: int, tier: str,
|
| 139 |
reliable: bool = True, speaker_note: str = "") -> str:
|
| 140 |
speakers = "" if not speaker_note else (
|
|
|
|
| 171 |
"fix grammar inside quoted speech."
|
| 172 |
f"{speakers}"
|
| 173 |
f"{audio_quality}\n\n"
|
| 174 |
+
f"{_FORMAT_GUIDE}"
|
|
|
|
|
|
|
|
|
|
| 175 |
)
|
| 176 |
|
| 177 |
|
|
|
|
| 223 |
f"{task_line}\n\n"
|
| 224 |
f"How to write the answer:\n"
|
| 225 |
f"- Reply in the SAME language as the user's question above.\n"
|
| 226 |
+
f"- Format for readability (per the system instructions): short paragraphs, with a "
|
| 227 |
+
f"bullet list or a brief heading when the answer is long or has parts. Don't prefix "
|
| 228 |
+
f"lines with timestamps.\n"
|
| 229 |
f"- Ground every statement only in the fragments; never merge two fragments into a "
|
| 230 |
f"single claim, and don't derive new claims from the supporting-context blocks alone.\n"
|
| 231 |
f"- Keep it to about {max_words} words at most; be shorter if the material is thin.\n"
|
|
|
|
| 343 |
"\"a bearded man\", \"the speaker\", \"a person on stage\") and say their identity "
|
| 344 |
"can't be determined from the video. Never guess a real person's name."
|
| 345 |
f"{speakers}{audio}\n\n"
|
| 346 |
+
f"{_FORMAT_GUIDE}"
|
|
|
|
| 347 |
)
|
| 348 |
|
| 349 |
|