Malina Claude Opus 4.8 commited on
Commit
9510df1
Β·
1 Parent(s): 6dd7da1

NINA: conversational chat endpoint (Phase 2)

Browse files

- app.py β†’ gr.Blocks exposing two API endpoints: /run (legacy single-shot, the
current tool page) and /chat (new). Keeps delete_cache privacy purge.
- chat.py: per-turn router (LLM decides ANALYZE vs ANSWER), context answering from
the stored transcript + conversation, manual force_analyze override. Streams
[answer, analyzed, transcript]; the website stores the transcript text (no media).
- pipeline: split run() into a token-checked wrapper + _run_core so chat can analyze
without re-consuming the single-use token; add transcript_text() for storage.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (3) hide show
  1. app.py +37 -23
  2. chat.py +144 -0
  3. pipeline.py +19 -0
app.py CHANGED
@@ -1,6 +1,6 @@
1
  # ─────────────────────────────────────────────
2
  # NINA β€” Neural Information & Narrative Abstraction
3
- # app.py | v0.81 | Made by Malina
4
  # ─────────────────────────────────────────────
5
 
6
  import logging
@@ -8,6 +8,7 @@ import os
8
 
9
  import gradio as gr
10
 
 
11
  from pipeline import run
12
 
13
 
@@ -29,28 +30,41 @@ class _DropClientDisconnect(logging.Filter):
29
  for _name in ("uvicorn.error", "uvicorn"):
30
  logging.getLogger(_name).addFilter(_DropClientDisconnect())
31
 
32
- demo = gr.Interface(
33
- fn=run,
34
- inputs=[
35
- gr.Video(label="Upload video "),
36
- gr.Textbox(
37
- label="What are you looking for?",
38
- placeholder='e.g. "What does the video say about...?" '
39
- '(add "briefly" or "in detail" to size the answer)',
40
- ),
41
- gr.Textbox(label="access token (set automatically by the website)", value=""),
42
- ],
43
- outputs=gr.Textbox(label="Results", lines=38),
44
- title="NINA v0.81 β€” Multimodal Video Search (audio + video via CLIP)",
45
- description=(
46
- "video -> a bit of magic (speech + vision) -> summary. "
47
- "Answer length adapts to your prompt. "
48
- "Use it at https://app.nnlabs.pl/nina."
49
- ),
50
- flagging_mode="never",
51
- # Privacy: purge uploaded files older than 5 min, every 5 min.
52
- delete_cache=(300, 300),
53
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
  demo.queue(
56
  default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
 
1
  # ─────────────────────────────────────────────
2
  # NINA β€” Neural Information & Narrative Abstraction
3
+ # app.py | v0.90 | Made by Malina
4
  # ─────────────────────────────────────────────
5
 
6
  import logging
 
8
 
9
  import gradio as gr
10
 
11
+ from chat import chat_stream
12
  from pipeline import run
13
 
14
 
 
30
  for _name in ("uvicorn.error", "uvicorn"):
31
  logging.getLogger(_name).addFilter(_DropClientDisconnect())
32
 
33
+ # The custom front-end at app.nnlabs.pl drives these two API endpoints directly
34
+ # (the Gradio UI itself is unused). Privacy: purge uploaded files every 5 min.
35
+ with gr.Blocks(title="NINA β€” Multimodal Video Chat", delete_cache=(300, 300)) as demo:
36
+ gr.Markdown(
37
+ "## NINA β€” Neural Information & Narrative Abstraction\n"
38
+ "API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
39
+ "(single-shot analysis) and `/chat` (conversation)."
40
+ )
41
+
42
+ # ── /run : single-shot analysis (legacy tool page) ───────────────────────────
43
+ with gr.Tab("Analyze"):
44
+ run_video = gr.Video(label="Upload video")
45
+ run_prompt = gr.Textbox(label="What are you looking for?")
46
+ run_token = gr.Textbox(label="access token (set by the website)", value="")
47
+ run_out = gr.Textbox(label="Results", lines=24)
48
+ gr.Button("Analyze").click(
49
+ run, inputs=[run_video, run_prompt, run_token], outputs=run_out, api_name="run"
50
+ )
51
+
52
+ # ── /chat : conversational endpoint (new chat UI) ────────────────────────────
53
+ # payload (JSON): {messages:[{role,content,quote?}], transcript, allow_analyze,
54
+ # force_analyze}. Streams [answer, analyzed, transcript].
55
+ with gr.Tab("Chat"):
56
+ chat_video = gr.Video(label="Video (optional β€” only on upload turns)")
57
+ chat_payload = gr.Textbox(label="payload (JSON)")
58
+ chat_token = gr.Textbox(label="access token", value="")
59
+ chat_answer = gr.Textbox(label="answer", lines=18)
60
+ chat_analyzed = gr.Checkbox(label="analyzed")
61
+ chat_transcript = gr.Textbox(label="transcript", lines=6)
62
+ gr.Button("Send").click(
63
+ chat_stream,
64
+ inputs=[chat_video, chat_payload, chat_token],
65
+ outputs=[chat_answer, chat_analyzed, chat_transcript],
66
+ api_name="chat",
67
+ )
68
 
69
  demo.queue(
70
  default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
chat.py ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ─────────────────────────────────────────────
2
+ # NINA β€” conversational chat endpoint
3
+ # ─────────────────────────────────────────────
4
+ # Turns NINA into a chat: each turn the model decides whether to RE-WATCH the video
5
+ # (retrieval + VLM on the uploaded/cached clip) or ANSWER from the conversation +
6
+ # the stored transcript. The video itself is never stored server-side beyond the
7
+ # short processing cache; the website stores only the derived transcript text.
8
+ #
9
+ # Streams a 3-tuple [answer_text, analyzed_bool, transcript_text]:
10
+ # β€’ answer_text β€” grows as the model streams
11
+ # β€’ analyzed_bool β€” True if this turn re-watched the video (counts toward usage)
12
+ # β€’ transcript_text β€” filled on the final event after an analysis (for storage)
13
+
14
+ import json
15
+
16
+ from auth import verify_usage_token
17
+ from config import LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE
18
+ from models import llm_client
19
+ from synthesis import _detect_language
20
+
21
+
22
+ def _last_user(messages: list) -> dict:
23
+ return next((m for m in reversed(messages) if m.get("role") == "user"), {})
24
+
25
+
26
+ def _render_history(messages: list, max_turns: int = 12) -> str:
27
+ lines = []
28
+ for m in messages[-max_turns:]:
29
+ who = "User" if m.get("role") == "user" else "NINA"
30
+ text = m.get("content", "")
31
+ if m.get("quote"):
32
+ text = f'(referring to: "{m["quote"]}")\n{text}'
33
+ lines.append(f"{who}: {text}")
34
+ return "\n\n".join(lines)
35
+
36
+
37
+ def _route(messages: list, has_video: bool) -> str:
38
+ """Decide 'analyze' (re-watch the video) vs 'answer' (from context). The LLM
39
+ chooses; first turn / no prior answer defaults to analyze."""
40
+ if not any(m.get("role") == "assistant" for m in messages):
41
+ return "analyze"
42
+ try:
43
+ r = llm_client.chat_completion(
44
+ model=LLM_MODEL,
45
+ messages=[
46
+ {"role": "system", "content":
47
+ "Decide whether answering the user's latest message needs RE-WATCHING the "
48
+ "video for visual/audio detail not yet covered, or can be answered from the "
49
+ "conversation so far. Reply with exactly one word: ANALYZE or ANSWER."},
50
+ {"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
51
+ ],
52
+ max_tokens=3, temperature=0.0, stream=False,
53
+ )
54
+ out = (r.choices[0].message.content or "").strip().upper()
55
+ return "analyze" if "ANALYZE" in out else "answer"
56
+ except Exception:
57
+ return "answer"
58
+
59
+
60
+ def _answer_from_context(messages: list, transcript: str):
61
+ """Stream a conversational answer grounded in the history + stored transcript."""
62
+ last = _last_user(messages)
63
+ prompt = (last.get("content") or "").strip()
64
+ language = _detect_language(prompt)
65
+ system = (
66
+ f"You are NINA, a video-understanding assistant continuing a conversation. "
67
+ f"Write your answer in {language}. Ground it in the conversation and the stored video "
68
+ f"transcript below β€” do not invent facts that aren't there. If the answer needs visual "
69
+ f"detail you don't have, say so and suggest re-analyzing the video."
70
+ )
71
+ user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
72
+ if last.get("quote"):
73
+ user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
74
+ user += "\n\nAnswer the user's latest message."
75
+ acc = ""
76
+ try:
77
+ stream = llm_client.chat_completion(
78
+ model=LLM_MODEL,
79
+ messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
80
+ max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
81
+ )
82
+ for ev in stream:
83
+ if not ev.choices:
84
+ continue
85
+ d = ev.choices[0].delta.content or ""
86
+ if d:
87
+ acc += d
88
+ yield acc.strip()
89
+ if not acc.strip():
90
+ yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
91
+ except Exception as e:
92
+ yield f"Couldn't answer from context ({e})."
93
+
94
+
95
+ def chat_stream(video_path, payload_json: str, token: str = ""):
96
+ """Gradio chat endpoint. payload_json: {messages, transcript, allow_analyze, force_analyze}."""
97
+ if not verify_usage_token(token):
98
+ yield ["β›” Not authorized. Please use NINA at https://app.nnlabs.pl (sign in first).",
99
+ False, ""]
100
+ return
101
+
102
+ try:
103
+ payload = json.loads(payload_json or "{}")
104
+ except Exception:
105
+ payload = {}
106
+ messages = payload.get("messages") or []
107
+ transcript = payload.get("transcript") or ""
108
+ allow_analyze = bool(payload.get("allow_analyze", True))
109
+ force = bool(payload.get("force_analyze", False))
110
+
111
+ last = _last_user(messages)
112
+ prompt = (last.get("content") or "").strip()
113
+ quote = last.get("quote")
114
+ if not prompt and video_path is None:
115
+ yield ["Please type a message.", False, transcript]
116
+ return
117
+
118
+ # Decide: analyze (re-watch) vs answer (from context).
119
+ decision = "analyze" if force else _route(messages, video_path is not None)
120
+ do_analyze = decision == "analyze" and video_path is not None and allow_analyze
121
+
122
+ if do_analyze:
123
+ from pipeline import _run_core, transcript_text
124
+ ask = (f'Regarding "{quote}": ' if quote else "") + (prompt or "Summarize the video.")
125
+ last_out = ""
126
+ for out in _run_core(video_path, ask):
127
+ last_out = out
128
+ yield [out, True, ""]
129
+ yield [last_out, True, transcript_text(video_path)]
130
+ return
131
+
132
+ # Answer from context. If the model wanted to re-watch but there's no video this
133
+ # turn (media already gone) or no budget, note it.
134
+ note = ""
135
+ if decision == "analyze" and video_path is None:
136
+ note = "\n\n_(To re-watch the video for new detail, re-upload it.)_"
137
+ elif decision == "analyze" and not allow_analyze:
138
+ note = "\n\n_(You've used your video-analysis budget for now, so I answered from our chat.)_"
139
+ acc = ""
140
+ for partial in _answer_from_context(messages, transcript):
141
+ acc = partial
142
+ yield [partial, False, transcript]
143
+ if note:
144
+ yield [acc + note, False, transcript]
pipeline.py CHANGED
@@ -21,6 +21,7 @@ from enhance import isolate_vocals
21
  from frames import keyframes_for
22
  from ocr import ocr_burned_in
23
  from subtitles import extract_embedded_subs
 
24
  from retrieval import (
25
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
26
  merge_regions, pad_with_neighbours, rerank_and_score,
@@ -178,6 +179,7 @@ def _synthesize_streaming(regions: list[dict], prompt: str,
178
  # ── Main pipeline ──────────────────────────────────────────────────────────────
179
 
180
  def run(video_path: str, prompt: str, token: str = ""):
 
181
  if not verify_usage_token(token):
182
  yield (
183
  "β›” This request is not authorized.\n\n"
@@ -185,6 +187,23 @@ def run(video_path: str, prompt: str, token: str = ""):
185
  "Direct API access is disabled, and each account has a usage limit."
186
  )
187
  return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  if video_path is None:
189
  yield "Please upload a video."
190
  return
 
21
  from frames import keyframes_for
22
  from ocr import ocr_burned_in
23
  from subtitles import extract_embedded_subs
24
+ from utils import fmt_time
25
  from retrieval import (
26
  add_entities_to_chunks, bi_encode_retrieve, classify_query,
27
  merge_regions, pad_with_neighbours, rerank_and_score,
 
179
  # ── Main pipeline ──────────────────────────────────────────────────────────────
180
 
181
  def run(video_path: str, prompt: str, token: str = ""):
182
+ """Public API endpoint: verify the single-use token, then run the pipeline."""
183
  if not verify_usage_token(token):
184
  yield (
185
  "β›” This request is not authorized.\n\n"
 
187
  "Direct API access is disabled, and each account has a usage limit."
188
  )
189
  return
190
+ yield from _run_core(video_path, prompt)
191
+
192
+
193
+ def transcript_text(video_path: str) -> str:
194
+ """The cached transcript for a video as timestamped text (for chat storage)."""
195
+ entry = _cache.get(video_path)
196
+ if not entry:
197
+ return ""
198
+ return "\n".join(
199
+ f"[{fmt_time(s['start'])}] {s['text']}"
200
+ for s in (entry.get("segments") or []) if s.get("text")
201
+ )
202
+
203
+
204
+ def _run_core(video_path: str, prompt: str):
205
+ """The pipeline without the token check β€” reused by the chat endpoint, which has
206
+ already authorized the turn (so it must not re-consume the single-use token)."""
207
  if video_path is None:
208
  yield "Please upload a video."
209
  return