Spaces:
Paused
Paused
Malina Claude Opus 4.8 commited on
Commit Β·
9510df1
1
Parent(s): 6dd7da1
NINA: conversational chat endpoint (Phase 2)
Browse files- app.py β gr.Blocks exposing two API endpoints: /run (legacy single-shot, the
current tool page) and /chat (new). Keeps delete_cache privacy purge.
- chat.py: per-turn router (LLM decides ANALYZE vs ANSWER), context answering from
the stored transcript + conversation, manual force_analyze override. Streams
[answer, analyzed, transcript]; the website stores the transcript text (no media).
- pipeline: split run() into a token-checked wrapper + _run_core so chat can analyze
without re-consuming the single-use token; add transcript_text() for storage.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
- app.py +37 -23
- chat.py +144 -0
- pipeline.py +19 -0
app.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 2 |
# NINA β Neural Information & Narrative Abstraction
|
| 3 |
-
# app.py | v0.
|
| 4 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 5 |
|
| 6 |
import logging
|
|
@@ -8,6 +8,7 @@ import os
|
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
|
|
|
|
| 11 |
from pipeline import run
|
| 12 |
|
| 13 |
|
|
@@ -29,28 +30,41 @@ class _DropClientDisconnect(logging.Filter):
|
|
| 29 |
for _name in ("uvicorn.error", "uvicorn"):
|
| 30 |
logging.getLogger(_name).addFilter(_DropClientDisconnect())
|
| 31 |
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
#
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
demo.queue(
|
| 56 |
default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
|
|
|
|
| 1 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 2 |
# NINA β Neural Information & Narrative Abstraction
|
| 3 |
+
# app.py | v0.90 | Made by Malina
|
| 4 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 5 |
|
| 6 |
import logging
|
|
|
|
| 8 |
|
| 9 |
import gradio as gr
|
| 10 |
|
| 11 |
+
from chat import chat_stream
|
| 12 |
from pipeline import run
|
| 13 |
|
| 14 |
|
|
|
|
| 30 |
for _name in ("uvicorn.error", "uvicorn"):
|
| 31 |
logging.getLogger(_name).addFilter(_DropClientDisconnect())
|
| 32 |
|
| 33 |
+
# The custom front-end at app.nnlabs.pl drives these two API endpoints directly
|
| 34 |
+
# (the Gradio UI itself is unused). Privacy: purge uploaded files every 5 min.
|
| 35 |
+
with gr.Blocks(title="NINA β Multimodal Video Chat", delete_cache=(300, 300)) as demo:
|
| 36 |
+
gr.Markdown(
|
| 37 |
+
"## NINA β Neural Information & Narrative Abstraction\n"
|
| 38 |
+
"API for the app at **https://app.nnlabs.pl/nina**. Two endpoints: `/run` "
|
| 39 |
+
"(single-shot analysis) and `/chat` (conversation)."
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
# ββ /run : single-shot analysis (legacy tool page) βββββββββββββββββββββββββββ
|
| 43 |
+
with gr.Tab("Analyze"):
|
| 44 |
+
run_video = gr.Video(label="Upload video")
|
| 45 |
+
run_prompt = gr.Textbox(label="What are you looking for?")
|
| 46 |
+
run_token = gr.Textbox(label="access token (set by the website)", value="")
|
| 47 |
+
run_out = gr.Textbox(label="Results", lines=24)
|
| 48 |
+
gr.Button("Analyze").click(
|
| 49 |
+
run, inputs=[run_video, run_prompt, run_token], outputs=run_out, api_name="run"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
# ββ /chat : conversational endpoint (new chat UI) ββββββββββββββββββββββββββββ
|
| 53 |
+
# payload (JSON): {messages:[{role,content,quote?}], transcript, allow_analyze,
|
| 54 |
+
# force_analyze}. Streams [answer, analyzed, transcript].
|
| 55 |
+
with gr.Tab("Chat"):
|
| 56 |
+
chat_video = gr.Video(label="Video (optional β only on upload turns)")
|
| 57 |
+
chat_payload = gr.Textbox(label="payload (JSON)")
|
| 58 |
+
chat_token = gr.Textbox(label="access token", value="")
|
| 59 |
+
chat_answer = gr.Textbox(label="answer", lines=18)
|
| 60 |
+
chat_analyzed = gr.Checkbox(label="analyzed")
|
| 61 |
+
chat_transcript = gr.Textbox(label="transcript", lines=6)
|
| 62 |
+
gr.Button("Send").click(
|
| 63 |
+
chat_stream,
|
| 64 |
+
inputs=[chat_video, chat_payload, chat_token],
|
| 65 |
+
outputs=[chat_answer, chat_analyzed, chat_transcript],
|
| 66 |
+
api_name="chat",
|
| 67 |
+
)
|
| 68 |
|
| 69 |
demo.queue(
|
| 70 |
default_concurrency_limit=int(os.getenv("NINA_CONCURRENCY", "3")),
|
chat.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 2 |
+
# NINA β conversational chat endpoint
|
| 3 |
+
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
+
# Turns NINA into a chat: each turn the model decides whether to RE-WATCH the video
|
| 5 |
+
# (retrieval + VLM on the uploaded/cached clip) or ANSWER from the conversation +
|
| 6 |
+
# the stored transcript. The video itself is never stored server-side beyond the
|
| 7 |
+
# short processing cache; the website stores only the derived transcript text.
|
| 8 |
+
#
|
| 9 |
+
# Streams a 3-tuple [answer_text, analyzed_bool, transcript_text]:
|
| 10 |
+
# β’ answer_text β grows as the model streams
|
| 11 |
+
# β’ analyzed_bool β True if this turn re-watched the video (counts toward usage)
|
| 12 |
+
# β’ transcript_text β filled on the final event after an analysis (for storage)
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
|
| 16 |
+
from auth import verify_usage_token
|
| 17 |
+
from config import LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE
|
| 18 |
+
from models import llm_client
|
| 19 |
+
from synthesis import _detect_language
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _last_user(messages: list) -> dict:
|
| 23 |
+
return next((m for m in reversed(messages) if m.get("role") == "user"), {})
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _render_history(messages: list, max_turns: int = 12) -> str:
|
| 27 |
+
lines = []
|
| 28 |
+
for m in messages[-max_turns:]:
|
| 29 |
+
who = "User" if m.get("role") == "user" else "NINA"
|
| 30 |
+
text = m.get("content", "")
|
| 31 |
+
if m.get("quote"):
|
| 32 |
+
text = f'(referring to: "{m["quote"]}")\n{text}'
|
| 33 |
+
lines.append(f"{who}: {text}")
|
| 34 |
+
return "\n\n".join(lines)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _route(messages: list, has_video: bool) -> str:
|
| 38 |
+
"""Decide 'analyze' (re-watch the video) vs 'answer' (from context). The LLM
|
| 39 |
+
chooses; first turn / no prior answer defaults to analyze."""
|
| 40 |
+
if not any(m.get("role") == "assistant" for m in messages):
|
| 41 |
+
return "analyze"
|
| 42 |
+
try:
|
| 43 |
+
r = llm_client.chat_completion(
|
| 44 |
+
model=LLM_MODEL,
|
| 45 |
+
messages=[
|
| 46 |
+
{"role": "system", "content":
|
| 47 |
+
"Decide whether answering the user's latest message needs RE-WATCHING the "
|
| 48 |
+
"video for visual/audio detail not yet covered, or can be answered from the "
|
| 49 |
+
"conversation so far. Reply with exactly one word: ANALYZE or ANSWER."},
|
| 50 |
+
{"role": "user", "content": _render_history(messages) + "\n\nReply ANALYZE or ANSWER."},
|
| 51 |
+
],
|
| 52 |
+
max_tokens=3, temperature=0.0, stream=False,
|
| 53 |
+
)
|
| 54 |
+
out = (r.choices[0].message.content or "").strip().upper()
|
| 55 |
+
return "analyze" if "ANALYZE" in out else "answer"
|
| 56 |
+
except Exception:
|
| 57 |
+
return "answer"
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _answer_from_context(messages: list, transcript: str):
|
| 61 |
+
"""Stream a conversational answer grounded in the history + stored transcript."""
|
| 62 |
+
last = _last_user(messages)
|
| 63 |
+
prompt = (last.get("content") or "").strip()
|
| 64 |
+
language = _detect_language(prompt)
|
| 65 |
+
system = (
|
| 66 |
+
f"You are NINA, a video-understanding assistant continuing a conversation. "
|
| 67 |
+
f"Write your answer in {language}. Ground it in the conversation and the stored video "
|
| 68 |
+
f"transcript below β do not invent facts that aren't there. If the answer needs visual "
|
| 69 |
+
f"detail you don't have, say so and suggest re-analyzing the video."
|
| 70 |
+
)
|
| 71 |
+
user = f"Stored video transcript:\n{transcript or '(none yet)'}\n\nConversation:\n{_render_history(messages)}"
|
| 72 |
+
if last.get("quote"):
|
| 73 |
+
user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
|
| 74 |
+
user += "\n\nAnswer the user's latest message."
|
| 75 |
+
acc = ""
|
| 76 |
+
try:
|
| 77 |
+
stream = llm_client.chat_completion(
|
| 78 |
+
model=LLM_MODEL,
|
| 79 |
+
messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
| 80 |
+
max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
|
| 81 |
+
)
|
| 82 |
+
for ev in stream:
|
| 83 |
+
if not ev.choices:
|
| 84 |
+
continue
|
| 85 |
+
d = ev.choices[0].delta.content or ""
|
| 86 |
+
if d:
|
| 87 |
+
acc += d
|
| 88 |
+
yield acc.strip()
|
| 89 |
+
if not acc.strip():
|
| 90 |
+
yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
|
| 91 |
+
except Exception as e:
|
| 92 |
+
yield f"Couldn't answer from context ({e})."
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def chat_stream(video_path, payload_json: str, token: str = ""):
|
| 96 |
+
"""Gradio chat endpoint. payload_json: {messages, transcript, allow_analyze, force_analyze}."""
|
| 97 |
+
if not verify_usage_token(token):
|
| 98 |
+
yield ["β Not authorized. Please use NINA at https://app.nnlabs.pl (sign in first).",
|
| 99 |
+
False, ""]
|
| 100 |
+
return
|
| 101 |
+
|
| 102 |
+
try:
|
| 103 |
+
payload = json.loads(payload_json or "{}")
|
| 104 |
+
except Exception:
|
| 105 |
+
payload = {}
|
| 106 |
+
messages = payload.get("messages") or []
|
| 107 |
+
transcript = payload.get("transcript") or ""
|
| 108 |
+
allow_analyze = bool(payload.get("allow_analyze", True))
|
| 109 |
+
force = bool(payload.get("force_analyze", False))
|
| 110 |
+
|
| 111 |
+
last = _last_user(messages)
|
| 112 |
+
prompt = (last.get("content") or "").strip()
|
| 113 |
+
quote = last.get("quote")
|
| 114 |
+
if not prompt and video_path is None:
|
| 115 |
+
yield ["Please type a message.", False, transcript]
|
| 116 |
+
return
|
| 117 |
+
|
| 118 |
+
# Decide: analyze (re-watch) vs answer (from context).
|
| 119 |
+
decision = "analyze" if force else _route(messages, video_path is not None)
|
| 120 |
+
do_analyze = decision == "analyze" and video_path is not None and allow_analyze
|
| 121 |
+
|
| 122 |
+
if do_analyze:
|
| 123 |
+
from pipeline import _run_core, transcript_text
|
| 124 |
+
ask = (f'Regarding "{quote}": ' if quote else "") + (prompt or "Summarize the video.")
|
| 125 |
+
last_out = ""
|
| 126 |
+
for out in _run_core(video_path, ask):
|
| 127 |
+
last_out = out
|
| 128 |
+
yield [out, True, ""]
|
| 129 |
+
yield [last_out, True, transcript_text(video_path)]
|
| 130 |
+
return
|
| 131 |
+
|
| 132 |
+
# Answer from context. If the model wanted to re-watch but there's no video this
|
| 133 |
+
# turn (media already gone) or no budget, note it.
|
| 134 |
+
note = ""
|
| 135 |
+
if decision == "analyze" and video_path is None:
|
| 136 |
+
note = "\n\n_(To re-watch the video for new detail, re-upload it.)_"
|
| 137 |
+
elif decision == "analyze" and not allow_analyze:
|
| 138 |
+
note = "\n\n_(You've used your video-analysis budget for now, so I answered from our chat.)_"
|
| 139 |
+
acc = ""
|
| 140 |
+
for partial in _answer_from_context(messages, transcript):
|
| 141 |
+
acc = partial
|
| 142 |
+
yield [partial, False, transcript]
|
| 143 |
+
if note:
|
| 144 |
+
yield [acc + note, False, transcript]
|
pipeline.py
CHANGED
|
@@ -21,6 +21,7 @@ from enhance import isolate_vocals
|
|
| 21 |
from frames import keyframes_for
|
| 22 |
from ocr import ocr_burned_in
|
| 23 |
from subtitles import extract_embedded_subs
|
|
|
|
| 24 |
from retrieval import (
|
| 25 |
add_entities_to_chunks, bi_encode_retrieve, classify_query,
|
| 26 |
merge_regions, pad_with_neighbours, rerank_and_score,
|
|
@@ -178,6 +179,7 @@ def _synthesize_streaming(regions: list[dict], prompt: str,
|
|
| 178 |
# ββ Main pipeline ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 179 |
|
| 180 |
def run(video_path: str, prompt: str, token: str = ""):
|
|
|
|
| 181 |
if not verify_usage_token(token):
|
| 182 |
yield (
|
| 183 |
"β This request is not authorized.\n\n"
|
|
@@ -185,6 +187,23 @@ def run(video_path: str, prompt: str, token: str = ""):
|
|
| 185 |
"Direct API access is disabled, and each account has a usage limit."
|
| 186 |
)
|
| 187 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
if video_path is None:
|
| 189 |
yield "Please upload a video."
|
| 190 |
return
|
|
|
|
| 21 |
from frames import keyframes_for
|
| 22 |
from ocr import ocr_burned_in
|
| 23 |
from subtitles import extract_embedded_subs
|
| 24 |
+
from utils import fmt_time
|
| 25 |
from retrieval import (
|
| 26 |
add_entities_to_chunks, bi_encode_retrieve, classify_query,
|
| 27 |
merge_regions, pad_with_neighbours, rerank_and_score,
|
|
|
|
| 179 |
# ββ Main pipeline ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 180 |
|
| 181 |
def run(video_path: str, prompt: str, token: str = ""):
|
| 182 |
+
"""Public API endpoint: verify the single-use token, then run the pipeline."""
|
| 183 |
if not verify_usage_token(token):
|
| 184 |
yield (
|
| 185 |
"β This request is not authorized.\n\n"
|
|
|
|
| 187 |
"Direct API access is disabled, and each account has a usage limit."
|
| 188 |
)
|
| 189 |
return
|
| 190 |
+
yield from _run_core(video_path, prompt)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def transcript_text(video_path: str) -> str:
|
| 194 |
+
"""The cached transcript for a video as timestamped text (for chat storage)."""
|
| 195 |
+
entry = _cache.get(video_path)
|
| 196 |
+
if not entry:
|
| 197 |
+
return ""
|
| 198 |
+
return "\n".join(
|
| 199 |
+
f"[{fmt_time(s['start'])}] {s['text']}"
|
| 200 |
+
for s in (entry.get("segments") or []) if s.get("text")
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def _run_core(video_path: str, prompt: str):
|
| 205 |
+
"""The pipeline without the token check β reused by the chat endpoint, which has
|
| 206 |
+
already authorized the turn (so it must not re-consume the single-use token)."""
|
| 207 |
if video_path is None:
|
| 208 |
yield "Please upload a video."
|
| 209 |
return
|