Spaces:
Paused
Follow-up z kontekstu szybkim modelem, z limitem na sprawdzenia i z logami
Browse files2026-09-19 jeden follow-up ("Make me a quiz from this video") trwal 3 min 48 s. Nic
sie nie wysypalo, odpowiedz doszla i zapisala sie w bazie, ale dla uzytkownika
wygladalo to jak awaria: najpierw pol minuty samych kropek, potem tekst sacznacy sie
przez prawie cztery minuty.
Follow-up robi trzy wywolania modelu po kolei: route (czy ogladac jeszcze raz),
on_topic (czy pytanie dotyczy filmu) i sama odpowiedz. Wszystkie trzy szly modelem
tekstowym Qwen2.5-72B. Samo route, trzy tokeny, uderzylo w 30-sekundowy limit
Workera. W tym samym Space, kilka minut wczesniej, Qwen3-VL-30B-A3B napisal pierwsza
odpowiedz w 8 s. Zmierzony wprost generuje okolo 4,5 raza szybciej (~450 wobec ~100
znakow/s) i to on pisze kazda analize, wiec glos NINY sie nie zmienia.
- CHAT_MODEL = VLM_MODEL dla wszystkich trzech wywolan czatu
- oba jednowyrazowe sprawdzenia przez osobnego klienta z limitem 10 s
(CHAT_CHECK_TIMEOUT); maja bezpieczne domyslne (answer / on-topic), wiec wolny
dostawca nie moze juz trzymac tury
- logi z czasami na kazdym etapie. Do tej pory ta sciezka nie drukowala nic, wiec
w logach Space tura trwajaca cztery minuty wygladala dokladnie jak tura, ktora
padla, i dokladnie jak tura, ktora zadzialala
Sprawdzone na atrapach klientow: wszystkie trzy wywolania ida szybkim modelem,
sprawdzenia przez klienta z krotkim limitem, a przy padnietym dostawcy oba biora
bezpieczne domyslne.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
|
@@ -12,12 +12,19 @@
|
|
| 12 |
# β’ transcript_text β filled on the final event after an analysis (for storage)
|
| 13 |
|
| 14 |
import json
|
|
|
|
| 15 |
|
| 16 |
from auth import verify_usage_token
|
| 17 |
-
from config import
|
| 18 |
-
from models import llm_client
|
| 19 |
from synthesis import _detect_language
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
# Control signal: the model wants to re-watch the video but the client didn't send
|
| 22 |
# it this turn. The client detects this, uploads the chat's held video, and retries.
|
| 23 |
NEED_VIDEO = "\x01NEED_VIDEO\x01"
|
|
@@ -43,9 +50,10 @@ def _route(messages: list, has_video: bool) -> str:
|
|
| 43 |
chooses; first turn / no prior answer defaults to analyze."""
|
| 44 |
if not any(m.get("role") == "assistant" for m in messages):
|
| 45 |
return "analyze"
|
|
|
|
| 46 |
try:
|
| 47 |
-
r =
|
| 48 |
-
model=
|
| 49 |
messages=[
|
| 50 |
{"role": "system", "content":
|
| 51 |
"You route turns for a video-chat assistant. Decide whether answering the "
|
|
@@ -60,8 +68,11 @@ def _route(messages: list, has_video: bool) -> str:
|
|
| 60 |
max_tokens=3, temperature=0.0, stream=False,
|
| 61 |
)
|
| 62 |
out = (r.choices[0].message.content or "").strip().upper()
|
| 63 |
-
|
| 64 |
-
|
|
|
|
|
|
|
|
|
|
| 65 |
return "answer"
|
| 66 |
|
| 67 |
|
|
@@ -74,9 +85,10 @@ def _on_topic(messages: list) -> bool:
|
|
| 74 |
last = _last_user(messages)
|
| 75 |
if not (last.get("content") or "").strip():
|
| 76 |
return True
|
|
|
|
| 77 |
try:
|
| 78 |
-
r =
|
| 79 |
-
model=
|
| 80 |
messages=[
|
| 81 |
{"role": "system", "content":
|
| 82 |
"You guard a video-understanding assistant. The user uploaded a video and is "
|
|
@@ -89,8 +101,11 @@ def _on_topic(messages: list) -> bool:
|
|
| 89 |
],
|
| 90 |
max_tokens=2, temperature=0.0, stream=False,
|
| 91 |
)
|
| 92 |
-
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
| 94 |
return True
|
| 95 |
|
| 96 |
|
|
@@ -102,6 +117,7 @@ def route_decision(payload_json: str, token: str = "") -> str:
|
|
| 102 |
to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
|
| 103 |
video again" popup."""
|
| 104 |
if not verify_usage_token(token):
|
|
|
|
| 105 |
return "answer"
|
| 106 |
try:
|
| 107 |
payload = json.loads(payload_json or "{}")
|
|
@@ -143,9 +159,12 @@ def _answer_from_context(messages: list, transcript: str):
|
|
| 143 |
user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
|
| 144 |
user += "\n\nAnswer the user's latest message."
|
| 145 |
acc = ""
|
|
|
|
|
|
|
|
|
|
| 146 |
try:
|
| 147 |
stream = llm_client.chat_completion(
|
| 148 |
-
model=
|
| 149 |
messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
| 150 |
max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
|
| 151 |
)
|
|
@@ -154,11 +173,16 @@ def _answer_from_context(messages: list, transcript: str):
|
|
| 154 |
continue
|
| 155 |
d = ev.choices[0].delta.content or ""
|
| 156 |
if d:
|
|
|
|
|
|
|
| 157 |
acc += d
|
| 158 |
yield acc.strip()
|
|
|
|
|
|
|
| 159 |
if not acc.strip():
|
| 160 |
yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
|
| 161 |
except Exception as e:
|
|
|
|
| 162 |
yield f"Couldn't answer from context ({e})."
|
| 163 |
|
| 164 |
|
|
@@ -183,6 +207,8 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
|
|
| 183 |
last = _last_user(messages)
|
| 184 |
prompt = (last.get("content") or "").strip()
|
| 185 |
quote = last.get("quote")
|
|
|
|
|
|
|
| 186 |
if not prompt and video_path is None:
|
| 187 |
yield ["Please type a message.", False, transcript]
|
| 188 |
return
|
|
|
|
| 12 |
# β’ transcript_text β filled on the final event after an analysis (for storage)
|
| 13 |
|
| 14 |
import json
|
| 15 |
+
import time
|
| 16 |
|
| 17 |
from auth import verify_usage_token
|
| 18 |
+
from config import CHAT_MODEL, LLM_MAX_TOKENS, LLM_TEMPERATURE
|
| 19 |
+
from models import check_client, llm_client
|
| 20 |
from synthesis import _detect_language
|
| 21 |
|
| 22 |
+
# Every step of a chat turn logs how long it took. Until 2026-09-19 this path printed
|
| 23 |
+
# nothing at all, so a turn that took four minutes looked, in the Space logs, exactly
|
| 24 |
+
# like a turn that crashed - and exactly like one that worked.
|
| 25 |
+
def _log(msg: str) -> None:
|
| 26 |
+
print(f"[NINA chat] {msg}", flush=True)
|
| 27 |
+
|
| 28 |
# Control signal: the model wants to re-watch the video but the client didn't send
|
| 29 |
# it this turn. The client detects this, uploads the chat's held video, and retries.
|
| 30 |
NEED_VIDEO = "\x01NEED_VIDEO\x01"
|
|
|
|
| 50 |
chooses; first turn / no prior answer defaults to analyze."""
|
| 51 |
if not any(m.get("role") == "assistant" for m in messages):
|
| 52 |
return "analyze"
|
| 53 |
+
t0 = time.time()
|
| 54 |
try:
|
| 55 |
+
r = check_client.chat_completion(
|
| 56 |
+
model=CHAT_MODEL,
|
| 57 |
messages=[
|
| 58 |
{"role": "system", "content":
|
| 59 |
"You route turns for a video-chat assistant. Decide whether answering the "
|
|
|
|
| 68 |
max_tokens=3, temperature=0.0, stream=False,
|
| 69 |
)
|
| 70 |
out = (r.choices[0].message.content or "").strip().upper()
|
| 71 |
+
decision = "analyze" if "ANALYZE" in out else "answer"
|
| 72 |
+
_log(f"route -> {decision} ({time.time() - t0:.1f}s)")
|
| 73 |
+
return decision
|
| 74 |
+
except Exception as e:
|
| 75 |
+
_log(f"route check failed after {time.time() - t0:.1f}s ({type(e).__name__}) - defaulting to answer")
|
| 76 |
return "answer"
|
| 77 |
|
| 78 |
|
|
|
|
| 85 |
last = _last_user(messages)
|
| 86 |
if not (last.get("content") or "").strip():
|
| 87 |
return True
|
| 88 |
+
t0 = time.time()
|
| 89 |
try:
|
| 90 |
+
r = check_client.chat_completion(
|
| 91 |
+
model=CHAT_MODEL,
|
| 92 |
messages=[
|
| 93 |
{"role": "system", "content":
|
| 94 |
"You guard a video-understanding assistant. The user uploaded a video and is "
|
|
|
|
| 101 |
],
|
| 102 |
max_tokens=2, temperature=0.0, stream=False,
|
| 103 |
)
|
| 104 |
+
on = not (r.choices[0].message.content or "").strip().upper().startswith("OFF")
|
| 105 |
+
_log(f"on-topic -> {'on' if on else 'OFF'} ({time.time() - t0:.1f}s)")
|
| 106 |
+
return on
|
| 107 |
+
except Exception as e:
|
| 108 |
+
_log(f"on-topic check failed after {time.time() - t0:.1f}s ({type(e).__name__}) - treating as on-topic")
|
| 109 |
return True
|
| 110 |
|
| 111 |
|
|
|
|
| 117 |
to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
|
| 118 |
video again" popup."""
|
| 119 |
if not verify_usage_token(token):
|
| 120 |
+
_log("route: token rejected")
|
| 121 |
return "answer"
|
| 122 |
try:
|
| 123 |
payload = json.loads(payload_json or "{}")
|
|
|
|
| 159 |
user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
|
| 160 |
user += "\n\nAnswer the user's latest message."
|
| 161 |
acc = ""
|
| 162 |
+
t0 = time.time()
|
| 163 |
+
first = None
|
| 164 |
+
_log(f"answering from context ({len(messages)} messages, transcript {len(transcript or '')} chars)")
|
| 165 |
try:
|
| 166 |
stream = llm_client.chat_completion(
|
| 167 |
+
model=CHAT_MODEL,
|
| 168 |
messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
|
| 169 |
max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
|
| 170 |
)
|
|
|
|
| 173 |
continue
|
| 174 |
d = ev.choices[0].delta.content or ""
|
| 175 |
if d:
|
| 176 |
+
if first is None:
|
| 177 |
+
first = time.time() - t0
|
| 178 |
acc += d
|
| 179 |
yield acc.strip()
|
| 180 |
+
took = time.time() - t0
|
| 181 |
+
_log(f"answer done: {len(acc)} chars in {took:.1f}s (first token {first or 0:.1f}s)")
|
| 182 |
if not acc.strip():
|
| 183 |
yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
|
| 184 |
except Exception as e:
|
| 185 |
+
_log(f"answer failed after {time.time() - t0:.1f}s: {type(e).__name__}: {e}")
|
| 186 |
yield f"Couldn't answer from context ({e})."
|
| 187 |
|
| 188 |
|
|
|
|
| 207 |
last = _last_user(messages)
|
| 208 |
prompt = (last.get("content") or "").strip()
|
| 209 |
quote = last.get("quote")
|
| 210 |
+
_log(f'turn: "{prompt[:80]}" force_analyze={force} force_answer={force_answer} '
|
| 211 |
+
f'video={"yes" if video_path is not None else "no"}')
|
| 212 |
if not prompt and video_path is None:
|
| 213 |
yield ["Please type a message.", False, transcript]
|
| 214 |
return
|
|
@@ -95,6 +95,19 @@ VLM_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct" # live via novita/featherle
|
|
| 95 |
VLM_MAX_FRAMES = 10 # keyframes sent to the VLM (more = more image tokens)
|
| 96 |
VLM_FRAME_MAX_PX = 768 # downscale each frame's long side (caps image tokens)
|
| 97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
# ββ Upload limits & server-side compression βββββββββββββββββββββββββββββββββββ
|
| 99 |
# The website rejects > MAX_UPLOAD_MB before upload; we re-check here because the
|
| 100 |
# client check is bypassable. Files over COMPRESS_TRIGGER_MB are transcoded down
|
|
|
|
| 95 |
VLM_MAX_FRAMES = 10 # keyframes sent to the VLM (more = more image tokens)
|
| 96 |
VLM_FRAME_MAX_PX = 768 # downscale each frame's long side (caps image tokens)
|
| 97 |
|
| 98 |
+
# ββ Chat follow-ups βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 99 |
+
# Follow-up turns answer from the conversation + stored transcript, and before that
|
| 100 |
+
# run two one-word checks (_route, _on_topic). All three used LLM_MODEL, the 72B
|
| 101 |
+
# text model, and a single follow-up took 3 min 48 s on 2026-09-19: the route check
|
| 102 |
+
# alone ran into the website's 30 s cap, and the answer streamed a 3.5k-char quiz in
|
| 103 |
+
# nearly four minutes. In the same Space, minutes earlier, VLM_MODEL wrote the first
|
| 104 |
+
# answer in 8 s. Measured directly it generates ~4.5x faster (~450 vs ~100 chars/s),
|
| 105 |
+
# and it is the model that already writes every analysis, so the voice stays the same.
|
| 106 |
+
CHAT_MODEL = VLM_MODEL
|
| 107 |
+
# The two checks have safe defaults (route -> answer, on_topic -> on), so a slow
|
| 108 |
+
# provider must not be allowed to hold a turn hostage. Past this, take the default.
|
| 109 |
+
CHAT_CHECK_TIMEOUT = 10
|
| 110 |
+
|
| 111 |
# ββ Upload limits & server-side compression βββββββββββββββββββββββββββββββββββ
|
| 112 |
# The website rejects > MAX_UPLOAD_MB before upload; we re-check here because the
|
| 113 |
# client check is bypassable. Files over COMPRESS_TRIGGER_MB are transcoded down
|
|
@@ -8,6 +8,7 @@ from huggingface_hub import InferenceClient
|
|
| 8 |
from sentence_transformers import SentenceTransformer, CrossEncoder
|
| 9 |
|
| 10 |
from config import (
|
|
|
|
| 11 |
CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
|
| 12 |
VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
|
| 13 |
)
|
|
@@ -57,6 +58,8 @@ else:
|
|
| 57 |
print("[NINA] HF_TOKEN not set β LLM calls will fail.")
|
| 58 |
|
| 59 |
llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
|
|
|
|
|
|
|
| 60 |
|
| 61 |
|
| 62 |
# ββ Lazy: precise Whisper βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 8 |
from sentence_transformers import SentenceTransformer, CrossEncoder
|
| 9 |
|
| 10 |
from config import (
|
| 11 |
+
CHAT_CHECK_TIMEOUT,
|
| 12 |
CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
|
| 13 |
VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
|
| 14 |
)
|
|
|
|
| 58 |
print("[NINA] HF_TOKEN not set β LLM calls will fail.")
|
| 59 |
|
| 60 |
llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
|
| 61 |
+
# Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT.
|
| 62 |
+
check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT)
|
| 63 |
|
| 64 |
|
| 65 |
# ββ Lazy: precise Whisper βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|