QuerryRaspBerry Claude Opus 5 (1M context) commited on
Commit
077fd06
Β·
1 Parent(s): 857c67b

Follow-up z kontekstu szybkim modelem, z limitem na sprawdzenia i z logami

Browse files

2026-09-19 jeden follow-up ("Make me a quiz from this video") trwal 3 min 48 s. Nic
sie nie wysypalo, odpowiedz doszla i zapisala sie w bazie, ale dla uzytkownika
wygladalo to jak awaria: najpierw pol minuty samych kropek, potem tekst sacznacy sie
przez prawie cztery minuty.

Follow-up robi trzy wywolania modelu po kolei: route (czy ogladac jeszcze raz),
on_topic (czy pytanie dotyczy filmu) i sama odpowiedz. Wszystkie trzy szly modelem
tekstowym Qwen2.5-72B. Samo route, trzy tokeny, uderzylo w 30-sekundowy limit
Workera. W tym samym Space, kilka minut wczesniej, Qwen3-VL-30B-A3B napisal pierwsza
odpowiedz w 8 s. Zmierzony wprost generuje okolo 4,5 raza szybciej (~450 wobec ~100
znakow/s) i to on pisze kazda analize, wiec glos NINY sie nie zmienia.

- CHAT_MODEL = VLM_MODEL dla wszystkich trzech wywolan czatu
- oba jednowyrazowe sprawdzenia przez osobnego klienta z limitem 10 s
(CHAT_CHECK_TIMEOUT); maja bezpieczne domyslne (answer / on-topic), wiec wolny
dostawca nie moze juz trzymac tury
- logi z czasami na kazdym etapie. Do tej pory ta sciezka nie drukowala nic, wiec
w logach Space tura trwajaca cztery minuty wygladala dokladnie jak tura, ktora
padla, i dokladnie jak tura, ktora zadzialala

Sprawdzone na atrapach klientow: wszystkie trzy wywolania ida szybkim modelem,
sprawdzenia przez klienta z krotkim limitem, a przy padnietym dostawcy oba biora
bezpieczne domyslne.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Files changed (3) hide show
  1. chat.py +37 -11
  2. config.py +13 -0
  3. models.py +3 -0
chat.py CHANGED
@@ -12,12 +12,19 @@
12
  # β€’ transcript_text β€” filled on the final event after an analysis (for storage)
13
 
14
  import json
 
15
 
16
  from auth import verify_usage_token
17
- from config import LLM_MAX_TOKENS, LLM_MODEL, LLM_TEMPERATURE
18
- from models import llm_client
19
  from synthesis import _detect_language
20
 
 
 
 
 
 
 
21
  # Control signal: the model wants to re-watch the video but the client didn't send
22
  # it this turn. The client detects this, uploads the chat's held video, and retries.
23
  NEED_VIDEO = "\x01NEED_VIDEO\x01"
@@ -43,9 +50,10 @@ def _route(messages: list, has_video: bool) -> str:
43
  chooses; first turn / no prior answer defaults to analyze."""
44
  if not any(m.get("role") == "assistant" for m in messages):
45
  return "analyze"
 
46
  try:
47
- r = llm_client.chat_completion(
48
- model=LLM_MODEL,
49
  messages=[
50
  {"role": "system", "content":
51
  "You route turns for a video-chat assistant. Decide whether answering the "
@@ -60,8 +68,11 @@ def _route(messages: list, has_video: bool) -> str:
60
  max_tokens=3, temperature=0.0, stream=False,
61
  )
62
  out = (r.choices[0].message.content or "").strip().upper()
63
- return "analyze" if "ANALYZE" in out else "answer"
64
- except Exception:
 
 
 
65
  return "answer"
66
 
67
 
@@ -74,9 +85,10 @@ def _on_topic(messages: list) -> bool:
74
  last = _last_user(messages)
75
  if not (last.get("content") or "").strip():
76
  return True
 
77
  try:
78
- r = llm_client.chat_completion(
79
- model=LLM_MODEL,
80
  messages=[
81
  {"role": "system", "content":
82
  "You guard a video-understanding assistant. The user uploaded a video and is "
@@ -89,8 +101,11 @@ def _on_topic(messages: list) -> bool:
89
  ],
90
  max_tokens=2, temperature=0.0, stream=False,
91
  )
92
- return not (r.choices[0].message.content or "").strip().upper().startswith("OFF")
93
- except Exception:
 
 
 
94
  return True
95
 
96
 
@@ -102,6 +117,7 @@ def route_decision(payload_json: str, token: str = "") -> str:
102
  to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
103
  video again" popup."""
104
  if not verify_usage_token(token):
 
105
  return "answer"
106
  try:
107
  payload = json.loads(payload_json or "{}")
@@ -143,9 +159,12 @@ def _answer_from_context(messages: list, transcript: str):
143
  user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
144
  user += "\n\nAnswer the user's latest message."
145
  acc = ""
 
 
 
146
  try:
147
  stream = llm_client.chat_completion(
148
- model=LLM_MODEL,
149
  messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
150
  max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
151
  )
@@ -154,11 +173,16 @@ def _answer_from_context(messages: list, transcript: str):
154
  continue
155
  d = ev.choices[0].delta.content or ""
156
  if d:
 
 
157
  acc += d
158
  yield acc.strip()
 
 
159
  if not acc.strip():
160
  yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
161
  except Exception as e:
 
162
  yield f"Couldn't answer from context ({e})."
163
 
164
 
@@ -183,6 +207,8 @@ def chat_stream(video_path, payload_json: str, token: str = ""):
183
  last = _last_user(messages)
184
  prompt = (last.get("content") or "").strip()
185
  quote = last.get("quote")
 
 
186
  if not prompt and video_path is None:
187
  yield ["Please type a message.", False, transcript]
188
  return
 
12
  # β€’ transcript_text β€” filled on the final event after an analysis (for storage)
13
 
14
  import json
15
+ import time
16
 
17
  from auth import verify_usage_token
18
+ from config import CHAT_MODEL, LLM_MAX_TOKENS, LLM_TEMPERATURE
19
+ from models import check_client, llm_client
20
  from synthesis import _detect_language
21
 
22
+ # Every step of a chat turn logs how long it took. Until 2026-09-19 this path printed
23
+ # nothing at all, so a turn that took four minutes looked, in the Space logs, exactly
24
+ # like a turn that crashed - and exactly like one that worked.
25
+ def _log(msg: str) -> None:
26
+ print(f"[NINA chat] {msg}", flush=True)
27
+
28
  # Control signal: the model wants to re-watch the video but the client didn't send
29
  # it this turn. The client detects this, uploads the chat's held video, and retries.
30
  NEED_VIDEO = "\x01NEED_VIDEO\x01"
 
50
  chooses; first turn / no prior answer defaults to analyze."""
51
  if not any(m.get("role") == "assistant" for m in messages):
52
  return "analyze"
53
+ t0 = time.time()
54
  try:
55
+ r = check_client.chat_completion(
56
+ model=CHAT_MODEL,
57
  messages=[
58
  {"role": "system", "content":
59
  "You route turns for a video-chat assistant. Decide whether answering the "
 
68
  max_tokens=3, temperature=0.0, stream=False,
69
  )
70
  out = (r.choices[0].message.content or "").strip().upper()
71
+ decision = "analyze" if "ANALYZE" in out else "answer"
72
+ _log(f"route -> {decision} ({time.time() - t0:.1f}s)")
73
+ return decision
74
+ except Exception as e:
75
+ _log(f"route check failed after {time.time() - t0:.1f}s ({type(e).__name__}) - defaulting to answer")
76
  return "answer"
77
 
78
 
 
85
  last = _last_user(messages)
86
  if not (last.get("content") or "").strip():
87
  return True
88
+ t0 = time.time()
89
  try:
90
+ r = check_client.chat_completion(
91
+ model=CHAT_MODEL,
92
  messages=[
93
  {"role": "system", "content":
94
  "You guard a video-understanding assistant. The user uploaded a video and is "
 
101
  ],
102
  max_tokens=2, temperature=0.0, stream=False,
103
  )
104
+ on = not (r.choices[0].message.content or "").strip().upper().startswith("OFF")
105
+ _log(f"on-topic -> {'on' if on else 'OFF'} ({time.time() - t0:.1f}s)")
106
+ return on
107
+ except Exception as e:
108
+ _log(f"on-topic check failed after {time.time() - t0:.1f}s ({type(e).__name__}) - treating as on-topic")
109
  return True
110
 
111
 
 
117
  to re-watch). The website turns 'confirm' into the "NINA wants to analyze the
118
  video again" popup."""
119
  if not verify_usage_token(token):
120
+ _log("route: token rejected")
121
  return "answer"
122
  try:
123
  payload = json.loads(payload_json or "{}")
 
159
  user += f'\n\nThe user is asking about this excerpt: "{last["quote"]}"'
160
  user += "\n\nAnswer the user's latest message."
161
  acc = ""
162
+ t0 = time.time()
163
+ first = None
164
+ _log(f"answering from context ({len(messages)} messages, transcript {len(transcript or '')} chars)")
165
  try:
166
  stream = llm_client.chat_completion(
167
+ model=CHAT_MODEL,
168
  messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
169
  max_tokens=LLM_MAX_TOKENS, temperature=LLM_TEMPERATURE, stream=True,
170
  )
 
173
  continue
174
  d = ev.choices[0].delta.content or ""
175
  if d:
176
+ if first is None:
177
+ first = time.time() - t0
178
  acc += d
179
  yield acc.strip()
180
+ took = time.time() - t0
181
+ _log(f"answer done: {len(acc)} chars in {took:.1f}s (first token {first or 0:.1f}s)")
182
  if not acc.strip():
183
  yield "I couldn't form an answer from the conversation. Try re-analyzing the video."
184
  except Exception as e:
185
+ _log(f"answer failed after {time.time() - t0:.1f}s: {type(e).__name__}: {e}")
186
  yield f"Couldn't answer from context ({e})."
187
 
188
 
 
207
  last = _last_user(messages)
208
  prompt = (last.get("content") or "").strip()
209
  quote = last.get("quote")
210
+ _log(f'turn: "{prompt[:80]}" force_analyze={force} force_answer={force_answer} '
211
+ f'video={"yes" if video_path is not None else "no"}')
212
  if not prompt and video_path is None:
213
  yield ["Please type a message.", False, transcript]
214
  return
config.py CHANGED
@@ -95,6 +95,19 @@ VLM_MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct" # live via novita/featherle
95
  VLM_MAX_FRAMES = 10 # keyframes sent to the VLM (more = more image tokens)
96
  VLM_FRAME_MAX_PX = 768 # downscale each frame's long side (caps image tokens)
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  # ── Upload limits & server-side compression ───────────────────────────────────
99
  # The website rejects > MAX_UPLOAD_MB before upload; we re-check here because the
100
  # client check is bypassable. Files over COMPRESS_TRIGGER_MB are transcoded down
 
95
  VLM_MAX_FRAMES = 10 # keyframes sent to the VLM (more = more image tokens)
96
  VLM_FRAME_MAX_PX = 768 # downscale each frame's long side (caps image tokens)
97
 
98
+ # ── Chat follow-ups ───────────────────────────────────────────────────────────
99
+ # Follow-up turns answer from the conversation + stored transcript, and before that
100
+ # run two one-word checks (_route, _on_topic). All three used LLM_MODEL, the 72B
101
+ # text model, and a single follow-up took 3 min 48 s on 2026-09-19: the route check
102
+ # alone ran into the website's 30 s cap, and the answer streamed a 3.5k-char quiz in
103
+ # nearly four minutes. In the same Space, minutes earlier, VLM_MODEL wrote the first
104
+ # answer in 8 s. Measured directly it generates ~4.5x faster (~450 vs ~100 chars/s),
105
+ # and it is the model that already writes every analysis, so the voice stays the same.
106
+ CHAT_MODEL = VLM_MODEL
107
+ # The two checks have safe defaults (route -> answer, on_topic -> on), so a slow
108
+ # provider must not be allowed to hold a turn hostage. Past this, take the default.
109
+ CHAT_CHECK_TIMEOUT = 10
110
+
111
  # ── Upload limits & server-side compression ───────────────────────────────────
112
  # The website rejects > MAX_UPLOAD_MB before upload; we re-check here because the
113
  # client check is bypassable. Files over COMPRESS_TRIGGER_MB are transcoded down
models.py CHANGED
@@ -8,6 +8,7 @@ from huggingface_hub import InferenceClient
8
  from sentence_transformers import SentenceTransformer, CrossEncoder
9
 
10
  from config import (
 
11
  CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
12
  VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
13
  )
@@ -57,6 +58,8 @@ else:
57
  print("[NINA] HF_TOKEN not set β€” LLM calls will fail.")
58
 
59
  llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
 
 
60
 
61
 
62
  # ── Lazy: precise Whisper ─────────────────────────────────────────────────────
 
8
  from sentence_transformers import SentenceTransformer, CrossEncoder
9
 
10
  from config import (
11
+ CHAT_CHECK_TIMEOUT,
12
  CLIP_MODEL, LLM_TIMEOUT, LLM_MODEL,
13
  VISUAL_CONCEPTS, WHISPER_CPU_THREADS, WHISPER_FAST, WHISPER_PRECISE,
14
  )
 
58
  print("[NINA] HF_TOKEN not set β€” LLM calls will fail.")
59
 
60
  llm_client = InferenceClient(token=HF_TOKEN, timeout=LLM_TIMEOUT)
61
+ # Short leash for the one-word checks in chat.py; see CHAT_CHECK_TIMEOUT.
62
+ check_client = InferenceClient(token=HF_TOKEN, timeout=CHAT_CHECK_TIMEOUT)
63
 
64
 
65
  # ── Lazy: precise Whisper ─────────────────────────────────────────────────────