Sharon2897 commited on
Commit
d37f0d9
·
verified ·
1 Parent(s): 6d63f49

Update app/utils/llm.py

Browse files
Files changed (1) hide show
  1. app/utils/llm.py +788 -195
app/utils/llm.py CHANGED
@@ -1,218 +1,811 @@
1
- # app/utils/llm.py
2
  from __future__ import annotations
3
- import os
4
- from typing import Optional
5
- from transformers import pipeline
6
- from dotenv import load_dotenv
7
- import threading
8
-
9
- # ───────────────────────────────────────────────────────────────────────────────
10
- # ENV (אופציונלי)
11
- # ───────────────────────────────────────────────────────────────────────────────
12
- if os.path.exists(".env"):
13
- load_dotenv()
14
-
15
- HF_TOKEN = os.getenv("HF_TOKEN", None) # מומלץ להתחבר עם huggingface_hub.login במקום להעביר token בפרמטרים
16
- CPU_FALLBACK_MODEL = os.getenv("CPU_FALLBACK_MODEL", "microsoft/Phi-3-mini-4k-instruct")
17
- ALLOW_HEAVY_ON_CPU = os.getenv("ALLOW_HEAVY_ON_CPU", "0") in {"1", "true", "True", "YES", "yes"}
18
-
19
- # ───────────────────────────────────────────────────────────────────────────────
20
- # Hardware detection
21
- # ───────────────────────────────────────────────────────────────────────────────
22
- def _has_cuda() -> bool:
23
- try:
24
- import torch
25
- return bool(torch.cuda.is_available())
26
- except Exception:
27
- return False
28
-
29
- # מודלים "כבדים" (7B) שב-CPU עדיף להפיל לפולבאק אלא אם ALLOW_HEAVY_ON_CPU=1
30
- _HEAVY_7B = {
31
- "mistralai/Mistral-7B-Instruct-v0.2",
32
- "mistralai/Mistral-7B-Instruct-v0.3",
33
- "Qwen/Qwen2.5-7B-Instruct",
34
- "tiiuae/falcon-7b-instruct",
 
 
 
 
 
 
 
 
 
 
 
35
  }
36
 
37
- # זיהוי שגיאת meta/accelerate אופיינית (ליתר ביטחון בפולבאק מאוחר)
38
- _META_MARKERS = (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  "is on the meta device, we need a `value` to put in on",
40
  "meta device",
41
- "accelerate/hooks.py",
42
- "set_module_tensor_to_device",
 
 
 
 
 
43
  "device_map",
44
- "tensor.item() cannot be called on meta tensors", # ← added
45
  )
46
- def _looks_like_meta_cpu_error(exc: Exception) -> bool:
47
- msg = f"{exc}".lower()
48
- return any(m in msg for m in (s.lower() for s in _META_MARKERS))
49
-
50
- # ───────────────────────────────────────────────────────────────────────────────
51
- # Generator cache + locks
52
- # ───────────────────────────────────────────────────────────────────────────────
53
- _GEN_CACHE = {} # requested_model -> pipeline instance
54
- _GEN_LOCKS = {} # requested_model -> threading.Lock
55
- _GEN_LOCKS_GLOBAL = threading.Lock()
56
-
57
- def _get_lock(key: str) -> threading.Lock:
58
- with _GEN_LOCKS_GLOBAL:
59
- if key not in _GEN_LOCKS:
60
- _GEN_LOCKS[key] = threading.Lock()
61
- return _GEN_LOCKS[key]
62
-
63
- def _contains_meta_params(m) -> bool:
64
- try:
65
- for _, p in m.named_parameters(recurse=True):
66
- if getattr(p, "is_meta", False):
67
- return True
68
- return False
69
- except Exception:
70
- return False
71
 
72
- def _build_generator_unlocked(requested_model: str):
73
- """
74
- בונה pipeline בצורה בטוחה *ללא* נעילה (הקריאה מבחוץ נעולה):
75
- - GPU: device_map='auto', torch_dtype=float16, low_cpu_mem_usage=True
76
- - CPU: device=-1 בלבד; בלי device_map/low_cpu_mem_usage
77
- ואם נבחר מודל 7B והמשתנה ALLOW_HEAVY_ON_CPU לא מופעל ⇒ פולבאק ל-Phi-3-mini.
78
- חשוב: לא להעביר trust_remote_code בכלל כדי למנוע 'multiple values'.
79
- """
80
- import torch
81
 
82
- use_model = requested_model
83
- kwargs = {}
 
 
 
 
 
 
 
 
 
84
 
85
- if _has_cuda():
86
- kwargs.update({
87
- "device_map": "auto",
88
- "torch_dtype": torch.float16,
89
- "low_cpu_mem_usage": True,
90
- })
91
- else:
92
- if (requested_model in _HEAVY_7B) and not ALLOW_HEAVY_ON_CPU:
93
- use_model = CPU_FALLBACK_MODEL
94
- kwargs.update({
95
- "device": -1,
96
- })
97
-
98
- gen = pipeline(
99
- task="text-generation",
100
- model=use_model,
101
- tokenizer=use_model,
102
- framework="pt",
103
- **kwargs
104
- )
105
-
106
- mdl = getattr(gen, "model", None)
107
- if mdl is not None and _contains_meta_params(mdl):
108
- raise RuntimeError(
109
- f"Model `{use_model}` contains meta tensors after load"
110
- )
111
 
112
- return gen
 
 
 
113
 
114
- def _build_generator(requested_model: str):
115
- """
116
- בנייה עם נעילה פר-מודל + double-checked cache.
117
- """
118
- lock = _get_lock(requested_model)
119
- if requested_model in _GEN_CACHE:
120
- return _GEN_CACHE[requested_model]
121
- with lock:
122
- if requested_model in _GEN_CACHE:
123
- return _GEN_CACHE[requested_model]
124
- gen = _build_generator_unlocked(requested_model)
125
- _GEN_CACHE[requested_model] = gen
126
- return gen
127
-
128
- def _safe_get_generator(requested_model: str):
129
- """
130
- ניסיון לבנות את ה-pipeline; אם נקבל שגיאת meta/accelerate — ננקה cache וננסה פולבאק/שחזור פעם אחת.
131
- """
132
  try:
133
- return _build_generator(requested_model)
 
134
  except Exception as e:
135
- if (not _has_cuda()) and _looks_like_meta_cpu_error(e):
136
- _GEN_CACHE.pop(requested_model, None)
137
- if requested_model != CPU_FALLBACK_MODEL:
138
- _GEN_CACHE.pop(CPU_FALLBACK_MODEL, None)
139
- return _build_generator(CPU_FALLBACK_MODEL)
140
- raise
141
-
142
- # ───────────────────────────────────────────────────────────────────────────────
143
- # Public API (שומר חתימות כפי שביקשת)
144
- # ───────────────────────────────────────────────────────────────────────────────
145
- def get_generator(model_name: str = "mistralai/Mistral-7B-Instruct-v0.2"):
146
- """
147
- זהה לחתימה הישנה, בטוח ל-CPU/‏GPU, בלי trust_remote_code כפול.
148
- """
149
- return _safe_get_generator(model_name)
150
 
151
- def ask_question(context: str, question: str, model_name: str = "mistralai/Mistral-7B-Instruct-v0.2") -> str:
152
- """
153
- מקבל קונט��סט/שאלה/שם מודל ומחזיר תשובה. שומר על ההיפר-פרמטרים שלך.
154
- """
155
- prompt = (
156
- f"You are a helpful and concise assistant.\n\n"
157
- f"Context:\n\"\"\"{context}\"\"\"\n\n"
158
- f"Question:\n\"\"\"{question}\"\"\"\n\nAnswer:"
159
- )
160
  try:
161
- generator = get_generator(model_name)
162
- result = generator(
163
- prompt,
164
- max_new_tokens=256,
165
- temperature=0.7,
166
- top_p=0.95,
167
- do_sample=True,
168
- repetition_penalty=1.1,
169
- )
170
- text = result[0].get("generated_text", "")
171
- return text.split("Answer:", 1)[-1].strip() if "Answer:" in text else text.strip()
172
  except Exception as e:
173
- if (not _has_cuda()) and _looks_like_meta_cpu_error(e) and model_name != CPU_FALLBACK_MODEL:
 
 
174
  try:
175
- _GEN_CACHE.pop(model_name, None)
176
- _GEN_CACHE.pop(CPU_FALLBACK_MODEL, None)
177
- generator = get_generator(CPU_FALLBACK_MODEL)
178
- result = generator(
179
- prompt,
180
- max_new_tokens=256,
181
- temperature=0.7,
182
- top_p=0.95,
183
- do_sample=True,
184
- repetition_penalty=1.1,
185
- )
186
- text = result[0].get("generated_text", "")
187
- return text.split("Answer:", 1)[-1].strip() if "Answer:" in text else text.strip()
188
  except Exception as e2:
189
- return f"Error (fallback failed): {str(e2)}"
190
- return f"Error: {str(e)}"
 
191
 
192
- def ask_arxiv_question(metadata: dict, question: str, model_key: Optional[str] = None, mode: str = "short") -> str:
193
- """
194
- בונה קונטקסט מסיכום המטא־דאטה של מאמר arXiv וקורא ל-ask_question.
195
- שומר על החתימה שלך: הפרמטר model_key הוא שם המודל שיוזן ל-ask_question.
196
- """
197
- title = (metadata or {}).get("title", "")
198
- abstract = (metadata or {}).get("abstract", "")
199
- authors = (metadata or {}).get("authors", "")
200
-
201
- if mode == "detailed":
202
- context = (
203
- f"Answer the question thoroughly using this arXiv paper.\n\n"
204
- f"Title: {title}\n"
205
- f"Authors: {authors}\n"
206
- f"Abstract: {abstract}"
207
- )
208
- aug_q = question + "\nPlease provide a detailed explanation."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
  else:
210
- context = (
211
- f"Answer the question briefly and concisely using this arXiv paper.\n\n"
212
- f"Title: {title}\n"
213
- f"Authors: {authors}\n"
214
- f"Abstract: {abstract}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
215
  )
216
- aug_q = question
217
 
218
- return ask_question(context, aug_q, model_key or "mistralai/Mistral-7B-Instruct-v0.2")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from __future__ import annotations
2
+
3
+ # =========================
4
+ # Imports
5
+ # =========================
6
+ from fastapi import FastAPI, UploadFile, File, HTTPException, Form, Query, Request
7
+ from fastapi.responses import JSONResponse
8
+ from fastapi.middleware.cors import CORSMiddleware
9
+ from pydantic import BaseModel
10
+ from enum import Enum
11
+ from typing import Optional, Dict, Any, List, Tuple
12
+ import os, time, re, logging, concurrent.futures, traceback
13
+
14
+ import gradio as gr
15
+ from gradio.routes import mount_gradio_app
16
+
17
+ # חשוב: נשמור שימוש בפונקציות שלך כמו שהן
18
+ from app.utils.llm import ask_question, ask_arxiv_question
19
+
20
+ # =========================
21
+ # Config via ENV
22
+ # =========================
23
+ FILE_MAX_MB: int = int(os.getenv("FILE_MAX_MB", "5"))
24
+
25
+ # זמן מקסימלי לחישובי זמן/Latency (hard clamp)
26
+ LATENCY_TIMEOUT_MS: int = int(os.getenv("LATENCY_TIMEOUT_MS", "900000")) # 15 דקות
27
+ # ברירת מחדל לבדיקת latency (ניתן לבחור ב-UI), ומקסימום לבדיקות latency
28
+ LATENCY_DEFAULT_MS: int = int(os.getenv("LATENCY_DEFAULT_MS", "60000")) # 1 דקה
29
+ LATENCY_MAX_MS: int = int(os.getenv("LATENCY_MAX_MS", "600000")) # 10 דקות
30
+
31
+ # ברירת מחדל ל-timeout של בקשות LLM (שאילתות)
32
+ REQUEST_TIMEOUT_MS: int = int(os.getenv("REQUEST_TIMEOUT_MS", "15000")) # 15 שניות (ברירת מחדל)
33
+ CORS_ORIGINS: str = os.getenv("CORS_ORIGINS", "*")
34
+
35
+ # מודל פולבאק בטוח ל-CPU
36
+ CPU_FALLBACK_MODEL: str = os.getenv("CPU_FALLBACK_MODEL", "microsoft/Phi-3-mini-4k-instruct")
37
+
38
+ DEFAULT_MAX_INPUT_TOKENS = int(os.getenv("DEFAULT_MAX_INPUT_TOKENS", "8000"))
39
+ MODEL_MAX_INPUT_TOKENS: Dict[str, int] = {
40
+ "microsoft/Phi-3-mini-4k-instruct": 4000,
41
+ "tiiuae/falcon-7b-instruct": 4096,
42
+ "Qwen/Qwen2.5-7B-Instruct": 8000,
43
+ "mistralai/Mistral-7B-Instruct-v0.2": 8000,
44
+ "mistralai/Mistral-7B-Instruct-v0.3": 8000,
45
  }
46
 
47
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
48
+ log = logging.getLogger("llm_api")
49
+
50
+ # =========================
51
+ # Hardware info (לוג בלבד; לא מסנן מודלים – הכל מוצג)
52
+ # =========================
53
+ try:
54
+ import torch
55
+ _HAS_CUDA = bool(torch.cuda.is_available())
56
+ except Exception:
57
+ torch = None
58
+ _HAS_CUDA = False
59
+ log.info("CUDA available: %s", _HAS_CUDA)
60
+
61
+ # =========================
62
+ # Enums / models
63
+ # =========================
64
+ class Mode(str, Enum):
65
+ short = "short"
66
+ detailed = "detailed"
67
+
68
+ class ModelName(str, Enum):
69
+ mistral_v02 = "mistralai/Mistral-7B-Instruct-v0.2"
70
+ mistral_v03 = "mistralai/Mistral-7B-Instruct-v0.3"
71
+ phi3_mini = "microsoft/Phi-3-mini-4k-instruct"
72
+ qwen_7b = "Qwen/Qwen2.5-7B-Instruct"
73
+ falcon_7b = "tiiuae/falcon-7b-instruct"
74
+
75
+ # === ALWAYS show ALL models in UI ===
76
+ MODEL_IDS: List[str] = [m.value for m in ModelName]
77
+
78
+ MODEL_INFO: Dict[str, Dict[str, Any]] = {
79
+ "mistralai/Mistral-7B-Instruct-v0.2": {
80
+ "nickname": "Mistral v0.2 (7B)",
81
+ "best_for": ["General QA (short–medium)", "Brief summaries", "Balanced quality/speed"],
82
+ "input_recommendation": "Short to medium texts; focused questions.",
83
+ "tone": "Concise and fairly accurate.",
84
+ "notes": "Stable daily-driver.",
85
+ "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
86
+ },
87
+ "mistralai/Mistral-7B-Instruct-v0.3": {
88
+ "nickname": "Mistral v0.3 (7B)",
89
+ "best_for": ["More complex instructions", "Summary + short rationale"],
90
+ "input_recommendation": "Medium input; a few steps.",
91
+ "tone": "Follows instructions well.",
92
+ "notes": "Improved phrasing vs. v0.2.",
93
+ "link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3",
94
+ },
95
+ "microsoft/Phi-3-mini-4k-instruct": {
96
+ "nickname": "Phi-3 Mini (4k)",
97
+ "best_for": ["Very short questions", "Quick chat", "Low-resource setups"],
98
+ "input_recommendation": "Short snippets.",
99
+ "tone": "Fast and economical.",
100
+ "notes": "Short context window.",
101
+ "link": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct",
102
+ },
103
+ "Qwen/Qwen2.5-7B-Instruct": {
104
+ "nickname": "Qwen2.5 (7B)",
105
+ "best_for": ["Bullet points", "Light tech/code", "Structured outputs"],
106
+ "input_recommendation": "Medium input; structured tasks.",
107
+ "tone": "Organized.",
108
+ "notes": "Strong for a 7B model.",
109
+ "link": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct",
110
+ },
111
+ "tiiuae/falcon-7b-instruct": {
112
+ "nickname": "Falcon 7B Instruct",
113
+ "best_for": ["Basic use", "General short questions", "Fallback/variety"],
114
+ "input_recommendation": "Short–medium input.",
115
+ "tone": "Simple and to the point.",
116
+ "notes": "Older model; variable quality.",
117
+ "link": "https://huggingface.co/tiiuae/falcon-7b-instruct",
118
+ },
119
+ }
120
+
121
+ # =========================
122
+ # Data models
123
+ # =========================
124
+ class ArxivMetadata(BaseModel):
125
+ title: str
126
+ abstract: Optional[str] = ""
127
+ authors: Optional[str] = ""
128
+
129
+ class LLMQueryInput(BaseModel):
130
+ paper: Optional[str] = None
131
+ metadata: Optional[ArxivMetadata] = None
132
+ query: str
133
+
134
+ class AnswerResponse(BaseModel):
135
+ answer: str
136
+ meta: Optional[Dict[str, Any]] = None
137
+
138
+ class ErrorResponse(BaseModel):
139
+ error: str
140
+ details: Optional[str] = None
141
+
142
+ # =========================
143
+ # Token + RAG helpers
144
+ # =========================
145
+ def estimate_tokens(text: str) -> int:
146
+ return max(1, len(text) // 4) if text else 0
147
+
148
+ def model_max_ctx(model_id: str) -> int:
149
+ return MODEL_MAX_INPUT_TOKENS.get(model_id, DEFAULT_MAX_INPUT_TOKENS)
150
+
151
+ def output_reserve(mode: Mode) -> int:
152
+ return 800 if mode == Mode.detailed else 400
153
+
154
+ def instruction_reserve() -> int:
155
+ return 200
156
+
157
+ def split_into_chunks(text: str, chunk_chars: int = 1000, overlap: int = 150) -> List[str]:
158
+ if not text:
159
+ return []
160
+ paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
161
+ chunks, buf = [], ""
162
+ for p in paras:
163
+ if len(buf) + len(p) + 1 <= chunk_chars:
164
+ buf = (buf + "\n" + p).strip()
165
+ else:
166
+ if buf:
167
+ chunks.append(buf)
168
+ buf = ((buf[-overlap:] + "\n" + p).strip() if buf else p)
169
+ while len(buf) > chunk_chars:
170
+ chunks.append(buf[:chunk_chars])
171
+ buf = buf[chunk_chars - overlap:]
172
+ if buf:
173
+ chunks.append(buf)
174
+ return chunks
175
+
176
+ def simple_scores(question: str, chunks: List[str]) -> List[Tuple[int, float]]:
177
+ q = re.findall(r"\w+", (question or "").lower())
178
+ q_set = set(q)
179
+ scores: List[Tuple[int, float]] = []
180
+ for i, c in enumerate(chunks):
181
+ t = c.lower()
182
+ tf = sum(t.count(term) for term in q_set if len(term) > 2)
183
+ overlap = len(q_set.intersection(set(re.findall(r"\w+", t))))
184
+ scores.append((i, tf + 0.3 * overlap))
185
+ scores.sort(key=lambda x: x[1], reverse=True)
186
+ return scores
187
+
188
+ def select_top_chunks_fit(chunks: List[str], question: str, budget_tokens: int, base_k: int = 5) -> List[Tuple[int, str]]:
189
+ ranked = simple_scores(question, chunks)
190
+ for k in range(base_k, 0, -1):
191
+ sel, used = [], 0
192
+ for i, _ in ranked[:k]:
193
+ c = chunks[i]
194
+ t = estimate_tokens(c) + 10
195
+ if used + t <= budget_tokens:
196
+ sel.append((i, c))
197
+ used += t
198
+ if sel:
199
+ return sel
200
+ if ranked:
201
+ i, _ = ranked[0]
202
+ c = chunks[i]
203
+ max_chars = max(200, budget_tokens * 4 - 100)
204
+ return [(i, c[:max_chars])]
205
+ return []
206
+
207
+ def build_snippets_text(chunks_with_idx: List[Tuple[int, str]]) -> str:
208
+ return "\n\n".join(f"[Snippet {i}]\n{c}" for i, c in chunks_with_idx)
209
+
210
+ def build_augmented_question(user_q: str) -> str:
211
+ return ("Use ONLY the snippets provided to answer. "
212
+ "Cite snippet numbers like [Snippet 3]. Be concise.\n\n"
213
+ f"Question: {user_q}")
214
+
215
+ def prepare_context_from_paper(paper: str, question: str, model_id: str, mode: Mode) -> Dict[str, Any]:
216
+ max_ctx = model_max_ctx(model_id)
217
+ prompt_budget = max_ctx - output_reserve(mode) - instruction_reserve()
218
+ if prompt_budget <= 300:
219
+ raise HTTPException(status_code=413, detail=f"Model context too small (max {max_ctx}).")
220
+ chunks = split_into_chunks(paper, 1000, 150)
221
+ if not chunks:
222
+ raise HTTPException(status_code=400, detail="Empty paper content.")
223
+
224
+ selected = select_top_chunks_fit(chunks, question, int(0.85 * prompt_budget), 5)
225
+ snippets = build_snippets_text(selected)
226
+ aug_q = build_augmented_question(question)
227
+
228
+ t_ctx = estimate_tokens(snippets)
229
+ t_q = estimate_tokens(aug_q)
230
+ t_total = t_ctx + t_q
231
+ diag = {
232
+ "max_ctx": max_ctx, "prompt_budget": prompt_budget,
233
+ "tokens_context": t_ctx, "tokens_question": t_q, "tokens_total": t_total,
234
+ "tokens_reserved_output": output_reserve(mode),
235
+ "selected_snippets": [i for i, _ in selected], "selected_count": len(selected),
236
+ "cannot_fit": t_total > prompt_budget
237
+ }
238
+ if t_total > prompt_budget:
239
+ max_chars = max(1000, (prompt_budget - t_q) * 4 - 100)
240
+ snippets = snippets[:max_chars]
241
+ t_ctx = estimate_tokens(snippets)
242
+ t_total = t_ctx + t_q
243
+ diag["tokens_context"] = t_ctx
244
+ diag["tokens_total"] = t_total
245
+ if t_total > prompt_budget:
246
+ raise HTTPException(status_code=413, detail=f"Input too long: ~{t_total} > {prompt_budget} tokens.")
247
+ return {"snippets_context": snippets, "augmented_question": aug_q, "diagnostics": diag}
248
+
249
+ # =========================
250
+ # Infra helpers (threading + timeouts)
251
+ # =========================
252
+ _executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(4, len(MODEL_IDS)))
253
+
254
+ def _run_with_timeout(fn, timeout_ms: int, *args, **kwargs):
255
+ future = _executor.submit(fn, *args, **kwargs)
256
+ try:
257
+ return future.result(timeout=timeout_ms / 1000.0)
258
+ except concurrent.futures.TimeoutError:
259
+ raise TimeoutError(f"operation timed out after {timeout_ms} ms")
260
+
261
+ def clamp_timeout(ms: int) -> int:
262
+ """
263
+ מבטיח זמן מינימלי 1s ומקסימלי לפי LATENCY_TIMEOUT_MS (hard clamp).
264
+ """
265
+ return 1000 if ms < 1000 else LATENCY_TIMEOUT_MS if ms > LATENCY_TIMEOUT_MS else ms
266
+
267
+ # =========================
268
+ # Safe call wrappers: auto-fallback on CPU/meta errors
269
+ # =========================
270
+ _META_ERR_SNIPPETS = (
271
  "is on the meta device, we need a `value` to put in on",
272
  "meta device",
273
+ "Tensor.item() cannot be called on meta tensors", # ← added
274
+ )
275
+ _CPU_7B_HINTS = (
276
+ "out of memory",
277
+ "CUDA out of memory",
278
+ "no cuda",
279
+ "CUDA is not available",
280
  "device_map",
 
281
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
282
 
283
+ def _looks_like_meta_cpu_error(exc: Exception) -> bool:
284
+ msg = f"{exc}"
285
+ if any(s in msg for s in _META_ERR_SNIPPETS):
286
+ return True
287
+ if any(s in msg.lower() for s in _CPU_7B_HINTS):
288
+ return True
289
+ tb = "".join(traceback.format_exception_only(type(exc), exc))
290
+ return "accelerate/hooks.py" in tb or "set_module_tensor_to_device" in tb
 
291
 
292
+ def _call_ask_question_once(context: str, question: str, model_id: Optional[str], mode: Optional[str]):
293
+ try:
294
+ return ask_question(context=context, question=question, model_key=model_id, mode=mode)
295
+ except TypeError:
296
+ try:
297
+ return ask_question(context=context, question=question, model_name=model_id, mode=mode)
298
+ except TypeError:
299
+ try:
300
+ return ask_question(context=context, question=question, model=model_id, mode=mode)
301
+ except TypeError:
302
+ return ask_question(context=context, question=question)
303
 
304
+ def _call_ask_arxiv_once(metadata: Dict[str, Any], question: str, model_id: Optional[str], mode: Optional[str]):
305
+ try:
306
+ return ask_arxiv_question(metadata=metadata, question=question, model_key=model_id, mode=mode)
307
+ except TypeError:
308
+ try:
309
+ return ask_arxiv_question(metadata=metadata, question=question, model_name=model_id, mode=mode)
310
+ except TypeError:
311
+ try:
312
+ return ask_arxiv_question(metadata=metadata, question=question, model=model_id, mode=mode)
313
+ except TypeError:
314
+ return ask_arxiv_question(metadata=metadata, question=question)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
 
316
+ def _call_ask_question(context: str, question: str, model_id: Optional[str] = None, mode: Optional[str] = None) -> Dict[str, Any]:
317
+ fallback_used = False
318
+ fallback_reason = ""
319
+ chosen_model = model_id or CPU_FALLBACK_MODEL
320
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
321
  try:
322
+ ans = _call_ask_question_once(context, question, chosen_model, mode)
323
+ return {"answer": ans, "fallback_used": fallback_used, "fallback_reason": fallback_reason, "chosen_model": chosen_model}
324
  except Exception as e:
325
+ if _looks_like_meta_cpu_error(e):
326
+ fallback_used = True
327
+ fallback_reason = f"auto-fallback due to CPU/meta error: {str(e)[:140]}"
328
+ try:
329
+ ans = _call_ask_question_once(context, question, CPU_FALLBACK_MODEL, mode)
330
+ return {"answer": ans, "fallback_used": fallback_used, "fallback_reason": fallback_reason, "chosen_model": CPU_FALLBACK_MODEL}
331
+ except Exception as e2:
332
+ raise e2
333
+ else:
334
+ raise e
 
 
 
 
 
335
 
336
+ def _call_ask_arxiv(metadata: Dict[str, Any], question: str, model_id: Optional[str] = None, mode: Optional[str] = None) -> Dict[str, Any]:
337
+ fallback_used = False
338
+ fallback_reason = ""
339
+ chosen_model = model_id or CPU_FALLBACK_MODEL
 
 
 
 
 
340
  try:
341
+ ans = _call_ask_arxiv_once(metadata, question, chosen_model, mode)
342
+ return {"answer": ans, "fallback_used": fallback_used, "fallback_reason": fallback_reason, "chosen_model": chosen_model}
 
 
 
 
 
 
 
 
 
343
  except Exception as e:
344
+ if _looks_like_meta_cpu_error(e):
345
+ fallback_used = True
346
+ fallback_reason = f"auto-fallback due to CPU/meta error: {str(e)[:140]}"
347
  try:
348
+ ans = _call_ask_arxiv_once(metadata, question, CPU_FALLBACK_MODEL, mode)
349
+ return {"answer": ans, "fallback_used": fallback_used, "fallback_reason": fallback_reason, "chosen_model": CPU_FALLBACK_MODEL}
 
 
 
 
 
 
 
 
 
 
 
350
  except Exception as e2:
351
+ raise e2
352
+ else:
353
+ raise e
354
 
355
+ def sanitize_model_id(val: str) -> str:
356
+ return val if val in MODEL_IDS else ModelName.mistral_v02.value
357
+
358
+ # =========================
359
+ # Latency functions (עם זמן נבחר)
360
+ # =========================
361
+ def _ping_once(model_id: str, timeout_ms: int) -> Optional[float]:
362
+ try:
363
+ t0 = time.perf_counter()
364
+ _ = _run_with_timeout(_call_ask_question, clamp_timeout(timeout_ms), "", "ping", model_id, "short")
365
+ return time.perf_counter() - t0
366
+ except Exception:
367
+ return None
368
+
369
+ def measure_latency(model_id: str, max_wait_ms: int = LATENCY_DEFAULT_MS) -> Optional[float]:
370
+ mid = sanitize_model_id(model_id)
371
+ return _ping_once(mid, max_wait_ms)
372
+
373
+ def measure_latencies_all(model_ids: List[str],
374
+ soft_timeout_ms: int = LATENCY_DEFAULT_MS,
375
+ ensure_one_model: Optional[str] = None,
376
+ ensure_timeout_ms: int = LATENCY_MAX_MS) -> Dict[str, Optional[float]]:
377
+ results: Dict[str, Optional[float]] = {}
378
+ if not model_ids:
379
+ return results
380
+ max_workers = min(8, max(4, len(model_ids)))
381
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
382
+ fut_map = {pool.submit(_ping_once, mid, soft_timeout_ms): mid for mid in model_ids}
383
+ for fut in concurrent.futures.as_completed(fut_map):
384
+ mid = fut_map[fut]
385
+ try:
386
+ results[mid] = fut.result()
387
+ except Exception:
388
+ results[mid] = None
389
+ if all(v is None for v in results.values()):
390
+ target = ensure_one_model or ModelName.mistral_v02.value
391
+ results[target] = _ping_once(target, ensure_timeout_ms)
392
+ return results
393
+
394
+ # =========================
395
+ # Help & latency text builders (מקבלים timeout מה-UI)
396
+ # =========================
397
+ def build_help_markdown_static() -> str:
398
+ lines = []
399
+ lines.append("# 📘 Model Help & Comparison\n")
400
+ lines.append(f"Hardware detected: **{'GPU' if _HAS_CUDA else 'CPU-only'}**. All models are selectable.\n")
401
+ lines.append(f"If a heavy model fails to load on CPU, the app auto-falls back to `{CPU_FALLBACK_MODEL}` and notes it in the run report.\n")
402
+ lines.append("| Model | Nickname | Best for | Recommended Input | Tone | Notes |")
403
+ lines.append("|---|---|---|---|---|---|")
404
+ for mid in MODEL_IDS:
405
+ info = MODEL_INFO.get(mid, {})
406
+ name_cell = f"[`{mid}`]({info.get('link', '#')})"
407
+ best_for = " • ".join(info.get("best_for", []))
408
+ lines.append(f"| {name_cell} | {info.get('nickname','')} | {best_for} | "
409
+ f"{info.get('input_recommendation','')} | {info.get('tone','')} | {info.get('notes','')} |")
410
+ return "\n".join(lines)
411
+
412
+ def build_help_with_latency(model_id: str, wait_ms: int) -> str:
413
+ base = build_help_markdown_static()
414
+ mid = sanitize_model_id(model_id)
415
+ lat = measure_latency(mid, wait_ms)
416
+ if lat is None:
417
+ note = f"**Live latency for `{mid}`:** timeout / unavailable (tried ~{int(wait_ms/1000)}s)"
418
  else:
419
+ note = f"**Live latency for `{mid}`:** {lat:.2f} s (timeout set to {int(wait_ms/1000)}s)"
420
+ return base + "\n\n" + note
421
+
422
+ def build_help_with_all_latencies(wait_ms: int) -> str:
423
+ base = build_help_markdown_static()
424
+ res = measure_latencies_all(MODEL_IDS, soft_timeout_ms=wait_ms)
425
+ lines = []
426
+ lines.append("\n\n**Live latency (s):**\n")
427
+ lines.append("| Model | Latency |")
428
+ lines.append("|---|---|")
429
+ for mid in MODEL_IDS:
430
+ v = res.get(mid)
431
+ cell = f"{v:.2f}" if isinstance(v, float) and v is not None else "timeout"
432
+ lines.append(f"| `{mid}` | {cell} |")
433
+ return base + "\n" + "\n".join(lines)
434
+
435
+ # =========================
436
+ # FastAPI app
437
+ # =========================
438
+ app = FastAPI(title="LLM Query API", version="1.8", docs_url="/docs", redoc_url="/redoc")
439
+
440
+ origins = [o.strip() for o in CORS_ORIGINS.split(",")] if CORS_ORIGINS != "*" else ["*"]
441
+ app.add_middleware(
442
+ CORSMiddleware, allow_origins=origins, allow_credentials=True, allow_methods=["*"], allow_headers=["*"],
443
+ )
444
+
445
+ @app.exception_handler(HTTPException)
446
+ async def http_exception_handler(request: Request, exc: HTTPException):
447
+ return JSONResponse(status_code=exc.status_code, content=ErrorResponse(error=str(exc.detail)).model_dump())
448
+
449
+ @app.exception_handler(Exception)
450
+ async def unhandled_exception_handler(request: Request, exc: Exception):
451
+ log.exception("Unhandled error")
452
+ return JSONResponse(status_code=500, content=ErrorResponse(error="Internal server error", details=str(exc)[:200]).model_dump())
453
+
454
+ @app.get("/", response_model=dict)
455
+ def root():
456
+ return {"message": "API is running!"}
457
+
458
+ @app.get("/health", response_model=dict)
459
+ def health():
460
+ return {"status": "ok"}
461
+
462
+ @app.get("/info", response_model=dict)
463
+ def info():
464
+ return {
465
+ "cuda": bool(_HAS_CUDA),
466
+ "available_models": MODEL_IDS,
467
+ "cpu_fallback_model": CPU_FALLBACK_MODEL,
468
+ "defaults": {
469
+ "request_timeout_ms": REQUEST_TIMEOUT_MS,
470
+ "latency_default_ms": LATENCY_DEFAULT_MS,
471
+ "latency_max_ms": LATENCY_MAX_MS,
472
+ }
473
+ }
474
+
475
+ # =========================
476
+ # Endpoints
477
+ # =========================
478
+ def _format_run_report(meta: Dict[str, Any]) -> str:
479
+ t = meta.get("timing_ms", {})
480
+ tok = meta.get("tokens", {})
481
+ lines = []
482
+ lines.append("### Run report")
483
+ lines.append(f"- Path: **{meta.get('path','')}**")
484
+ lines.append(f"- Requested model: **{meta.get('requested_model','')}**")
485
+ lines.append(f"- Used model: **{meta.get('model_id','')}**")
486
+ if meta.get("fallback_used"):
487
+ lines.append(f"- Fallback used: **Yes**")
488
+ if meta.get("fallback_reason"):
489
+ lines.append(f" - Reason: {meta.get('fallback_reason')}")
490
+ else:
491
+ lines.append(f"- Fallback used: **No**")
492
+ lines.append(f"- Mode: **{meta.get('mode','')}**")
493
+ lines.append(f"- Timeout used: **{meta.get('request_timeout_ms_used','')} ms** (~{meta.get('timeout_seconds_used','?')} s)")
494
+ if t:
495
+ lines.append(f"- Timing: prep **{t.get('prep',0)} ms**, LLM **{t.get('llm',0)} ms**, total **{t.get('total',0)} ms**")
496
+ if tok:
497
+ if "prompt_budget" in tok:
498
+ lines.append(f"- Tokens: ctx ≈ **{tok.get('tokens_context','?')}**, question ≈ **{tok.get('tokens_question','?')}**, total ≈ **{tok.get('tokens_total','?')}** / budget **{tok.get('prompt_budget','?')}** (max ctx **{tok.get('max_ctx','?')}**, reserved output **{tok.get('tokens_reserved_output','?')}**)")
499
+ elif "approx_input_tokens" in tok:
500
+ lines.append(f"- Tokens (metadata): input ≈ **{tok.get('approx_input_tokens','?')}**, budget **{tok.get('prompt_budget','?')}**, max ctx **{tok.get('max_ctx','?')}**")
501
+ if "selected_snippets" in tok:
502
+ lines.append(f"- Selected snippets: {tok.get('selected_snippets',[])} (count {tok.get('selected_count',0)})")
503
+ if tok.get("cannot_fit"):
504
+ lines.append("⚠️ Some content was trimmed to fit the prompt budget.")
505
+ return "\n".join(lines)
506
+
507
+ from fastapi import Body
508
+
509
+ @app.post("/uploadfile", include_in_schema=True, response_model=AnswerResponse)
510
+ async def upload_file(file: UploadFile = File(..., description="Upload .txt file"), query: str = Query(...)):
511
+ if not file.filename.lower().endswith(".txt"):
512
+ raise HTTPException(status_code=400, detail="Only .txt files are supported")
513
+ contents = await file.read()
514
+ if len(contents) > FILE_MAX_MB * 1024 * 1024:
515
+ raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
516
+ text = contents.decode("utf-8", errors="ignore")
517
+
518
+ requested_model = ModelName.mistral_v02.value
519
+ model_id = sanitize_model_id(requested_model)
520
+ mode_val = Mode.short
521
+ timeout = clamp_timeout(REQUEST_TIMEOUT_MS)
522
+
523
+ t0 = time.perf_counter()
524
+ prep = prepare_context_from_paper(text, query, model_id=model_id, mode=mode_val)
525
+ t1 = time.perf_counter()
526
+ call_res = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
527
+ t2 = time.perf_counter()
528
+
529
+ used_model = call_res.get("chosen_model", model_id)
530
+ meta = {
531
+ "path": "uploadfile",
532
+ "requested_model": requested_model,
533
+ "model_id": used_model,
534
+ "mode": mode_val.value,
535
+ "fallback_used": call_res.get("fallback_used", False),
536
+ "fallback_reason": call_res.get("fallback_reason", ""),
537
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
538
+ "timing_ms": {"prep": int((t1 - t0) * 1000), "llm": int((t2 - t1) * 1000), "total": int((t2 - t0) * 1000)},
539
+ "tokens": prep["diagnostics"],
540
+ }
541
+ return AnswerResponse(answer=str(call_res["answer"]), meta=meta)
542
+
543
+ @app.post("/query", include_in_schema=True, response_model=AnswerResponse)
544
+ async def query_llm(
545
+ input: LLMQueryInput = Body(...),
546
+ mode: Mode = Query(Mode.short, description="Answer length"),
547
+ model_name: ModelName = Query(ModelName.mistral_v02, description="Choose the LLM"),
548
+ request_timeout_ms: int = Query(REQUEST_TIMEOUT_MS, ge=1000, le=900000, description="Per-request timeout in ms")
549
+ ):
550
+ requested_model = model_name.value
551
+ model_id = sanitize_model_id(requested_model)
552
+ mode_val = mode
553
+ timeout = clamp_timeout(request_timeout_ms)
554
+
555
+ if input.metadata:
556
+ t0 = time.perf_counter()
557
+ md = input.metadata.model_dump()
558
+ call_res = _run_with_timeout(_call_ask_arxiv, timeout, md, input.query, model_id, mode_val.value)
559
+ t1 = time.perf_counter()
560
+ used_model = call_res.get("chosen_model", model_id)
561
+ budget = model_max_ctx(used_model) - output_reserve(mode_val) - instruction_reserve()
562
+ meta = {
563
+ "path": "metadata", "requested_model": requested_model, "model_id": used_model, "mode": mode_val.value,
564
+ "fallback_used": call_res.get("fallback_used", False),
565
+ "fallback_reason": call_res.get("fallback_reason", ""),
566
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
567
+ "timing_ms": {"prep": 0, "llm": int((t1 - t0) * 1000), "total": int((t1 - t0) * 1000)},
568
+ "tokens": {
569
+ "approx_input_tokens": estimate_tokens(" ".join([md.get("title", ""), md.get("abstract", ""), md.get("authors", "")])),
570
+ "max_ctx": model_max_ctx(used_model), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val),
571
+ },
572
+ }
573
+ return AnswerResponse(answer=str(call_res["answer"]), meta=meta)
574
+
575
+ if input.paper:
576
+ t0 = time.perf_counter()
577
+ prep = prepare_context_from_paper(input.paper, input.query, model_id=model_id, mode=mode_val)
578
+ t1 = time.perf_counter()
579
+ call_res = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
580
+ t2 = time.perf_counter()
581
+ used_model = call_res.get("chosen_model", model_id)
582
+ meta = {
583
+ "path": "paper", "requested_model": requested_model, "model_id": used_model, "mode": mode_val.value,
584
+ "fallback_used": call_res.get("fallback_used", False),
585
+ "fallback_reason": call_res.get("fallback_reason", ""),
586
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
587
+ "timing_ms": {"prep": int((t1 - t0) * 1000), "llm": int((t2 - t1) * 1000), "total": int((t2 - t0) * 1000)},
588
+ "tokens": prep["diagnostics"],
589
+ }
590
+ return AnswerResponse(answer=str(call_res["answer"]), meta=meta)
591
+
592
+ raise HTTPException(status_code=400, detail="Provide either 'paper' or 'metadata'")
593
+
594
+ @app.post("/query_form", response_model=AnswerResponse)
595
+ async def query_form(
596
+ query: str = Form(..., description="Your question"),
597
+ mode: Mode = Form(Mode.short, description="Answer length"),
598
+ model_name: ModelName = Form(ModelName.mistral_v02, description="Choose the LLM"),
599
+ paper: Optional[str] = Form(None, description="Full paper text (optional)"),
600
+ metadata_title: Optional[str] = Form(None, description="arXiv Title"),
601
+ metadata_abstract: Optional[str] = Form("", description="arXiv Abstract"),
602
+ metadata_authors: Optional[str] = Form("", description="arXiv Authors"),
603
+ file: Optional[UploadFile] = File(None, description="Upload .txt file"),
604
+ request_timeout_ms: int = Form(REQUEST_TIMEOUT_MS),
605
+ ):
606
+ if file is not None:
607
+ if not file.filename.lower().endswith(".txt"):
608
+ raise HTTPException(status_code=400, detail="Only .txt files are supported")
609
+ contents = await file.read()
610
+ if len(contents) > FILE_MAX_MB * 1024 * 1024:
611
+ raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
612
+ paper = contents.decode("utf-8", errors="ignore")
613
+
614
+ requested_model = model_name.value
615
+ model_id = sanitize_model_id(requested_model)
616
+ mode_val = mode
617
+ timeout = clamp_timeout(request_timeout_ms)
618
+
619
+ if metadata_title:
620
+ t0 = time.perf_counter()
621
+ md = {"title": metadata_title, "abstract": metadata_abstract or "", "authors": metadata_authors or ""}
622
+ call_res = _run_with_timeout(_call_ask_arxiv, timeout, md, query, model_id, mode_val.value)
623
+ t1 = time.perf_counter()
624
+ used_model = call_res.get("chosen_model", model_id)
625
+ budget = model_max_ctx(used_model) - output_reserve(mode_val) - instruction_reserve()
626
+ meta = {
627
+ "path": "metadata(form)", "requested_model": requested_model, "model_id": used_model, "mode": mode_val.value,
628
+ "fallback_used": call_res.get("fallback_used", False),
629
+ "fallback_reason": call_res.get("fallback_reason", ""),
630
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
631
+ "timing_ms": {"prep": 0, "llm": int((t1 - t0) * 1000), "total": int((t1 - t0) * 1000)},
632
+ "tokens": {
633
+ "approx_input_tokens": estimate_tokens(" ".join([md["title"], md["abstract"], md["authors"]])),
634
+ "max_ctx": model_max_ctx(used_model), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val),
635
+ },
636
+ }
637
+ return AnswerResponse(answer=str(call_res["answer"]), meta=meta)
638
+
639
+ if paper:
640
+ t0 = time.perf_counter()
641
+ prep = prepare_context_from_paper(paper, query, model_id=model_id, mode=mode_val)
642
+ t1 = time.perf_counter()
643
+ call_res = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
644
+ t2 = time.perf_counter()
645
+ used_model = call_res.get("chosen_model", model_id)
646
+ meta = {
647
+ "path": "paper(form)", "requested_model": requested_model, "model_id": used_model, "mode": mode_val.value,
648
+ "fallback_used": call_res.get("fallback_used", False),
649
+ "fallback_reason": call_res.get("fallback_reason", ""),
650
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
651
+ "timing_ms": {"prep": int((t1 - t0) * 1000), "llm": int((t2 - t1) * 1000), "total": int((t2 - t0) * 1000)},
652
+ "tokens": prep["diagnostics"],
653
+ }
654
+ return AnswerResponse(answer=str(call_res["answer"]), meta=meta)
655
+
656
+ raise HTTPException(status_code=400, detail="Provide either paper text/file or metadata (title/abstract/authors).")
657
+
658
+ # =========================
659
+ # Gradio UI
660
+ # =========================
661
+ def _token_info_for_paper(paper: str, question: str, model_id: str, mode: str) -> str:
662
+ try:
663
+ m = Mode(mode) if isinstance(mode, str) else mode
664
+ prep = prepare_context_from_paper(paper or "", question or "", model_id=sanitize_model_id(model_id), mode=m)
665
+ d = prep["diagnostics"]
666
+ return (f"**Token budget**: max ctx ≈ {d['max_ctx']} | prompt budget ≈ {d['prompt_budget']} | "
667
+ f"context ≈ {d['tokens_context']} | question ≈ {d['tokens_question']} | total ≈ {d['tokens_total']} "
668
+ f"| selected {d['selected_count']} snippets {d['selected_snippets']}")
669
+ except HTTPException as e:
670
+ return f"⚠️ {e.detail}"
671
+
672
+ def ui_submit(title, abstract, authors, paper, question, mode, model_id, request_timeout_ms):
673
+ try:
674
+ timeout = clamp_timeout(int(request_timeout_ms))
675
+ model_id = sanitize_model_id(model_id)
676
+
677
+ if title:
678
+ metadata = {"title": title, "abstract": abstract or "", "authors": authors or ""}
679
+ t0 = time.perf_counter()
680
+ call_res = _run_with_timeout(_call_ask_arxiv, timeout, metadata, question, model_id, mode)
681
+ t1 = time.perf_counter()
682
+ used_model = call_res.get("chosen_model", model_id)
683
+ meta = {
684
+ "path": "metadata(ui)", "requested_model": model_id, "model_id": used_model, "mode": mode,
685
+ "fallback_used": call_res.get("fallback_used", False),
686
+ "fallback_reason": call_res.get("fallback_reason", ""),
687
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
688
+ "timing_ms": {"prep": 0, "llm": int((t1 - t0) * 1000), "total": int((t1 - t0) * 1000)},
689
+ "tokens": {
690
+ "approx_input_tokens": estimate_tokens(" ".join([metadata["title"], metadata["abstract"], metadata["authors"]])),
691
+ "max_ctx": model_max_ctx(used_model),
692
+ "prompt_budget": model_max_ctx(used_model) - output_reserve(Mode(mode)) - instruction_reserve(),
693
+ "tokens_reserved_output": output_reserve(Mode(mode)),
694
+ },
695
+ }
696
+ return str(call_res["answer"]), _format_run_report(meta)
697
+
698
+ if paper:
699
+ t0 = time.perf_counter()
700
+ prep = prepare_context_from_paper(paper, question, model_id=model_id, mode=Mode(mode))
701
+ t1 = time.perf_counter()
702
+ call_res = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode)
703
+ t2 = time.perf_counter()
704
+ used_model = call_res.get("chosen_model", model_id)
705
+ meta = {
706
+ "path": "paper(ui)", "requested_model": model_id, "model_id": used_model, "mode": mode,
707
+ "fallback_used": call_res.get("fallback_used", False),
708
+ "fallback_reason": call_res.get("fallback_reason", ""),
709
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout / 1000, 3),
710
+ "timing_ms": {"prep": int((t1 - t0) * 1000), "llm": int((t2 - t1) * 1000), "total": int((t2 - t0) * 1000)},
711
+ "tokens": prep["diagnostics"],
712
+ }
713
+ return str(call_res["answer"]), _format_run_report(meta)
714
+
715
+ return "Please provide either arXiv metadata (title) or paper text.", ""
716
+ except HTTPException as e:
717
+ return f"[Input error: {str(e.detail)}]", ""
718
+ except Exception as e:
719
+ return f"[Error: {str(e)[:200]}]", ""
720
+
721
+ def build_help_markdown_static_wrapper():
722
+ return build_help_markdown_static()
723
+
724
+ # NEW: helpers that *show* the panel when clicked
725
+ def _show_help():
726
+ return gr.update(value=build_help_markdown_static(), visible=True)
727
+
728
+ def _show_latency_selected(model_id, wait_ms):
729
+ return gr.update(value=build_help_with_latency(model_id, wait_ms), visible=True)
730
+
731
+ def _show_latency_all(wait_ms):
732
+ return gr.update(value=build_help_with_all_latencies(wait_ms), visible=True)
733
+
734
+ def build_ui():
735
+ with gr.Blocks(title="Query LLM") as demo:
736
+ gr.Markdown("# Query LLM")
737
+
738
+ with gr.Row():
739
+ mode = gr.Radio(["short", "detailed"], value="short", label="Answer length", scale=1)
740
+ model_id = gr.Dropdown(MODEL_IDS, value=ModelName.mistral_v02.value, label="Choose the LLM", scale=1)
741
+ request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)", scale=2)
742
+ latency_timeout_ms = gr.Slider(5000, LATENCY_MAX_MS, value=LATENCY_DEFAULT_MS, step=5000, label="Latency check (ms)", scale=2)
743
+
744
+ with gr.Row():
745
+ help_btn = gr.Button("HELP", variant="secondary", scale=1)
746
+ latency_sel_btn = gr.Button("Refresh latency (selected)", variant="secondary", scale=1)
747
+ latency_all_btn = gr.Button("Refresh latency (all)", variant="secondary", scale=1)
748
+
749
+ # CHANGED: start hidden
750
+ help_output = gr.Markdown(value="", visible=False)
751
+
752
+ with gr.Tab("arXiv metadata"):
753
+ title = gr.Textbox(label="Title")
754
+ abstract = gr.Textbox(label="Abstract", lines=6)
755
+ authors = gr.Textbox(label="Authors")
756
+
757
+ with gr.Tab("Paper text"):
758
+ paper = gr.Textbox(label="Paper text", lines=12)
759
+ file_upl = gr.File(label="Upload .txt file", file_types=[".txt"], file_count="single", type="filepath")
760
+
761
+ def _on_file_change(path):
762
+ if not path:
763
+ return ""
764
+ try:
765
+ if os.path.getsize(path) > FILE_MAX_MB * 1024 * 1024:
766
+ return f"[Error reading file: file too large > {FILE_MAX_MB} MB]"
767
+ except Exception:
768
+ pass
769
+ return _load_file_to_text(path)
770
+
771
+ file_upl.change(fn=_on_file_change, inputs=file_upl, outputs=paper)
772
+
773
+ question = gr.Textbox(label="Your question", lines=3)
774
+
775
+ token_info = gr.Markdown(value="Token info will appear here…", visible=True)
776
+ run_report = gr.Markdown(value="", visible=True)
777
+
778
+ recalc_btn = gr.Button("Update token info", variant="secondary")
779
+ recalc_btn.click(fn=_token_info_for_paper, inputs=[paper, question, model_id, mode], outputs=[token_info])
780
+
781
+ out = gr.Textbox(label="Answer", lines=10, interactive=False, show_copy_button=True,
782
+ placeholder="Answer will appear here...")
783
+
784
+ gr.Button("Ask").click(
785
+ fn=ui_submit,
786
+ inputs=[title, abstract, authors, paper, question, mode, model_id, request_timeout_ms],
787
+ outputs=[out, run_report],
788
  )
 
789
 
790
+ # Wire buttons — now they also reveal the panel
791
+ help_btn.click(fn=_show_help, inputs=None, outputs=[help_output])
792
+ latency_sel_btn.click(fn=_show_latency_selected, inputs=[model_id, latency_timeout_ms], outputs=[help_output])
793
+ latency_all_btn.click(fn=_show_latency_all, inputs=[latency_timeout_ms], outputs=[help_output])
794
+
795
+ return demo
796
+
797
+ def _load_file_to_text(path: Optional[str]) -> str:
798
+ if not path:
799
+ return ""
800
+ try:
801
+ size_bytes = os.path.getsize(path)
802
+ if size_bytes > FILE_MAX_MB * 1024 * 1024:
803
+ return f"[Error reading file: file too large > {FILE_MAX_MB} MB]"
804
+ with open(path, "r", encoding="utf-8", errors="ignore") as f:
805
+ return f.read()
806
+ except Exception as e:
807
+ return f"[Error reading file: {e}]"
808
+
809
+ # Build and mount UI
810
+ demo = build_ui()
811
+ app = mount_gradio_app(app, demo, path="/ui")