Sharon2897 commited on
Commit
ed7bebe
·
verified ·
1 Parent(s): d735a35

Update app/main.py

Browse files
Files changed (1) hide show
  1. app/main.py +79 -23
app/main.py CHANGED
@@ -9,14 +9,12 @@ from typing import Optional, Dict, Any, List, Tuple
9
  import os, time, re, logging, concurrent.futures
10
  import gradio as gr
11
  from gradio.routes import mount_gradio_app
12
-
13
- # ====== Your LLM functions (unchanged) ======
14
  from app.utils.llm import ask_question, ask_arxiv_question
15
 
16
- # ====== Config via ENV ======
17
  FILE_MAX_MB: int = int(os.getenv("FILE_MAX_MB", "5"))
18
- LATENCY_TIMEOUT_MS: int = int(os.getenv("LATENCY_TIMEOUT_MS", "900000"))
19
- REQUEST_TIMEOUT_MS: int = int(os.getenv("REQUEST_TIMEOUT_MS", "15000"))
20
  CORS_ORIGINS: str = os.getenv("CORS_ORIGINS", "*")
21
 
22
  DEFAULT_MAX_INPUT_TOKENS = int(os.getenv("DEFAULT_MAX_INPUT_TOKENS", "8000"))
@@ -31,7 +29,7 @@ MODEL_MAX_INPUT_TOKENS: Dict[str, int] = {
31
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
32
  log = logging.getLogger("llm_api")
33
 
34
- # ===== Enums =====
35
  class Mode(str, Enum):
36
  short = "short"
37
  detailed = "detailed"
@@ -45,7 +43,7 @@ class ModelName(str, Enum):
45
 
46
  MODEL_IDS: List[str] = [m.value for m in ModelName]
47
 
48
- # === HELP data ===
49
  MODEL_INFO: Dict[str, Dict[str, Any]] = {
50
  "mistralai/Mistral-7B-Instruct-v0.2": {
51
  "nickname": "Mistral v0.2 (7B)",
@@ -89,7 +87,7 @@ MODEL_INFO: Dict[str, Dict[str, Any]] = {
89
  },
90
  }
91
 
92
- # ===== Response models =====
93
  class ArxivMetadata(BaseModel):
94
  title: str
95
  abstract: Optional[str] = ""
@@ -108,7 +106,8 @@ class ErrorResponse(BaseModel):
108
  error: str
109
  details: Optional[str] = None
110
 
111
- # ===== Token + RAG helpers =====
 
112
  def estimate_tokens(text: str) -> int:
113
  return max(1, len(text)//4) if text else 0
114
 
@@ -201,13 +200,15 @@ def prepare_context_from_paper(paper: str, question: str, model_id: str, mode: M
201
  raise HTTPException(status_code=413, detail=f"Input too long: ~{t_total} > {prompt_budget} tokens.")
202
  return {"snippets_context": snippets, "augmented_question": aug_q, "diagnostics": diag}
203
 
204
- # ===== Infra helpers =====
205
  _executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(4, len(MODEL_IDS)))
206
 
207
  def _run_with_timeout(fn, timeout_ms: int, *args, **kwargs):
208
  future = _executor.submit(fn, *args, **kwargs)
209
- try: return future.result(timeout=timeout_ms/1000.0)
210
- except concurrent.futures.TimeoutError: raise TimeoutError(f"operation timed out after {timeout_ms} ms")
 
 
211
 
212
  def _call_ask_question(context: str, question: str, model_id: Optional[str]=None, mode: Optional[str]=None):
213
  try: return ask_question(context=context, question=question, model_key=model_id, mode=mode)
@@ -241,8 +242,53 @@ def build_help_markdown() -> str:
241
  # keep simple; avoid heavy probes to reduce latency
242
  return build_help_markdown_static() + "\n\n*(Live latency disabled here for stability.)*"
243
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
244
  # ===== FastAPI app =====
245
- app = FastAPI(title="LLM Query API", version="1.4", docs_url="/docs", redoc_url="/redoc")
246
 
247
  origins = [o.strip() for o in CORS_ORIGINS.split(",")] if CORS_ORIGINS != "*" else ["*"]
248
  app.add_middleware(
@@ -264,7 +310,7 @@ def root(): return {"message": "API is running!"}
264
  @app.get("/health", response_model=dict)
265
  def health(): return {"status": "ok"}
266
 
267
- # ---- Endpoints ----
268
  @app.post("/uploadfile", include_in_schema=True, response_model=AnswerResponse)
269
  async def upload_file(file: UploadFile = File(..., description="Upload .txt file"), query: str = Query(...)):
270
  if not file.filename.lower().endswith(".txt"):
@@ -282,7 +328,8 @@ async def upload_file(file: UploadFile = File(..., description="Upload .txt file
282
  t2 = time.perf_counter()
283
 
284
  meta = {
285
- "path": "uploadfile", "model_id": model_id, "mode": mode_val.value, "request_timeout_ms_used": timeout,
 
286
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
287
  "tokens": prep["diagnostics"],
288
  }
@@ -304,7 +351,8 @@ async def query_llm(
304
  t1 = time.perf_counter()
305
  budget = model_max_ctx(model_id) - output_reserve(mode_val) - instruction_reserve()
306
  meta = {
307
- "path": "metadata", "model_id": model_id, "mode": mode_val.value, "request_timeout_ms_used": timeout,
 
308
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
309
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([md.get("title",""), md.get("abstract",""), md.get("authors","")])),
310
  "max_ctx": model_max_ctx(model_id), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val)}
@@ -318,7 +366,8 @@ async def query_llm(
318
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
319
  t2 = time.perf_counter()
320
  meta = {
321
- "path": "paper", "model_id": model_id, "mode": mode_val.value, "request_timeout_ms_used": timeout,
 
322
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
323
  "tokens": prep["diagnostics"],
324
  }
@@ -353,7 +402,8 @@ async def query_form(
353
  t1 = time.perf_counter()
354
  budget = model_max_ctx(model_id) - output_reserve(mode_val) - instruction_reserve()
355
  meta = {
356
- "path": "metadata(form)", "model_id": model_id, "mode": mode_val.value, "request_timeout_ms_used": timeout,
 
357
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
358
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([md["title"], md["abstract"], md["authors"]])),
359
  "max_ctx": model_max_ctx(model_id), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val)}
@@ -367,7 +417,8 @@ async def query_form(
367
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
368
  t2 = time.perf_counter()
369
  meta = {
370
- "path": "paper(form)", "model_id": model_id, "mode": mode_val.value, "request_timeout_ms_used": timeout,
 
371
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
372
  "tokens": prep["diagnostics"],
373
  }
@@ -394,11 +445,12 @@ def _format_run_report(meta: Dict[str, Any]) -> str:
394
  lines.append("### Run report")
395
  lines.append(f"- Path: **{meta.get('path','')}**")
396
  lines.append(f"- Model: **{meta.get('model_id','')}**, Mode: **{meta.get('mode','')}**")
397
- lines.append(f"- Timeout used: **{meta.get('request_timeout_ms_used','')} ms**")
398
  if t: lines.append(f"- Timing: prep **{t.get('prep',0)} ms**, LLM **{t.get('llm',0)} ms**, total **{t.get('total',0)} ms**")
399
  if tok:
400
  if "prompt_budget" in tok:
401
- lines.append(f"- Tokens: ctx ≈ **{tok.get('tokens_context','?')}**, question ≈ **{tok.get('tokens_question','?')}**, total ≈ **{tok.get('tokens_total','?')}** / budget **{tok.get('prompt_budget','?')}** (max ctx **{tok.get('max_ctx','?')}**, reserved output **{tok.get('tokens_reserved_output','?')}**)")
 
402
  elif "approx_input_tokens" in tok:
403
  lines.append(f"- Tokens (metadata): input ≈ **{tok.get('approx_input_tokens','?')}**, budget **{tok.get('prompt_budget','?')}**, max ctx **{tok.get('max_ctx','?')}**")
404
  if "selected_snippets" in tok:
@@ -416,7 +468,8 @@ def ui_submit(title, abstract, authors, paper, question, mode, model_id, request
416
  answer = _run_with_timeout(ask_arxiv_question, timeout, metadata=metadata, question=question, model_key=model_id, mode=mode)
417
  t1 = time.perf_counter()
418
  meta = {
419
- "path": "metadata(ui)", "model_id": model_id, "mode": mode, "request_timeout_ms_used": timeout,
 
420
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
421
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([metadata["title"], metadata["abstract"], metadata["authors"]])),
422
  "max_ctx": model_max_ctx(model_id), "prompt_budget": model_max_ctx(model_id)-output_reserve(Mode(mode))-instruction_reserve(),
@@ -431,7 +484,8 @@ def ui_submit(title, abstract, authors, paper, question, mode, model_id, request
431
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode)
432
  t2 = time.perf_counter()
433
  meta = {
434
- "path": "paper(ui)", "model_id": model_id, "mode": mode, "request_timeout_ms_used": timeout,
 
435
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
436
  "tokens": prep["diagnostics"],
437
  }
@@ -452,6 +506,7 @@ def build_ui():
452
  model_id = gr.Dropdown(MODEL_IDS, value=ModelName.mistral_v02.value, label="Choose the LLM")
453
  request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)")
454
  help_btn = gr.Button("HELP", variant="secondary")
 
455
 
456
  help_output = gr.Markdown(value="(Click HELP to load model comparison…)", visible=True)
457
 
@@ -492,6 +547,7 @@ def build_ui():
492
  )
493
 
494
  help_btn.click(fn=build_help_markdown, inputs=None, outputs=[help_output])
 
495
 
496
  return demo
497
 
 
9
  import os, time, re, logging, concurrent.futures
10
  import gradio as gr
11
  from gradio.routes import mount_gradio_app
 
 
12
  from app.utils.llm import ask_question, ask_arxiv_question
13
 
14
+ # Config via ENV
15
  FILE_MAX_MB: int = int(os.getenv("FILE_MAX_MB", "5"))
16
+ LATENCY_TIMEOUT_MS: int = int(os.getenv("LATENCY_TIMEOUT_MS", "900000")) # 15 min
17
+ REQUEST_TIMEOUT_MS: int = int(os.getenv("REQUEST_TIMEOUT_MS", "15000")) # 15 s
18
  CORS_ORIGINS: str = os.getenv("CORS_ORIGINS", "*")
19
 
20
  DEFAULT_MAX_INPUT_TOKENS = int(os.getenv("DEFAULT_MAX_INPUT_TOKENS", "8000"))
 
29
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
30
  log = logging.getLogger("llm_api")
31
 
32
+ # Enums
33
  class Mode(str, Enum):
34
  short = "short"
35
  detailed = "detailed"
 
43
 
44
  MODEL_IDS: List[str] = [m.value for m in ModelName]
45
 
46
+ # HELP data
47
  MODEL_INFO: Dict[str, Dict[str, Any]] = {
48
  "mistralai/Mistral-7B-Instruct-v0.2": {
49
  "nickname": "Mistral v0.2 (7B)",
 
87
  },
88
  }
89
 
90
+ # Response models
91
  class ArxivMetadata(BaseModel):
92
  title: str
93
  abstract: Optional[str] = ""
 
106
  error: str
107
  details: Optional[str] = None
108
 
109
+ # Token + RAG helpers
110
+
111
  def estimate_tokens(text: str) -> int:
112
  return max(1, len(text)//4) if text else 0
113
 
 
200
  raise HTTPException(status_code=413, detail=f"Input too long: ~{t_total} > {prompt_budget} tokens.")
201
  return {"snippets_context": snippets, "augmented_question": aug_q, "diagnostics": diag}
202
 
203
+ # Infra helpers
204
  _executor = concurrent.futures.ThreadPoolExecutor(max_workers=max(4, len(MODEL_IDS)))
205
 
206
  def _run_with_timeout(fn, timeout_ms: int, *args, **kwargs):
207
  future = _executor.submit(fn, *args, **kwargs)
208
+ try:
209
+ return future.result(timeout=timeout_ms/1000.0)
210
+ except concurrent.futures.TimeoutError:
211
+ raise TimeoutError(f"operation timed out after {timeout_ms} ms")
212
 
213
  def _call_ask_question(context: str, question: str, model_id: Optional[str]=None, mode: Optional[str]=None):
214
  try: return ask_question(context=context, question=question, model_key=model_id, mode=mode)
 
242
  # keep simple; avoid heavy probes to reduce latency
243
  return build_help_markdown_static() + "\n\n*(Live latency disabled here for stability.)*"
244
 
245
+ # Live latency helpers (all models)
246
+ def _ping_once(model_id: str, timeout_ms: int) -> float | None:
247
+ """Minimal-cost ping: empty context, 'ping' question, short mode."""
248
+ try:
249
+ t0 = time.perf_counter()
250
+ _run_with_timeout(_call_ask_question, timeout_ms, "", "ping", model_id, "short")
251
+ return time.perf_counter() - t0
252
+ except Exception:
253
+ return None
254
+
255
+ def measure_latencies_all(model_ids: List[str],
256
+ soft_timeout_ms: int = 2500,
257
+ ensure_one_model: Optional[str] = None,
258
+ ensure_timeout_ms: int = 6000) -> Dict[str, Optional[float]]:
259
+ results: Dict[str, Optional[float]] = {}
260
+ max_workers = min(8, max(4, len(model_ids)))
261
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
262
+ fut_map = {pool.submit(_ping_once, mid, soft_timeout_ms): mid for mid in model_ids}
263
+ for fut in concurrent.futures.as_completed(fut_map):
264
+ mid = fut_map[fut]
265
+ try:
266
+ results[mid] = fut.result()
267
+ except Exception:
268
+ results[mid] = None
269
+
270
+ # Ensure at least one value
271
+ if all(v is None for v in results.values()):
272
+ target = ensure_one_model or ModelName.mistral_v02.value
273
+ results[target] = _ping_once(target, ensure_timeout_ms)
274
+
275
+ return results
276
+
277
+ def build_help_with_all_latencies() -> str:
278
+ base = build_help_markdown_static()
279
+ res = measure_latencies_all(MODEL_IDS)
280
+ lines = []
281
+ lines.append("\n\n**Live latency (s):**\n")
282
+ lines.append("| Model | Latency |")
283
+ lines.append("|---|---|")
284
+ for mid in MODEL_IDS:
285
+ v = res.get(mid)
286
+ cell = f"{v:.2f}" if isinstance(v, float) and v is not None else "timeout"
287
+ lines.append(f"| `{mid}` | {cell} |")
288
+ return base + "\n" + "\n".join(lines)
289
+
290
  # ===== FastAPI app =====
291
+ app = FastAPI(title="LLM Query API", version="1.5", docs_url="/docs", redoc_url="/redoc")
292
 
293
  origins = [o.strip() for o in CORS_ORIGINS.split(",")] if CORS_ORIGINS != "*" else ["*"]
294
  app.add_middleware(
 
310
  @app.get("/health", response_model=dict)
311
  def health(): return {"status": "ok"}
312
 
313
+ # Endpoints
314
  @app.post("/uploadfile", include_in_schema=True, response_model=AnswerResponse)
315
  async def upload_file(file: UploadFile = File(..., description="Upload .txt file"), query: str = Query(...)):
316
  if not file.filename.lower().endswith(".txt"):
 
328
  t2 = time.perf_counter()
329
 
330
  meta = {
331
+ "path": "uploadfile", "model_id": model_id, "mode": mode_val.value,
332
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
333
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
334
  "tokens": prep["diagnostics"],
335
  }
 
351
  t1 = time.perf_counter()
352
  budget = model_max_ctx(model_id) - output_reserve(mode_val) - instruction_reserve()
353
  meta = {
354
+ "path": "metadata", "model_id": model_id, "mode": mode_val.value,
355
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
356
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
357
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([md.get("title",""), md.get("abstract",""), md.get("authors","")])),
358
  "max_ctx": model_max_ctx(model_id), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val)}
 
366
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
367
  t2 = time.perf_counter()
368
  meta = {
369
+ "path": "paper", "model_id": model_id, "mode": mode_val.value,
370
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
371
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
372
  "tokens": prep["diagnostics"],
373
  }
 
402
  t1 = time.perf_counter()
403
  budget = model_max_ctx(model_id) - output_reserve(mode_val) - instruction_reserve()
404
  meta = {
405
+ "path": "metadata(form)", "model_id": model_id, "mode": mode_val.value,
406
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
407
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
408
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([md["title"], md["abstract"], md["authors"]])),
409
  "max_ctx": model_max_ctx(model_id), "prompt_budget": budget, "tokens_reserved_output": output_reserve(mode_val)}
 
417
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode_val.value)
418
  t2 = time.perf_counter()
419
  meta = {
420
+ "path": "paper(form)", "model_id": model_id, "mode": mode_val.value,
421
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
422
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
423
  "tokens": prep["diagnostics"],
424
  }
 
445
  lines.append("### Run report")
446
  lines.append(f"- Path: **{meta.get('path','')}**")
447
  lines.append(f"- Model: **{meta.get('model_id','')}**, Mode: **{meta.get('mode','')}**")
448
+ lines.append(f"- Timeout used: **{meta.get('request_timeout_ms_used','')} ms** (~{meta.get('timeout_seconds_used','?')} s)")
449
  if t: lines.append(f"- Timing: prep **{t.get('prep',0)} ms**, LLM **{t.get('llm',0)} ms**, total **{t.get('total',0)} ms**")
450
  if tok:
451
  if "prompt_budget" in tok:
452
+ lines.append(f"- Tokens: ctx ≈ **{tok.get('tokens_context','?')}**, question ≈ **{tok.get('tokens_question','?')}**, total ≈ **{tok.get('tokens_total','?')}** / budget **{tok.get('prompt_budget','?')}** (max ctx **{tok.get('max_ctx','?')}**, reserved output **{tok.get('tokens_reserved_output','?')}**)"
453
+ )
454
  elif "approx_input_tokens" in tok:
455
  lines.append(f"- Tokens (metadata): input ≈ **{tok.get('approx_input_tokens','?')}**, budget **{tok.get('prompt_budget','?')}**, max ctx **{tok.get('max_ctx','?')}**")
456
  if "selected_snippets" in tok:
 
468
  answer = _run_with_timeout(ask_arxiv_question, timeout, metadata=metadata, question=question, model_key=model_id, mode=mode)
469
  t1 = time.perf_counter()
470
  meta = {
471
+ "path": "metadata(ui)", "model_id": model_id, "mode": mode,
472
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
473
  "timing_ms": {"prep": 0, "llm": int((t1-t0)*1000), "total": int((t1-t0)*1000)},
474
  "tokens": {"approx_input_tokens": estimate_tokens(" ".join([metadata["title"], metadata["abstract"], metadata["authors"]])),
475
  "max_ctx": model_max_ctx(model_id), "prompt_budget": model_max_ctx(model_id)-output_reserve(Mode(mode))-instruction_reserve(),
 
484
  answer = _run_with_timeout(_call_ask_question, timeout, prep["snippets_context"], prep["augmented_question"], model_id, mode)
485
  t2 = time.perf_counter()
486
  meta = {
487
+ "path": "paper(ui)", "model_id": model_id, "mode": mode,
488
+ "request_timeout_ms_used": timeout, "timeout_seconds_used": round(timeout/1000, 3),
489
  "timing_ms": {"prep": int((t1-t0)*1000), "llm": int((t2-t1)*1000), "total": int((t2-t0)*1000)},
490
  "tokens": prep["diagnostics"],
491
  }
 
506
  model_id = gr.Dropdown(MODEL_IDS, value=ModelName.mistral_v02.value, label="Choose the LLM")
507
  request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)")
508
  help_btn = gr.Button("HELP", variant="secondary")
509
+ latency_all_btn = gr.Button("Refresh latency (all)", variant="secondary")
510
 
511
  help_output = gr.Markdown(value="(Click HELP to load model comparison…)", visible=True)
512
 
 
547
  )
548
 
549
  help_btn.click(fn=build_help_markdown, inputs=None, outputs=[help_output])
550
+ latency_all_btn.click(fn=build_help_with_all_latencies, inputs=None, outputs=[help_output])
551
 
552
  return demo
553