Sharon2897 commited on
Commit
51fa647
·
verified ·
1 Parent(s): 8c67184

Update app/main.py

Browse files
Files changed (1) hide show
  1. app/main.py +60 -9
app/main.py CHANGED
@@ -41,6 +41,30 @@ MODEL_MAX_INPUT_TOKENS: Dict[str, int] = {
41
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
42
  log = logging.getLogger("llm_api")
43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  # =========================
45
  # Enums / models
46
  # =========================
@@ -55,7 +79,8 @@ class ModelName(str, Enum):
55
  qwen_7b = "Qwen/Qwen2.5-7B-Instruct"
56
  falcon_7b = "tiiuae/falcon-7b-instruct"
57
 
58
- MODEL_IDS: List[str] = [m.value for m in ModelName]
 
59
 
60
  MODEL_INFO: Dict[str, Dict[str, Any]] = {
61
  "mistralai/Mistral-7B-Instruct-v0.2": {
@@ -242,7 +267,7 @@ def _run_with_timeout(fn, timeout_ms: int, *args, **kwargs):
242
 
243
  def _call_ask_question(context: str, question: str, model_id: Optional[str] = None, mode: Optional[str] = None):
244
  """
245
- עוטף את ask_question כדי לתמוך בשמות פרמטרים שונים (model_key/model_name/model) כפי שקיים בקוד שלך.
246
  """
247
  try:
248
  return ask_question(context=context, question=question, model_key=model_id, mode=mode)
@@ -262,7 +287,13 @@ def clamp_timeout(ms: int) -> int:
262
  return 1000 if ms < 1000 else LATENCY_TIMEOUT_MS if ms > LATENCY_TIMEOUT_MS else ms
263
 
264
  def sanitize_model_id(val: str) -> str:
265
- return val if val in MODEL_IDS else ModelName.mistral_v02.value
 
 
 
 
 
 
266
 
267
  # =========================
268
  # Latency functions (עם זמן נבחר)
@@ -283,6 +314,8 @@ def measure_latencies_all(model_ids: List[str],
283
  soft_timeout_ms: int = LATENCY_DEFAULT_MS,
284
  ensure_one_model: Optional[str] = None,
285
  ensure_timeout_ms: int = LATENCY_MAX_MS) -> Dict[str, Optional[float]]:
 
 
286
  results: Dict[str, Optional[float]] = {}
287
  max_workers = min(8, max(4, len(model_ids)))
288
  with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
@@ -293,8 +326,8 @@ def measure_latencies_all(model_ids: List[str],
293
  results[mid] = fut.result()
294
  except Exception:
295
  results[mid] = None
296
- if all(v is None for v in results.values()):
297
- target = ensure_one_model or ModelName.mistral_v02.value
298
  results[target] = _ping_once(target, ensure_timeout_ms)
299
  return results
300
 
@@ -304,6 +337,7 @@ def measure_latencies_all(model_ids: List[str],
304
  def build_help_markdown_static() -> str:
305
  lines = []
306
  lines.append("# 📘 Model Help & Comparison\n")
 
307
  lines.append("Static comparison. Use **Refresh latency** with your chosen timeout.\n")
308
  lines.append("| Model | Nickname | Best for | Recommended Input | Tone | Notes |")
309
  lines.append("|---|---|---|---|---|---|")
@@ -365,6 +399,18 @@ def root():
365
  def health():
366
  return {"status": "ok"}
367
 
 
 
 
 
 
 
 
 
 
 
 
 
368
  # =========================
369
  # Endpoints
370
  # =========================
@@ -376,7 +422,8 @@ async def upload_file(file: UploadFile = File(..., description="Upload .txt file
376
  if len(contents) > FILE_MAX_MB * 1024 * 1024:
377
  raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
378
  text = contents.decode("utf-8", errors="ignore")
379
- model_id = ModelName.mistral_v02.value
 
380
  mode_val = Mode.short
381
  timeout = clamp_timeout(REQUEST_TIMEOUT_MS)
382
 
@@ -401,7 +448,8 @@ async def query_llm(
401
  model_name: ModelName = Query(ModelName.mistral_v02, description="Choose the LLM"),
402
  request_timeout_ms: int = Query(REQUEST_TIMEOUT_MS, ge=1000, le=900000, description="Per-request timeout in ms")
403
  ):
404
- model_id = model_name.value
 
405
  mode_val = mode
406
  timeout = clamp_timeout(request_timeout_ms)
407
 
@@ -458,7 +506,7 @@ async def query_form(
458
  raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
459
  paper = contents.decode("utf-8", errors="ignore")
460
 
461
- model_id = model_name.value
462
  mode_val = mode
463
  timeout = clamp_timeout(request_timeout_ms)
464
 
@@ -578,7 +626,10 @@ def build_ui():
578
 
579
  with gr.Row():
580
  mode = gr.Radio(["short", "detailed"], value="short", label="Answer length", scale=1)
581
- model_id = gr.Dropdown(MODEL_IDS, value=ModelName.mistral_v02.value, label="Choose the LLM", scale=1)
 
 
 
582
  request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)", scale=2)
583
  latency_timeout_ms = gr.Slider(5000, LATENCY_MAX_MS, value=LATENCY_DEFAULT_MS, step=5000, label="Latency check (ms)", scale=2)
584
 
 
41
  logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
42
  log = logging.getLogger("llm_api")
43
 
44
+ # =========================
45
+ # Hardware detection & available models
46
+ # =========================
47
+ try:
48
+ import torch
49
+ _HAS_CUDA = bool(torch.cuda.is_available())
50
+ except Exception:
51
+ torch = None
52
+ _HAS_CUDA = False
53
+
54
+ GPU_FRIENDLY: List[str] = [
55
+ "mistralai/Mistral-7B-Instruct-v0.2",
56
+ "mistralai/Mistral-7B-Instruct-v0.3",
57
+ "Qwen/Qwen2.5-7B-Instruct",
58
+ "tiiuae/falcon-7b-instruct",
59
+ "microsoft/Phi-3-mini-4k-instruct",
60
+ ]
61
+ CPU_SAFE: List[str] = [
62
+ "microsoft/Phi-3-mini-4k-instruct", # ברירת מחדל בטוחה ב-CPU
63
+ # אפשר להוסיף כאן דגמים קטנים נוספים אם תרצי
64
+ ]
65
+
66
+ AVAILABLE_MODEL_IDS: List[str] = GPU_FRIENDLY if _HAS_CUDA else CPU_SAFE
67
+
68
  # =========================
69
  # Enums / models
70
  # =========================
 
79
  qwen_7b = "Qwen/Qwen2.5-7B-Instruct"
80
  falcon_7b = "tiiuae/falcon-7b-instruct"
81
 
82
+ # רק מה שזמין בחומרה הנוכחית
83
+ MODEL_IDS: List[str] = AVAILABLE_MODEL_IDS
84
 
85
  MODEL_INFO: Dict[str, Dict[str, Any]] = {
86
  "mistralai/Mistral-7B-Instruct-v0.2": {
 
267
 
268
  def _call_ask_question(context: str, question: str, model_id: Optional[str] = None, mode: Optional[str] = None):
269
  """
270
+ עוטף את ask_question כדי לתמוך בשמות פרמטרים שונים (model_key/model_name/model).
271
  """
272
  try:
273
  return ask_question(context=context, question=question, model_key=model_id, mode=mode)
 
287
  return 1000 if ms < 1000 else LATENCY_TIMEOUT_MS if ms > LATENCY_TIMEOUT_MS else ms
288
 
289
  def sanitize_model_id(val: str) -> str:
290
+ """
291
+ ברירת מחדל בטוחה: ב-CPU -> Phi-3-mini; ב-GPU -> Mistral v0.2.
292
+ """
293
+ if val in MODEL_IDS:
294
+ return val
295
+ return ("mistralai/Mistral-7B-Instruct-v0.2" if _HAS_CUDA
296
+ else "microsoft/Phi-3-mini-4k-instruct")
297
 
298
  # =========================
299
  # Latency functions (עם זמן נבחר)
 
314
  soft_timeout_ms: int = LATENCY_DEFAULT_MS,
315
  ensure_one_model: Optional[str] = None,
316
  ensure_timeout_ms: int = LATENCY_MAX_MS) -> Dict[str, Optional[float]]:
317
+ # מודדים רק מודלים זמינים בסביבה הנוכחית
318
+ model_ids = [m for m in model_ids if m in MODEL_IDS]
319
  results: Dict[str, Optional[float]] = {}
320
  max_workers = min(8, max(4, len(model_ids)))
321
  with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
 
326
  results[mid] = fut.result()
327
  except Exception:
328
  results[mid] = None
329
+ if results and all(v is None for v in results.values()):
330
+ target = ensure_one_model or sanitize_model_id(ModelName.mistral_v02.value)
331
  results[target] = _ping_once(target, ensure_timeout_ms)
332
  return results
333
 
 
337
  def build_help_markdown_static() -> str:
338
  lines = []
339
  lines.append("# 📘 Model Help & Comparison\n")
340
+ lines.append(f"Hardware: **{'GPU' if _HAS_CUDA else 'CPU-only'}** • Only available models are shown below.\n")
341
  lines.append("Static comparison. Use **Refresh latency** with your chosen timeout.\n")
342
  lines.append("| Model | Nickname | Best for | Recommended Input | Tone | Notes |")
343
  lines.append("|---|---|---|---|---|---|")
 
399
  def health():
400
  return {"status": "ok"}
401
 
402
+ @app.get("/info", response_model=dict)
403
+ def info():
404
+ return {
405
+ "cuda": bool(_HAS_CUDA),
406
+ "available_models": MODEL_IDS,
407
+ "defaults": {
408
+ "request_timeout_ms": REQUEST_TIMEOUT_MS,
409
+ "latency_default_ms": LATENCY_DEFAULT_MS,
410
+ "latency_max_ms": LATENCY_MAX_MS,
411
+ }
412
+ }
413
+
414
  # =========================
415
  # Endpoints
416
  # =========================
 
422
  if len(contents) > FILE_MAX_MB * 1024 * 1024:
423
  raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
424
  text = contents.decode("utf-8", errors="ignore")
425
+ # ברירת מחדל בטוחה לפי חו��רה
426
+ model_id = sanitize_model_id(ModelName.mistral_v02.value)
427
  mode_val = Mode.short
428
  timeout = clamp_timeout(REQUEST_TIMEOUT_MS)
429
 
 
448
  model_name: ModelName = Query(ModelName.mistral_v02, description="Choose the LLM"),
449
  request_timeout_ms: int = Query(REQUEST_TIMEOUT_MS, ge=1000, le=900000, description="Per-request timeout in ms")
450
  ):
451
+ # שימוש ב-sanitize כדי לא לבחור 7B ב-CPU
452
+ model_id = sanitize_model_id(model_name.value)
453
  mode_val = mode
454
  timeout = clamp_timeout(request_timeout_ms)
455
 
 
506
  raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
507
  paper = contents.decode("utf-8", errors="ignore")
508
 
509
+ model_id = sanitize_model_id(model_name.value)
510
  mode_val = mode
511
  timeout = clamp_timeout(request_timeout_ms)
512
 
 
626
 
627
  with gr.Row():
628
  mode = gr.Radio(["short", "detailed"], value="short", label="Answer length", scale=1)
629
+ # ברירת מחדל בטוחה לפי חומרה
630
+ model_id = gr.Dropdown(MODEL_IDS,
631
+ value=sanitize_model_id(ModelName.mistral_v02.value),
632
+ label="Choose the LLM", scale=1)
633
  request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)", scale=2)
634
  latency_timeout_ms = gr.Slider(5000, LATENCY_MAX_MS, value=LATENCY_DEFAULT_MS, step=5000, label="Latency check (ms)", scale=2)
635