Spaces:
Sleeping
Sleeping
Update app/main.py
Browse files- app/main.py +60 -9
app/main.py
CHANGED
|
@@ -41,6 +41,30 @@ MODEL_MAX_INPUT_TOKENS: Dict[str, int] = {
|
|
| 41 |
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 42 |
log = logging.getLogger("llm_api")
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
# =========================
|
| 45 |
# Enums / models
|
| 46 |
# =========================
|
|
@@ -55,7 +79,8 @@ class ModelName(str, Enum):
|
|
| 55 |
qwen_7b = "Qwen/Qwen2.5-7B-Instruct"
|
| 56 |
falcon_7b = "tiiuae/falcon-7b-instruct"
|
| 57 |
|
| 58 |
-
|
|
|
|
| 59 |
|
| 60 |
MODEL_INFO: Dict[str, Dict[str, Any]] = {
|
| 61 |
"mistralai/Mistral-7B-Instruct-v0.2": {
|
|
@@ -242,7 +267,7 @@ def _run_with_timeout(fn, timeout_ms: int, *args, **kwargs):
|
|
| 242 |
|
| 243 |
def _call_ask_question(context: str, question: str, model_id: Optional[str] = None, mode: Optional[str] = None):
|
| 244 |
"""
|
| 245 |
-
עוטף את ask_question כדי לתמוך בשמות פרמטרים שונים (model_key/model_name/model)
|
| 246 |
"""
|
| 247 |
try:
|
| 248 |
return ask_question(context=context, question=question, model_key=model_id, mode=mode)
|
|
@@ -262,7 +287,13 @@ def clamp_timeout(ms: int) -> int:
|
|
| 262 |
return 1000 if ms < 1000 else LATENCY_TIMEOUT_MS if ms > LATENCY_TIMEOUT_MS else ms
|
| 263 |
|
| 264 |
def sanitize_model_id(val: str) -> str:
|
| 265 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
# =========================
|
| 268 |
# Latency functions (עם זמן נבחר)
|
|
@@ -283,6 +314,8 @@ def measure_latencies_all(model_ids: List[str],
|
|
| 283 |
soft_timeout_ms: int = LATENCY_DEFAULT_MS,
|
| 284 |
ensure_one_model: Optional[str] = None,
|
| 285 |
ensure_timeout_ms: int = LATENCY_MAX_MS) -> Dict[str, Optional[float]]:
|
|
|
|
|
|
|
| 286 |
results: Dict[str, Optional[float]] = {}
|
| 287 |
max_workers = min(8, max(4, len(model_ids)))
|
| 288 |
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
|
|
@@ -293,8 +326,8 @@ def measure_latencies_all(model_ids: List[str],
|
|
| 293 |
results[mid] = fut.result()
|
| 294 |
except Exception:
|
| 295 |
results[mid] = None
|
| 296 |
-
if all(v is None for v in results.values()):
|
| 297 |
-
target = ensure_one_model or ModelName.mistral_v02.value
|
| 298 |
results[target] = _ping_once(target, ensure_timeout_ms)
|
| 299 |
return results
|
| 300 |
|
|
@@ -304,6 +337,7 @@ def measure_latencies_all(model_ids: List[str],
|
|
| 304 |
def build_help_markdown_static() -> str:
|
| 305 |
lines = []
|
| 306 |
lines.append("# 📘 Model Help & Comparison\n")
|
|
|
|
| 307 |
lines.append("Static comparison. Use **Refresh latency** with your chosen timeout.\n")
|
| 308 |
lines.append("| Model | Nickname | Best for | Recommended Input | Tone | Notes |")
|
| 309 |
lines.append("|---|---|---|---|---|---|")
|
|
@@ -365,6 +399,18 @@ def root():
|
|
| 365 |
def health():
|
| 366 |
return {"status": "ok"}
|
| 367 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 368 |
# =========================
|
| 369 |
# Endpoints
|
| 370 |
# =========================
|
|
@@ -376,7 +422,8 @@ async def upload_file(file: UploadFile = File(..., description="Upload .txt file
|
|
| 376 |
if len(contents) > FILE_MAX_MB * 1024 * 1024:
|
| 377 |
raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
|
| 378 |
text = contents.decode("utf-8", errors="ignore")
|
| 379 |
-
|
|
|
|
| 380 |
mode_val = Mode.short
|
| 381 |
timeout = clamp_timeout(REQUEST_TIMEOUT_MS)
|
| 382 |
|
|
@@ -401,7 +448,8 @@ async def query_llm(
|
|
| 401 |
model_name: ModelName = Query(ModelName.mistral_v02, description="Choose the LLM"),
|
| 402 |
request_timeout_ms: int = Query(REQUEST_TIMEOUT_MS, ge=1000, le=900000, description="Per-request timeout in ms")
|
| 403 |
):
|
| 404 |
-
|
|
|
|
| 405 |
mode_val = mode
|
| 406 |
timeout = clamp_timeout(request_timeout_ms)
|
| 407 |
|
|
@@ -458,7 +506,7 @@ async def query_form(
|
|
| 458 |
raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
|
| 459 |
paper = contents.decode("utf-8", errors="ignore")
|
| 460 |
|
| 461 |
-
model_id = model_name.value
|
| 462 |
mode_val = mode
|
| 463 |
timeout = clamp_timeout(request_timeout_ms)
|
| 464 |
|
|
@@ -578,7 +626,10 @@ def build_ui():
|
|
| 578 |
|
| 579 |
with gr.Row():
|
| 580 |
mode = gr.Radio(["short", "detailed"], value="short", label="Answer length", scale=1)
|
| 581 |
-
|
|
|
|
|
|
|
|
|
|
| 582 |
request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)", scale=2)
|
| 583 |
latency_timeout_ms = gr.Slider(5000, LATENCY_MAX_MS, value=LATENCY_DEFAULT_MS, step=5000, label="Latency check (ms)", scale=2)
|
| 584 |
|
|
|
|
| 41 |
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 42 |
log = logging.getLogger("llm_api")
|
| 43 |
|
| 44 |
+
# =========================
|
| 45 |
+
# Hardware detection & available models
|
| 46 |
+
# =========================
|
| 47 |
+
try:
|
| 48 |
+
import torch
|
| 49 |
+
_HAS_CUDA = bool(torch.cuda.is_available())
|
| 50 |
+
except Exception:
|
| 51 |
+
torch = None
|
| 52 |
+
_HAS_CUDA = False
|
| 53 |
+
|
| 54 |
+
GPU_FRIENDLY: List[str] = [
|
| 55 |
+
"mistralai/Mistral-7B-Instruct-v0.2",
|
| 56 |
+
"mistralai/Mistral-7B-Instruct-v0.3",
|
| 57 |
+
"Qwen/Qwen2.5-7B-Instruct",
|
| 58 |
+
"tiiuae/falcon-7b-instruct",
|
| 59 |
+
"microsoft/Phi-3-mini-4k-instruct",
|
| 60 |
+
]
|
| 61 |
+
CPU_SAFE: List[str] = [
|
| 62 |
+
"microsoft/Phi-3-mini-4k-instruct", # ברירת מחדל בטוחה ב-CPU
|
| 63 |
+
# אפשר להוסיף כאן דגמים קטנים נוספים אם תרצי
|
| 64 |
+
]
|
| 65 |
+
|
| 66 |
+
AVAILABLE_MODEL_IDS: List[str] = GPU_FRIENDLY if _HAS_CUDA else CPU_SAFE
|
| 67 |
+
|
| 68 |
# =========================
|
| 69 |
# Enums / models
|
| 70 |
# =========================
|
|
|
|
| 79 |
qwen_7b = "Qwen/Qwen2.5-7B-Instruct"
|
| 80 |
falcon_7b = "tiiuae/falcon-7b-instruct"
|
| 81 |
|
| 82 |
+
# רק מה שזמין בחומרה הנוכחית
|
| 83 |
+
MODEL_IDS: List[str] = AVAILABLE_MODEL_IDS
|
| 84 |
|
| 85 |
MODEL_INFO: Dict[str, Dict[str, Any]] = {
|
| 86 |
"mistralai/Mistral-7B-Instruct-v0.2": {
|
|
|
|
| 267 |
|
| 268 |
def _call_ask_question(context: str, question: str, model_id: Optional[str] = None, mode: Optional[str] = None):
|
| 269 |
"""
|
| 270 |
+
עוטף את ask_question כדי לתמוך בשמות פרמטרים שונים (model_key/model_name/model).
|
| 271 |
"""
|
| 272 |
try:
|
| 273 |
return ask_question(context=context, question=question, model_key=model_id, mode=mode)
|
|
|
|
| 287 |
return 1000 if ms < 1000 else LATENCY_TIMEOUT_MS if ms > LATENCY_TIMEOUT_MS else ms
|
| 288 |
|
| 289 |
def sanitize_model_id(val: str) -> str:
|
| 290 |
+
"""
|
| 291 |
+
ברירת מחדל בטוחה: ב-CPU -> Phi-3-mini; ב-GPU -> Mistral v0.2.
|
| 292 |
+
"""
|
| 293 |
+
if val in MODEL_IDS:
|
| 294 |
+
return val
|
| 295 |
+
return ("mistralai/Mistral-7B-Instruct-v0.2" if _HAS_CUDA
|
| 296 |
+
else "microsoft/Phi-3-mini-4k-instruct")
|
| 297 |
|
| 298 |
# =========================
|
| 299 |
# Latency functions (עם זמן נבחר)
|
|
|
|
| 314 |
soft_timeout_ms: int = LATENCY_DEFAULT_MS,
|
| 315 |
ensure_one_model: Optional[str] = None,
|
| 316 |
ensure_timeout_ms: int = LATENCY_MAX_MS) -> Dict[str, Optional[float]]:
|
| 317 |
+
# מודדים רק מודלים זמינים בסביבה הנוכחית
|
| 318 |
+
model_ids = [m for m in model_ids if m in MODEL_IDS]
|
| 319 |
results: Dict[str, Optional[float]] = {}
|
| 320 |
max_workers = min(8, max(4, len(model_ids)))
|
| 321 |
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
|
|
|
|
| 326 |
results[mid] = fut.result()
|
| 327 |
except Exception:
|
| 328 |
results[mid] = None
|
| 329 |
+
if results and all(v is None for v in results.values()):
|
| 330 |
+
target = ensure_one_model or sanitize_model_id(ModelName.mistral_v02.value)
|
| 331 |
results[target] = _ping_once(target, ensure_timeout_ms)
|
| 332 |
return results
|
| 333 |
|
|
|
|
| 337 |
def build_help_markdown_static() -> str:
|
| 338 |
lines = []
|
| 339 |
lines.append("# 📘 Model Help & Comparison\n")
|
| 340 |
+
lines.append(f"Hardware: **{'GPU' if _HAS_CUDA else 'CPU-only'}** • Only available models are shown below.\n")
|
| 341 |
lines.append("Static comparison. Use **Refresh latency** with your chosen timeout.\n")
|
| 342 |
lines.append("| Model | Nickname | Best for | Recommended Input | Tone | Notes |")
|
| 343 |
lines.append("|---|---|---|---|---|---|")
|
|
|
|
| 399 |
def health():
|
| 400 |
return {"status": "ok"}
|
| 401 |
|
| 402 |
+
@app.get("/info", response_model=dict)
|
| 403 |
+
def info():
|
| 404 |
+
return {
|
| 405 |
+
"cuda": bool(_HAS_CUDA),
|
| 406 |
+
"available_models": MODEL_IDS,
|
| 407 |
+
"defaults": {
|
| 408 |
+
"request_timeout_ms": REQUEST_TIMEOUT_MS,
|
| 409 |
+
"latency_default_ms": LATENCY_DEFAULT_MS,
|
| 410 |
+
"latency_max_ms": LATENCY_MAX_MS,
|
| 411 |
+
}
|
| 412 |
+
}
|
| 413 |
+
|
| 414 |
# =========================
|
| 415 |
# Endpoints
|
| 416 |
# =========================
|
|
|
|
| 422 |
if len(contents) > FILE_MAX_MB * 1024 * 1024:
|
| 423 |
raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
|
| 424 |
text = contents.decode("utf-8", errors="ignore")
|
| 425 |
+
# ברירת מחדל בטוחה לפי חו��רה
|
| 426 |
+
model_id = sanitize_model_id(ModelName.mistral_v02.value)
|
| 427 |
mode_val = Mode.short
|
| 428 |
timeout = clamp_timeout(REQUEST_TIMEOUT_MS)
|
| 429 |
|
|
|
|
| 448 |
model_name: ModelName = Query(ModelName.mistral_v02, description="Choose the LLM"),
|
| 449 |
request_timeout_ms: int = Query(REQUEST_TIMEOUT_MS, ge=1000, le=900000, description="Per-request timeout in ms")
|
| 450 |
):
|
| 451 |
+
# שימוש ב-sanitize כדי לא לבחור 7B ב-CPU
|
| 452 |
+
model_id = sanitize_model_id(model_name.value)
|
| 453 |
mode_val = mode
|
| 454 |
timeout = clamp_timeout(request_timeout_ms)
|
| 455 |
|
|
|
|
| 506 |
raise HTTPException(status_code=413, detail=f"File too large (> {FILE_MAX_MB} MB)")
|
| 507 |
paper = contents.decode("utf-8", errors="ignore")
|
| 508 |
|
| 509 |
+
model_id = sanitize_model_id(model_name.value)
|
| 510 |
mode_val = mode
|
| 511 |
timeout = clamp_timeout(request_timeout_ms)
|
| 512 |
|
|
|
|
| 626 |
|
| 627 |
with gr.Row():
|
| 628 |
mode = gr.Radio(["short", "detailed"], value="short", label="Answer length", scale=1)
|
| 629 |
+
# ברירת מחדל בטוחה לפי חומרה
|
| 630 |
+
model_id = gr.Dropdown(MODEL_IDS,
|
| 631 |
+
value=sanitize_model_id(ModelName.mistral_v02.value),
|
| 632 |
+
label="Choose the LLM", scale=1)
|
| 633 |
request_timeout_ms = gr.Slider(1000, 900000, value=REQUEST_TIMEOUT_MS, step=500, label="Timeout (ms)", scale=2)
|
| 634 |
latency_timeout_ms = gr.Slider(5000, LATENCY_MAX_MS, value=LATENCY_DEFAULT_MS, step=5000, label="Latency check (ms)", scale=2)
|
| 635 |
|