Spaces:
Running on Zero
Running on Zero
Run CPU benchmarks asynchronously
Browse files
app.py
CHANGED
|
@@ -203,6 +203,8 @@ _BENCH_MODELS = [
|
|
| 203 |
_BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench")
|
| 204 |
_BENCH_RESULTS = []
|
| 205 |
_BENCH_LOCK = threading.Lock()
|
|
|
|
|
|
|
| 206 |
|
| 207 |
|
| 208 |
def _ensure_llama():
|
|
@@ -358,6 +360,49 @@ def _run_cpu_smoke(model_filter):
|
|
| 358 |
return result
|
| 359 |
|
| 360 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
MODEL_CACHE = {}
|
| 362 |
|
| 363 |
def load_model(name):
|
|
@@ -420,21 +465,20 @@ def _patched_create_app(blocks, **kwargs):
|
|
| 420 |
except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
|
| 421 |
|
| 422 |
@fa_app.get("/respite/benchmark/smoke")
|
| 423 |
-
def _smoke(model: str="Gemma 4 E4B"):
|
| 424 |
-
|
| 425 |
-
with
|
| 426 |
-
return JSONResponse(
|
| 427 |
-
|
| 428 |
-
|
| 429 |
|
| 430 |
@fa_app.get("/respite/benchmark")
|
| 431 |
-
def _b(force: bool=False, model: str=""):
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
except Exception as e: return JSONResponse({"error": str(e)}, status_code=500)
|
| 438 |
|
| 439 |
_log("Routes injected via create_app monkey-patch")
|
| 440 |
return fa_app
|
|
|
|
| 203 |
_BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench")
|
| 204 |
_BENCH_RESULTS = []
|
| 205 |
_BENCH_LOCK = threading.Lock()
|
| 206 |
+
_BENCH_JOBS = {}
|
| 207 |
+
_BENCH_JOBS_LOCK = threading.Lock()
|
| 208 |
|
| 209 |
|
| 210 |
def _ensure_llama():
|
|
|
|
| 360 |
return result
|
| 361 |
|
| 362 |
|
| 363 |
+
def _run_benchmark_job(job_id, kind, model_filter=None, force=False):
|
| 364 |
+
try:
|
| 365 |
+
if kind == "smoke":
|
| 366 |
+
with _BENCH_LOCK:
|
| 367 |
+
result = _run_cpu_smoke(model_filter)
|
| 368 |
+
payload = {"status": "complete", "result": result}
|
| 369 |
+
else:
|
| 370 |
+
if force:
|
| 371 |
+
global _BENCH_RESULTS
|
| 372 |
+
_BENCH_RESULTS = []
|
| 373 |
+
payload = {
|
| 374 |
+
"status": "complete",
|
| 375 |
+
"models": _run_bench(model_filter=model_filter),
|
| 376 |
+
"system": {
|
| 377 |
+
"cpu_cores": _parse_cpu_count(),
|
| 378 |
+
"ram_bytes": _parse_ram(),
|
| 379 |
+
"threads": _parse_cpu_count() or 8,
|
| 380 |
+
},
|
| 381 |
+
}
|
| 382 |
+
with _BENCH_JOBS_LOCK:
|
| 383 |
+
_BENCH_JOBS[job_id] = payload
|
| 384 |
+
except Exception as exc:
|
| 385 |
+
with _BENCH_JOBS_LOCK:
|
| 386 |
+
_BENCH_JOBS[job_id] = {"status": "error", "error": str(exc)}
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _start_benchmark_job(kind, model_filter=None, force=False):
|
| 390 |
+
job_id = f"bench-{int(time.time() * 1000)}-{threading.get_ident()}"
|
| 391 |
+
with _BENCH_JOBS_LOCK:
|
| 392 |
+
_BENCH_JOBS[job_id] = {
|
| 393 |
+
"status": "running",
|
| 394 |
+
"kind": kind,
|
| 395 |
+
"model": model_filter,
|
| 396 |
+
"started_at": time.time(),
|
| 397 |
+
}
|
| 398 |
+
threading.Thread(
|
| 399 |
+
target=_run_benchmark_job,
|
| 400 |
+
args=(job_id, kind, model_filter, force),
|
| 401 |
+
daemon=True,
|
| 402 |
+
).start()
|
| 403 |
+
return job_id
|
| 404 |
+
|
| 405 |
+
|
| 406 |
MODEL_CACHE = {}
|
| 407 |
|
| 408 |
def load_model(name):
|
|
|
|
| 465 |
except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
|
| 466 |
|
| 467 |
@fa_app.get("/respite/benchmark/smoke")
|
| 468 |
+
def _smoke(model: str="Gemma 4 E4B", job_id: str=""):
|
| 469 |
+
if job_id:
|
| 470 |
+
with _BENCH_JOBS_LOCK:
|
| 471 |
+
return JSONResponse(_BENCH_JOBS.get(job_id, {"status": "not_found"}))
|
| 472 |
+
job = _start_benchmark_job("smoke", model.strip())
|
| 473 |
+
return JSONResponse({"status": "accepted", "job_id": job, "poll": "/respite/benchmark/smoke?job_id=" + job}, status_code=202)
|
| 474 |
|
| 475 |
@fa_app.get("/respite/benchmark")
|
| 476 |
+
def _b(force: bool=False, model: str="", job_id: str=""):
|
| 477 |
+
if job_id:
|
| 478 |
+
with _BENCH_JOBS_LOCK:
|
| 479 |
+
return JSONResponse(_BENCH_JOBS.get(job_id, {"status": "not_found"}))
|
| 480 |
+
job = _start_benchmark_job("benchmark", model.strip() or None, force)
|
| 481 |
+
return JSONResponse({"status": "accepted", "job_id": job, "poll": "/respite/benchmark?job_id=" + job}, status_code=202)
|
|
|
|
| 482 |
|
| 483 |
_log("Routes injected via create_app monkey-patch")
|
| 484 |
return fa_app
|