cazyundee commited on
Commit
318e7cb
·
verified ·
1 Parent(s): 50b3226

Run CPU benchmarks asynchronously

Browse files
Files changed (1) hide show
  1. app.py +57 -13
app.py CHANGED
@@ -203,6 +203,8 @@ _BENCH_MODELS = [
203
  _BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench")
204
  _BENCH_RESULTS = []
205
  _BENCH_LOCK = threading.Lock()
 
 
206
 
207
 
208
  def _ensure_llama():
@@ -358,6 +360,49 @@ def _run_cpu_smoke(model_filter):
358
  return result
359
 
360
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
361
  MODEL_CACHE = {}
362
 
363
  def load_model(name):
@@ -420,21 +465,20 @@ def _patched_create_app(blocks, **kwargs):
420
  except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
421
 
422
  @fa_app.get("/respite/benchmark/smoke")
423
- def _smoke(model: str="Gemma 4 E4B"):
424
- try:
425
- with _BENCH_LOCK:
426
- return JSONResponse(_run_cpu_smoke(model.strip()))
427
- except Exception as e:
428
- return JSONResponse({"error": str(e)}, status_code=500)
429
 
430
  @fa_app.get("/respite/benchmark")
431
- def _b(force: bool=False, model: str=""):
432
- try:
433
- global _BENCH_RESULTS
434
- if force: _BENCH_RESULTS = []
435
- results = _run_bench(model_filter=model.strip() or None)
436
- return JSONResponse({"status":"complete","models":results,"system":{"cpu_cores":_parse_cpu_count(),"ram_bytes":_parse_ram(),"threads":_parse_cpu_count() or 8}})
437
- except Exception as e: return JSONResponse({"error": str(e)}, status_code=500)
438
 
439
  _log("Routes injected via create_app monkey-patch")
440
  return fa_app
 
203
  _BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench")
204
  _BENCH_RESULTS = []
205
  _BENCH_LOCK = threading.Lock()
206
+ _BENCH_JOBS = {}
207
+ _BENCH_JOBS_LOCK = threading.Lock()
208
 
209
 
210
  def _ensure_llama():
 
360
  return result
361
 
362
 
363
+ def _run_benchmark_job(job_id, kind, model_filter=None, force=False):
364
+ try:
365
+ if kind == "smoke":
366
+ with _BENCH_LOCK:
367
+ result = _run_cpu_smoke(model_filter)
368
+ payload = {"status": "complete", "result": result}
369
+ else:
370
+ if force:
371
+ global _BENCH_RESULTS
372
+ _BENCH_RESULTS = []
373
+ payload = {
374
+ "status": "complete",
375
+ "models": _run_bench(model_filter=model_filter),
376
+ "system": {
377
+ "cpu_cores": _parse_cpu_count(),
378
+ "ram_bytes": _parse_ram(),
379
+ "threads": _parse_cpu_count() or 8,
380
+ },
381
+ }
382
+ with _BENCH_JOBS_LOCK:
383
+ _BENCH_JOBS[job_id] = payload
384
+ except Exception as exc:
385
+ with _BENCH_JOBS_LOCK:
386
+ _BENCH_JOBS[job_id] = {"status": "error", "error": str(exc)}
387
+
388
+
389
+ def _start_benchmark_job(kind, model_filter=None, force=False):
390
+ job_id = f"bench-{int(time.time() * 1000)}-{threading.get_ident()}"
391
+ with _BENCH_JOBS_LOCK:
392
+ _BENCH_JOBS[job_id] = {
393
+ "status": "running",
394
+ "kind": kind,
395
+ "model": model_filter,
396
+ "started_at": time.time(),
397
+ }
398
+ threading.Thread(
399
+ target=_run_benchmark_job,
400
+ args=(job_id, kind, model_filter, force),
401
+ daemon=True,
402
+ ).start()
403
+ return job_id
404
+
405
+
406
  MODEL_CACHE = {}
407
 
408
  def load_model(name):
 
465
  except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
466
 
467
  @fa_app.get("/respite/benchmark/smoke")
468
+ def _smoke(model: str="Gemma 4 E4B", job_id: str=""):
469
+ if job_id:
470
+ with _BENCH_JOBS_LOCK:
471
+ return JSONResponse(_BENCH_JOBS.get(job_id, {"status": "not_found"}))
472
+ job = _start_benchmark_job("smoke", model.strip())
473
+ return JSONResponse({"status": "accepted", "job_id": job, "poll": "/respite/benchmark/smoke?job_id=" + job}, status_code=202)
474
 
475
  @fa_app.get("/respite/benchmark")
476
+ def _b(force: bool=False, model: str="", job_id: str=""):
477
+ if job_id:
478
+ with _BENCH_JOBS_LOCK:
479
+ return JSONResponse(_BENCH_JOBS.get(job_id, {"status": "not_found"}))
480
+ job = _start_benchmark_job("benchmark", model.strip() or None, force)
481
+ return JSONResponse({"status": "accepted", "job_id": job, "poll": "/respite/benchmark?job_id=" + job}, status_code=202)
 
482
 
483
  _log("Routes injected via create_app monkey-patch")
484
  return fa_app