Spaces:
Running
Running
Fix local progress and Colab handoff
Browse files
app.py
CHANGED
|
@@ -37,6 +37,7 @@ from starlette.concurrency import run_in_threadpool
|
|
| 37 |
|
| 38 |
load_dotenv(Path(__file__).with_name(".env"))
|
| 39 |
jobs = {}
|
|
|
|
| 40 |
executor = ThreadPoolExecutor(max_workers=1)
|
| 41 |
logger = logging.getLogger("sporalize")
|
| 42 |
POSE_MODEL_FILE = "Weights/rtmpose-x-halpe26.onnx"
|
|
@@ -189,6 +190,68 @@ def run_pipeline_on_colab(worker_url, client_id, work, cameras, job):
|
|
| 189 |
logger.info("Temporary Colab handoff cleanup skipped for %s: %s", remote_path, error)
|
| 190 |
|
| 191 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
@lru_cache(maxsize=1)
|
| 193 |
def runtime():
|
| 194 |
root = Path(__file__).resolve().parent.parent / "Core"
|
|
@@ -506,19 +569,35 @@ def run_job(client_id, work, cameras):
|
|
| 506 |
if reports is None:
|
| 507 |
if job.get("cancel"):
|
| 508 |
raise InterruptedError()
|
| 509 |
-
job.update(status="running", progress=5, phase="Analyzing session", stage="analysis")
|
| 510 |
pipeline, weights = runtime()
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
total=
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
if job.get("cancel"):
|
| 523 |
raise InterruptedError()
|
| 524 |
(work / "results.json").write_text(json.dumps(json_value(reports), allow_nan=False), encoding="utf-8")
|
|
@@ -726,16 +805,19 @@ def active(request: Request):
|
|
| 726 |
@app.post("/api/cancel/{client_id}")
|
| 727 |
def cancel(client_id: str, request: Request):
|
| 728 |
job = owned_job(client_id, request.state.user_id, request.state.is_admin)
|
| 729 |
-
|
| 730 |
-
job["
|
|
|
|
| 731 |
return {"status": job["status"], "clientId": client_id}
|
| 732 |
|
| 733 |
|
| 734 |
def cancel_session_job(session_id: UUID):
|
| 735 |
matching = [(client_id, job) for client_id, job in list(jobs.items()) if job.get("sessionId") == str(session_id)]
|
| 736 |
for client_id, job in matching:
|
|
|
|
|
|
|
|
|
|
| 737 |
if job.get("status") in {"uploading", "queued", "running"}:
|
| 738 |
-
job["cancel"] = True
|
| 739 |
future = job.get("future")
|
| 740 |
if future is not None:
|
| 741 |
try:
|
|
|
|
| 37 |
|
| 38 |
load_dotenv(Path(__file__).with_name(".env"))
|
| 39 |
jobs = {}
|
| 40 |
+
jobs_lock = threading.RLock()
|
| 41 |
executor = ThreadPoolExecutor(max_workers=1)
|
| 42 |
logger = logging.getLogger("sporalize")
|
| 43 |
POSE_MODEL_FILE = "Weights/rtmpose-x-halpe26.onnx"
|
|
|
|
| 190 |
logger.info("Temporary Colab handoff cleanup skipped for %s: %s", remote_path, error)
|
| 191 |
|
| 192 |
|
| 193 |
+
def monitor_colab_worker(job, stop_event):
|
| 194 |
+
if len(os.environ.get("COLAB_WORKER_SECRET", "")) < 32:
|
| 195 |
+
return
|
| 196 |
+
while not stop_event.wait(10):
|
| 197 |
+
if job.get("cancel") or job.get("stage") != "analysis":
|
| 198 |
+
return
|
| 199 |
+
worker_url = registered_colab_worker()
|
| 200 |
+
if stop_event.is_set():
|
| 201 |
+
return
|
| 202 |
+
if not worker_url:
|
| 203 |
+
continue
|
| 204 |
+
with jobs_lock:
|
| 205 |
+
if stop_event.is_set() or job.get("cancel") or job.get("stage") != "analysis":
|
| 206 |
+
return
|
| 207 |
+
job.update(
|
| 208 |
+
cancel=True,
|
| 209 |
+
cancelReason="colab_handoff",
|
| 210 |
+
handoffWorkerUrl=worker_url,
|
| 211 |
+
phase="Colab connected; restarting analysis from the beginning",
|
| 212 |
+
updatedAt=int(time.time() * 1000),
|
| 213 |
+
)
|
| 214 |
+
logger.info("Switching analysis %s from the primary server to Colab; restarting from the beginning", job["clientId"])
|
| 215 |
+
return
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def run_local_pipeline(pipeline, cameras, utils_paths, sizes, job, allow_colab_handoff=True):
|
| 219 |
+
stop_event = threading.Event()
|
| 220 |
+
monitor = None
|
| 221 |
+
if allow_colab_handoff:
|
| 222 |
+
monitor = threading.Thread(target=monitor_colab_worker, args=(job, stop_event), daemon=True)
|
| 223 |
+
monitor.start()
|
| 224 |
+
|
| 225 |
+
def progress(action, total, frame, frames):
|
| 226 |
+
action_number = action + 1
|
| 227 |
+
frame_total = max(1, frames)
|
| 228 |
+
with jobs_lock:
|
| 229 |
+
job.update(progress=5 + 85 * (action + (frame + 1) / frame_total) / max(total, 1),
|
| 230 |
+
phase=f"Analyzing action {action_number}/{total}", step=frame + 1,
|
| 231 |
+
total=frame_total, action=action_number, actionsTotal=total,
|
| 232 |
+
updatedAt=int(time.time() * 1000))
|
| 233 |
+
return not job.get("cancel")
|
| 234 |
+
|
| 235 |
+
try:
|
| 236 |
+
reports = pipeline(cameras, utils_paths, sizes, progress_cb=progress)
|
| 237 |
+
except InterruptedError:
|
| 238 |
+
with jobs_lock:
|
| 239 |
+
if job.get("cancelReason") != "colab_handoff":
|
| 240 |
+
raise
|
| 241 |
+
return None, job.get("handoffWorkerUrl")
|
| 242 |
+
finally:
|
| 243 |
+
stop_event.set()
|
| 244 |
+
if monitor is not None:
|
| 245 |
+
monitor.join()
|
| 246 |
+
|
| 247 |
+
with jobs_lock:
|
| 248 |
+
if job.get("cancelReason") == "colab_handoff":
|
| 249 |
+
return None, job.get("handoffWorkerUrl")
|
| 250 |
+
if job.get("cancel"):
|
| 251 |
+
raise InterruptedError("Processing cancelled by user")
|
| 252 |
+
return reports, None
|
| 253 |
+
|
| 254 |
+
|
| 255 |
@lru_cache(maxsize=1)
|
| 256 |
def runtime():
|
| 257 |
root = Path(__file__).resolve().parent.parent / "Core"
|
|
|
|
| 569 |
if reports is None:
|
| 570 |
if job.get("cancel"):
|
| 571 |
raise InterruptedError()
|
|
|
|
| 572 |
pipeline, weights = runtime()
|
| 573 |
+
utils_paths = {**weights, "CALIBRATION_PATH": str(work / "calibration.npz"), "ACTIONS_PATH": str(work / "actions.json")}
|
| 574 |
+
sizes = {"YOLO_IMGSZ": YOLO_IMGSZ}
|
| 575 |
+
allow_handoff = worker_url is None
|
| 576 |
+
|
| 577 |
+
while True:
|
| 578 |
+
job.update(status="running", progress=5, phase="Analyzing session", stage="analysis",
|
| 579 |
+
step=0, total=0, action=0, actionsTotal=0)
|
| 580 |
+
reports, handoff_url = run_local_pipeline(
|
| 581 |
+
pipeline, cameras, utils_paths, sizes, job, allow_colab_handoff=allow_handoff,
|
| 582 |
+
)
|
| 583 |
+
if not handoff_url:
|
| 584 |
+
break
|
| 585 |
+
|
| 586 |
+
with jobs_lock:
|
| 587 |
+
if job.get("cancelReason") != "colab_handoff":
|
| 588 |
+
raise InterruptedError("Processing cancelled by user")
|
| 589 |
+
job.update(cancel=False, cancelReason=None, handoffWorkerUrl=None,
|
| 590 |
+
status="running", progress=5, phase="Restarting analysis on Colab from the beginning",
|
| 591 |
+
stage="analysis", step=0, total=0, action=0, actionsTotal=0,
|
| 592 |
+
updatedAt=int(time.time() * 1000))
|
| 593 |
+
logger.info("Restarting analysis %s on Colab from the original uploaded videos", client_id)
|
| 594 |
+
reports = run_pipeline_on_colab(handoff_url, client_id, work, cameras, job)
|
| 595 |
+
if reports is not None:
|
| 596 |
+
break
|
| 597 |
+
if job.get("cancel"):
|
| 598 |
+
raise InterruptedError()
|
| 599 |
+
logger.warning("Colab handoff failed for analysis %s; restarting on the primary server", client_id)
|
| 600 |
+
allow_handoff = False
|
| 601 |
if job.get("cancel"):
|
| 602 |
raise InterruptedError()
|
| 603 |
(work / "results.json").write_text(json.dumps(json_value(reports), allow_nan=False), encoding="utf-8")
|
|
|
|
| 805 |
@app.post("/api/cancel/{client_id}")
|
| 806 |
def cancel(client_id: str, request: Request):
|
| 807 |
job = owned_job(client_id, request.state.user_id, request.state.is_admin)
|
| 808 |
+
with jobs_lock:
|
| 809 |
+
if job["status"] not in {"completed", "failed", "cancelled", "sync_failed"}:
|
| 810 |
+
job.update(cancel=True, cancelReason="user")
|
| 811 |
return {"status": job["status"], "clientId": client_id}
|
| 812 |
|
| 813 |
|
| 814 |
def cancel_session_job(session_id: UUID):
|
| 815 |
matching = [(client_id, job) for client_id, job in list(jobs.items()) if job.get("sessionId") == str(session_id)]
|
| 816 |
for client_id, job in matching:
|
| 817 |
+
with jobs_lock:
|
| 818 |
+
if job.get("status") in {"uploading", "queued", "running"}:
|
| 819 |
+
job.update(cancel=True, cancelReason="delete")
|
| 820 |
if job.get("status") in {"uploading", "queued", "running"}:
|
|
|
|
| 821 |
future = job.get("future")
|
| 822 |
if future is not None:
|
| 823 |
try:
|