Shoraky commited on
Commit
3eee31a
·
1 Parent(s): bbb8ffa

Fix local progress and Colab handoff

Browse files
Files changed (1) hide show
  1. app.py +97 -15
app.py CHANGED
@@ -37,6 +37,7 @@ from starlette.concurrency import run_in_threadpool
37
 
38
  load_dotenv(Path(__file__).with_name(".env"))
39
  jobs = {}
 
40
  executor = ThreadPoolExecutor(max_workers=1)
41
  logger = logging.getLogger("sporalize")
42
  POSE_MODEL_FILE = "Weights/rtmpose-x-halpe26.onnx"
@@ -189,6 +190,68 @@ def run_pipeline_on_colab(worker_url, client_id, work, cameras, job):
189
  logger.info("Temporary Colab handoff cleanup skipped for %s: %s", remote_path, error)
190
 
191
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
  @lru_cache(maxsize=1)
193
  def runtime():
194
  root = Path(__file__).resolve().parent.parent / "Core"
@@ -506,19 +569,35 @@ def run_job(client_id, work, cameras):
506
  if reports is None:
507
  if job.get("cancel"):
508
  raise InterruptedError()
509
- job.update(status="running", progress=5, phase="Analyzing session", stage="analysis")
510
  pipeline, weights = runtime()
511
-
512
- def progress(action, total, frame, frames):
513
- action_number = action + 1
514
- frame_total = max(1, frames)
515
- job.update(progress=5 + 85 * (action + (frame + 1) / frame_total) / max(total, 1),
516
- phase=f"Analyzing action {action_number}/{total}", step=frame + 1,
517
- total=frame_total, action=action_number, actionsTotal=total,
518
- updatedAt=int(time.time() * 1000))
519
- return not job.get("cancel")
520
-
521
- reports = pipeline(cameras, {**weights, "CALIBRATION_PATH": str(work / "calibration.npz"), "ACTIONS_PATH": str(work / "actions.json")}, {"YOLO_IMGSZ": YOLO_IMGSZ}, progress)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
522
  if job.get("cancel"):
523
  raise InterruptedError()
524
  (work / "results.json").write_text(json.dumps(json_value(reports), allow_nan=False), encoding="utf-8")
@@ -726,16 +805,19 @@ def active(request: Request):
726
  @app.post("/api/cancel/{client_id}")
727
  def cancel(client_id: str, request: Request):
728
  job = owned_job(client_id, request.state.user_id, request.state.is_admin)
729
- if job["status"] not in {"completed", "failed", "cancelled", "sync_failed"}:
730
- job["cancel"] = True
 
731
  return {"status": job["status"], "clientId": client_id}
732
 
733
 
734
  def cancel_session_job(session_id: UUID):
735
  matching = [(client_id, job) for client_id, job in list(jobs.items()) if job.get("sessionId") == str(session_id)]
736
  for client_id, job in matching:
 
 
 
737
  if job.get("status") in {"uploading", "queued", "running"}:
738
- job["cancel"] = True
739
  future = job.get("future")
740
  if future is not None:
741
  try:
 
37
 
38
  load_dotenv(Path(__file__).with_name(".env"))
39
  jobs = {}
40
+ jobs_lock = threading.RLock()
41
  executor = ThreadPoolExecutor(max_workers=1)
42
  logger = logging.getLogger("sporalize")
43
  POSE_MODEL_FILE = "Weights/rtmpose-x-halpe26.onnx"
 
190
  logger.info("Temporary Colab handoff cleanup skipped for %s: %s", remote_path, error)
191
 
192
 
193
+ def monitor_colab_worker(job, stop_event):
194
+ if len(os.environ.get("COLAB_WORKER_SECRET", "")) < 32:
195
+ return
196
+ while not stop_event.wait(10):
197
+ if job.get("cancel") or job.get("stage") != "analysis":
198
+ return
199
+ worker_url = registered_colab_worker()
200
+ if stop_event.is_set():
201
+ return
202
+ if not worker_url:
203
+ continue
204
+ with jobs_lock:
205
+ if stop_event.is_set() or job.get("cancel") or job.get("stage") != "analysis":
206
+ return
207
+ job.update(
208
+ cancel=True,
209
+ cancelReason="colab_handoff",
210
+ handoffWorkerUrl=worker_url,
211
+ phase="Colab connected; restarting analysis from the beginning",
212
+ updatedAt=int(time.time() * 1000),
213
+ )
214
+ logger.info("Switching analysis %s from the primary server to Colab; restarting from the beginning", job["clientId"])
215
+ return
216
+
217
+
218
+ def run_local_pipeline(pipeline, cameras, utils_paths, sizes, job, allow_colab_handoff=True):
219
+ stop_event = threading.Event()
220
+ monitor = None
221
+ if allow_colab_handoff:
222
+ monitor = threading.Thread(target=monitor_colab_worker, args=(job, stop_event), daemon=True)
223
+ monitor.start()
224
+
225
+ def progress(action, total, frame, frames):
226
+ action_number = action + 1
227
+ frame_total = max(1, frames)
228
+ with jobs_lock:
229
+ job.update(progress=5 + 85 * (action + (frame + 1) / frame_total) / max(total, 1),
230
+ phase=f"Analyzing action {action_number}/{total}", step=frame + 1,
231
+ total=frame_total, action=action_number, actionsTotal=total,
232
+ updatedAt=int(time.time() * 1000))
233
+ return not job.get("cancel")
234
+
235
+ try:
236
+ reports = pipeline(cameras, utils_paths, sizes, progress_cb=progress)
237
+ except InterruptedError:
238
+ with jobs_lock:
239
+ if job.get("cancelReason") != "colab_handoff":
240
+ raise
241
+ return None, job.get("handoffWorkerUrl")
242
+ finally:
243
+ stop_event.set()
244
+ if monitor is not None:
245
+ monitor.join()
246
+
247
+ with jobs_lock:
248
+ if job.get("cancelReason") == "colab_handoff":
249
+ return None, job.get("handoffWorkerUrl")
250
+ if job.get("cancel"):
251
+ raise InterruptedError("Processing cancelled by user")
252
+ return reports, None
253
+
254
+
255
  @lru_cache(maxsize=1)
256
  def runtime():
257
  root = Path(__file__).resolve().parent.parent / "Core"
 
569
  if reports is None:
570
  if job.get("cancel"):
571
  raise InterruptedError()
 
572
  pipeline, weights = runtime()
573
+ utils_paths = {**weights, "CALIBRATION_PATH": str(work / "calibration.npz"), "ACTIONS_PATH": str(work / "actions.json")}
574
+ sizes = {"YOLO_IMGSZ": YOLO_IMGSZ}
575
+ allow_handoff = worker_url is None
576
+
577
+ while True:
578
+ job.update(status="running", progress=5, phase="Analyzing session", stage="analysis",
579
+ step=0, total=0, action=0, actionsTotal=0)
580
+ reports, handoff_url = run_local_pipeline(
581
+ pipeline, cameras, utils_paths, sizes, job, allow_colab_handoff=allow_handoff,
582
+ )
583
+ if not handoff_url:
584
+ break
585
+
586
+ with jobs_lock:
587
+ if job.get("cancelReason") != "colab_handoff":
588
+ raise InterruptedError("Processing cancelled by user")
589
+ job.update(cancel=False, cancelReason=None, handoffWorkerUrl=None,
590
+ status="running", progress=5, phase="Restarting analysis on Colab from the beginning",
591
+ stage="analysis", step=0, total=0, action=0, actionsTotal=0,
592
+ updatedAt=int(time.time() * 1000))
593
+ logger.info("Restarting analysis %s on Colab from the original uploaded videos", client_id)
594
+ reports = run_pipeline_on_colab(handoff_url, client_id, work, cameras, job)
595
+ if reports is not None:
596
+ break
597
+ if job.get("cancel"):
598
+ raise InterruptedError()
599
+ logger.warning("Colab handoff failed for analysis %s; restarting on the primary server", client_id)
600
+ allow_handoff = False
601
  if job.get("cancel"):
602
  raise InterruptedError()
603
  (work / "results.json").write_text(json.dumps(json_value(reports), allow_nan=False), encoding="utf-8")
 
805
  @app.post("/api/cancel/{client_id}")
806
  def cancel(client_id: str, request: Request):
807
  job = owned_job(client_id, request.state.user_id, request.state.is_admin)
808
+ with jobs_lock:
809
+ if job["status"] not in {"completed", "failed", "cancelled", "sync_failed"}:
810
+ job.update(cancel=True, cancelReason="user")
811
  return {"status": job["status"], "clientId": client_id}
812
 
813
 
814
  def cancel_session_job(session_id: UUID):
815
  matching = [(client_id, job) for client_id, job in list(jobs.items()) if job.get("sessionId") == str(session_id)]
816
  for client_id, job in matching:
817
+ with jobs_lock:
818
+ if job.get("status") in {"uploading", "queued", "running"}:
819
+ job.update(cancel=True, cancelReason="delete")
820
  if job.get("status") in {"uploading", "queued", "running"}:
 
821
  future = job.get("future")
822
  if future is not None:
823
  try: