Download hf_sync.py from wanhanisah/Biventricular_Segmentation_AngII: direct link, hf CLI and curl.
- Browser
- Download file 15 kB
-
https://huggingface.co/spaces/wanhanisah/Biventricular_Segmentation_AngII/resolve/main/hf_sync.py
- Command line
-
hf download hf://spaces/wanhanisah/Biventricular_Segmentation_AngII/hf_sync.py
-
curl -L -o hf_sync.py https://huggingface.co/spaces/wanhanisah/Biventricular_Segmentation_AngII/resolve/main/hf_sync.py
15 kB
| # hf_sync.py | |
| # Hugging Face review copy of Horos_Roundel_Mouse_2026_final. | |
| # | |
| # The Space's own disk is wiped on every restart, so the case data lives in a | |
| # private Hugging Face dataset and this module: | |
| # - downloads it once per container start (prepare_workspace), | |
| # - builds the reviewer's working folders (cache/sv, results/sv) from the | |
| # reviewer's last saved files in the dataset (sv/...) or, for a case the | |
| # reviewer has not saved yet, from Wan's segmentation (wan/...), | |
| # - uploads the reviewer's files back to sv/... on every save (push_*). | |
| # Wan's own files (wan/..., data/wan/...) are only ever read, never written. | |
| # | |
| # Dataset layout: | |
| # data/wan/image___<uid>.nii.gz, masks___<uid>.nii.gz, saxdf___<uid>.csv | |
| # wan/cache/config___<uid>.json, masks___<uid>.npz Wan's segmentation | |
| # wan/results/masks/<uid>.nii.gz, edited_sax_df/<uid>.csv | |
| # sv/... same layout, written here | |
| # | |
| # Environment: | |
| # HF_TOKEN write token (Space secret) | |
| # HF_DATASET_REPO default wanhanisah/Horos_Roundel_Mouse_2026_review | |
| # ROUNDEL_LOCAL_DATASET local folder used instead of the Hub (local testing) | |
| import datetime | |
| import json | |
| import os | |
| import shutil | |
| import tempfile | |
| import threading | |
| import time | |
| from pathlib import Path | |
| from zoneinfo import ZoneInfo | |
| import numpy as np | |
| import streamlit as st | |
| APP_DIR = Path(__file__).resolve().parent | |
| ORIGINAL = "wan" # Wan's segmentation (read-only) | |
| REVIEWER = "sv" # supervisor's working copy | |
| DATASET_REPO = os.environ.get("HF_DATASET_REPO", "wanhanisah/Horos_Roundel_Mouse_2026_review") | |
| LOCAL_DATASET = os.environ.get("ROUNDEL_LOCAL_DATASET", "").strip() | |
| CACHE_DIR = APP_DIR / "cache" / REVIEWER | |
| RESULTS_DIR = APP_DIR / "results" / REVIEWER | |
| def _token(): | |
| return os.environ.get("HF_TOKEN") or None | |
| def _copy_tree_files(src: Path, dst: Path, pattern: str): | |
| dst.mkdir(parents=True, exist_ok=True) | |
| if src.is_dir(): | |
| for p in src.glob(pattern): | |
| shutil.copy2(p, dst / p.name) | |
| def prepare_workspace() -> str: | |
| """Download the dataset and build cache/sv + results/sv. Returns the | |
| dataset folder (the images are read from <dataset>/data/wan directly).""" | |
| if LOCAL_DATASET: | |
| ds = Path(LOCAL_DATASET).resolve() | |
| else: | |
| from huggingface_hub import snapshot_download | |
| ds = Path(snapshot_download( | |
| repo_id=DATASET_REPO, repo_type="dataset", token=_token(), | |
| local_dir=str(APP_DIR / "_hf_dataset"), | |
| )) | |
| if not (ds / "data" / ORIGINAL).is_dir(): | |
| raise FileNotFoundError(f"No data/{ORIGINAL} folder in the dataset ({ds}).") | |
| # Configs (ED/ES frames), final masks and metrics: Wan's first, then the | |
| # reviewer's saved ones on top. The big .npy edit masks are unpacked per | |
| # case on first open (ensure_case_cache), not all at start-up. | |
| for who in (ORIGINAL, REVIEWER): | |
| _copy_tree_files(ds / who / "cache", CACHE_DIR, "config___*.json") | |
| _copy_tree_files(ds / who / "results" / "masks", RESULTS_DIR / "masks", "*.nii.gz") | |
| _copy_tree_files(ds / who / "results" / "edited_sax_df", RESULTS_DIR / "edited_sax_df", "*.csv") | |
| # Notes / Status: the reviewer's own only (Wan's notes are not copied, | |
| # so every case starts with an empty note and To-do status). | |
| sv_db = ds / REVIEWER / "results" / "case_db.json" | |
| if sv_db.exists(): | |
| shutil.copy2(sv_db, RESULTS_DIR / "case_db.json") | |
| elif not (RESULTS_DIR / "case_db.json").exists(): | |
| (RESULTS_DIR / "case_db.json").write_text("{}") | |
| # The last edited case -- after a restart the app reopens on it (the | |
| # app's own "last case" file lives on the wiped disk). | |
| sv_last = ds / REVIEWER / "results" / "last_edited.json" | |
| if sv_last.exists(): | |
| shutil.copy2(sv_last, LAST_EDITED_PATH) | |
| last_selection = APP_DIR / "cache" / "last_selection.json" | |
| if not last_selection.exists(): | |
| uid = json.loads(LAST_EDITED_PATH.read_text()).get("uid") | |
| if uid: | |
| last_selection.write_text(json.dumps({"dataset_key": REVIEWER, "sax_series_uid": uid})) | |
| return str(ds) | |
| LAST_EDITED_PATH = RESULTS_DIR / "last_edited.json" | |
| def _record_edit(uid: str): | |
| """Remember this as the last edited case (uploaded with the next save).""" | |
| LAST_EDITED_PATH.parent.mkdir(parents=True, exist_ok=True) | |
| LAST_EDITED_PATH.write_text(json.dumps({"uid": uid, "time": time.time()})) | |
| def last_edited(): | |
| """(uid, unix time) of the last edited case, or (None, None).""" | |
| try: | |
| d = json.loads(LAST_EDITED_PATH.read_text()) | |
| return d.get("uid"), d.get("time") | |
| except Exception: | |
| return None, None | |
| UK_TZ = ZoneInfo("Europe/London") | |
| def uk_time(ts: float, fmt: str = "%d %b, %H:%M %Z") -> str: | |
| """Unix time -> UK local time text (GMT in winter, BST in summer).""" | |
| return datetime.datetime.fromtimestamp(ts, UK_TZ).strftime(fmt) | |
| def _dataset_dir() -> Path: | |
| return Path(prepare_workspace()) | |
| def _load_npz_mask(path: Path) -> np.ndarray: | |
| with np.load(path) as z: | |
| return z["mask"] | |
| def ensure_case_cache(uid: str): | |
| """Unpack this case's edit mask into cache/sv/masks___<uid>.npy (the file | |
| the app reads and autosaves to) unless it is already there.""" | |
| out = CACHE_DIR / f"masks___{uid}.npy" | |
| if out.exists(): | |
| return | |
| ds = _dataset_dir() | |
| for who in (REVIEWER, ORIGINAL): | |
| src = ds / who / "cache" / f"masks___{uid}.npz" | |
| if src.exists(): | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| np.save(out, _load_npz_mask(src)) | |
| return | |
| def load_original_mask(uid: str): | |
| """Wan's saved segmentation for this case (what Reset Current Mask goes | |
| back to), or None if there is none.""" | |
| src = _dataset_dir() / ORIGINAL / "cache" / f"masks___{uid}.npz" | |
| return _load_npz_mask(src) if src.exists() else None | |
| def _upload(files, message): | |
| """files: list of (local Path, path in dataset). Returns (ok, message).""" | |
| files = [(Path(a), b) for a, b in files if Path(a).exists()] | |
| if not files: | |
| return False, "nothing to upload" | |
| try: | |
| if LOCAL_DATASET: | |
| ds = _dataset_dir() | |
| for local, remote in files: | |
| (ds / remote).parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copy2(local, ds / remote) | |
| else: | |
| from huggingface_hub import CommitOperationAdd, HfApi | |
| HfApi(token=_token()).create_commit( | |
| repo_id=DATASET_REPO, repo_type="dataset", commit_message=message, | |
| operations=[CommitOperationAdd(path_in_repo=remote, path_or_fileobj=str(local)) | |
| for local, remote in files], | |
| ) | |
| return True, f"uploaded {len(files)} file(s)" | |
| except Exception as exc: | |
| return False, str(exc) | |
| def push_case(uid: str): | |
| """Upload one case's saved mask, ED/ES config, final mask, metrics and the | |
| notes file to sv/ in the dataset.""" | |
| _record_edit(uid) | |
| with tempfile.TemporaryDirectory() as tmp: | |
| files = [] | |
| npy = CACHE_DIR / f"masks___{uid}.npy" | |
| if npy.exists(): | |
| npz = Path(tmp) / f"masks___{uid}.npz" | |
| np.savez_compressed(npz, mask=np.load(npy)) | |
| files.append((npz, f"{REVIEWER}/cache/masks___{uid}.npz")) | |
| files += [ | |
| (CACHE_DIR / f"config___{uid}.json", f"{REVIEWER}/cache/config___{uid}.json"), | |
| (RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"), | |
| (RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"), | |
| (RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json"), | |
| (LAST_EDITED_PATH, f"{REVIEWER}/results/last_edited.json"), | |
| ] | |
| if _hooks["rebuild_combined"]: | |
| _hooks["rebuild_combined"]() | |
| files.append((RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv", | |
| f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv")) | |
| with _autosave_lock: # this upload includes the edited mask too | |
| _pending.pop(uid, None) | |
| ok, msg = _upload(files, f"Supervisor review: save {uid}") | |
| if ok: | |
| with _autosave_lock: | |
| _status[uid] = ("saved", time.time(), msg) | |
| else: | |
| mark_dirty(uid, now_please=True) | |
| return ok, msg | |
| # ----------------------------------------------------------------------------- | |
| # Autosave of Mask Editor edits (Save Contour, Clear Slice, Delete, Undo/Redo, | |
| # Reset Current Mask all write cache/sv/masks___<uid>.npy through | |
| # _save_current_combined_mask_to_cache, which calls mark_dirty). Once edits | |
| # pause for AUTOSAVE_QUIET_S (or at the latest AUTOSAVE_MAX_WAIT_S after the | |
| # first unsent edit) a background thread does what "Save Masks and Metrics" | |
| # does -- final mask + metrics for the case, plus the combined metrics table -- | |
| # and uploads them with the edited mask and ED/ES config in one commit, so | |
| # rapid clicks become one upload and the dataset is not hit with a commit per | |
| # click. The final mask/metrics code lives in roundel_biventricular_utils.py | |
| # (which imports this module), so it is registered here with set_hooks(). | |
| # ----------------------------------------------------------------------------- | |
| _hooks = {"build_final": None, "rebuild_combined": None} | |
| def set_hooks(build_final, rebuild_combined): | |
| """build_final(uid): write results/sv/masks/<uid>.nii.gz and | |
| edited_sax_df/<uid>.csv from the edit cache; rebuild_combined(): rewrite | |
| edited_sax_df/all_cases_metrics.csv from the per-case files.""" | |
| _hooks["build_final"] = build_final | |
| _hooks["rebuild_combined"] = rebuild_combined | |
| AUTOSAVE_QUIET_S = 15 | |
| AUTOSAVE_MAX_WAIT_S = 60 | |
| _autosave_lock = threading.Lock() | |
| _pending = {} # uid -> (first_unsent_edit_time, last_edit_time) | |
| _status = {} # uid -> ("pending" | "uploading" | "saved" | "error", time, message) | |
| _worker = None | |
| def _push_edit_mask(uid: str): | |
| files = [] | |
| if _hooks["build_final"]: | |
| _hooks["build_final"](uid) | |
| _hooks["rebuild_combined"]() | |
| files += [ | |
| (RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"), | |
| (RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"), | |
| (RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv", | |
| f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv"), | |
| ] | |
| with tempfile.TemporaryDirectory() as tmp: | |
| npz = Path(tmp) / f"masks___{uid}.npz" | |
| np.savez_compressed(npz, mask=np.load(CACHE_DIR / f"masks___{uid}.npy")) | |
| files += [ | |
| (npz, f"{REVIEWER}/cache/masks___{uid}.npz"), | |
| (CACHE_DIR / f"config___{uid}.json", f"{REVIEWER}/cache/config___{uid}.json"), | |
| (LAST_EDITED_PATH, f"{REVIEWER}/results/last_edited.json"), | |
| ] | |
| return _upload(files, f"Supervisor review: autosave {uid} (mask, final mask, metrics)") | |
| def _autosave_loop(): | |
| while True: | |
| time.sleep(2) | |
| now = time.time() | |
| with _autosave_lock: | |
| due = [uid for uid, (first, last) in _pending.items() | |
| if now - last >= AUTOSAVE_QUIET_S or now - first >= AUTOSAVE_MAX_WAIT_S] | |
| for uid in due: | |
| _pending.pop(uid) | |
| _status[uid] = ("uploading", now, "") | |
| for uid in due: | |
| try: | |
| ok, msg = _push_edit_mask(uid) | |
| except Exception as exc: # e.g. the .npy was being rewritten mid-read | |
| ok, msg = False, str(exc) | |
| with _autosave_lock: | |
| if ok: | |
| _status[uid] = ("saved", time.time(), msg) | |
| else: | |
| # retry on the next pass, keep the error visible meanwhile | |
| first, last = _pending.get(uid, (time.time(), time.time())) | |
| _pending[uid] = (first, last) | |
| _status[uid] = ("error", time.time(), msg) | |
| def mark_dirty(uid: str, now_please: bool = False): | |
| """Schedule an upload of this case's edited mask (called after every edit; | |
| now_please=True uploads on the next pass, e.g. when leaving the case).""" | |
| global _worker | |
| t = time.time() | |
| if not now_please: | |
| _record_edit(uid) | |
| with _autosave_lock: | |
| first, _ = _pending.get(uid, (t, t)) | |
| _pending[uid] = (first - AUTOSAVE_MAX_WAIT_S if now_please else first, t) | |
| if _status.get(uid, ("",))[0] != "uploading": | |
| _status[uid] = ("pending", t, "") | |
| if _worker is None or not _worker.is_alive(): | |
| _worker = threading.Thread(target=_autosave_loop, daemon=True) | |
| _worker.start() | |
| def flush_soon(uid: str): | |
| """Leaving a case: upload its unsent edits on the next pass, if any.""" | |
| with _autosave_lock: | |
| has_pending = uid in _pending | |
| if has_pending: | |
| mark_dirty(uid, now_please=True) | |
| def autosave_busy(uid: str) -> bool: | |
| """True while this case has an upload waiting, running or retrying.""" | |
| with _autosave_lock: | |
| state = _status.get(uid) | |
| return uid in _pending or bool(state and state[0] in ("pending", "uploading", "error")) | |
| def autosave_status(uid: str) -> str: | |
| with _autosave_lock: | |
| state = _status.get(uid) | |
| if not state: | |
| return "" | |
| kind, t, msg = state | |
| clock = uk_time(t, "%H:%M:%S %Z") | |
| if kind == "pending": | |
| return "☁️ Edits will be saved to Hugging Face in a few seconds…" | |
| if kind == "uploading": | |
| return "☁️ Saving mask, final mask and metrics to Hugging Face…" | |
| if kind == "saved": | |
| return f"☁️ Mask, final mask and metrics saved to Hugging Face at {clock}" | |
| return f"⚠️ Could not save edits to Hugging Face yet (will retry): {msg[:200]}" | |
| def push_case_db(uid: str = ""): | |
| """Upload the notes/status file only.""" | |
| return _upload([(RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json")], | |
| f"Supervisor review: notes/status {uid}".strip()) | |
| def push_all_results(uids): | |
| """Upload every case's final mask + metrics, the combined metrics CSV and | |
| the notes file (after Save All Masks and Metrics).""" | |
| files = [(RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json"), | |
| (RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv", | |
| f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv")] | |
| for uid in uids: | |
| files += [ | |
| (RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"), | |
| (RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"), | |
| ] | |
| return _upload(files, "Supervisor review: save all masks and metrics") | |