wanhanisah's picture
Upload folder using huggingface_hub
7e37e63 verified
Raw History Blame Contribute Delete
15 kB
# hf_sync.py
# Hugging Face review copy of Horos_Roundel_Mouse_2026_final.
#
# The Space's own disk is wiped on every restart, so the case data lives in a
# private Hugging Face dataset and this module:
# - downloads it once per container start (prepare_workspace),
# - builds the reviewer's working folders (cache/sv, results/sv) from the
# reviewer's last saved files in the dataset (sv/...) or, for a case the
# reviewer has not saved yet, from Wan's segmentation (wan/...),
# - uploads the reviewer's files back to sv/... on every save (push_*).
# Wan's own files (wan/..., data/wan/...) are only ever read, never written.
#
# Dataset layout:
# data/wan/image___<uid>.nii.gz, masks___<uid>.nii.gz, saxdf___<uid>.csv
# wan/cache/config___<uid>.json, masks___<uid>.npz Wan's segmentation
# wan/results/masks/<uid>.nii.gz, edited_sax_df/<uid>.csv
# sv/... same layout, written here
#
# Environment:
# HF_TOKEN write token (Space secret)
# HF_DATASET_REPO default wanhanisah/Horos_Roundel_Mouse_2026_review
# ROUNDEL_LOCAL_DATASET local folder used instead of the Hub (local testing)
import datetime
import json
import os
import shutil
import tempfile
import threading
import time
from pathlib import Path
from zoneinfo import ZoneInfo
import numpy as np
import streamlit as st
APP_DIR = Path(__file__).resolve().parent
ORIGINAL = "wan" # Wan's segmentation (read-only)
REVIEWER = "sv" # supervisor's working copy
DATASET_REPO = os.environ.get("HF_DATASET_REPO", "wanhanisah/Horos_Roundel_Mouse_2026_review")
LOCAL_DATASET = os.environ.get("ROUNDEL_LOCAL_DATASET", "").strip()
CACHE_DIR = APP_DIR / "cache" / REVIEWER
RESULTS_DIR = APP_DIR / "results" / REVIEWER
def _token():
return os.environ.get("HF_TOKEN") or None
def _copy_tree_files(src: Path, dst: Path, pattern: str):
dst.mkdir(parents=True, exist_ok=True)
if src.is_dir():
for p in src.glob(pattern):
shutil.copy2(p, dst / p.name)
@st.cache_resource(show_spinner="Downloading the case data (first start only)...")
def prepare_workspace() -> str:
"""Download the dataset and build cache/sv + results/sv. Returns the
dataset folder (the images are read from <dataset>/data/wan directly)."""
if LOCAL_DATASET:
ds = Path(LOCAL_DATASET).resolve()
else:
from huggingface_hub import snapshot_download
ds = Path(snapshot_download(
repo_id=DATASET_REPO, repo_type="dataset", token=_token(),
local_dir=str(APP_DIR / "_hf_dataset"),
))
if not (ds / "data" / ORIGINAL).is_dir():
raise FileNotFoundError(f"No data/{ORIGINAL} folder in the dataset ({ds}).")
# Configs (ED/ES frames), final masks and metrics: Wan's first, then the
# reviewer's saved ones on top. The big .npy edit masks are unpacked per
# case on first open (ensure_case_cache), not all at start-up.
for who in (ORIGINAL, REVIEWER):
_copy_tree_files(ds / who / "cache", CACHE_DIR, "config___*.json")
_copy_tree_files(ds / who / "results" / "masks", RESULTS_DIR / "masks", "*.nii.gz")
_copy_tree_files(ds / who / "results" / "edited_sax_df", RESULTS_DIR / "edited_sax_df", "*.csv")
# Notes / Status: the reviewer's own only (Wan's notes are not copied,
# so every case starts with an empty note and To-do status).
sv_db = ds / REVIEWER / "results" / "case_db.json"
if sv_db.exists():
shutil.copy2(sv_db, RESULTS_DIR / "case_db.json")
elif not (RESULTS_DIR / "case_db.json").exists():
(RESULTS_DIR / "case_db.json").write_text("{}")
# The last edited case -- after a restart the app reopens on it (the
# app's own "last case" file lives on the wiped disk).
sv_last = ds / REVIEWER / "results" / "last_edited.json"
if sv_last.exists():
shutil.copy2(sv_last, LAST_EDITED_PATH)
last_selection = APP_DIR / "cache" / "last_selection.json"
if not last_selection.exists():
uid = json.loads(LAST_EDITED_PATH.read_text()).get("uid")
if uid:
last_selection.write_text(json.dumps({"dataset_key": REVIEWER, "sax_series_uid": uid}))
return str(ds)
LAST_EDITED_PATH = RESULTS_DIR / "last_edited.json"
def _record_edit(uid: str):
"""Remember this as the last edited case (uploaded with the next save)."""
LAST_EDITED_PATH.parent.mkdir(parents=True, exist_ok=True)
LAST_EDITED_PATH.write_text(json.dumps({"uid": uid, "time": time.time()}))
def last_edited():
"""(uid, unix time) of the last edited case, or (None, None)."""
try:
d = json.loads(LAST_EDITED_PATH.read_text())
return d.get("uid"), d.get("time")
except Exception:
return None, None
UK_TZ = ZoneInfo("Europe/London")
def uk_time(ts: float, fmt: str = "%d %b, %H:%M %Z") -> str:
"""Unix time -> UK local time text (GMT in winter, BST in summer)."""
return datetime.datetime.fromtimestamp(ts, UK_TZ).strftime(fmt)
def _dataset_dir() -> Path:
return Path(prepare_workspace())
def _load_npz_mask(path: Path) -> np.ndarray:
with np.load(path) as z:
return z["mask"]
def ensure_case_cache(uid: str):
"""Unpack this case's edit mask into cache/sv/masks___<uid>.npy (the file
the app reads and autosaves to) unless it is already there."""
out = CACHE_DIR / f"masks___{uid}.npy"
if out.exists():
return
ds = _dataset_dir()
for who in (REVIEWER, ORIGINAL):
src = ds / who / "cache" / f"masks___{uid}.npz"
if src.exists():
out.parent.mkdir(parents=True, exist_ok=True)
np.save(out, _load_npz_mask(src))
return
def load_original_mask(uid: str):
"""Wan's saved segmentation for this case (what Reset Current Mask goes
back to), or None if there is none."""
src = _dataset_dir() / ORIGINAL / "cache" / f"masks___{uid}.npz"
return _load_npz_mask(src) if src.exists() else None
def _upload(files, message):
"""files: list of (local Path, path in dataset). Returns (ok, message)."""
files = [(Path(a), b) for a, b in files if Path(a).exists()]
if not files:
return False, "nothing to upload"
try:
if LOCAL_DATASET:
ds = _dataset_dir()
for local, remote in files:
(ds / remote).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(local, ds / remote)
else:
from huggingface_hub import CommitOperationAdd, HfApi
HfApi(token=_token()).create_commit(
repo_id=DATASET_REPO, repo_type="dataset", commit_message=message,
operations=[CommitOperationAdd(path_in_repo=remote, path_or_fileobj=str(local))
for local, remote in files],
)
return True, f"uploaded {len(files)} file(s)"
except Exception as exc:
return False, str(exc)
def push_case(uid: str):
"""Upload one case's saved mask, ED/ES config, final mask, metrics and the
notes file to sv/ in the dataset."""
_record_edit(uid)
with tempfile.TemporaryDirectory() as tmp:
files = []
npy = CACHE_DIR / f"masks___{uid}.npy"
if npy.exists():
npz = Path(tmp) / f"masks___{uid}.npz"
np.savez_compressed(npz, mask=np.load(npy))
files.append((npz, f"{REVIEWER}/cache/masks___{uid}.npz"))
files += [
(CACHE_DIR / f"config___{uid}.json", f"{REVIEWER}/cache/config___{uid}.json"),
(RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"),
(RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"),
(RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json"),
(LAST_EDITED_PATH, f"{REVIEWER}/results/last_edited.json"),
]
if _hooks["rebuild_combined"]:
_hooks["rebuild_combined"]()
files.append((RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv",
f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv"))
with _autosave_lock: # this upload includes the edited mask too
_pending.pop(uid, None)
ok, msg = _upload(files, f"Supervisor review: save {uid}")
if ok:
with _autosave_lock:
_status[uid] = ("saved", time.time(), msg)
else:
mark_dirty(uid, now_please=True)
return ok, msg
# -----------------------------------------------------------------------------
# Autosave of Mask Editor edits (Save Contour, Clear Slice, Delete, Undo/Redo,
# Reset Current Mask all write cache/sv/masks___<uid>.npy through
# _save_current_combined_mask_to_cache, which calls mark_dirty). Once edits
# pause for AUTOSAVE_QUIET_S (or at the latest AUTOSAVE_MAX_WAIT_S after the
# first unsent edit) a background thread does what "Save Masks and Metrics"
# does -- final mask + metrics for the case, plus the combined metrics table --
# and uploads them with the edited mask and ED/ES config in one commit, so
# rapid clicks become one upload and the dataset is not hit with a commit per
# click. The final mask/metrics code lives in roundel_biventricular_utils.py
# (which imports this module), so it is registered here with set_hooks().
# -----------------------------------------------------------------------------
_hooks = {"build_final": None, "rebuild_combined": None}
def set_hooks(build_final, rebuild_combined):
"""build_final(uid): write results/sv/masks/<uid>.nii.gz and
edited_sax_df/<uid>.csv from the edit cache; rebuild_combined(): rewrite
edited_sax_df/all_cases_metrics.csv from the per-case files."""
_hooks["build_final"] = build_final
_hooks["rebuild_combined"] = rebuild_combined
AUTOSAVE_QUIET_S = 15
AUTOSAVE_MAX_WAIT_S = 60
_autosave_lock = threading.Lock()
_pending = {} # uid -> (first_unsent_edit_time, last_edit_time)
_status = {} # uid -> ("pending" | "uploading" | "saved" | "error", time, message)
_worker = None
def _push_edit_mask(uid: str):
files = []
if _hooks["build_final"]:
_hooks["build_final"](uid)
_hooks["rebuild_combined"]()
files += [
(RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"),
(RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"),
(RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv",
f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv"),
]
with tempfile.TemporaryDirectory() as tmp:
npz = Path(tmp) / f"masks___{uid}.npz"
np.savez_compressed(npz, mask=np.load(CACHE_DIR / f"masks___{uid}.npy"))
files += [
(npz, f"{REVIEWER}/cache/masks___{uid}.npz"),
(CACHE_DIR / f"config___{uid}.json", f"{REVIEWER}/cache/config___{uid}.json"),
(LAST_EDITED_PATH, f"{REVIEWER}/results/last_edited.json"),
]
return _upload(files, f"Supervisor review: autosave {uid} (mask, final mask, metrics)")
def _autosave_loop():
while True:
time.sleep(2)
now = time.time()
with _autosave_lock:
due = [uid for uid, (first, last) in _pending.items()
if now - last >= AUTOSAVE_QUIET_S or now - first >= AUTOSAVE_MAX_WAIT_S]
for uid in due:
_pending.pop(uid)
_status[uid] = ("uploading", now, "")
for uid in due:
try:
ok, msg = _push_edit_mask(uid)
except Exception as exc: # e.g. the .npy was being rewritten mid-read
ok, msg = False, str(exc)
with _autosave_lock:
if ok:
_status[uid] = ("saved", time.time(), msg)
else:
# retry on the next pass, keep the error visible meanwhile
first, last = _pending.get(uid, (time.time(), time.time()))
_pending[uid] = (first, last)
_status[uid] = ("error", time.time(), msg)
def mark_dirty(uid: str, now_please: bool = False):
"""Schedule an upload of this case's edited mask (called after every edit;
now_please=True uploads on the next pass, e.g. when leaving the case)."""
global _worker
t = time.time()
if not now_please:
_record_edit(uid)
with _autosave_lock:
first, _ = _pending.get(uid, (t, t))
_pending[uid] = (first - AUTOSAVE_MAX_WAIT_S if now_please else first, t)
if _status.get(uid, ("",))[0] != "uploading":
_status[uid] = ("pending", t, "")
if _worker is None or not _worker.is_alive():
_worker = threading.Thread(target=_autosave_loop, daemon=True)
_worker.start()
def flush_soon(uid: str):
"""Leaving a case: upload its unsent edits on the next pass, if any."""
with _autosave_lock:
has_pending = uid in _pending
if has_pending:
mark_dirty(uid, now_please=True)
def autosave_busy(uid: str) -> bool:
"""True while this case has an upload waiting, running or retrying."""
with _autosave_lock:
state = _status.get(uid)
return uid in _pending or bool(state and state[0] in ("pending", "uploading", "error"))
def autosave_status(uid: str) -> str:
with _autosave_lock:
state = _status.get(uid)
if not state:
return ""
kind, t, msg = state
clock = uk_time(t, "%H:%M:%S %Z")
if kind == "pending":
return "☁️ Edits will be saved to Hugging Face in a few seconds…"
if kind == "uploading":
return "☁️ Saving mask, final mask and metrics to Hugging Face…"
if kind == "saved":
return f"☁️ Mask, final mask and metrics saved to Hugging Face at {clock}"
return f"⚠️ Could not save edits to Hugging Face yet (will retry): {msg[:200]}"
def push_case_db(uid: str = ""):
"""Upload the notes/status file only."""
return _upload([(RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json")],
f"Supervisor review: notes/status {uid}".strip())
def push_all_results(uids):
"""Upload every case's final mask + metrics, the combined metrics CSV and
the notes file (after Save All Masks and Metrics)."""
files = [(RESULTS_DIR / "case_db.json", f"{REVIEWER}/results/case_db.json"),
(RESULTS_DIR / "edited_sax_df" / "all_cases_metrics.csv",
f"{REVIEWER}/results/edited_sax_df/all_cases_metrics.csv")]
for uid in uids:
files += [
(RESULTS_DIR / "masks" / f"{uid}.nii.gz", f"{REVIEWER}/results/masks/{uid}.nii.gz"),
(RESULTS_DIR / "edited_sax_df" / f"{uid}.csv", f"{REVIEWER}/results/edited_sax_df/{uid}.csv"),
]
return _upload(files, "Supervisor review: save all masks and metrics")