Spaces:
Running
Running
Download pkg/src/strokecopilot/compute/remote.py from parsa-hke/stroke-copilot: direct link, hf CLI and curl.
- Browser
- Download file 6.16 kB
-
https://huggingface.co/spaces/parsa-hke/stroke-copilot/resolve/main/pkg/src/strokecopilot/compute/remote.py
- Command line
-
hf download hf://spaces/parsa-hke/stroke-copilot/pkg/src/strokecopilot/compute/remote.py
-
curl -L -o remote.py https://huggingface.co/spaces/parsa-hke/stroke-copilot/resolve/main/pkg/src/strokecopilot/compute/remote.py
6.16 kB
| """Shell scripts that run on the pod. Kept as strings so tests can check them and `pod.py` can send them over SSH. | |
| Layout on the pod (/root/sc): data/ (the tarball), repo/ (the package at the pinned commit), runs/EXP-<id>/ (outputs, | |
| job.log, job.exit), watchdog.log. | |
| """ | |
| ROOT = "/root/sc" | |
| # SSH sessions do not always inherit the image's PATH; the PyTorch images keep python in /opt/conda/bin. | |
| PATH = "export PATH=/opt/conda/bin:/usr/local/bin:$PATH" | |
| # Run first, before any copy: rsync and tmux are not in every image, and the PyTorch runtime image has no C compiler, | |
| # which torch.compile (on by default in nnU-Net 2.8) needs (EXP-13, 5 Nov 2026: "Failed to find C compiler"). | |
| BOOTSTRAP = r"""set -e | |
| mkdir -p /root/sc/data /root/sc/repo /root/sc/runs | |
| if ! command -v rsync >/dev/null || ! command -v tmux >/dev/null || ! command -v gcc >/dev/null; then | |
| (apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq rsync tmux curl build-essential) >/dev/null 2>&1 | |
| fi | |
| gcc --version | head -1 | |
| nvidia-smi --query-gpu=name,memory.total --format=csv,noheader | |
| nproc | |
| free -g | awk '/Mem:/ {print $2 " GiB RAM"}' | |
| df -BG /root | awk 'NR==2 {print $4 " free on /root"}' | |
| """ | |
| # Install the package from the archive sent up from the Mac, and nnU-Net from PyPI. No token is involved. | |
| SETUP = r"""set -e | |
| export PATH=/opt/conda/bin:/usr/local/bin:$PATH | |
| cd /root/sc/repo | |
| python -m pip install -q --upgrade pip >/dev/null | |
| python -m pip install -q ".[isles]" "nnunetv2=={nnunet}" >/root/sc/pip.log 2>&1 || (tail -20 /root/sc/pip.log; exit 1) | |
| python -c "import torch, strokecopilot; print('torch', torch.__version__, 'cuda', torch.cuda.is_available())" | |
| """ | |
| # The watchdog destroys the pod after the hour limit with the instance's own key, which Vast sets in every instance | |
| # and which can only act on that instance. No key of ours is on the pod. If the destroy fails, it stops the container, | |
| # which ends GPU billing (storage billing continues until `down`). | |
| WATCHDOG = r"""#!/bin/bash | |
| LIMIT_S="$1" | |
| getvar() { local v="${!1}"; if [ -z "$v" ]; then v=$(tr '\0' '\n' </proc/1/environ | sed -n "s/^$1=//p"); fi; echo "$v"; } | |
| echo "watchdog armed for ${LIMIT_S} s at $(date -u +%FT%TZ)" >> /root/sc/watchdog.log | |
| sleep "$LIMIT_S" | |
| ID=$(getvar CONTAINER_ID); KEY=$(getvar CONTAINER_API_KEY) | |
| echo "hour limit reached at $(date -u +%FT%TZ); destroying instance ${ID}" >> /root/sc/watchdog.log | |
| curl -s -o /dev/null -w "destroy HTTP %{http_code}\n" -X DELETE -H "Authorization: Bearer ${KEY}" \ | |
| "https://console.vast.ai/api/v0/instances/${ID}/" >> /root/sc/watchdog.log 2>&1 | |
| sleep 120 | |
| echo "still alive after destroy; stopping the container" >> /root/sc/watchdog.log | |
| kill -TERM 1 | |
| """ | |
| def start_watchdog(limit_s): | |
| return (f"cat > /root/sc/watchdog.sh <<'WD'\n{WATCHDOG}WD\nchmod +x /root/sc/watchdog.sh\n" | |
| f"nohup setsid /root/sc/watchdog.sh {int(limit_s)} >/dev/null 2>&1 < /dev/null &\n" | |
| "sleep 1; pgrep -f watchdog.sh >/dev/null && echo watchdog running") | |
| def start_job(exp, command): | |
| """Start the job detached under tmux; its exit code lands in job.exit when it ends.""" | |
| run = f"/root/sc/runs/{exp}" | |
| inner = f"{PATH}; cd /root/sc && ( {command} ) > {run}/job.log 2>&1; echo $? > {run}/job.exit" | |
| return (f"mkdir -p {run} && rm -f {run}/job.exit && tmux new-session -d -s job {quote(inner)} " | |
| f"&& sleep 2 && tmux has-session -t job && echo job started") | |
| def status(exp): | |
| run = f"/root/sc/runs/{exp}" | |
| return (f"if [ -f {run}/job.exit ]; then echo EXIT=$(cat {run}/job.exit); else echo RUNNING; fi; " | |
| f"tail -n 5 {run}/job.log 2>/dev/null; " | |
| # job.log lags (nnU-Net's output is block-buffered into it); nnU-Net's own log is current. | |
| # A resumed run writes a new log, so read them all, oldest first. | |
| f"t=$(ls -tr {run}/nnUNet_results/*/*/fold_*/training_log_*.txt 2>/dev/null); " | |
| f"[ -n \"$t\" ] && echo \"epochs done: $(cat $t | grep -c 'Epoch time')\" && cat $t | grep -E 'Epoch time|Pseudo dice' | tail -2; " | |
| f"tail -n 2 /root/sc/watchdog.log 2>/dev/null") | |
| # A serve pod (jobs.SERVE_*): vLLM from PyPI, no package of ours, no data. HF_HOME keeps the weights on the pod's disk. | |
| SERVE_SETUP = r"""set -e | |
| export PATH=/opt/conda/bin:/usr/local/bin:$PATH | |
| mkdir -p /root/sc/hf | |
| python -m pip install -q --upgrade pip >/dev/null | |
| python -m pip install -q vllm >/root/sc/pip.log 2>&1 || (tail -20 /root/sc/pip.log; exit 1) | |
| python -c "import vllm, torch; print('vllm', vllm.__version__, 'torch', torch.__version__, 'cuda', torch.cuda.is_available())" | |
| """ | |
| def hf_env(token): | |
| """Write the Hugging Face token to a file only root can read, for gated weights (jobs.SERVE_POLICY allow_gated). | |
| The script travels on SSH's stdin, so the token is in no command line; it is never echoed or logged.""" | |
| if not token or "\n" in token or "'" in token: | |
| raise ValueError("The Hugging Face token is missing or malformed.") | |
| return f"umask 077; cat > /root/sc/hf_env <<'HF'\nexport HF_TOKEN='{token}'\nHF\necho token stored" | |
| def start_serve(argv): | |
| """Start `vllm serve` (open_models.serve_command) under tmux, replacing any server already running.""" | |
| inner = (f"{PATH}; export HF_HOME=/root/sc/hf; [ -f /root/sc/hf_env ] && . /root/sc/hf_env; " | |
| f"exec {' '.join(quote(a) for a in argv)} > /root/sc/serve.log 2>&1") | |
| return (f"tmux kill-session -t serve 2>/dev/null; sleep 1; tmux new-session -d -s serve {quote(inner)} " | |
| f"&& sleep 2 && tmux has-session -t serve && echo serve started") | |
| # Remove the Hugging Face token from the pod once the last gated model's weights are on disk (DEC-21, amended 16 Nov | |
| # 2026): a running vLLM server needs it no more, and the pod keeps no key of ours for the rest of the run. | |
| FORGET_TOKEN = "rm -f /root/sc/hf_env; test ! -e /root/sc/hf_env && echo token removed" | |
| STOP_SERVE = ("tmux kill-session -t serve 2>/dev/null; sleep 8; " | |
| "nvidia-smi --query-gpu=memory.used --format=csv,noheader; echo serve stopped") | |
| SERVE_LOG = "tail -n 25 /root/sc/serve.log 2>/dev/null; true" | |
| def quote(s): | |
| return "'" + s.replace("'", "'\"'\"'") + "'" | |