Spaces:
Running
Running
File size: 19,935 Bytes
974048c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 | """The pod lifecycle: up, status, collect, down (DEC-21, plan §4.6). Entry point: scripts/pod.
scripts/pod offers EXP-13 # search only: the cheapest offers that pass the filters
scripts/pod up EXP-13 --offer 123 --go # create (after Parsa approves the offer, price and hour limit)
scripts/pod status EXP-13 # job state, last log lines, watchdog
scripts/pod collect EXP-13 # results back to the Mac and Drive, then down
scripts/pod down EXP-13 # destroy and confirm by listing instances
State lives in runs/EXP-<id>/pod.json on the Mac (runs/ is never committed). No token is ever sent to the pod:
the tarball and an archive of the repo at the pinned commit go up over SSH, and results come back the same way.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
import shlex
import subprocess
import sys
import time
from pathlib import Path
from strokecopilot.compute import jobs, keyscan, remote, vast
REPO = Path(__file__).resolve().parents[3]
RUNS = REPO / "runs"
SSH_KEY = Path(os.environ.get("POD_SSH_KEY", "~/.ssh/vast_pod")).expanduser()
IMAGE = "pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime"
DISK_GB = 60
BALANCE_MARGIN = 1.5
TARBALL = "isles24_slim_v1.tar"
TARBALL_SHA256 = "b0afd16e4cbfb16a69fb5190083fc3869f5dcc6c92f86f10135b0de1f59f4728"
class PodError(RuntimeError):
pass
def provider(name="vast"):
if name == "vast":
from strokecopilot.compute.vast import Vast
return Vast()
if name == "runpod":
from strokecopilot.compute.runpod import RunPod
return RunPod()
raise PodError(f"Unknown provider {name!r}")
def state_path(exp):
return RUNS / exp / "pod.json"
def load_state(exp):
p = state_path(exp)
return json.loads(p.read_text()) if p.exists() else None
def save_state(exp, state):
state_path(exp).parent.mkdir(parents=True, exist_ok=True)
state_path(exp).write_text(json.dumps(state, indent=1))
def label(exp):
return f"strokecopilot-{exp}"
# --- local helpers --------------------------------------------------------------------------------
def drive_root():
"""The StrokeCopilot folder of Google Drive for desktop on this Mac, or None."""
base = Path("~/Library/CloudStorage").expanduser()
for d in sorted(base.glob("GoogleDrive-*")) if base.exists() else []:
for sub in ("My Drive/StrokeCopilot", "MyDrive/StrokeCopilot"):
if (d / sub).is_dir():
return d / sub
return None
def tarball_path():
local = REPO / "data" / TARBALL
if local.exists():
return local
root = drive_root()
if root and (root / "slim" / TARBALL).exists():
return root / "slim" / TARBALL
raise PodError(f"{TARBALL} is neither in data/ nor in Drive for desktop. Install Drive for desktop or download it to data/.")
def input_files(exp):
"""[(local path, pod path)] for the files `exp` needs sent up, read from Drive for Desktop. Raises before any
pod exists if one is missing."""
wanted = jobs.inputs(exp)
if not wanted:
return []
root = drive_root()
if root is None:
raise PodError(f"{exp} needs {len(wanted)} input file(s) from Drive, and Drive for desktop was not found.")
missing = [src for src in wanted if not (root / src).is_file()]
if missing:
raise PodError(f"{exp} needs input files that are not on Drive: {', '.join(missing)}.")
return [(root / src, dst) for src, dst in wanted.items()]
def sha256(path, chunk=16 * 2**20):
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(chunk), b""):
h.update(block)
return h.hexdigest()
def head_commit():
return subprocess.run(["git", "rev-parse", "HEAD"], cwd=REPO, capture_output=True, text=True, check=True).stdout.strip()
def uncommitted(paths=("src", "pyproject.toml", "schemas", "splits", ":!src/strokecopilot/compute")):
"""Files under `paths` that differ from HEAD or are untracked: the pod runs HEAD, so these would be missing.
compute/ is left out: it runs on the Mac only."""
out = subprocess.run(["git", "status", "--porcelain", "--", *paths], cwd=REPO, capture_output=True, text=True, check=True).stdout
return [line[3:] for line in out.splitlines() if line.strip()]
def repo_archive(commit, out):
subprocess.run(["git", "archive", "--format=tar", "-o", str(out), commit], cwd=REPO, check=True)
return out
def ssh_base(host, port, user):
return ["ssh", "-i", str(SSH_KEY), "-p", str(port), "-o", "StrictHostKeyChecking=accept-new",
"-o", "UserKnownHostsFile=" + str(RUNS / ".pod_known_hosts"), "-o", "ServerAliveInterval=30",
"-o", "ConnectTimeout=20", "-o", "BatchMode=yes", f"{user}@{host}"]
def run(cmd, check=True, capture=False, timeout=None):
out = subprocess.run(cmd, capture_output=capture, text=True, timeout=timeout)
if check and out.returncode:
raise PodError(f"Command failed ({out.returncode}): {' '.join(cmd[:3])} … {(out.stderr or '')[-400:]}")
return out
def remote_sh(state, script, capture=True, timeout=600):
cmd = ssh_base(state["host"], state["port"], state["user"]) + ["bash -s"]
out = subprocess.run(cmd, input=script, capture_output=capture, text=True, timeout=timeout)
if out.returncode:
raise PodError(f"Remote script failed ({out.returncode}): {(out.stderr or out.stdout or '')[-600:]}")
return out.stdout
def tunnel_argv(state, local_port, remote_port):
base = ssh_base(state["host"], state["port"], state["user"])
return base[:-1] + ["-N", "-o", "ExitOnForwardFailure=yes", "-L", f"127.0.0.1:{int(local_port)}:127.0.0.1:{int(remote_port)}",
base[-1]]
class Tunnel:
"""`with Tunnel(state, 8000):` forwards localhost:8000 on the Mac to the pod's 127.0.0.1:8000, where vLLM listens.
Nothing on the pod is open to the internet, and no key of ours is on the pod."""
def __init__(self, state, local_port, remote_port=None):
self.argv = tunnel_argv(state, local_port, remote_port or local_port)
self.proc = None
def __enter__(self):
self.proc = subprocess.Popen(self.argv, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
time.sleep(3)
if self.proc.poll() is not None:
raise PodError(f"The SSH tunnel did not start: {(self.proc.stderr.read() or '')[-300:]}")
return self
def __exit__(self, *exc):
if self.proc and self.proc.poll() is None:
self.proc.terminate()
try:
self.proc.wait(timeout=10)
except subprocess.TimeoutExpired:
self.proc.kill()
return False
UPLOAD_LIMIT_S = 1800
def rsync_up(state, local, remote_path, timeout=None):
e = " ".join(shlex.quote(x) for x in ssh_base(state["host"], state["port"], state["user"])[:-1])
# --timeout ends a transfer that moves no data for 5 min; on 12 Nov 2026 an upload to one host sat at 0 kB/s for
# 18 min while the pod billed. A failed copy makes `up` destroy the pod.
run(["rsync", "-a", "--partial", "--timeout=300", "-e", e, str(local), f"{state['user']}@{state['host']}:{remote_path}"], timeout=timeout)
def rsync_down(state, remote_path, local, includes):
e = " ".join(shlex.quote(x) for x in ssh_base(state["host"], state["port"], state["user"])[:-1])
filters = []
for pattern in includes:
filters += ["--include", pattern]
filters += ["--include", "*/", "--exclude", "*"]
run(["rsync", "-a", "--prune-empty-dirs", *filters, "-e", e, f"{state['user']}@{state['host']}:{remote_path}", str(local)])
def wait_ssh(state, tries=8, pause=15):
for _ in range(tries):
out = subprocess.run(ssh_base(state["host"], state["port"], state["user"]) + ["echo ok"], capture_output=True, text=True)
if out.returncode == 0 and "ok" in out.stdout:
return
time.sleep(pause)
raise PodError("SSH never answered.")
# --- the verbs ------------------------------------------------------------------------------------
# DEC-21 prefers hosts in the US or EU; Vast's geolocation ends with the country code ("Texas, US").
PREFERRED = {"US", "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT",
"LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE"}
def preferred(offer):
return (offer.get("geolocation") or "").rsplit(",", 1)[-1].strip().upper() in PREFERRED
def _offers(p, filters):
return p.offers(limit=64) if filters is None else p.offers(filters=filters, limit=64)
def search(p, rounds=6, filters=None):
"""Every offer seen over several searches, cheapest first. Vast answers each search with a varying sample of
about 15 matching offers (checked 5 Nov 2026), so one search can miss an offer that is still for rent.
`filters` replaces the provider's own (a run's jobs spec may carry them, such as a 48 GB GPU for a serve run)."""
seen = {}
for _ in range(rounds):
for o in _offers(p, filters):
seen[str(o["id"])] = o
return sorted(seen.values(), key=lambda o: o.get("dph_total", 1e9))
def find_offer(p, offer_id, rounds=25, filters=None):
"""The offer if it still passes the filters. Searches until it appears, because a search shows a varying sample:
on 12 Nov 2026 a rentable offer appeared in only 2 of 10 searches."""
for _ in range(rounds):
for o in _offers(p, filters):
if str(o["id"]) == str(offer_id):
return o
return None
def offers(exp, provider_name="vast", log=print, show=6):
"""The cheapest offers that pass the filters, US and EU hosts first."""
spec = jobs.get(exp)
p = provider(provider_name)
found = search(p, filters=spec.get("filters"))
found = ([o for o in found if preferred(o)] + [o for o in found if not preferred(o)])[:show]
balance = p.balance()
log(f"{exp}: {spec['what']}; hour limit {spec['hours']} h; estimated {spec['est_hours']} h. Balance ${balance:.2f}.")
if not found:
log("No offer passes the DEC-21 filters right now.")
for o in found:
log(" " + p.describe(o) + f" -> estimated ${o['dph_total'] * spec['est_hours']:.2f}, at most ${o['dph_total'] * spec['hours']:.2f}")
return found
def up(exp, offer_id, go=False, provider_name="vast", commit=None, log=print):
"""Create the pod and start the job. Refuses without --go, with a secret in the repo, with a pod already up for
this run, or when the balance does not cover the run's estimated cost with a margin."""
spec = jobs.get(exp)
keyscan.require_clean(REPO)
if load_state(exp) and load_state(exp).get("instance_id") and not load_state(exp).get("destroyed"):
raise PodError(f"{exp} already has a pod ({load_state(exp)['instance_id']}). Run collect or down first.")
p = provider(provider_name)
if any(i.get("label") == label(exp) for i in p.instances()):
raise PodError(f"An instance labelled {label(exp)} already exists. Run down first.")
offer = find_offer(p, offer_id, filters=spec.get("filters"))
if offer is None:
raise PodError(f"Offer {offer_id} no longer passes the filters or is gone. Run offers again.")
if provider_name == "vast" and vast.contract_hours(offer) < spec["hours"] + 1:
raise PodError(f"Offer {offer_id}'s host contract ends in {vast.contract_hours(offer):.1f} h, under the "
f"{spec['hours']} h limit plus 1 h. Vast stops the pod when it ends. Pick another offer.")
need = offer["dph_total"] * spec["est_hours"] * BALANCE_MARGIN
if p.balance() < need:
raise PodError(f"Balance ${p.balance():.2f} is under ${need:.2f} (1.5 x the estimated cost). Top up first.")
if not go:
raise PodError(f"Not created: add --go once Parsa has approved {p.describe(offer)} with an hour limit of {spec['hours']} h.")
serve = spec.get("serve") # a serve pod runs no code of ours and gets no data: only vLLM (jobs.SERVE_*)
hf_token = None
if serve and serve["allow_gated"]:
from strokecopilot.nlp.reasoner.providers import secret
hf_token = secret("HF_TOKEN")
if not hf_token:
raise PodError("HF_TOKEN is not set, and the models served are gated.")
dirty = [] if serve else uncommitted()
if dirty and commit is None:
raise PodError(f"{len(dirty)} file(s) under src/ or the package config are not committed, and the pod runs the "
f"committed code: {', '.join(dirty[:5])}. Commit first.")
if not serve:
tar = tarball_path()
if sha256(tar) != TARBALL_SHA256:
raise PodError("The local tarball's sha256 does not match the recorded one.")
sources = [] if serve else input_files(exp)
commit = commit or head_commit()
instance_id = p.create(offer["id"], spec.get("image", IMAGE), spec.get("disk_gb", DISK_GB), label(exp))
state = {"exp": exp, "provider": provider_name, "instance_id": instance_id, "offer": offer["id"],
"dph": offer["dph_total"], "machine_id": offer.get("machine_id"), "created_utc": dt.datetime.now(dt.timezone.utc).isoformat(), "commit": commit,
"hours_limit": spec["hours"], "runs": [] if serve else jobs.chain(exp),
"inputs_sha256": {dst: sha256(src) for src, dst in sources}, "destroyed": False}
save_state(exp, state)
log(f"Created instance {instance_id} ({p.describe(offer)}). Waiting for it to run.")
try:
inst = p.wait_running(instance_id)
state["host"], state["port"], state["user"] = p.ssh(inst)
save_state(exp, state)
wait_ssh(state)
log(remote_sh(state, remote.BOOTSTRAP).strip())
log(remote_sh(state, remote.start_watchdog(spec["hours"] * 3600)).strip())
if serve:
log(remote_sh(state, remote.SERVE_SETUP, timeout=1800).strip())
if hf_token:
log(remote_sh(state, remote.hf_env(hf_token)).strip())
state["serve"] = {"models": list(serve["models"]), "hf_token_on_pod": bool(hf_token)}
state["job_started_utc"] = dt.datetime.now(dt.timezone.utc).isoformat()
save_state(exp, state)
log(f"{exp} is ready to serve on {state['host']}:{state['port']} (reached through an SSH tunnel).")
return state
started = time.time()
# A host whose inbound link crawls costs pod time while nothing runs: on 12 Nov 2026 one took 38 min for 1.9 of
# 3.1 GB (EXP-14 took 15 min for all of it). Past UPLOAD_LIMIT_S the copy fails and the pod is destroyed.
rsync_up(state, tar, f"{remote.ROOT}/data/{TARBALL}", timeout=UPLOAD_LIMIT_S)
archive = repo_archive(commit, RUNS / exp / "repo.tar")
rsync_up(state, archive, f"{remote.ROOT}/repo.tar")
archive.unlink(missing_ok=True)
for src, dst in sources:
remote_sh(state, f"mkdir -p {Path(dst).parent}")
rsync_up(state, src, dst)
log(f"Copied the tarball, {len(sources)} input file(s) and the repo at {commit[:7]} in {time.time() - started:.0f} s.")
remote_sh(state, "cd /root/sc && tar -xf repo.tar -C repo && rm repo.tar")
log(remote_sh(state, remote.SETUP.replace("{nnunet}", jobs.NNUNET_VERSION), timeout=1800).strip())
log(remote_sh(state, remote.start_job(exp, jobs.command(exp, commit))).strip())
except Exception:
log("Setup failed; destroying the pod so nothing is left running.")
down(exp, log=log)
raise
state["job_started_utc"] = dt.datetime.now(dt.timezone.utc).isoformat()
save_state(exp, state)
log(f"{exp} is running on {state['host']}:{state['port']}. Watch it with: scripts/pod status {exp}")
return state
def status(exp, log=print):
state = load_state(exp)
if not state or state.get("destroyed"):
log(f"{exp}: no pod.")
return None
out = remote_sh(state, remote.status(exp), timeout=60)
log(out.rstrip())
return out
def collect(exp, force=False, log=print):
"""Copy the run's outputs back (config, metrics, per-case scores, best and final checkpoint, logs), put them in
Drive, then destroy the pod. Refuses while the job is still running unless force."""
state = load_state(exp)
if not state or state.get("destroyed"):
raise PodError(f"{exp}: no pod to collect from.")
out = remote_sh(state, remote.status(exp), timeout=60)
if "EXIT=" not in out and not force:
raise PodError(f"{exp} is still running. Use --force to collect anyway (the job is then lost).")
local = RUNS / exp
root = drive_root()
for e in state.get("runs", [exp]):
rsync_down(state, f"{remote.ROOT}/runs/{e}/", RUNS / e, jobs.COLLECT)
if e != exp:
log(f"{e}: " + (remote_sh(state, remote.status(e), timeout=60).strip().splitlines() or ["no status"])[0])
rsync_down(state, f"{remote.ROOT}/", local / "pod_logs", ["watchdog.log", "pip.log"])
for e in state.get("runs", [exp]):
if root:
dest = root / "runs" / jobs.get(e)["run_dir"]
run(["rsync", "-a", "--exclude", "pod.json", "--exclude", "repo.tar", str(RUNS / e) + "/", str(dest) + "/"])
log(f"Copied to Drive: {dest}")
if not root:
log("Drive for desktop not found: outputs are in runs/ on this Mac only.")
down(exp, log=log)
return local
def down(exp, log=print):
"""Destroy the pod and confirm it is gone by listing instances. Records the measured cost."""
state = load_state(exp)
if not state or not state.get("instance_id"):
log(f"{exp}: no pod recorded.")
return
p = provider(state["provider"])
if p.get(state["instance_id"]) is not None:
p.destroy(state["instance_id"])
for _ in range(12):
if p.get(state["instance_id"]) is None and not any(i.get("label") == label(exp) for i in p.instances()):
break
time.sleep(10)
else:
raise PodError(f"Instance {state['instance_id']} is still listed after destroy. Check the console now.")
created = dt.datetime.fromisoformat(state["created_utc"])
hours = (dt.datetime.now(dt.timezone.utc) - created).total_seconds() / 3600
state.update(destroyed=True, destroyed_utc=dt.datetime.now(dt.timezone.utc).isoformat(), hours_up=round(hours, 3),
cost_usd_estimate=round(hours * state["dph"], 3))
save_state(exp, state)
log(f"{exp}: instance {state['instance_id']} destroyed and gone from the list. Up {hours:.2f} h, about ${hours * state['dph']:.2f}.")
def main(argv=None):
ap = argparse.ArgumentParser(prog="scripts/pod", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("verb", choices=["offers", "up", "status", "collect", "down", "monitor"])
ap.add_argument("exp")
ap.add_argument("--offer")
ap.add_argument("--go", action="store_true")
ap.add_argument("--force", action="store_true")
ap.add_argument("--provider", default="vast")
a = ap.parse_args(argv)
try:
if a.verb == "offers":
offers(a.exp, a.provider)
elif a.verb == "up":
if not a.offer:
raise PodError("up needs --offer <id> from `scripts/pod offers`.")
up(a.exp, a.offer, a.go, a.provider)
elif a.verb == "status":
status(a.exp)
elif a.verb == "monitor":
from strokecopilot.compute import monitor
monitor.run(a.exp)
elif a.verb == "collect":
collect(a.exp, a.force)
else:
down(a.exp)
except (PodError, keyscan.SecretInRepo) as e:
print(f"Error: {e}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
|