Download scripts/train/third_party_run.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 26.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train/third_party_run.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/train/third_party_run.py
-
curl -L -o third_party_run.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train/third_party_run.py
26.1 kB
| #!/usr/bin/env python3 | |
| """Detached, fail-closed, single-GPU runner for an accepted third-party experiment. | |
| This runner never calls the historical HFD training shell. It only signals the | |
| verified selected-GPU keeper and its own training process group. SIGKILL, host failure, | |
| and a terminated container cannot be recovered by a process-local finally block. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| import signal | |
| import socket | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| PROJECT_ROOT = Path(__file__).resolve().parents[2] | |
| MONITOR_ROOT = PROJECT_ROOT.parent / "monitor" | |
| GPU = 3 | |
| DEFAULT_KEEPER_LOCKS = { | |
| "MONITOR_LOCK_FILE": "/tmp/yetinghong_gpu_keeper.lock", | |
| "MONITOR_SUPERVISOR_LOCK_FILE": "/tmp/yetinghong_gpu_keeper_supervisor.lock", | |
| "MONITOR_SUPERVISOR_PID_FILE": "/tmp/yetinghong_gpu_keeper_supervisor.pid", | |
| } | |
| KEEPER_ENV = { | |
| "CUDA_VISIBLE_DEVICES": "3", | |
| "MONITOR_GPU_INDICES": "3", | |
| "MONITOR_LOCK_FILE": "/tmp/movens_scale_preview_gpu3_keeper.lock", | |
| "MONITOR_SUPERVISOR_LOCK_FILE": "/tmp/movens_scale_preview_gpu3_keeper_supervisor.lock", | |
| "MONITOR_SUPERVISOR_PID_FILE": "/tmp/movens_scale_preview_gpu3_keeper_supervisor.pid", | |
| } | |
| class SafetyError(RuntimeError): | |
| pass | |
| def configure_gpu(gpu: int) -> None: | |
| """Only explicitly coordinated physical GPUs; one runner per process.""" | |
| global GPU, KEEPER_ENV | |
| if type(gpu) is not int or gpu not in (0, 3): | |
| raise SafetyError("Only physical gpu=0 or gpu=3 are supported") | |
| GPU = gpu | |
| locks = DEFAULT_KEEPER_LOCKS if gpu == 0 else { | |
| "MONITOR_LOCK_FILE": "/tmp/movens_scale_preview_gpu3_keeper.lock", | |
| "MONITOR_SUPERVISOR_LOCK_FILE": "/tmp/movens_scale_preview_gpu3_keeper_supervisor.lock", | |
| "MONITOR_SUPERVISOR_PID_FILE": "/tmp/movens_scale_preview_gpu3_keeper_supervisor.pid", | |
| } | |
| KEEPER_ENV = {"CUDA_VISIBLE_DEVICES": str(gpu), "MONITOR_GPU_INDICES": str(gpu), **locks} | |
| def digest(path: Path) -> str: | |
| result = hashlib.sha256() | |
| with path.open("rb") as stream: | |
| for chunk in iter(lambda: stream.read(1024 * 1024), b""): | |
| result.update(chunk) | |
| return result.hexdigest() | |
| def atomic_json(path: Path, value: dict[str, Any]) -> None: | |
| with tempfile.NamedTemporaryFile("w", dir=path.parent, delete=False) as stream: | |
| temporary = Path(stream.name) | |
| json.dump(value, stream, ensure_ascii=False, indent=2) | |
| stream.write("\n") | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| os.replace(temporary, path) | |
| def local_path(value: str, *, must_exist: bool = True) -> Path: | |
| path = Path(value).expanduser() | |
| if not path.is_absolute(): | |
| path = PROJECT_ROOT / path | |
| path = path.resolve() | |
| if not path.is_relative_to(PROJECT_ROOT): | |
| raise SafetyError(f"Path must remain under HFD_lzk: {path}") | |
| if must_exist and not path.is_file(): | |
| raise SafetyError(f"Required file does not exist: {path}") | |
| return path | |
| def checked_argv(value: Any, label: str) -> list[str]: | |
| if not isinstance(value, list) or len(value) < 2 or not all( | |
| isinstance(item, str) and item and "\x00" not in item for item in value | |
| ): | |
| raise SafetyError(f"{label} must be a Python executable + local .py script argv list") | |
| executable = Path(value[0]) | |
| if not executable.is_absolute() or not executable.is_file() or not os.access(executable, os.X_OK): | |
| raise SafetyError(f"{label}: provide an absolute executable Python path") | |
| if not executable.name.startswith("python"): | |
| raise SafetyError(f"{label}: shell commands are not accepted") | |
| script = local_path(value[1]) | |
| if script.suffix != ".py" or script == Path(__file__).resolve(): | |
| raise SafetyError(f"{label}: expected a dedicated local Python entrypoint") | |
| return [str(executable), str(script), *value[2:]] | |
| def load_spec(path: Path) -> dict[str, Any]: | |
| spec = json.loads(path.read_text()) | |
| if spec.get("schema") != 1: | |
| raise SafetyError("Only schema=1 is supported") | |
| configure_gpu(spec.get("gpu")) | |
| if Path(spec.get("project_root", PROJECT_ROOT)).resolve() != PROJECT_ROOT: | |
| raise SafetyError("project_root does not match this isolated HFD_lzk checkout") | |
| if spec.get("expected_host", socket.gethostname()) != socket.gethostname(): | |
| raise SafetyError("Host differs from expected_host; GPU ownership must be reviewed again") | |
| spec["run_dir"] = str(local_path(spec["run_dir"], must_exist=False)) | |
| if not Path(spec["run_dir"]).is_relative_to(PROJECT_ROOT / "runs"): | |
| raise SafetyError("run_dir must be below HFD_lzk/runs") | |
| if Path(spec["run_dir"]) == PROJECT_ROOT / "runs": | |
| raise SafetyError("Use an experiment subdirectory, not runs itself") | |
| spec["manifest_path"] = str(local_path(spec["manifest_path"])) | |
| spec["readiness_file"] = str(local_path(spec["readiness_file"])) | |
| spec["argv"] = checked_argv(spec["argv"], "argv") | |
| spec["preflight_command"] = checked_argv(spec["preflight_command"], "preflight_command") | |
| for key in ("argv", "preflight_command"): | |
| arguments = spec[key] | |
| if arguments.count("--manifest") != 1: | |
| raise SafetyError(f"{key} must explicitly name --manifest once") | |
| position = arguments.index("--manifest") | |
| if position + 1 >= len(arguments) or local_path(arguments[position + 1]) != Path(spec["manifest_path"]): | |
| raise SafetyError(f"{key} --manifest differs from the accepted manifest") | |
| arguments = spec["argv"] | |
| if arguments.count("--run-dir") != 1: | |
| raise SafetyError("argv must explicitly name --run-dir once") | |
| position = arguments.index("--run-dir") | |
| if position + 1 >= len(arguments) or local_path(arguments[position + 1], must_exist=False) != Path(spec["run_dir"]): | |
| raise SafetyError("argv --run-dir differs from the launch run_dir") | |
| extra_env = spec.get("env", {}) | |
| if not isinstance(extra_env, dict) or not all( | |
| isinstance(k, str) and isinstance(v, str) and "\x00" not in k + v | |
| and "=" not in k for k, v in extra_env.items() | |
| ): | |
| raise SafetyError("env must be a string-to-string dictionary") | |
| if set(extra_env) & (set(KEEPER_ENV) | {"HFD_NPROC_PER_NODE", "PYTHONPATH", "LD_PRELOAD"}): | |
| raise SafetyError("env cannot override GPU, keeper, launcher or import-path isolation") | |
| for key, default, upper in (("restore_timeout_seconds", 120, 300), ("terminate_timeout_seconds", 120, 300)): | |
| value = spec.get(key, default) | |
| if not isinstance(value, (int, float)) or not 1 <= value <= upper: | |
| raise SafetyError(f"{key} must be between 1 and {upper}") | |
| spec[key] = value | |
| validate_acceptance(spec) | |
| return spec | |
| def validate_acceptance(spec: dict[str, Any]) -> None: | |
| report = json.loads(Path(spec["readiness_file"]).read_text()) | |
| count = 300 | |
| if spec.get("scope") == "full_third_party": | |
| manifest = json.loads(Path(spec["manifest_path"]).read_text()) | |
| ids = manifest.get("session_ids", []) | |
| if (report.get("format") != "hfd_third_party_scale_readiness_v1" | |
| or manifest.get("scaleup") is not True or manifest.get("split") != "train" | |
| or len(ids) < 300 or len(set(ids)) != len(ids)): | |
| raise SafetyError("Acceptance failed: invalid full third-party selection") | |
| count = len(ids) | |
| elif spec.get("scope") not in (None, "pilot"): | |
| raise SafetyError("Unknown third-party scope") | |
| required = {"passed": True, "train_count": count, "visual_inputs": 0, | |
| "other_task_inputs": 0, "heldout_inputs": 0} | |
| for key, expected in required.items(): | |
| if key not in report or type(report[key]) is not type(expected) or report[key] != expected: | |
| raise SafetyError(f"Acceptance failed: {key} must be {expected!r}") | |
| if report.get("manifest_sha256") != digest(Path(spec["manifest_path"])): | |
| raise SafetyError("Acceptance manifest_sha256 does not match the current manifest") | |
| class ProcessIdentity: | |
| pid: int | |
| start_ticks: int | |
| pgrp: int | |
| session: int | |
| argv: tuple[str, ...] | |
| def identity(pid: int) -> ProcessIdentity | None: | |
| try: | |
| directory = Path("/proc") / str(pid) | |
| if directory.stat().st_uid != os.getuid(): | |
| raise SafetyError(f"Refusing to operate on PID {pid}: different Unix owner") | |
| fields = (directory / "stat").read_text().rsplit(")", 1)[1].split() | |
| if fields[0] == "Z": | |
| return None | |
| argv = tuple(part.decode() for part in (directory / "cmdline").read_bytes().split(b"\0") if part) | |
| return ProcessIdentity(pid, int(fields[19]), int(fields[2]), int(fields[3]), argv) | |
| except (FileNotFoundError, ProcessLookupError): | |
| return None | |
| def process_env(pid: int) -> dict[str, str]: | |
| """Read only keeper configuration keys; never log process credentials.""" | |
| allowed = set(KEEPER_ENV) | |
| result = {} | |
| for item in (Path("/proc") / str(pid) / "environ").read_bytes().split(b"\0"): | |
| key, separator, value = item.partition(b"=") | |
| name = key.decode(errors="replace") | |
| if separator and name in allowed: | |
| result[name] = value.decode(errors="replace") | |
| return result | |
| def locked_pid(path: Path) -> int | None: | |
| try: | |
| value = path.read_text().strip() | |
| except FileNotFoundError: | |
| return None | |
| if not value: | |
| return None | |
| if not value.isdecimal() or int(value) <= 1: | |
| raise SafetyError(f"Malformed keeper PID file: {path}") | |
| return int(value) | |
| def keeper_processes() -> dict[str, ProcessIdentity]: | |
| result = {} | |
| for role, key, script in ( | |
| ("supervisor", "MONITOR_SUPERVISOR_PID_FILE", "monitor.sh"), | |
| ("worker", "MONITOR_LOCK_FILE", "monitor.py"), | |
| ): | |
| pid = locked_pid(Path(KEEPER_ENV[key])) | |
| process = identity(pid) if pid else None | |
| if process is None: | |
| continue | |
| if str(MONITOR_ROOT / script) not in process.argv: | |
| raise SafetyError(f"Keeper {role} PID {pid} is not the expected exact program") | |
| environment = process_env(pid) | |
| # monitor.sh selects physical GPUs with MONITOR_GPU_INDICES and only | |
| # then exports CUDA_VISIBLE_DEVICES for monitor.py. /proc/<bash>/environ | |
| # reflects its initial environment, not later shell exports. Check the | |
| # supervisor's allowlist/locks, and the actual CUDA mask on the worker. | |
| required_environment = { | |
| name: value for name, value in KEEPER_ENV.items() | |
| if role == "worker" or name != "CUDA_VISIBLE_DEVICES" | |
| } | |
| # The existing GPU-0 keeper uses monitor.sh's documented default lock | |
| # paths; absence of those exported keys is not a different lock. GPU | |
| # masks must always be explicit and single-card. GPU-3 stays strict. | |
| defaults = DEFAULT_KEEPER_LOCKS if GPU == 0 else {} | |
| mismatched = [name for name, value in required_environment.items() | |
| if environment.get(name, defaults.get(name)) != value] | |
| if mismatched: | |
| raise SafetyError(f"Keeper {role} PID {pid} has a different GPU/lock configuration " | |
| f"(mismatched keys: {', '.join(mismatched)})") | |
| if role == "worker": | |
| arguments = list(process.argv) | |
| try: | |
| position = arguments.index("--lock-file") | |
| valid_lock = arguments[position + 1] == KEEPER_ENV["MONITOR_LOCK_FILE"] | |
| except (ValueError, IndexError): | |
| valid_lock = False | |
| if not valid_lock: | |
| raise SafetyError(f"Keeper worker PID {pid} has a different lock-file argument") | |
| result[role] = process | |
| return result | |
| def gpu_compute_pids() -> set[int]: | |
| result = subprocess.run( | |
| ["nvidia-smi", "-i", str(GPU), "--query-compute-apps=pid", "--format=csv,noheader,nounits"], | |
| check=True, text=True, capture_output=True, timeout=20, | |
| ) | |
| pids = set() | |
| for row in csv.reader(result.stdout.splitlines()): | |
| if not row: | |
| continue | |
| if len(row) != 1 or not row[0].strip().isdecimal(): | |
| raise SafetyError("Unrecognized nvidia-smi process output; refusing GPU handoff") | |
| pids.add(int(row[0].strip())) | |
| return pids | |
| def assert_gpu_available(keepers: dict[str, ProcessIdentity]) -> None: | |
| worker = keepers.get("worker") | |
| allowed = {worker.pid} if worker else set() | |
| unexpected = gpu_compute_pids() - allowed | |
| if unexpected: | |
| raise SafetyError(f"GPU {GPU} has non-keeper compute process(es): {sorted(unexpected)}") | |
| def signal_verified(process: ProcessIdentity, sig: int) -> bool: | |
| # A pidfd binds the signal to this process, closing the PID-reuse race. | |
| try: | |
| descriptor = os.pidfd_open(process.pid) | |
| except ProcessLookupError: | |
| return False | |
| try: | |
| current = identity(process.pid) | |
| if current is None: | |
| return False | |
| if current != process: | |
| raise SafetyError(f"PID {process.pid} identity changed; refusing to signal it") | |
| signal.pidfd_send_signal(descriptor, sig) | |
| return True | |
| finally: | |
| os.close(descriptor) | |
| def wait_gone(processes: list[ProcessIdentity], timeout: float) -> bool: | |
| deadline = time.monotonic() + timeout | |
| while time.monotonic() < deadline: | |
| if all(identity(item.pid) != item for item in processes): | |
| return True | |
| time.sleep(0.25) | |
| return all(identity(item.pid) != item for item in processes) | |
| def stop_keeper(keepers: dict[str, ProcessIdentity]) -> None: | |
| # Stop the supervisor first, so a worker failure cannot create a replacement. | |
| supervisor = keepers.get("supervisor") | |
| if supervisor: | |
| signal_verified(supervisor, signal.SIGTERM) | |
| worker = keepers.get("worker") | |
| if worker: | |
| signal_verified(worker, signal.SIGTERM) | |
| if not wait_gone(list(keepers.values()), 35): | |
| raise SafetyError("Keeper did not stop within 35 seconds; training was not started") | |
| # A newly appearing keeper or another user's process is not ours to kill. | |
| assert_gpu_available({}) | |
| def restore_keeper(timeout: float) -> dict[str, Any]: | |
| keeper_processes() # Reject stale lock PIDs pointing at another program. | |
| environment = {**os.environ, **KEEPER_ENV} | |
| command = subprocess.run( | |
| ["bash", str(MONITOR_ROOT / "cmd.sh")], env=environment, | |
| check=False, text=True, capture_output=True, timeout=30, | |
| ) | |
| print(command.stdout, end="", flush=True) | |
| if command.stderr: | |
| print(command.stderr, file=sys.stderr, end="", flush=True) | |
| result: dict[str, Any] = {"keeper_restore_command_returncode": command.returncode, | |
| "keeper_restore_verified": False} | |
| if command.returncode: | |
| return result | |
| deadline = time.monotonic() + timeout | |
| while time.monotonic() < deadline: | |
| processes = keeper_processes() | |
| worker = processes.get("worker") | |
| if worker and worker.pid in gpu_compute_pids(): | |
| result.update(keeper_restore_verified=True, keeper_pid=worker.pid) | |
| return result | |
| time.sleep(2) | |
| result["keeper_restore_note"] = "Launcher accepted; GPU allocation not yet verified. Inspect monitor.log." | |
| return result | |
| def own_group_alive(pgid: int) -> bool: | |
| try: | |
| os.killpg(pgid, 0) | |
| return True | |
| except ProcessLookupError: | |
| return False | |
| def terminate_group(child: subprocess.Popen[Any], timeout: float) -> None: | |
| """Only this runner's independently-created session/process group is touched.""" | |
| try: | |
| os.killpg(child.pid, signal.SIGTERM) | |
| except ProcessLookupError: | |
| child.wait() | |
| return | |
| deadline = time.monotonic() + timeout | |
| while time.monotonic() < deadline: | |
| child.poll() # Reap a terminated leader, even while descendants exit. | |
| if not own_group_alive(child.pid): | |
| return | |
| time.sleep(0.25) | |
| try: | |
| os.killpg(child.pid, signal.SIGKILL) | |
| except ProcessLookupError: | |
| pass | |
| child.wait(timeout=10) | |
| class Runner: | |
| def __init__(self, spec: dict[str, Any]): | |
| configure_gpu(spec["gpu"]) | |
| self.spec = spec | |
| self.directory = Path(spec["run_dir"]) | |
| self.child: subprocess.Popen[Any] | None = None | |
| self.requested_signal: int | None = None | |
| self.status: dict[str, Any] = { | |
| "schema": 1, "host": socket.gethostname(), "gpu": GPU, | |
| "launcher_pid": os.getpid(), "launcher_start_ticks": identity(os.getpid()).start_ticks, | |
| "started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), | |
| "run_dir": str(self.directory), "manifest_sha256": digest(Path(spec["manifest_path"])), | |
| "training_returncode": None, "keeper_restore_verified": None, | |
| } | |
| def record(self, phase: str, **fields: Any) -> None: | |
| timestamp = time.strftime("%Y-%m-%dT%H:%M:%S%z") | |
| self.status.update(phase=phase, updated_at=timestamp, **fields) | |
| atomic_json(self.directory / "run_status.json", self.status) | |
| with (self.directory / "launcher_events.jsonl").open("a") as stream: | |
| stream.write(json.dumps({"time": timestamp, "phase": phase, **fields}, ensure_ascii=False) + "\n") | |
| print(f"[{timestamp}] {phase} {json.dumps(fields, ensure_ascii=False)}", flush=True) | |
| def on_signal(self, sig: int, _frame: Any) -> None: | |
| self.requested_signal = sig | |
| if self.child and self.child.poll() is None: | |
| try: | |
| os.killpg(self.child.pid, signal.SIGTERM) | |
| except ProcessLookupError: | |
| pass | |
| def check_cancelled(self) -> None: | |
| if self.requested_signal: | |
| raise InterruptedError(f"Termination signal {self.requested_signal} requested") | |
| def wait_child(self) -> int: | |
| assert self.child is not None | |
| while self.child.poll() is None: | |
| if self.requested_signal: | |
| terminate_group(self.child, self.spec["terminate_timeout_seconds"]) | |
| return 128 + self.requested_signal | |
| time.sleep(0.25) | |
| return self.child.returncode | |
| def run(self) -> int: | |
| self.directory.mkdir(parents=True, exist_ok=True) | |
| lock = (PROJECT_ROOT / "runs" / f".third_party_gpu{GPU}.lock").open("a+") | |
| try: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| except BlockingIOError: | |
| lock.close() | |
| raise SafetyError(f"Another third-party launcher owns the GPU-{GPU} launch lock") | |
| previous = self.directory / "run_status.json" | |
| if previous.exists(): | |
| lock.close() | |
| raise SafetyError("run_dir already contains a launch status; choose a fresh run directory") | |
| old_handlers = {sig: signal.signal(sig, self.on_signal) for sig in (signal.SIGTERM, signal.SIGINT)} | |
| touched_keeper = False | |
| returncode = 1 | |
| try: | |
| self.record("preflight") | |
| validate_acceptance(self.spec) | |
| environment = {**os.environ, **self.spec.get("env", {}), "PYTHONUNBUFFERED": "1", | |
| "PYTHONDONTWRITEBYTECODE": "1", "HFD_NPROC_PER_NODE": "1", | |
| "PYTHONPATH": str(PROJECT_ROOT / "src") + os.pathsep + str(PROJECT_ROOT)} | |
| self.child = subprocess.Popen( | |
| self.spec["preflight_command"], cwd=PROJECT_ROOT, | |
| env={**environment, "CUDA_VISIBLE_DEVICES": "-1"}, start_new_session=True, | |
| ) | |
| preflight_rc = self.wait_child() | |
| if own_group_alive(self.child.pid): | |
| terminate_group(self.child, self.spec["terminate_timeout_seconds"]) | |
| self.child = None | |
| self.check_cancelled() | |
| if preflight_rc: | |
| raise SafetyError(f"CPU preflight failed with return code {preflight_rc}") | |
| validate_acceptance(self.spec) | |
| if not (MONITOR_ROOT / "cmd.sh").is_file(): | |
| raise SafetyError("GPU keeper restore command does not exist") | |
| keepers = keeper_processes() | |
| assert_gpu_available(keepers) | |
| self.check_cancelled() | |
| self.record("keeper_handoff", keeper_pids={key: value.pid for key, value in keepers.items()}) | |
| # Restore even if stopping only partly succeeds, or if initially idle. | |
| touched_keeper = True | |
| stop_keeper(keepers) | |
| self.check_cancelled() | |
| validate_acceptance(self.spec) | |
| self.child = subprocess.Popen( | |
| self.spec["argv"], cwd=PROJECT_ROOT, | |
| env={**environment, "CUDA_VISIBLE_DEVICES": str(GPU)}, start_new_session=True, | |
| ) | |
| self.record("training", training_pid=self.child.pid) | |
| returncode = self.wait_child() | |
| self.record("training_finished", training_returncode=returncode) | |
| except Exception as exc: | |
| returncode = 128 + self.requested_signal if self.requested_signal else 1 | |
| self.record("failed", error=f"{type(exc).__name__}: {exc}") | |
| finally: | |
| if self.child is not None: | |
| try: | |
| if self.child.poll() is None or own_group_alive(self.child.pid): | |
| terminate_group(self.child, self.spec["terminate_timeout_seconds"]) | |
| except Exception as exc: | |
| self.record("cleanup_error", cleanup_error=f"{type(exc).__name__}: {exc}") | |
| if returncode == 0: | |
| returncode = 1 | |
| if touched_keeper: | |
| self.record("restoring_keeper") | |
| try: | |
| self.status.update(restore_keeper(self.spec["restore_timeout_seconds"])) | |
| except Exception as exc: | |
| self.status.update(keeper_restore_verified=False, | |
| keeper_restore_error=f"{type(exc).__name__}: {exc}") | |
| self.record("completed" if returncode == 0 else "failed", launcher_returncode=returncode, | |
| termination_signal=self.requested_signal, | |
| finished_at=time.strftime("%Y-%m-%dT%H:%M:%S%z")) | |
| for sig, handler in old_handlers.items(): | |
| signal.signal(sig, handler) | |
| lock.close() | |
| return returncode | |
| def start(spec_path: Path) -> int: | |
| spec = load_spec(spec_path) | |
| directory = Path(spec["run_dir"]) | |
| directory.mkdir(parents=True, exist_ok=True) | |
| if (directory / "run_status.json").exists() or (directory / "launch_spec.json").exists(): | |
| raise SafetyError("This run directory was already launched; use a fresh run directory") | |
| # O_EXCL prevents simultaneous starts using the same output directory. | |
| with (directory / "launch_spec.json").open("x") as stream: | |
| json.dump(spec, stream, ensure_ascii=False, indent=2) | |
| with (directory / "train.log").open("a") as logfile: | |
| child = subprocess.Popen( | |
| [sys.executable, str(Path(__file__).resolve()), "_run", "--spec", str(directory / "launch_spec.json")], | |
| cwd=PROJECT_ROOT, stdin=subprocess.DEVNULL, stdout=logfile, stderr=subprocess.STDOUT, | |
| start_new_session=True, close_fds=True, | |
| env={**os.environ, "PYTHONUNBUFFERED": "1", "PYTHONDONTWRITEBYTECODE": "1"}, | |
| ) | |
| deadline = time.monotonic() + 3 | |
| while time.monotonic() < deadline and not (directory / "run_status.json").exists() and child.poll() is None: | |
| time.sleep(0.1) | |
| print(json.dumps({"launcher_pid": child.pid, "run_dir": str(directory), | |
| "log": str(directory / "train.log"), "status": str(directory / "run_status.json")}, indent=2)) | |
| if child.poll() is not None and child.returncode: | |
| raise SafetyError(f"Background launcher exited with {child.returncode}; inspect train.log") | |
| return 0 | |
| def show_status(directory: Path) -> int: | |
| value = json.loads((directory / "run_status.json").read_text()) | |
| process = identity(value["launcher_pid"]) | |
| value["launcher_alive"] = bool(process and process.start_ticks == value["launcher_start_ticks"]) | |
| print(json.dumps(value, ensure_ascii=False, indent=2)) | |
| return 0 | |
| def stop(directory: Path) -> int: | |
| value = json.loads((directory / "run_status.json").read_text()) | |
| process = identity(value["launcher_pid"]) | |
| if process is None or process.start_ticks != value["launcher_start_ticks"]: | |
| raise SafetyError("Recorded launcher no longer exists; no process was signalled") | |
| expected_spec = str(directory / "launch_spec.json") | |
| if str(Path(__file__).resolve()) not in process.argv or "_run" not in process.argv or expected_spec not in process.argv: | |
| raise SafetyError("Recorded PID is not this run's launcher; no process was signalled") | |
| signal_verified(process, signal.SIGTERM) | |
| print("Stop requested. Wait for run_status.json to record completion and keeper restoration.") | |
| return 0 | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| commands = parser.add_subparsers(dest="command", required=True) | |
| for command in ("start", "_run"): | |
| sub = commands.add_parser(command) | |
| sub.add_argument("--spec", type=Path, required=True) | |
| for command in ("status", "stop"): | |
| sub = commands.add_parser(command) | |
| sub.add_argument("--run-dir", type=Path, required=True) | |
| args = parser.parse_args() | |
| try: | |
| if args.command == "start": | |
| return start(local_path(str(args.spec))) | |
| if args.command == "_run": | |
| return Runner(load_spec(local_path(str(args.spec)))).run() | |
| directory = local_path(str(args.run_dir), must_exist=False) | |
| return show_status(directory) if args.command == "status" else stop(directory) | |
| except (SafetyError, OSError, ValueError, KeyError) as exc: | |
| print(f"Refused: {exc}", file=sys.stderr) | |
| return 2 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |