Download test_training_launcher.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 3.99 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_training_launcher.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_training_launcher.py
-
curl -L -o test_training_launcher.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_training_launcher.py
3.99 kB
| """CPU-only regression tests for graceful and forced supervised shutdown.""" | |
| import os | |
| from pathlib import Path | |
| import signal | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| import pytest | |
| ROOT = Path(__file__).resolve().parent | |
| WORKSPACE = ROOT.parent | |
| LAUNCHER = ROOT / "ops/launch_training.py" | |
| def workspace_tmp(): | |
| directory = WORKSPACE / "tmp" | |
| directory.mkdir(exist_ok=True) | |
| with tempfile.TemporaryDirectory(prefix="launcher-test-", dir=directory) as temporary: | |
| yield Path(temporary) | |
| def test_stop_reaches_or_cleans_up_worker_in_another_session(workspace_tmp, cooperative): | |
| stop_file = workspace_tmp / "stop-request" | |
| ready_file = workspace_tmp / "ready" | |
| observed_file = workspace_tmp / "observed" | |
| worker = workspace_tmp / "worker.py" | |
| worker.write_text( | |
| "import os, pathlib, signal, sys, time\n" | |
| f"cooperative = {cooperative!r}\n" | |
| "if not cooperative: signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" | |
| "stop, ready, observed = map(pathlib.Path, sys.argv[1:])\n" | |
| "ready.write_text(str(os.getpid()))\n" | |
| "while not cooperative or not stop.exists(): time.sleep(0.02)\n" | |
| "observed.write_text('graceful stop')\n" | |
| ) | |
| agent = workspace_tmp / "agent.py" | |
| agent.write_text( | |
| "import signal, subprocess, sys\n" | |
| f"if not {cooperative!r}: signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" | |
| "child = subprocess.Popen([sys.executable, *sys.argv[1:]], start_new_session=True)\n" | |
| "sys.exit(child.wait())\n" | |
| ) | |
| harness = ( | |
| "import importlib.util, os, pathlib, sys; " | |
| "spec = importlib.util.spec_from_file_location('launcher', sys.argv[1]); " | |
| "module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); " | |
| "sys.exit(module.supervise([sys.executable, *sys.argv[3:]], pathlib.Path(sys.argv[2]), " | |
| "os.environ.copy(), grace_seconds=0.3, terminate_seconds=0.3))" | |
| ) | |
| environment = os.environ.copy() | |
| environment.update(TMPDIR=str(WORKSPACE / "tmp"), PYTHONDONTWRITEBYTECODE="1") | |
| command = [sys.executable, "-c", harness, str(LAUNCHER), str(stop_file), | |
| str(agent), str(worker), str(stop_file), str(ready_file), str(observed_file)] | |
| process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | |
| text=True, env=environment) | |
| try: | |
| deadline = time.monotonic() + 10 | |
| while not ready_file.exists() and process.poll() is None and time.monotonic() < deadline: | |
| time.sleep(0.02) | |
| assert ready_file.exists(), "Fake distributed worker did not start" | |
| worker_pid = int(ready_file.read_text()) | |
| assert os.getpgid(worker_pid) != os.getpgid(process.pid) | |
| process.send_signal(signal.SIGUSR1) | |
| stdout, stderr = process.communicate(timeout=10) | |
| assert stop_file.exists() | |
| if cooperative: | |
| assert process.returncode == 0, stdout + "\n" + stderr | |
| assert observed_file.read_text() == "graceful stop" | |
| else: | |
| assert process.returncode != 0, stdout + "\n" + stderr | |
| assert "Termination deadline exceeded" in stdout | |
| stat = Path(f"/proc/{worker_pid}/stat") | |
| assert not stat.exists() or stat.read_text().rsplit(")", 1)[1].split()[0] == "Z" | |
| finally: | |
| if process.poll() is None: | |
| process.send_signal(signal.SIGTERM) | |
| try: | |
| process.wait(timeout=10) | |
| except subprocess.TimeoutExpired: | |
| process.kill() | |
| process.wait(timeout=5) | |
| def test_launcher_refuses_work_outside_workspace(): | |
| result = subprocess.run( | |
| [sys.executable, str(LAUNCHER), "--work-dir", "/tmp"], | |
| capture_output=True, text=True, timeout=10, | |
| ) | |
| assert result.returncode != 0 | |
| assert "Created paths must stay inside" in result.stderr | |