#!/usr/bin/env python3 """Read-only selected-GPU samples while one verified managed launcher is alive.""" import argparse import csv import importlib.util import json import subprocess import sys import time from pathlib import Path PROJECT = Path(__file__).resolve().parents[2] # scripts/train.py shadows this directory as a Python package. Load the # dependency by filename without importing the training entry point. _spec = importlib.util.spec_from_file_location( "hfd_telemetry_runner", PROJECT / "scripts/train/third_party_run.py") _runner = importlib.util.module_from_spec(_spec) sys.modules[_spec.name] = _runner _spec.loader.exec_module(_runner) identity = _runner.identity def launcher_alive(status): process = identity(status["launcher_pid"]) return bool(process and process.start_ticks == status["launcher_start_ticks"] and str(PROJECT / "scripts/train/third_party_run.py") in process.argv) def sample_gpu(gpu=3): if type(gpu) is not int or gpu not in (0, 3): raise ValueError("Telemetry requires the managed physical GPU") result = subprocess.run(["nvidia-smi", "-i", str(gpu), "--query-gpu=utilization.gpu,memory.used,power.draw", "--format=csv,noheader,nounits"], capture_output=True, text=True, check=True, timeout=15) row, = list(csv.reader(result.stdout.strip().splitlines())) if len(row) != 3: raise ValueError("Expected exactly three GPU measurements") return dict(zip(("gpu_utilization_percent", "memory_used_mib", "power_watts"), map(float, row))) def main(): parser = argparse.ArgumentParser() parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--worker", action="store_true") args = parser.parse_args() run = args.run_dir.resolve() if not run.is_relative_to(PROJECT / "runs") or run == PROJECT / "runs": raise ValueError("Telemetry output must be a managed local run") status = json.loads((run / "run_status.json").read_text()) if not launcher_alive(status): raise RuntimeError("Managed launcher is no longer alive") if not args.worker: with (run / "gpu_telemetry.log").open("x") as log: child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"], stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) print(json.dumps({"telemetry_pid": child.pid, "output": str(run / "gpu_usage.jsonl")})) return while launcher_alive(status): row = {"time": time.time(), "gpu": status["gpu"]} try: state = json.loads((run / "workflow_state.json").read_text()) row["workflow_phase"] = state["phase"] row.update(sample_gpu(status["gpu"])) except Exception as error: row["error"] = repr(error) with (run / "gpu_usage.jsonl").open("a") as handle: handle.write(json.dumps(row) + "\n") time.sleep(5) if __name__ == "__main__": main()