code-HFD_lzk / scripts /train /third_party_gpu_telemetry.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
8562e9b verified
Raw History Blame Contribute Delete
3.09 kB
#!/usr/bin/env python3
"""Read-only selected-GPU samples while one verified managed launcher is alive."""
import argparse
import csv
import importlib.util
import json
import subprocess
import sys
import time
from pathlib import Path
PROJECT = Path(__file__).resolve().parents[2]
# scripts/train.py shadows this directory as a Python package. Load the
# dependency by filename without importing the training entry point.
_spec = importlib.util.spec_from_file_location(
"hfd_telemetry_runner", PROJECT / "scripts/train/third_party_run.py")
_runner = importlib.util.module_from_spec(_spec)
sys.modules[_spec.name] = _runner
_spec.loader.exec_module(_runner)
identity = _runner.identity
def launcher_alive(status):
process = identity(status["launcher_pid"])
return bool(process and process.start_ticks == status["launcher_start_ticks"]
and str(PROJECT / "scripts/train/third_party_run.py") in process.argv)
def sample_gpu(gpu=3):
if type(gpu) is not int or gpu not in (0, 3):
raise ValueError("Telemetry requires the managed physical GPU")
result = subprocess.run(["nvidia-smi", "-i", str(gpu), "--query-gpu=utilization.gpu,memory.used,power.draw",
"--format=csv,noheader,nounits"], capture_output=True, text=True,
check=True, timeout=15)
row, = list(csv.reader(result.stdout.strip().splitlines()))
if len(row) != 3:
raise ValueError("Expected exactly three GPU measurements")
return dict(zip(("gpu_utilization_percent", "memory_used_mib", "power_watts"), map(float, row)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--worker", action="store_true")
args = parser.parse_args()
run = args.run_dir.resolve()
if not run.is_relative_to(PROJECT / "runs") or run == PROJECT / "runs":
raise ValueError("Telemetry output must be a managed local run")
status = json.loads((run / "run_status.json").read_text())
if not launcher_alive(status):
raise RuntimeError("Managed launcher is no longer alive")
if not args.worker:
with (run / "gpu_telemetry.log").open("x") as log:
child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"],
stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
print(json.dumps({"telemetry_pid": child.pid, "output": str(run / "gpu_usage.jsonl")}))
return
while launcher_alive(status):
row = {"time": time.time(), "gpu": status["gpu"]}
try:
state = json.loads((run / "workflow_state.json").read_text())
row["workflow_phase"] = state["phase"]
row.update(sample_gpu(status["gpu"]))
except Exception as error:
row["error"] = repr(error)
with (run / "gpu_usage.jsonl").open("a") as handle:
handle.write(json.dumps(row) + "\n")
time.sleep(5)
if __name__ == "__main__":
main()