File size: 3,090 Bytes
8562e9b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#!/usr/bin/env python3
"""Read-only selected-GPU samples while one verified managed launcher is alive."""
import argparse
import csv
import importlib.util
import json
import subprocess
import sys
import time
from pathlib import Path

PROJECT = Path(__file__).resolve().parents[2]
# scripts/train.py shadows this directory as a Python package. Load the
# dependency by filename without importing the training entry point.
_spec = importlib.util.spec_from_file_location(
    "hfd_telemetry_runner", PROJECT / "scripts/train/third_party_run.py")
_runner = importlib.util.module_from_spec(_spec)
sys.modules[_spec.name] = _runner
_spec.loader.exec_module(_runner)
identity = _runner.identity


def launcher_alive(status):
    process = identity(status["launcher_pid"])
    return bool(process and process.start_ticks == status["launcher_start_ticks"]
                and str(PROJECT / "scripts/train/third_party_run.py") in process.argv)


def sample_gpu(gpu=3):
    if type(gpu) is not int or gpu not in (0, 3):
        raise ValueError("Telemetry requires the managed physical GPU")
    result = subprocess.run(["nvidia-smi", "-i", str(gpu), "--query-gpu=utilization.gpu,memory.used,power.draw",
                             "--format=csv,noheader,nounits"], capture_output=True, text=True,
                            check=True, timeout=15)
    row, = list(csv.reader(result.stdout.strip().splitlines()))
    if len(row) != 3:
        raise ValueError("Expected exactly three GPU measurements")
    return dict(zip(("gpu_utilization_percent", "memory_used_mib", "power_watts"), map(float, row)))


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--worker", action="store_true")
    args = parser.parse_args()
    run = args.run_dir.resolve()
    if not run.is_relative_to(PROJECT / "runs") or run == PROJECT / "runs":
        raise ValueError("Telemetry output must be a managed local run")
    status = json.loads((run / "run_status.json").read_text())
    if not launcher_alive(status):
        raise RuntimeError("Managed launcher is no longer alive")
    if not args.worker:
        with (run / "gpu_telemetry.log").open("x") as log:
            child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"],
                stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
        print(json.dumps({"telemetry_pid": child.pid, "output": str(run / "gpu_usage.jsonl")}))
        return
    while launcher_alive(status):
        row = {"time": time.time(), "gpu": status["gpu"]}
        try:
            state = json.loads((run / "workflow_state.json").read_text())
            row["workflow_phase"] = state["phase"]
            row.update(sample_gpu(status["gpu"]))
        except Exception as error:
            row["error"] = repr(error)
        with (run / "gpu_usage.jsonl").open("a") as handle:
            handle.write(json.dumps(row) + "\n")
        time.sleep(5)


if __name__ == "__main__":
    main()