Download scripts/train/third_party_gpu_telemetry.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 3.09 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train/third_party_gpu_telemetry.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/train/third_party_gpu_telemetry.py
-
curl -L -o third_party_gpu_telemetry.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train/third_party_gpu_telemetry.py
3.09 kB
| #!/usr/bin/env python3 | |
| """Read-only selected-GPU samples while one verified managed launcher is alive.""" | |
| import argparse | |
| import csv | |
| import importlib.util | |
| import json | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| PROJECT = Path(__file__).resolve().parents[2] | |
| # scripts/train.py shadows this directory as a Python package. Load the | |
| # dependency by filename without importing the training entry point. | |
| _spec = importlib.util.spec_from_file_location( | |
| "hfd_telemetry_runner", PROJECT / "scripts/train/third_party_run.py") | |
| _runner = importlib.util.module_from_spec(_spec) | |
| sys.modules[_spec.name] = _runner | |
| _spec.loader.exec_module(_runner) | |
| identity = _runner.identity | |
| def launcher_alive(status): | |
| process = identity(status["launcher_pid"]) | |
| return bool(process and process.start_ticks == status["launcher_start_ticks"] | |
| and str(PROJECT / "scripts/train/third_party_run.py") in process.argv) | |
| def sample_gpu(gpu=3): | |
| if type(gpu) is not int or gpu not in (0, 3): | |
| raise ValueError("Telemetry requires the managed physical GPU") | |
| result = subprocess.run(["nvidia-smi", "-i", str(gpu), "--query-gpu=utilization.gpu,memory.used,power.draw", | |
| "--format=csv,noheader,nounits"], capture_output=True, text=True, | |
| check=True, timeout=15) | |
| row, = list(csv.reader(result.stdout.strip().splitlines())) | |
| if len(row) != 3: | |
| raise ValueError("Expected exactly three GPU measurements") | |
| return dict(zip(("gpu_utilization_percent", "memory_used_mib", "power_watts"), map(float, row))) | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--worker", action="store_true") | |
| args = parser.parse_args() | |
| run = args.run_dir.resolve() | |
| if not run.is_relative_to(PROJECT / "runs") or run == PROJECT / "runs": | |
| raise ValueError("Telemetry output must be a managed local run") | |
| status = json.loads((run / "run_status.json").read_text()) | |
| if not launcher_alive(status): | |
| raise RuntimeError("Managed launcher is no longer alive") | |
| if not args.worker: | |
| with (run / "gpu_telemetry.log").open("x") as log: | |
| child = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--run-dir", str(run), "--worker"], | |
| stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) | |
| print(json.dumps({"telemetry_pid": child.pid, "output": str(run / "gpu_usage.jsonl")})) | |
| return | |
| while launcher_alive(status): | |
| row = {"time": time.time(), "gpu": status["gpu"]} | |
| try: | |
| state = json.loads((run / "workflow_state.json").read_text()) | |
| row["workflow_phase"] = state["phase"] | |
| row.update(sample_gpu(status["gpu"])) | |
| except Exception as error: | |
| row["error"] = repr(error) | |
| with (run / "gpu_usage.jsonl").open("a") as handle: | |
| handle.write(json.dumps(row) + "\n") | |
| time.sleep(5) | |
| if __name__ == "__main__": | |
| main() | |