Buckets:
| # 中文源码导读(2026-09-15)|阅读顺序 066|本地扩展 | |
| # 来源:官方同路径无此文件。 | |
| # 作用:记录 GPU 显存及使用情况;显存占用率和计算利用率是不同指标。 | |
| # 对照基准:test-time-training/discover@6c40e82;完整路线见 docs/CODE_READING_ORDER_ZH.md。 | |
| """Read-only hardware samples, owned by the launcher (no detached monitor).""" | |
| from datetime import datetime, timezone | |
| import csv | |
| import io | |
| import json | |
| from pathlib import Path | |
| import subprocess | |
| import threading | |
| FIELDS = ("index", "uuid", "utilization.gpu", "utilization.memory", "memory.used", "memory.total", | |
| "power.draw", "temperature.gpu", "clocks.current.sm") | |
| class GpuTelemetry: | |
| def __init__(self, run_dir, gpus, interval=0): | |
| self.path = Path(run_dir) / "gpu_telemetry.jsonl" | |
| self.gpus = gpus | |
| self.interval = interval | |
| self.event = threading.Event() | |
| self.thread = None | |
| def start(self): | |
| if self.interval > 0: | |
| self.thread = threading.Thread(target=self._run, name="gpu-telemetry", daemon=True) | |
| self.thread.start() | |
| def stop(self): | |
| self.event.set() | |
| if self.thread: | |
| self.thread.join(timeout=7) | |
| def _run(self): | |
| with self.path.open("a", encoding="utf-8") as output: | |
| while not self.event.is_set(): | |
| row = {"time_utc": datetime.now(timezone.utc).isoformat()} | |
| try: | |
| text = subprocess.check_output([ | |
| "nvidia-smi", "-i", ",".join(map(str, self.gpus)), | |
| "--query-gpu=" + ",".join(FIELDS), "--format=csv,noheader,nounits", | |
| ], text=True, timeout=5, stderr=subprocess.STDOUT) | |
| row["gpus"] = [dict(zip(FIELDS, map(str.strip, values), strict=True)) | |
| for values in csv.reader(io.StringIO(text)) if values] | |
| for gpu in row["gpus"]: | |
| try: | |
| gpu["memory_used_percent"] = 100 * float(gpu["memory.used"]) / float(gpu["memory.total"]) | |
| except (ValueError, ZeroDivisionError): | |
| gpu["memory_used_percent"] = None | |
| except Exception as exc: | |
| row["error"] = str(exc) | |
| output.write(json.dumps(row) + "\n") | |
| output.flush() | |
| self.event.wait(self.interval) | |
Xet Storage Details
- Size:
- 2.46 kB
- Xet hash:
- 3783bce2587ecb5e0f85911f53b9bb7a5e04fbabcdb46d7806f6b139ee475be2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.