Download code/scripts/monitor_comparison.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 2.76 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/monitor_comparison.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/monitor_comparison.py
-
curl -L -o monitor_comparison.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/monitor_comparison.py
2.76 kB
| """Record experiment health every 30 minutes without retrying training updates.""" | |
| import argparse | |
| import fcntl | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import time | |
| from datetime import datetime, timezone | |
| ROOT = Path(__file__).resolve().parents[1] | |
| LOCAL = ROOT / '.release_local' | |
| def read(path): | |
| try: | |
| return json.loads(path.read_text()) | |
| except (OSError, ValueError): | |
| return None | |
| def command(args): | |
| try: | |
| return subprocess.run(args, capture_output=True, text=True, timeout=15).stdout.strip() | |
| except (OSError, subprocess.TimeoutExpired) as error: | |
| return type(error).__name__ | |
| def snapshot(): | |
| now = time.time() | |
| stages = {'driver': read(LOCAL / 'comparison-status.json'), | |
| 'prefix': read(Path('/tmp/robottt-comparison/prefix-cache/status.json'))} | |
| for name in ('comparison-lite', 'comparison-reference'): | |
| stages[name] = read(LOCAL / 'runs' / name / 'status.json') | |
| for value in stages.values(): | |
| if value and 'updated_unix_time' in value: | |
| value['age_seconds'] = round(now - value['updated_unix_time']) | |
| failures = {} | |
| for pattern in ('*failure*.json', 'runs/comparison-*/*failure*.json'): | |
| for path in LOCAL.glob(pattern): | |
| failures[str(path.relative_to(LOCAL))] = read(path) | |
| disks = {} | |
| for location in (ROOT, Path('/tmp')): | |
| stat = os.statvfs(location) | |
| disks[str(location)] = dict(free_bytes=stat.f_bavail * stat.f_frsize, | |
| free_inodes=stat.f_favail) | |
| result = dict(updated_utc=datetime.now(timezone.utc).isoformat(), | |
| interval_seconds=1800, stages=stages, failures=failures, | |
| gpu=command(['nvidia-smi', '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu', '--format=csv,noheader']), | |
| gpu_processes=command(['nvidia-smi', '--query-compute-apps=pid,used_memory', '--format=csv,noheader']), | |
| disks=disks, backup=read(LOCAL / 'hf-backup-status.json')) | |
| encoded = json.dumps(result, ensure_ascii=False) | |
| temporary = LOCAL / 'monitor-status.tmp' | |
| temporary.write_text(encoded + '\n') | |
| temporary.replace(LOCAL / 'monitor-status.json') | |
| with (LOCAL / 'monitor-history.jsonl').open('a') as output: | |
| output.write(encoded + '\n') | |
| print(encoded, flush=True) | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--watch', action='store_true') | |
| args = parser.parse_args() | |
| lock = (LOCAL / 'monitor.lock').open('a') | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| while True: | |
| snapshot() | |
| if not args.watch: | |
| return | |
| time.sleep(1800) | |
| if __name__ == '__main__': | |
| main() | |