robottt-pi05-base-comparison / code /scripts /monitor_comparison.py
jisue's picture
Experiment backup 2026-11-10T02:40:09.306159+00:00
34f7b5d verified
Raw History Blame Contribute Delete
2.76 kB
"""Record experiment health every 30 minutes without retrying training updates."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import time
from datetime import datetime, timezone
ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
def read(path):
try:
return json.loads(path.read_text())
except (OSError, ValueError):
return None
def command(args):
try:
return subprocess.run(args, capture_output=True, text=True, timeout=15).stdout.strip()
except (OSError, subprocess.TimeoutExpired) as error:
return type(error).__name__
def snapshot():
now = time.time()
stages = {'driver': read(LOCAL / 'comparison-status.json'),
'prefix': read(Path('/tmp/robottt-comparison/prefix-cache/status.json'))}
for name in ('comparison-lite', 'comparison-reference'):
stages[name] = read(LOCAL / 'runs' / name / 'status.json')
for value in stages.values():
if value and 'updated_unix_time' in value:
value['age_seconds'] = round(now - value['updated_unix_time'])
failures = {}
for pattern in ('*failure*.json', 'runs/comparison-*/*failure*.json'):
for path in LOCAL.glob(pattern):
failures[str(path.relative_to(LOCAL))] = read(path)
disks = {}
for location in (ROOT, Path('/tmp')):
stat = os.statvfs(location)
disks[str(location)] = dict(free_bytes=stat.f_bavail * stat.f_frsize,
free_inodes=stat.f_favail)
result = dict(updated_utc=datetime.now(timezone.utc).isoformat(),
interval_seconds=1800, stages=stages, failures=failures,
gpu=command(['nvidia-smi', '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu', '--format=csv,noheader']),
gpu_processes=command(['nvidia-smi', '--query-compute-apps=pid,used_memory', '--format=csv,noheader']),
disks=disks, backup=read(LOCAL / 'hf-backup-status.json'))
encoded = json.dumps(result, ensure_ascii=False)
temporary = LOCAL / 'monitor-status.tmp'
temporary.write_text(encoded + '\n')
temporary.replace(LOCAL / 'monitor-status.json')
with (LOCAL / 'monitor-history.jsonl').open('a') as output:
output.write(encoded + '\n')
print(encoded, flush=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--watch', action='store_true')
args = parser.parse_args()
lock = (LOCAL / 'monitor.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
while True:
snapshot()
if not args.watch:
return
time.sleep(1800)
if __name__ == '__main__':
main()