File size: 2,763 Bytes
34f7b5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
"""Record experiment health every 30 minutes without retrying training updates."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import time
from datetime import datetime, timezone

ROOT = Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'


def read(path):
    try:
        return json.loads(path.read_text())
    except (OSError, ValueError):
        return None


def command(args):
    try:
        return subprocess.run(args, capture_output=True, text=True, timeout=15).stdout.strip()
    except (OSError, subprocess.TimeoutExpired) as error:
        return type(error).__name__


def snapshot():
    now = time.time()
    stages = {'driver': read(LOCAL / 'comparison-status.json'),
              'prefix': read(Path('/tmp/robottt-comparison/prefix-cache/status.json'))}
    for name in ('comparison-lite', 'comparison-reference'):
        stages[name] = read(LOCAL / 'runs' / name / 'status.json')
    for value in stages.values():
        if value and 'updated_unix_time' in value:
            value['age_seconds'] = round(now - value['updated_unix_time'])
    failures = {}
    for pattern in ('*failure*.json', 'runs/comparison-*/*failure*.json'):
        for path in LOCAL.glob(pattern):
            failures[str(path.relative_to(LOCAL))] = read(path)
    disks = {}
    for location in (ROOT, Path('/tmp')):
        stat = os.statvfs(location)
        disks[str(location)] = dict(free_bytes=stat.f_bavail * stat.f_frsize,
                                    free_inodes=stat.f_favail)
    result = dict(updated_utc=datetime.now(timezone.utc).isoformat(),
                  interval_seconds=1800, stages=stages, failures=failures,
                  gpu=command(['nvidia-smi', '--query-gpu=name,utilization.gpu,memory.used,memory.total,temperature.gpu', '--format=csv,noheader']),
                  gpu_processes=command(['nvidia-smi', '--query-compute-apps=pid,used_memory', '--format=csv,noheader']),
                  disks=disks, backup=read(LOCAL / 'hf-backup-status.json'))
    encoded = json.dumps(result, ensure_ascii=False)
    temporary = LOCAL / 'monitor-status.tmp'
    temporary.write_text(encoded + '\n')
    temporary.replace(LOCAL / 'monitor-status.json')
    with (LOCAL / 'monitor-history.jsonl').open('a') as output:
        output.write(encoded + '\n')
    print(encoded, flush=True)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--watch', action='store_true')
    args = parser.parse_args()
    lock = (LOCAL / 'monitor.lock').open('a')
    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
    while True:
        snapshot()
        if not args.watch:
            return
        time.sleep(1800)


if __name__ == '__main__':
    main()