File size: 655 Bytes
13cc860
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
"""Repair live-log backup without interrupting the existing training process."""
import fcntl
import time
import train_reference_baseline_fresh as run

if __name__=='__main__':
    with (run.LOCAL/'backup_live.lock').open('a') as lock:
        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
        while True:
            try:
                run.backup_once()
            except Exception as exc:
                value=dict(error=repr(exc),time=time.time())
            else:
                value=dict(status='ok',time=time.time(),owner='reference_backup_live')
            run.save(run.LOCAL/'backup_live_status.json',value)
            time.sleep(60)