"""Repair live-log backup without interrupting the existing training process.""" import fcntl import time import train_reference_baseline_fresh as run if __name__=='__main__': with (run.LOCAL/'backup_live.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) while True: try: run.backup_once() except Exception as exc: value=dict(error=repr(exc),time=time.time()) else: value=dict(status='ok',time=time.time(),owner='reference_backup_live') run.save(run.LOCAL/'backup_live_status.json',value) time.sleep(60)