File size: 1,337 Bytes
b21cfae | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | """Bounded resumable overnight supervisor; never kills or restarts live services."""
import json,os,subprocess,sys,time
from pathlib import Path
ROOT=Path(__file__).resolve().parent
def main():
statefile=ROOT/'runner.json'
if statefile.exists():
old=json.loads(statefile.read_text());pid=old.get('pid',0)
try:
command=Path('/proc',str(pid),'cmdline').read_bytes()
if b'speaker_v3.runner' in command:raise SystemExit('An owned runner is already active')
except FileNotFoundError:pass
statefile.write_text(json.dumps({'pid':os.getpid(),'started':time.time()}))
with (ROOT/'overnight.log').open('ab',buffering=0) as log:
for attempt in range(4):
if (ROOT/'STOP').exists():return
child=subprocess.Popen([sys.executable,'-u','-m','speaker_v3.overnight'],cwd=ROOT.parent,stdout=log,stderr=log)
statefile.write_text(json.dumps({'pid':os.getpid(),'child_pid':child.pid,'attempt':attempt+1,'started':time.time()}))
code=child.wait()
if code==0:return
status=json.loads((ROOT/'progress.json').read_text());error=status.get('error','')
if any(s in error for s in ('Storage','space','budget','STOP','Nonfinite')):return
time.sleep(60*(attempt+1))
if __name__=='__main__':main()
|