File size: 3,578 Bytes
e947806 6b3533a e947806 6b3533a 55e0940 6b3533a e947806 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """Supervise the detached Plus driver without starting duplicate evaluators."""
import fcntl
import json
from pathlib import Path
import os
import subprocess
import time
import launch_comparison as pipeline
ROOT,LOCAL=pipeline.ROOT,pipeline.LOCAL
def read(path):return json.loads(path.read_text()) if path.exists() else {}
def running(script):
found=[]
for proc in Path('/proc').glob('[0-9]*/cmdline'):
try:
if any(arg.endswith(('/'+script).encode()) or arg==script.encode() for arg in proc.read_bytes().split(b'\0')):
found.append(int(proc.parent.name))
except (FileNotFoundError,PermissionError):pass
return found
def main():
lock=(LOCAL/'libero-plus-guard.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
missing=None
notification=read(LOCAL/'libero-plus-monitor.json')
while True:
state=read(LOCAL/'libero-plus-status.json');backup=read(LOCAL/'libero-plus-backup.json')
info=dict(pid=os.getpid(),updated_unix_time=time.time(),driver_stage=state.get('stage'))
if backup.get('remote_readback_verified'):
info['state']='completed_and_remote_verified';pipeline.write(LOCAL/'libero-plus-guard.json',info);return
if state.get('stage')=='failed':
info['state']='needs_diagnosis';info['error']=state.get('error')
elif running('run_libero_plus.py'):
missing=None;info['state']='monitoring'
elif running('evaluate_libero_plus.py'):
info['state']='waiting_for_orphan_evaluators'
else:
missing=missing or time.time();info['state']='waiting_to_resume'
if time.time()-missing>=120:
with (LOCAL/'audit/libero-plus-driver.log').open('a') as log:
child=subprocess.Popen([str(pipeline.PYTHON),'scripts/run_libero_plus.py'],cwd=ROOT,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
info.update(state='restarted',driver_pid=child.pid);missing=None
counts={label:len(list((LOCAL/'evaluations'/('libero-plus-'+label)).glob('task_*.json'))) for label in ('Original','L4','L18')}
info['completed_episodes']=counts
preflight_done=all(n>=7 for n in counts.values())
first_report=preflight_done and not notification.get('preflight_reported')
periodic=False # User requested milestone-only agent wakeups to reduce token use.
if 'last_notification' not in notification:notification['last_notification']=time.time()
if (first_report or periodic) and state.get('stage')!='failed':
c=read(LOCAL/'completion-watch.json')
message='[LIBERO-Plus 자동 감시] '+('세 모델 7개 유형 사전 검증 완료. ' if first_report else '')+'현재 완료 횟수 '+str(counts)+'. libero-plus-status.json과 평가 기록을 확인해 진행 상황 및 KST ETA를 보고하세요. 기존 driver와 evaluator를 먼저 확인하고 중복 실행하지 마세요.'
result=subprocess.run([c['codex'],'queue','--thread',c['thread_id'],'--message',message,'--resume'],cwd=ROOT,capture_output=True,text=True,timeout=30)
if result.returncode==0 or 'Queued message' in result.stdout:
notification.update(last_notification=time.time(),preflight_reported=preflight_done or notification.get('preflight_reported',False))
pipeline.write(LOCAL/'libero-plus-monitor.json',notification)
pipeline.write(LOCAL/'libero-plus-guard.json',info);time.sleep(30)
if __name__=='__main__':main()
|