File size: 3,578 Bytes
e947806
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6b3533a
e947806
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6b3533a
 
 
 
55e0940
6b3533a
 
 
 
 
 
 
 
e947806
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
"""Supervise the detached Plus driver without starting duplicate evaluators."""
import fcntl
import json
from pathlib import Path
import os
import subprocess
import time
import launch_comparison as pipeline
ROOT,LOCAL=pipeline.ROOT,pipeline.LOCAL


def read(path):return json.loads(path.read_text()) if path.exists() else {}


def running(script):
    found=[]
    for proc in Path('/proc').glob('[0-9]*/cmdline'):
        try:
            if any(arg.endswith(('/'+script).encode()) or arg==script.encode() for arg in proc.read_bytes().split(b'\0')):
                found.append(int(proc.parent.name))
        except (FileNotFoundError,PermissionError):pass
    return found


def main():
    lock=(LOCAL/'libero-plus-guard.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    missing=None
    notification=read(LOCAL/'libero-plus-monitor.json')
    while True:
        state=read(LOCAL/'libero-plus-status.json');backup=read(LOCAL/'libero-plus-backup.json')
        info=dict(pid=os.getpid(),updated_unix_time=time.time(),driver_stage=state.get('stage'))
        if backup.get('remote_readback_verified'):
            info['state']='completed_and_remote_verified';pipeline.write(LOCAL/'libero-plus-guard.json',info);return
        if state.get('stage')=='failed':
            info['state']='needs_diagnosis';info['error']=state.get('error')
        elif running('run_libero_plus.py'):
            missing=None;info['state']='monitoring'
        elif running('evaluate_libero_plus.py'):
            info['state']='waiting_for_orphan_evaluators'
        else:
            missing=missing or time.time();info['state']='waiting_to_resume'
            if time.time()-missing>=120:
                with (LOCAL/'audit/libero-plus-driver.log').open('a') as log:
                    child=subprocess.Popen([str(pipeline.PYTHON),'scripts/run_libero_plus.py'],cwd=ROOT,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
                info.update(state='restarted',driver_pid=child.pid);missing=None
        counts={label:len(list((LOCAL/'evaluations'/('libero-plus-'+label)).glob('task_*.json'))) for label in ('Original','L4','L18')}
        info['completed_episodes']=counts
        preflight_done=all(n>=7 for n in counts.values())
        first_report=preflight_done and not notification.get('preflight_reported')
        periodic=False  # User requested milestone-only agent wakeups to reduce token use.
        if 'last_notification' not in notification:notification['last_notification']=time.time()
        if (first_report or periodic) and state.get('stage')!='failed':
            c=read(LOCAL/'completion-watch.json')
            message='[LIBERO-Plus 자동 감시] '+('세 모델 7개 유형 사전 검증 완료. ' if first_report else '')+'현재 완료 횟수 '+str(counts)+'. libero-plus-status.json과 평가 기록을 확인해 진행 상황 및 KST ETA를 보고하세요. 기존 driver와 evaluator를 먼저 확인하고 중복 실행하지 마세요.'
            result=subprocess.run([c['codex'],'queue','--thread',c['thread_id'],'--message',message,'--resume'],cwd=ROOT,capture_output=True,text=True,timeout=30)
            if result.returncode==0 or 'Queued message' in result.stdout:
                notification.update(last_notification=time.time(),preflight_reported=preflight_done or notification.get('preflight_reported',False))
        pipeline.write(LOCAL/'libero-plus-monitor.json',notification)
        pipeline.write(LOCAL/'libero-plus-guard.json',info);time.sleep(30)

if __name__=='__main__':main()