robottt-pi05-base-comparison / code /scripts /watch_comparison.py
jisue's picture
Automate completion and error wakeups to the existing Codex session
e2d338b verified
Raw History Blame Contribute Delete
7.02 kB
"""Watch the existing driver and queue milestone/error prompts to its Codex thread.
This process never starts training, changes checkpoints, or retries failed jobs.
"""
import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import time
import fcntl
ROOT = pathlib.Path(__file__).resolve().parents[1]
LOCAL = ROOT / '.release_local'
def read(path, default=None):
try:
return json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError):
return default
def write(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + '.tmp')
temporary.write_text(json.dumps(value, indent=2) + '\n')
temporary.replace(path)
def driver_alive(pid):
if not isinstance(pid, int):
return False
try:
proc = pathlib.Path('/proc') / str(pid)
return b'resume_comparison.py' in (proc / 'cmdline').read_bytes()
except FileNotFoundError:
return False
def events(local, now, alive=driver_alive):
result = []
done = read(local / 'runs/comparison-reference/completed.json', {})
config = read(local / 'comparison-reference.json', {})
if done.get('status') == 'completed' and done.get('updates') == config.get('updates') and done.get('checkpoint', {}).get('sha256'):
result.append(('reference-trained:' + done['checkpoint']['sha256'],
'reference 학습 완료. 기존 driver가 flow 검증과 LIBERO 평가를 이어 실행 중인지 확인하고 학습 loss와 KST 평가 ETA를 사용자에게 알려주세요.'))
summaries = [read(local / 'evaluations' / ('libero-' + k) / 'summary.json', {}) for k in ('lite', 'reference')]
plan = read(local / 'libero-eval-plan.json', {})
complete = bool(plan.get('episodes')) and all(s.get('complete') is True and s.get('attempts') == len(plan['episodes']) for s in summaries)
if complete:
for kind, summary in zip(('lite', 'reference'), summaries):
run = read(local / 'runs' / ('comparison-' + kind) / 'completed.json', {})
identity = summary.get('identity', {})
cp = run.get('checkpoint', {}).get('sha256')
rp = local / ('comparison-' + kind + '.json')
if not cp or identity.get('checkpoint_sha256') != cp or not rp.is_file() or identity.get('run_config_sha256') != hashlib.sha256(rp.read_bytes()).hexdigest():
complete = False
if complete:
fingerprint = ':'.join(s['identity']['checkpoint_sha256'] for s in summaries)
result.append(('evaluation-completed:' + fingerprint,
'두 모델 LIBERO 평가 완료. scripts/summarize_comparison.py 결과를 확인하고 train/dev flow loss 및 task별/전체 성공률을 보고하세요. 최종 결과의 HF 백업도 확인하세요.'))
status = read(local / 'comparison-status.json', {})
stage = status.get('stage', 'unknown')
if status.get('state') == 'failed':
result.append(('driver-error:' + str(status.get('updated_unix_time')),
'자동 파이프라인 오류. comparison-status.json과 해당 audit 로그를 읽어 원인을 진단하고 사용자에게 알리세요. 수치 오류가 있는 학습을 맹목적으로 재시도하지 마세요.'))
elif not complete and not alive(status.get('pid')):
result.append(('driver-stopped:' + str(status.get('pid')),
'평가 완료 전 driver가 종료됐습니다. 프로세스/로그/저장 checkpoint를 확인하고 기존 복구 절차로 안전하게 이어갈 수 있는지 진단하세요. 중복 driver를 시작하지 마세요.'))
elif not complete:
latest = status.get('updated_unix_time', now)
training = read(local / 'runs/comparison-reference/status.json', {})
if stage == 'train-reference':
latest = max(latest, training.get('updated_unix_time', 0))
if stage.startswith(('flow-', 'libero-')):
paths = list((local / 'evaluations' / stage).glob('*.json'))
if paths:
latest = max(latest, max(p.stat().st_mtime for p in paths))
if now - latest > 1800:
result.append(('driver-stalled:' + stage + ':' + str(status.get('pid')),
'30분 이상 새 학습/평가 진행 기록이 없습니다. GPU·프로세스·로그·디스크 상태를 확인해 실제 정지인지 진단하고 사용자에게 알려주세요.'))
return result, complete
def check(local, config, send):
state_path = local / 'completion-watch-state.json'
state = read(state_path, {'sent': {}})
observed, complete = events(local, time.time())
for key, description in observed:
if key in state['sent']:
continue
message = '[RoboTTT 자동 감시] ' + description + ' 작업 위치: ' + str(ROOT) + '. 기존 학습·평가 프로세스를 먼저 확인하고 방해하지 마세요.'
delivery = send(config, message)
record = dict(event=key, time=time.time(), **delivery)
with (local / 'completion-watch-events.jsonl').open('a') as f:
f.write(json.dumps(record) + '\n')
if delivery['queued']:
state['sent'][key] = record
write(state_path, state)
write(local / 'completion-watch-status.json', dict(checked_unix_time=time.time(), pid=os.getpid(), evaluation_complete=complete, observed_events=[k for k, _ in observed], sent_events=list(state['sent'])))
return complete and all(k in state['sent'] for k, _ in observed)
def send(config, message):
try:
p = subprocess.run([config['codex'], 'queue', '--thread', config['thread_id'], '--message', message, '--resume'], cwd=ROOT, capture_output=True, text=True, timeout=30)
# --resume can warn but still exit 0 after successfully queueing.
return dict(queued=p.returncode == 0 and 'Queued message' in p.stdout,
resume_warning=bool(p.stderr.strip()), returncode=p.returncode)
except (OSError, subprocess.TimeoutExpired) as error:
return dict(queued=False, error=type(error).__name__)
def main():
p = argparse.ArgumentParser(); p.add_argument('--watch', action='store_true'); a = p.parse_args()
config = read(LOCAL / 'completion-watch.json')
if not config:
raise SystemExit('Completion watch config missing')
with (LOCAL / 'completion-watch.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
while True:
try:
finished = check(LOCAL, config, send)
except Exception as error:
write(LOCAL / 'completion-watch-status.json', dict(checked_unix_time=time.time(), error=type(error).__name__ + ': ' + str(error)))
finished = False
if finished or not a.watch:
return
time.sleep(config.get('interval_seconds', 60))
if __name__ == '__main__':
main()