"""Watch the existing driver and queue milestone/error prompts to its Codex thread. This process never starts training, changes checkpoints, or retries failed jobs. """ import argparse import hashlib import json import os import pathlib import subprocess import time import fcntl ROOT = pathlib.Path(__file__).resolve().parents[1] LOCAL = ROOT / '.release_local' def read(path, default=None): try: return json.loads(path.read_text()) except (FileNotFoundError, json.JSONDecodeError): return default def write(path, value): path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + '.tmp') temporary.write_text(json.dumps(value, indent=2) + '\n') temporary.replace(path) def driver_alive(pid): if not isinstance(pid, int): return False try: proc = pathlib.Path('/proc') / str(pid) return b'resume_comparison.py' in (proc / 'cmdline').read_bytes() except FileNotFoundError: return False def events(local, now, alive=driver_alive): result = [] done = read(local / 'runs/comparison-reference/completed.json', {}) config = read(local / 'comparison-reference.json', {}) if done.get('status') == 'completed' and done.get('updates') == config.get('updates') and done.get('checkpoint', {}).get('sha256'): result.append(('reference-trained:' + done['checkpoint']['sha256'], 'reference 학습 완료. 기존 driver가 flow 검증과 LIBERO 평가를 이어 실행 중인지 확인하고 학습 loss와 KST 평가 ETA를 사용자에게 알려주세요.')) summaries = [read(local / 'evaluations' / ('libero-' + k) / 'summary.json', {}) for k in ('lite', 'reference')] plan = read(local / 'libero-eval-plan.json', {}) complete = bool(plan.get('episodes')) and all(s.get('complete') is True and s.get('attempts') == len(plan['episodes']) for s in summaries) if complete: for kind, summary in zip(('lite', 'reference'), summaries): run = read(local / 'runs' / ('comparison-' + kind) / 'completed.json', {}) identity = summary.get('identity', {}) cp = run.get('checkpoint', {}).get('sha256') rp = local / ('comparison-' + kind + '.json') if not cp or identity.get('checkpoint_sha256') != cp or not rp.is_file() or identity.get('run_config_sha256') != hashlib.sha256(rp.read_bytes()).hexdigest(): complete = False if complete: fingerprint = ':'.join(s['identity']['checkpoint_sha256'] for s in summaries) result.append(('evaluation-completed:' + fingerprint, '두 모델 LIBERO 평가 완료. scripts/summarize_comparison.py 결과를 확인하고 train/dev flow loss 및 task별/전체 성공률을 보고하세요. 최종 결과의 HF 백업도 확인하세요.')) status = read(local / 'comparison-status.json', {}) stage = status.get('stage', 'unknown') if status.get('state') == 'failed': result.append(('driver-error:' + str(status.get('updated_unix_time')), '자동 파이프라인 오류. comparison-status.json과 해당 audit 로그를 읽어 원인을 진단하고 사용자에게 알리세요. 수치 오류가 있는 학습을 맹목적으로 재시도하지 마세요.')) elif not complete and not alive(status.get('pid')): result.append(('driver-stopped:' + str(status.get('pid')), '평가 완료 전 driver가 종료됐습니다. 프로세스/로그/저장 checkpoint를 확인하고 기존 복구 절차로 안전하게 이어갈 수 있는지 진단하세요. 중복 driver를 시작하지 마세요.')) elif not complete: latest = status.get('updated_unix_time', now) training = read(local / 'runs/comparison-reference/status.json', {}) if stage == 'train-reference': latest = max(latest, training.get('updated_unix_time', 0)) if stage.startswith(('flow-', 'libero-')): paths = list((local / 'evaluations' / stage).glob('*.json')) if paths: latest = max(latest, max(p.stat().st_mtime for p in paths)) if now - latest > 1800: result.append(('driver-stalled:' + stage + ':' + str(status.get('pid')), '30분 이상 새 학습/평가 진행 기록이 없습니다. GPU·프로세스·로그·디스크 상태를 확인해 실제 정지인지 진단하고 사용자에게 알려주세요.')) return result, complete def check(local, config, send): state_path = local / 'completion-watch-state.json' state = read(state_path, {'sent': {}}) observed, complete = events(local, time.time()) for key, description in observed: if key in state['sent']: continue message = '[RoboTTT 자동 감시] ' + description + ' 작업 위치: ' + str(ROOT) + '. 기존 학습·평가 프로세스를 먼저 확인하고 방해하지 마세요.' delivery = send(config, message) record = dict(event=key, time=time.time(), **delivery) with (local / 'completion-watch-events.jsonl').open('a') as f: f.write(json.dumps(record) + '\n') if delivery['queued']: state['sent'][key] = record write(state_path, state) write(local / 'completion-watch-status.json', dict(checked_unix_time=time.time(), pid=os.getpid(), evaluation_complete=complete, observed_events=[k for k, _ in observed], sent_events=list(state['sent']))) return complete and all(k in state['sent'] for k, _ in observed) def send(config, message): try: p = subprocess.run([config['codex'], 'queue', '--thread', config['thread_id'], '--message', message, '--resume'], cwd=ROOT, capture_output=True, text=True, timeout=30) # --resume can warn but still exit 0 after successfully queueing. return dict(queued=p.returncode == 0 and 'Queued message' in p.stdout, resume_warning=bool(p.stderr.strip()), returncode=p.returncode) except (OSError, subprocess.TimeoutExpired) as error: return dict(queued=False, error=type(error).__name__) def main(): p = argparse.ArgumentParser(); p.add_argument('--watch', action='store_true'); a = p.parse_args() config = read(LOCAL / 'completion-watch.json') if not config: raise SystemExit('Completion watch config missing') with (LOCAL / 'completion-watch.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) while True: try: finished = check(LOCAL, config, send) except Exception as error: write(LOCAL / 'completion-watch-status.json', dict(checked_unix_time=time.time(), error=type(error).__name__ + ': ' + str(error))) finished = False if finished or not a.watch: return time.sleep(config.get('interval_seconds', 60)) if __name__ == '__main__': main()