Download code/scripts/watch_comparison.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 7.02 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/watch_comparison.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/watch_comparison.py
-
curl -L -o watch_comparison.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/watch_comparison.py
7.02 kB
| """Watch the existing driver and queue milestone/error prompts to its Codex thread. | |
| This process never starts training, changes checkpoints, or retries failed jobs. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| import pathlib | |
| import subprocess | |
| import time | |
| import fcntl | |
| ROOT = pathlib.Path(__file__).resolve().parents[1] | |
| LOCAL = ROOT / '.release_local' | |
| def read(path, default=None): | |
| try: | |
| return json.loads(path.read_text()) | |
| except (FileNotFoundError, json.JSONDecodeError): | |
| return default | |
| def write(path, value): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_name(path.name + '.tmp') | |
| temporary.write_text(json.dumps(value, indent=2) + '\n') | |
| temporary.replace(path) | |
| def driver_alive(pid): | |
| if not isinstance(pid, int): | |
| return False | |
| try: | |
| proc = pathlib.Path('/proc') / str(pid) | |
| return b'resume_comparison.py' in (proc / 'cmdline').read_bytes() | |
| except FileNotFoundError: | |
| return False | |
| def events(local, now, alive=driver_alive): | |
| result = [] | |
| done = read(local / 'runs/comparison-reference/completed.json', {}) | |
| config = read(local / 'comparison-reference.json', {}) | |
| if done.get('status') == 'completed' and done.get('updates') == config.get('updates') and done.get('checkpoint', {}).get('sha256'): | |
| result.append(('reference-trained:' + done['checkpoint']['sha256'], | |
| 'reference 학습 완료. 기존 driver가 flow 검증과 LIBERO 평가를 이어 실행 중인지 확인하고 학습 loss와 KST 평가 ETA를 사용자에게 알려주세요.')) | |
| summaries = [read(local / 'evaluations' / ('libero-' + k) / 'summary.json', {}) for k in ('lite', 'reference')] | |
| plan = read(local / 'libero-eval-plan.json', {}) | |
| complete = bool(plan.get('episodes')) and all(s.get('complete') is True and s.get('attempts') == len(plan['episodes']) for s in summaries) | |
| if complete: | |
| for kind, summary in zip(('lite', 'reference'), summaries): | |
| run = read(local / 'runs' / ('comparison-' + kind) / 'completed.json', {}) | |
| identity = summary.get('identity', {}) | |
| cp = run.get('checkpoint', {}).get('sha256') | |
| rp = local / ('comparison-' + kind + '.json') | |
| if not cp or identity.get('checkpoint_sha256') != cp or not rp.is_file() or identity.get('run_config_sha256') != hashlib.sha256(rp.read_bytes()).hexdigest(): | |
| complete = False | |
| if complete: | |
| fingerprint = ':'.join(s['identity']['checkpoint_sha256'] for s in summaries) | |
| result.append(('evaluation-completed:' + fingerprint, | |
| '두 모델 LIBERO 평가 완료. scripts/summarize_comparison.py 결과를 확인하고 train/dev flow loss 및 task별/전체 성공률을 보고하세요. 최종 결과의 HF 백업도 확인하세요.')) | |
| status = read(local / 'comparison-status.json', {}) | |
| stage = status.get('stage', 'unknown') | |
| if status.get('state') == 'failed': | |
| result.append(('driver-error:' + str(status.get('updated_unix_time')), | |
| '자동 파이프라인 오류. comparison-status.json과 해당 audit 로그를 읽어 원인을 진단하고 사용자에게 알리세요. 수치 오류가 있는 학습을 맹목적으로 재시도하지 마세요.')) | |
| elif not complete and not alive(status.get('pid')): | |
| result.append(('driver-stopped:' + str(status.get('pid')), | |
| '평가 완료 전 driver가 종료됐습니다. 프로세스/로그/저장 checkpoint를 확인하고 기존 복구 절차로 안전하게 이어갈 수 있는지 진단하세요. 중복 driver를 시작하지 마세요.')) | |
| elif not complete: | |
| latest = status.get('updated_unix_time', now) | |
| training = read(local / 'runs/comparison-reference/status.json', {}) | |
| if stage == 'train-reference': | |
| latest = max(latest, training.get('updated_unix_time', 0)) | |
| if stage.startswith(('flow-', 'libero-')): | |
| paths = list((local / 'evaluations' / stage).glob('*.json')) | |
| if paths: | |
| latest = max(latest, max(p.stat().st_mtime for p in paths)) | |
| if now - latest > 1800: | |
| result.append(('driver-stalled:' + stage + ':' + str(status.get('pid')), | |
| '30분 이상 새 학습/평가 진행 기록이 없습니다. GPU·프로세스·로그·디스크 상태를 확인해 실제 정지인지 진단하고 사용자에게 알려주세요.')) | |
| return result, complete | |
| def check(local, config, send): | |
| state_path = local / 'completion-watch-state.json' | |
| state = read(state_path, {'sent': {}}) | |
| observed, complete = events(local, time.time()) | |
| for key, description in observed: | |
| if key in state['sent']: | |
| continue | |
| message = '[RoboTTT 자동 감시] ' + description + ' 작업 위치: ' + str(ROOT) + '. 기존 학습·평가 프로세스를 먼저 확인하고 방해하지 마세요.' | |
| delivery = send(config, message) | |
| record = dict(event=key, time=time.time(), **delivery) | |
| with (local / 'completion-watch-events.jsonl').open('a') as f: | |
| f.write(json.dumps(record) + '\n') | |
| if delivery['queued']: | |
| state['sent'][key] = record | |
| write(state_path, state) | |
| write(local / 'completion-watch-status.json', dict(checked_unix_time=time.time(), pid=os.getpid(), evaluation_complete=complete, observed_events=[k for k, _ in observed], sent_events=list(state['sent']))) | |
| return complete and all(k in state['sent'] for k, _ in observed) | |
| def send(config, message): | |
| try: | |
| p = subprocess.run([config['codex'], 'queue', '--thread', config['thread_id'], '--message', message, '--resume'], cwd=ROOT, capture_output=True, text=True, timeout=30) | |
| # --resume can warn but still exit 0 after successfully queueing. | |
| return dict(queued=p.returncode == 0 and 'Queued message' in p.stdout, | |
| resume_warning=bool(p.stderr.strip()), returncode=p.returncode) | |
| except (OSError, subprocess.TimeoutExpired) as error: | |
| return dict(queued=False, error=type(error).__name__) | |
| def main(): | |
| p = argparse.ArgumentParser(); p.add_argument('--watch', action='store_true'); a = p.parse_args() | |
| config = read(LOCAL / 'completion-watch.json') | |
| if not config: | |
| raise SystemExit('Completion watch config missing') | |
| with (LOCAL / 'completion-watch.lock').open('a') as lock: | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| while True: | |
| try: | |
| finished = check(LOCAL, config, send) | |
| except Exception as error: | |
| write(LOCAL / 'completion-watch-status.json', dict(checked_unix_time=time.time(), error=type(error).__name__ + ': ' + str(error))) | |
| finished = False | |
| if finished or not a.watch: | |
| return | |
| time.sleep(config.get('interval_seconds', 60)) | |
| if __name__ == '__main__': | |
| main() | |