"""Train the matched LIBERO-FT L18 ablation and evaluate the original data plan.""" import copy import fcntl import json import os from pathlib import Path import subprocess import time import numpy as np import torch import launch_comparison as pipeline ROOT = pipeline.ROOT LOCAL = pipeline.LOCAL CONFIG = LOCAL / 'legacy-ft-l18-learned.json' def stage(name, state='running', **extra): pipeline.write(LOCAL / 'legacy-l18-status.json', dict(stage=name, state=state, pid=os.getpid(), updated_unix_time=time.time(), **extra)) def run(name, command): stage(name) with (LOCAL / 'audit' / ('legacy-l18-' + name + '.log')).open('a') as log: subprocess.run(list(map(str, command)), cwd=ROOT, env=pipeline.ENV, stdout=log, stderr=subprocess.STDOUT, check=True) def notify(message): config = json.loads((LOCAL / 'completion-watch.json').read_text()) result = subprocess.run([config['codex'], 'queue', '--thread', config['thread_id'], '--message', message, '--resume'], cwd=ROOT, capture_output=True, text=True, timeout=30) pipeline.write(LOCAL / 'legacy-l18-notification.json', dict(returncode=result.returncode, queued='Queued message' in result.stdout, resume_warning=bool(result.stderr))) def main(): with (LOCAL / 'legacy-l18-driver.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) proof = json.loads((LOCAL / 'audit/legacy-l18-runtime-validation.json').read_text()) assert proof['gradient_exact'] and proof['original_L4_prediction_exact'] config = json.loads(CONFIG.read_text()) out = Path(config['paths']['output']) if (out / 'checkpoints/latest.pt').exists() and not (out / 'completed.json').exists(): run('recover-training', [pipeline.PYTHON, 'scripts/recover_checkpoint.py', '--run-config', CONFIG]) args = [pipeline.PYTHON, '-m', 'robottt_pi05', 'train', '--run-config', CONFIG] if (out / 'checkpoints/latest.pt').exists(): args.append('--resume') run('training', args) notify('[LIBERO FT L18 자동 알림] 동일 L4 초기값·데이터·noise의 L18 학습 1,024 updates 완료. legacy-l18-status.json과 runs/legacy-ft-l18-learned를 확인하세요. 기존 driver가 원본 FT/L4/L18 offline 평가를 진행 중입니다. 중복 프로세스를 시작하지 마세요.') manifest = json.loads(Path(config['paths']['asset_index']).read_text()) plan = [dict(key=f'e{eid:06d}_n{noise}', episode=eid, noise_id=noise, mode='persistent', timesteps=list(range(16))) for eid in manifest['train_ids']+manifest['dev_ids'] for noise in range(2)] plan_path = LOCAL / 'protocol-legacy-ft-l18/evaluation-plan.json' pipeline.write(plan_path, plan) checkpoints = dict(L18=out / 'checkpoints/latest.pt', L4=LOCAL / 'legacy-l4-handoff/bundle/learned_inner_lr_v1/runs/LEARN/step_001024.pt', Original=None) scores = {} for label, checkpoint in checkpoints.items(): eval_config = copy.deepcopy(config) eval_config['original'] = checkpoint is None eval_config['paths']['experiment_config'] = str(ROOT / ('configs/l18_learned_s4.json' if label == 'L18' else 'configs/l4_learned_s4.json')) eval_config['paths']['evaluation_plan'] = str(plan_path) eval_out = LOCAL / 'evaluations' / ('legacy-ft-' + label) eval_config['paths']['output'] = str(eval_out) if checkpoint is not None: eval_config['paths']['extension_checkpoint'] = str(checkpoint) eval_config['hashes']['extension_checkpoint'] = pipeline.digest(checkpoint) eval_config['limits'] = dict(model_load=4, expert=len(plan)*160, inner=len(plan)*160*18, rollout=len(plan)*16, backward=0, optimizer_update=0) path = LOCAL / ('legacy-ft-' + label + '-eval.json') pipeline.write(path, eval_config) run('evaluate-' + label, [pipeline.PYTHON, '-m', 'robottt_pi05', 'evaluate', '--run-config', path]) by_split = {} for split, ids in [('train', manifest['train_ids']), ('dev', manifest['dev_ids'])]: values = [] for eid in ids: targets = np.load(Path(config['paths']['data_root']) / manifest['episodes'][str(eid)]['folder'] / 'targets.npy', mmap_mode='r')[:16, :, :7] episode_values = [] for noise in range(2): saved = torch.load(eval_out / f'e{eid:06d}_n{noise}/sequence.pt', map_location='cpu', weights_only=True) prediction = saved['prediction'][0, :, :, :7].numpy() episode_values.append(float(np.mean((prediction.astype(np.float64)-targets.astype(np.float64))**2))) values.append(float(np.mean(episode_values))) by_split[split] = dict(episodes=len(ids), normalized_real7_action_mse=float(np.mean(values))) scores[label] = by_split pipeline.write(LOCAL / 'legacy-l18-results.json', dict(kind='matched_LIBERO_FT_T16_S4_sequential_ablation', conditions=scores)) stage('completed', 'completed') notify('[LIBERO FT L18 자동 알림] 원본 FT·기존 L4·신규 L18의 동일 데이터 offline 평가 완료. legacy-l18-results.json과 학습 로그를 확인하고 비교 결과 및 HF 백업 상태를 사용자에게 보고하세요.') if __name__ == '__main__': try: main() except BlockingIOError: raise SystemExit('Legacy L18 driver already active') except BaseException as error: stage('failed', 'failed', error=type(error).__name__ + ': ' + str(error)) notify('[LIBERO FT L18 자동 알림] 새 L18 driver 오류. legacy-l18-status.json과 audit/legacy-l18-*.log를 확인하고 원인을 진단하세요. 수치 오류가 있으면 맹목적으로 재시도하지 마세요.') raise