jisue's picture
Experiment backup 2026-11-11T06:43:55.654465+00:00
6e6c22f verified
Raw History Blame Contribute Delete
6.03 kB
"""Train the matched LIBERO-FT L18 ablation and evaluate the original data plan."""
import copy
import fcntl
import json
import os
from pathlib import Path
import subprocess
import time
import numpy as np
import torch
import launch_comparison as pipeline
ROOT = pipeline.ROOT
LOCAL = pipeline.LOCAL
CONFIG = LOCAL / 'legacy-ft-l18-learned.json'
def stage(name, state='running', **extra):
pipeline.write(LOCAL / 'legacy-l18-status.json', dict(stage=name, state=state, pid=os.getpid(), updated_unix_time=time.time(), **extra))
def run(name, command):
stage(name)
with (LOCAL / 'audit' / ('legacy-l18-' + name + '.log')).open('a') as log:
subprocess.run(list(map(str, command)), cwd=ROOT, env=pipeline.ENV, stdout=log, stderr=subprocess.STDOUT, check=True)
def notify(message):
config = json.loads((LOCAL / 'completion-watch.json').read_text())
result = subprocess.run([config['codex'], 'queue', '--thread', config['thread_id'], '--message', message, '--resume'],
cwd=ROOT, capture_output=True, text=True, timeout=30)
pipeline.write(LOCAL / 'legacy-l18-notification.json', dict(returncode=result.returncode, queued='Queued message' in result.stdout, resume_warning=bool(result.stderr)))
def main():
with (LOCAL / 'legacy-l18-driver.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
proof = json.loads((LOCAL / 'audit/legacy-l18-runtime-validation.json').read_text())
assert proof['gradient_exact'] and proof['original_L4_prediction_exact']
config = json.loads(CONFIG.read_text())
out = Path(config['paths']['output'])
if (out / 'checkpoints/latest.pt').exists() and not (out / 'completed.json').exists():
run('recover-training', [pipeline.PYTHON, 'scripts/recover_checkpoint.py', '--run-config', CONFIG])
args = [pipeline.PYTHON, '-m', 'robottt_pi05', 'train', '--run-config', CONFIG]
if (out / 'checkpoints/latest.pt').exists():
args.append('--resume')
run('training', args)
notify('[LIBERO FT L18 자동 알림] 동일 L4 초기값·데이터·noise의 L18 학습 1,024 updates 완료. legacy-l18-status.json과 runs/legacy-ft-l18-learned를 확인하세요. 기존 driver가 원본 FT/L4/L18 offline 평가를 진행 중입니다. 중복 프로세스를 시작하지 마세요.')
manifest = json.loads(Path(config['paths']['asset_index']).read_text())
plan = [dict(key=f'e{eid:06d}_n{noise}', episode=eid, noise_id=noise, mode='persistent', timesteps=list(range(16)))
for eid in manifest['train_ids']+manifest['dev_ids'] for noise in range(2)]
plan_path = LOCAL / 'protocol-legacy-ft-l18/evaluation-plan.json'
pipeline.write(plan_path, plan)
checkpoints = dict(L18=out / 'checkpoints/latest.pt',
L4=LOCAL / 'legacy-l4-handoff/bundle/learned_inner_lr_v1/runs/LEARN/step_001024.pt',
Original=None)
scores = {}
for label, checkpoint in checkpoints.items():
eval_config = copy.deepcopy(config)
eval_config['original'] = checkpoint is None
eval_config['paths']['experiment_config'] = str(ROOT / ('configs/l18_learned_s4.json' if label == 'L18' else 'configs/l4_learned_s4.json'))
eval_config['paths']['evaluation_plan'] = str(plan_path)
eval_out = LOCAL / 'evaluations' / ('legacy-ft-' + label)
eval_config['paths']['output'] = str(eval_out)
if checkpoint is not None:
eval_config['paths']['extension_checkpoint'] = str(checkpoint)
eval_config['hashes']['extension_checkpoint'] = pipeline.digest(checkpoint)
eval_config['limits'] = dict(model_load=4, expert=len(plan)*160, inner=len(plan)*160*18, rollout=len(plan)*16, backward=0, optimizer_update=0)
path = LOCAL / ('legacy-ft-' + label + '-eval.json')
pipeline.write(path, eval_config)
run('evaluate-' + label, [pipeline.PYTHON, '-m', 'robottt_pi05', 'evaluate', '--run-config', path])
by_split = {}
for split, ids in [('train', manifest['train_ids']), ('dev', manifest['dev_ids'])]:
values = []
for eid in ids:
targets = np.load(Path(config['paths']['data_root']) / manifest['episodes'][str(eid)]['folder'] / 'targets.npy', mmap_mode='r')[:16, :, :7]
episode_values = []
for noise in range(2):
saved = torch.load(eval_out / f'e{eid:06d}_n{noise}/sequence.pt', map_location='cpu', weights_only=True)
prediction = saved['prediction'][0, :, :, :7].numpy()
episode_values.append(float(np.mean((prediction.astype(np.float64)-targets.astype(np.float64))**2)))
values.append(float(np.mean(episode_values)))
by_split[split] = dict(episodes=len(ids), normalized_real7_action_mse=float(np.mean(values)))
scores[label] = by_split
pipeline.write(LOCAL / 'legacy-l18-results.json', dict(kind='matched_LIBERO_FT_T16_S4_sequential_ablation', conditions=scores))
stage('completed', 'completed')
notify('[LIBERO FT L18 자동 알림] 원본 FT·기존 L4·신규 L18의 동일 데이터 offline 평가 완료. legacy-l18-results.json과 학습 로그를 확인하고 비교 결과 및 HF 백업 상태를 사용자에게 보고하세요.')
if __name__ == '__main__':
try:
main()
except BlockingIOError:
raise SystemExit('Legacy L18 driver already active')
except BaseException as error:
stage('failed', 'failed', error=type(error).__name__ + ': ' + str(error))
notify('[LIBERO FT L18 자동 알림] 새 L18 driver 오류. legacy-l18-status.json과 audit/legacy-l18-*.log를 확인하고 원인을 진단하세요. 수치 오류가 있으면 맹목적으로 재시도하지 마세요.')
raise