"""Synchronous four-GPU Spatial PPO with a fixed global rollout/minibatch budget.""" from __future__ import annotations import argparse import datetime import fcntl import gc import hashlib import json import logging import multiprocessing import os from pathlib import Path import random import time import traceback # Each process sees one GPU; EGL retains the physical device index. RANK=int(os.environ.get('RANK','0')) WORLD=int(os.environ.get('WORLD_SIZE','1')) PHYSICAL_GPUS=os.environ.get('MOLMO_GPU_IDS','4,5,6,7').split(',') os.environ['CUDA_VISIBLE_DEVICES']=PHYSICAL_GPUS[RANK] os.environ['MUJOCO_EGL_DEVICE_ID']=PHYSICAL_GPUS[RANK] import torch.distributed as dist import gymnasium as gym import numpy as np import torch from lerobot.envs.libero import LiberoEnv from lerobot.envs.utils import preprocess_observation from lerobot.processor.env_processor import LiberoProcessorStep from libero.libero import benchmark from runtime.flow_ppo_pilot_v1 import ( load_sft_flow_adapter, MolmoSingleEnvCollector, MolmoChunkRollout, execute_environment_chunk, chunk_ppo_loss, ) ROOT=Path(__file__).resolve().parent CACHE=Path('/cache/minjaeoh/molmoact2') SFT=CACHE/'runs/fewshot_sft/checkpoints/001000/pretrained_model' FLOW=ROOT/'runtime/flow_ppo_pilot_v1.py' KST=datetime.timezone(datetime.timedelta(hours=9)) log=logging.getLogger('spatial_ppo') def now(): return datetime.datetime.now(KST).isoformat() def write_json(path,obj): path.parent.mkdir(parents=True,exist_ok=True) tmp=path.with_suffix(path.suffix+'.tmp');tmp.write_text(json.dumps(obj,indent=2)+'\n');tmp.replace(path) def atomic_save(path,obj): tmp=path.with_suffix(path.suffix+'.tmp');torch.save(obj,tmp);tmp.replace(path) def batch_arrays(value): if isinstance(value,dict):return {k:batch_arrays(v) for k,v in value.items()} if isinstance(value,np.ndarray):return np.expand_dims(value,0) return value def task_quotas(total): return [total//10+(i1e-3:raise RuntimeError(f'Worker old-policy replay mismatch: {gap.item()}') for begin in range(0,len(order),128): global_ids=order[begin:begin+128] if validation:global_ids=global_ids[:16] ids=global_ids[RANK::WORLD] self.optimizer.zero_grad(set_to_none=True) sums={'loss':0.,'policy_loss':0.,'value_loss':0.,'ratio_mean':0.,'clip_fraction':0.,'joint_kl':0.} for i in ids: batch=rollout.batch([i],device='cuda') replay=self.adapter(batch['packed'][0],batch['trajectory']) delta=replay['logprobs'].detach()-batch['old_logprobs'] if not torch.isfinite(delta).all() or delta.abs().max()>60:raise RuntimeError('Excessive/nonfinite joint density drift') loss,m=chunk_ppo_loss(replay['logprobs'],batch['old_logprobs'],all_advantages[i:i+1].to('cuda'),replay['values'],batch['returns']) (loss/len(global_ids)).backward() sums['loss']+=loss.item()/len(global_ids) for k,v in m.items():sums[k]+=v.item()/len(global_ids) sums['joint_kl']+=(delta.exp()-1-delta).mean().item()/len(global_ids) del loss,m,replay,batch values=torch.tensor(list(sums.values()),device='cuda',dtype=torch.float64) dist.all_reduce(values,op=dist.ReduceOp.SUM) sums=dict(zip(sums,values.tolist())) if sums['joint_kl']>.05 and not validation: self.optimizer.zero_grad(set_to_none=True) metrics.append({**sums,'early_stop':True});break # Every rank contributes its share of the GLOBAL mean loss. SUM gives # exactly the serial gradient; dividing again would shrink the update. for p in self.parameters: present=torch.tensor(int(p.grad is not None),device='cuda') dist.all_reduce(present,op=dist.ReduceOp.SUM) if present.item()==0:continue if p.grad is None:p.grad=torch.zeros_like(p) dist.all_reduce(p.grad,op=dist.ReduceOp.SUM) if not torch.isfinite(p.grad).all():raise RuntimeError('Nonfinite synchronized gradient') norm=torch.nn.utils.clip_grad_norm_(self.parameters,1.,error_if_nonfinite=True) self.optimizer.step();self.optimizer_steps+=1 for p in self.parameters: if not torch.isfinite(p).all():raise RuntimeError('Nonfinite updated parameter') sums.update(gradient_norm=float(norm),optimizer_steps=self.optimizer_steps) metrics.append(sums);self.mark('validation_optimizer' if validation else 'optimize',records_processed=min(begin+len(global_ids),len(order)),metrics=sums) if validation:break self.optimizer.zero_grad(set_to_none=True) self.check_replica_parity() return {'minibatches':metrics,'initial_logprob_gaps':initial_gaps,'seconds':time.monotonic()-start, 'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9,'world_size':WORLD,'replica_parity':True} @torch.no_grad() def check_replica_parity(self): for p in self.parameters: reference=p.detach().clone() dist.broadcast(reference,src=0) if not torch.equal(reference,p):raise RuntimeError('Actor/critic replicas diverged') @torch.no_grad() def evaluate(self,update,trials=None,label=''): existing=self.out/f'eval_{update:03d}{label}.json' trials=self.args.eval_trials if trials is None else trials if existing.exists():return json.loads(existing.read_text()) results=[];start=time.monotonic() self.mark('evaluate',eval_update=update) for task in range(10): if task%WORLD!=RANK:continue env,adapt=self.make_env(task) try: for episode in range(trials): raw,_=env.reset(seed=1000+episode) gen=torch.Generator(device='cuda').manual_seed(1000+episode) self.adapter.policy.reset() success=False;length=0 while True: actions,_=self.adapter.predict_action_batch(adapt(raw),processor_bridge=self.bridge,mode='eval',generator=gen) outcome=execute_environment_chunk(env,actions[0]);raw=outcome['observation'] length+=int(outcome['executed'].sum());success|=bool(outcome['info'].get('is_success',False)) if outcome['terminated'] or outcome['truncated']:break results.append({'task':task,'episode':episode,'seed':1000+episode,'initial_state':episode,'success':success,'length':length}) self.mark('evaluate',eval_update=update,trials=len(results),successes=sum(r['success'] for r in results)) finally:env.close() write_json(self.out/f'eval_shard_{RANK}.json',{'update':update,'label':label,'episodes':results}) dist.barrier() if RANK==0: results=[] for rank in range(WORLD): shard=json.loads((self.out/f'eval_shard_{rank}.json').read_text()) if shard['update']!=update or shard['label']!=label:raise RuntimeError('Stale evaluation shard') results.extend(shard['episodes']) results.sort(key=lambda x:(x['task'],x['episode'])) if len(results)!=trials*10:raise RuntimeError('Incomplete distributed evaluation') info={'update':update,'n_episodes':len(results),'n_success':sum(r['success'] for r in results), 'success_rate':sum(r['success'] for r in results)/len(results),'episodes':results, 'seconds':time.monotonic()-start,'finished':now(),'protocol':f'{trials*10} paired trials', 'world_size':WORLD} write_json(existing,info) dist.barrier() return json.loads(existing.read_text()) def validate(self): self.restore(self.args.source_checkpoint,legacy=True) initial=self.checkpoint(self.update) rollout,stats=self.collect(640,validation=True) if not any(ep['success'] and ep['reward']>0 and ep['terminated'] for ep in stats['episodes']): raise RuntimeError('Distributed preflight did not observe successful terminal reward') if not any(ep['length']==280 and ep['truncated'] for ep in stats['episodes']): raise RuntimeError('Distributed preflight did not observe production truncation') result=self.update_policy(rollout,validation=True) post=self.checkpoint(self.update+1) batch=rollout.batch([0],device='cuda') with torch.no_grad():expected=self.adapter(batch['packed'][0],batch['trajectory']) self.restore(post) with torch.no_grad():restored=self.adapter(batch['packed'][0],batch['trajectory']) for key in ['values','logprobs']:torch.testing.assert_close(expected[key],restored[key],rtol=0,atol=0) self.restore(initial) with torch.no_grad():replay=self.adapter(batch['packed'][0],batch['trajectory']) torch.testing.assert_close(replay['logprobs'],batch['old_logprobs'],rtol=0,atol=1e-3) self.check_replica_parity() if RANK==0: write_json(self.out/'validation.json',{'finished':now(),'config':self.config,'collection':stats, 'optimizer_validation':result,'passed':True,'world_size':WORLD, 'checkpoint_actor_critic_optimizer_roundtrip':True,'replica_parity':True}) self.mark('validated');dist.barrier() def train(self): validation=json.loads(Path(self.args.validation).read_text()) if not validation['passed'] or validation['config']!=self.config:raise RuntimeError('Matching successful preflight required') self.mark('starting_extended_training') last=self.out/'last_checkpoint.json' if self.args.resume and last.exists(): self.restore(json.loads(last.read_text())['path']) elif last.exists():raise RuntimeError('Existing pilot checkpoint; use explicit resume') else: self.restore(self.args.source_checkpoint,legacy=True) self.checkpoint(self.update) baseline=self.evaluate(0) if self.update in self.config['eval_updates'] and self.update>0: current=self.evaluate(self.update) if baseline['success_rate']-current['success_rate']>=.20: self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return for update in range(self.update+1,self.config['max_updates']+1): self.update=update-1 if self.args.bootstrap_rollout and update==int(Path(self.args.source_checkpoint).stem.split('_')[1])+1: state=torch.load(self.args.bootstrap_rollout,map_location='cpu',weights_only=True) if state['policy_version']!=self.update:raise RuntimeError('Bootstrap rollout policy version differs') rollout=MolmoChunkRollout.from_state_dict(state);del state stats={'chunks':len(rollout),'reused_single_gpu_rollout':self.args.bootstrap_rollout} if len(rollout)!=2048:raise RuntimeError('Expected 2048 bootstrap chunks') else: rollout,stats=self.collect() if RANK==0:write_json(self.out/f'rollout_{update:03d}.json',stats) result=self.update_policy(rollout) self.update=update;self.checkpoint(update) if RANK==0:write_json(self.out/f'update_{update:03d}.json',result) del rollout;gc.collect();torch.cuda.empty_cache() if update in self.config['eval_updates']: current=self.evaluate(update) if baseline['success_rate']-current['success_rate']>=.20: self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return self.evaluate(self.update,trials=50,label='_benchmark500') self.finish('complete') def finish(self,status,**fields): self.mark(status,**fields) def main(): parser=argparse.ArgumentParser() parser.add_argument('--output-dir',required=True) parser.add_argument('--validate-only',action='store_true') parser.add_argument('--validation') parser.add_argument('--source-checkpoint',required=True) parser.add_argument('--bootstrap-rollout') parser.add_argument('--resume',action='store_true') parser.add_argument('--max-updates',type=int,default=150) parser.add_argument('--eval-every',type=int,default=10) parser.add_argument('--eval-trials',type=int,default=10) args=parser.parse_args() if min(args.max_updates,args.eval_every,args.eval_trials)<=0 or args.eval_trials>50: parser.error('Positive budgets required; evaluation trials must be <=50/task') logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s',force=True) if WORLD!=4:raise RuntimeError('This driver requires exactly four synchronous workers') torch.cuda.set_device(0) dist.init_process_group('nccl',timeout=datetime.timedelta(hours=2),device_id=torch.device('cuda',0)) lock=None if RANK==0: lock=(ROOT/'spatial_ppo_launch.lock').open('w') fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) dist.barrier() pilot=None try: pilot=Pilot(args) if args.validate_only:pilot.validate() else:pilot.train() except BaseException as error: log.exception('PPO driver failed') if pilot is not None:pilot.mark('failed',error=repr(error)) raise finally: dist.destroy_process_group() if __name__=='__main__':main()