Download run_spatial_ppo_distributed.py from Riasok/PPO-run1: direct link, hf CLI and curl.
- Browser
- Download file 27.5 kB
-
https://huggingface.co/Riasok/PPO-run1/resolve/main/run_spatial_ppo_distributed.py
- Command line
-
hf download hf://Riasok/PPO-run1/run_spatial_ppo_distributed.py
-
curl -L -o run_spatial_ppo_distributed.py https://huggingface.co/Riasok/PPO-run1/resolve/main/run_spatial_ppo_distributed.py
27.5 kB
| """Synchronous four-GPU Spatial PPO with a fixed global rollout/minibatch budget.""" | |
| from __future__ import annotations | |
| import argparse | |
| import datetime | |
| import fcntl | |
| import gc | |
| import hashlib | |
| import json | |
| import logging | |
| import multiprocessing | |
| import os | |
| from pathlib import Path | |
| import random | |
| import time | |
| import traceback | |
| # Each process sees one GPU; EGL retains the physical device index. | |
| RANK=int(os.environ.get('RANK','0')) | |
| WORLD=int(os.environ.get('WORLD_SIZE','1')) | |
| PHYSICAL_GPUS=os.environ.get('MOLMO_GPU_IDS','4,5,6,7').split(',') | |
| os.environ['CUDA_VISIBLE_DEVICES']=PHYSICAL_GPUS[RANK] | |
| os.environ['MUJOCO_EGL_DEVICE_ID']=PHYSICAL_GPUS[RANK] | |
| import torch.distributed as dist | |
| import gymnasium as gym | |
| import numpy as np | |
| import torch | |
| from lerobot.envs.libero import LiberoEnv | |
| from lerobot.envs.utils import preprocess_observation | |
| from lerobot.processor.env_processor import LiberoProcessorStep | |
| from libero.libero import benchmark | |
| from runtime.flow_ppo_pilot_v1 import ( | |
| load_sft_flow_adapter, MolmoSingleEnvCollector, MolmoChunkRollout, | |
| execute_environment_chunk, chunk_ppo_loss, | |
| ) | |
| ROOT=Path(__file__).resolve().parent | |
| CACHE=Path('/cache/minjaeoh/molmoact2') | |
| SFT=CACHE/'runs/fewshot_sft/checkpoints/001000/pretrained_model' | |
| FLOW=ROOT/'runtime/flow_ppo_pilot_v1.py' | |
| KST=datetime.timezone(datetime.timedelta(hours=9)) | |
| log=logging.getLogger('spatial_ppo') | |
| def now(): return datetime.datetime.now(KST).isoformat() | |
| def write_json(path,obj): | |
| path.parent.mkdir(parents=True,exist_ok=True) | |
| tmp=path.with_suffix(path.suffix+'.tmp');tmp.write_text(json.dumps(obj,indent=2)+'\n');tmp.replace(path) | |
| def atomic_save(path,obj): | |
| tmp=path.with_suffix(path.suffix+'.tmp');torch.save(obj,tmp);tmp.replace(path) | |
| def batch_arrays(value): | |
| if isinstance(value,dict):return {k:batch_arrays(v) for k,v in value.items()} | |
| if isinstance(value,np.ndarray):return np.expand_dims(value,0) | |
| return value | |
| def task_quotas(total): | |
| return [total//10+(i<total%10) for i in range(10)] | |
| def normalized_advantages(values): | |
| return (values-values.mean())/values.std(unbiased=False).clamp_min(1e-8) | |
| def environment_worker(connection,task,init_index): | |
| """Own EGL in a process that never initializes torch distributed.""" | |
| os.environ['CUDA_VISIBLE_DEVICES']='5' | |
| os.environ['MUJOCO_EGL_DEVICE_ID']='5' | |
| env=None | |
| try: | |
| suite=benchmark.get_benchmark_dict()['libero_spatial']() | |
| base=LiberoEnv(task_suite=suite,task_id=task,task_suite_name='libero_spatial', | |
| obs_type='pixels_agent_pos',observation_height=256,observation_width=256, | |
| camera_name_mapping={'agentview_image':'image','robot0_eye_in_hand_image':'wrist_image'}, | |
| episode_index=init_index,num_steps_wait=50,control_mode='relative',hard_reset=True,control_freq=20) | |
| env=gym.wrappers.TimeLimit(base,max_episode_steps=280) | |
| connection.send(('ok',base.task_description)) | |
| while True: | |
| command,payload=connection.recv() | |
| if command=='reset': | |
| kwargs,index=payload | |
| if index is not None:env.unwrapped.init_state_id=index | |
| result=env.reset(**kwargs) | |
| elif command=='step':result=env.step(payload) | |
| elif command=='close':break | |
| else:raise ValueError(f'Unknown environment command: {command}') | |
| connection.send(('ok',result)) | |
| except BaseException: | |
| connection.send(('error',traceback.format_exc())) | |
| raise | |
| finally: | |
| if env is not None:env.close() | |
| connection.close() | |
| class EnvironmentProxy: | |
| def __init__(self,task,init_index): | |
| context=multiprocessing.get_context('spawn') | |
| self.connection,child=context.Pipe() | |
| self.process=context.Process(target=environment_worker,args=(child,task,init_index),daemon=True) | |
| self.process.start();child.close() | |
| self.task_description=self.receive();self.init_state_id=None | |
| self.unwrapped=self | |
| def receive(self): | |
| if not self.connection.poll(180):raise RuntimeError('Environment worker response timed out') | |
| status,result=self.connection.recv() | |
| if status!='ok':raise RuntimeError(result) | |
| return result | |
| def reset(self,**kwargs): | |
| self.connection.send(('reset',(kwargs,self.init_state_id))) | |
| return self.receive() | |
| def step(self,action): | |
| self.connection.send(('step',action)) | |
| return self.receive() | |
| def close(self): | |
| if self.process.is_alive(): | |
| self.connection.send(('close',None));self.process.join(15) | |
| if self.process.is_alive():self.process.terminate();self.process.join(5) | |
| self.connection.close() | |
| class Pilot: | |
| def __init__(self,args): | |
| self.args=args;self.out=Path(args.output_dir);self.out.mkdir(parents=True,exist_ok=True) | |
| self.config={'source_sft':str(SFT),'flow_sha256':hashlib.sha256(FLOW.read_bytes()).hexdigest(), | |
| 'execution':'synchronous replicated actor/critic; NCCL gradient sum across four GPUs', | |
| 'render_gpu':5,'world_size':WORLD,'physical_gpus':PHYSICAL_GPUS,'source_checkpoint':args.source_checkpoint, | |
| 'bootstrap_rollout':args.bootstrap_rollout, | |
| 'max_updates':args.max_updates,'rollout_chunks':2048,'minibatch_chunks':128,'microbatch_chunks':1, | |
| 'actor_lr':1e-6,'critic_lr':1e-4,'clip_ratio':.2,'value_coefficient':.5, | |
| 'gamma':.99,'gae_lambda':.95,'grad_clip':1.0,'epochs':1,'target_joint_kl':.05, | |
| 'noise_std':.05,'flow_steps':10,'chunk_size':10,'seed':42, | |
| 'eval_updates':sorted(set([0,args.max_updates]+list(range(args.eval_every,args.max_updates+1,args.eval_every)))),'eval_trials_per_task':args.eval_trials,'eval_seed':1000, | |
| 'regression_stop_absolute_drop':.20,'horizon':280,'settling_steps':50, | |
| 'eval_protocol':f'{args.eval_trials*10} paired trials; final benchmark separately recorded', | |
| 'driver_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), | |
| 'resume':'Only whole PPO-update boundaries; unfinished updates recollect from the last durable checkpoint'} | |
| existing=self.out/'config.json' | |
| if RANK==0: | |
| if existing.exists() and json.loads(existing.read_text())!=self.config: | |
| raise RuntimeError('Distributed configuration differs from saved run') | |
| write_json(existing,self.config) | |
| dist.barrier() | |
| torch.manual_seed(42);np.random.seed(42);random.seed(42) | |
| torch.backends.cuda.matmul.allow_tf32=False | |
| torch.backends.cudnn.allow_tf32=False | |
| self.rng=np.random.default_rng(42+RANK) | |
| self.generator=torch.Generator(device='cuda').manual_seed(42+RANK) | |
| self.optimizer_steps=0;self.episode_counter=0;self.update=0 | |
| self.mark('loading') | |
| self.adapter,self.bridge=load_sft_flow_adapter(str(SFT),noise_std=.05) | |
| self.adapter.to('cuda').train() | |
| groups=[{'params':list(self.adapter.actor.parameters()),'lr':1e-6,'name':'actor'}, | |
| {'params':list(self.adapter.value_head.parameters()),'lr':1e-4,'name':'critic'}] | |
| self.parameters=[p for group in groups for p in group['params']] | |
| if len({id(p) for p in self.parameters})!=len(self.parameters):raise RuntimeError('Duplicate optimizer parameter') | |
| assert {id(p) for p in self.parameters}=={id(p) for p in self.adapter.parameters() if p.requires_grad} | |
| self.optimizer=torch.optim.AdamW(groups,betas=(.9,.95),eps=1e-8,weight_decay=0.0) | |
| self.suite=benchmark.get_benchmark_dict()['libero_spatial']() | |
| self.env_processor=LiberoProcessorStep() | |
| self.mark('loaded') | |
| def mark(self,phase,**fields): | |
| self.status={'updated':now(),'pid':os.getpid(),'phase':phase,'update':self.update, | |
| 'optimizer_steps':self.optimizer_steps,'gpu':os.environ.get('CUDA_VISIBLE_DEVICES'),**fields} | |
| write_json(self.out/f'status_rank{RANK}.json',{**self.status,'rank':RANK,'world_size':WORLD}) | |
| if RANK==0:write_json(self.out/'status.json',{**self.status,'rank':RANK,'world_size':WORLD,'gpus':PHYSICAL_GPUS}) | |
| log.info('%s',json.dumps(self.status)) | |
| def make_env(self,task,init_index=0): | |
| env=EnvironmentProxy(task,init_index) | |
| def adapt(raw): | |
| p=self.env_processor._process_observation(preprocess_observation(batch_arrays(raw))) | |
| return {'main_images':(p[self.bridge.image_keys[0]]*255).round().to(torch.uint8).permute(0,2,3,1), | |
| 'wrist_images':(p[self.bridge.image_keys[1]]*255).round().to(torch.uint8).permute(0,2,3,1), | |
| 'states':p['observation.state'],'task_descriptions':[env.task_description]} | |
| return env,adapt | |
| def collect(self,count=2048,validation=False): | |
| rollout=MolmoChunkRollout(capacity=count,policy_version=self.update) | |
| episodes=[];steps=0;started=time.monotonic() | |
| quotas=task_quotas(count) | |
| for task,quota in enumerate(quotas): | |
| if not quota or task%WORLD!=RANK:continue | |
| env,adapt=self.make_env(task,int(self.rng.integers(0,50))) | |
| collector=MolmoSingleEnvCollector(env,self.adapter,self.bridge,adapt,env_id=task,policy_version=self.update) | |
| task_records=0;ep_steps=0;ep_reward=0.;ep_success=False | |
| try: | |
| while task_records<quota: | |
| if collector.needs_reset: | |
| env.unwrapped.init_state_id=int(self.rng.integers(0,50)) | |
| collector.reset(seed=100000+RANK*10000000+self.episode_counter) | |
| self.episode_counter+=1;ep_steps=0;ep_reward=0.;ep_success=False | |
| result=collector.collect(rollout,generator=self.generator) | |
| n=int(result['executed'].sum());steps+=n;ep_steps+=n | |
| ep_reward+=float(result['rewards'].sum());ep_success|=bool(result['info'].get('is_success',False)) | |
| task_records+=1 | |
| if result['terminated'] or result['truncated']: | |
| episodes.append({'task':task,'length':ep_steps,'reward':ep_reward,'success':ep_success, | |
| 'terminated':result['terminated'],'truncated':result['truncated']}) | |
| log.info('episode %s',json.dumps(episodes[-1])) | |
| if len(rollout)%32==0:self.mark('validation_collect' if validation else 'collect',chunks=len(rollout),target_chunks=count,environment_steps=steps) | |
| finally:env.close() | |
| rollout.finalize(gamma=.99,gae_lambda=.95) | |
| stats={'chunks':len(rollout),'environment_steps':steps,'episodes':episodes,'seconds':time.monotonic()-started, | |
| 'storage_tensor_bytes':sum(x.numel()*x.element_size() for r in rollout._records for x in r['packed'].values()), | |
| 'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9} | |
| shard=self.out/f'rollout_rank{RANK}.pt' | |
| atomic_save(shard,rollout.state_dict()) | |
| write_json(self.out/f'rollout_rank{RANK}.json',stats) | |
| del rollout;gc.collect() | |
| dist.barrier() | |
| merged=MolmoChunkRollout(capacity=count,policy_version=self.update) | |
| all_stats=[] | |
| for rank in range(WORLD): | |
| state=torch.load(self.out/f'rollout_rank{rank}.pt',map_location='cpu',weights_only=True) | |
| if state['policy_version']!=self.update:raise RuntimeError('Stale worker rollout') | |
| # Task streams belong to exactly one rank. Finalize after merging preserves GAE. | |
| merged._records.extend(state['records']) | |
| all_stats.append(json.loads((self.out/f'rollout_rank{rank}.json').read_text())) | |
| del state | |
| if len(merged)!=count:raise RuntimeError(f'Expected {count} global chunks, got {len(merged)}') | |
| merged.finalize(gamma=.99,gae_lambda=.95) | |
| stats={'chunks':count,'environment_steps':sum(x['environment_steps'] for x in all_stats), | |
| 'episodes':[ep for x in all_stats for ep in x['episodes']], | |
| 'seconds':time.monotonic()-started,'workers':all_stats, | |
| 'storage_tensor_bytes':sum(x['storage_tensor_bytes'] for x in all_stats)} | |
| return merged,stats | |
| def checkpoint(self,update): | |
| path=self.out/'checkpoints'/f'update_{update:03d}.pt' | |
| local={'episode_counter':self.episode_counter,'generator_state':self.generator.get_state(), | |
| 'numpy_generator_state':self.rng.bit_generator.state,'torch_rng':torch.get_rng_state(), | |
| 'cuda_rng':torch.cuda.get_rng_state()} | |
| rank_rng=[None]*WORLD | |
| dist.all_gather_object(rank_rng,local) | |
| if RANK==0: | |
| path.parent.mkdir(exist_ok=True) | |
| atomic_save(path,{'config':self.config,'update':update,'actor':self.adapter.actor.state_dict(), | |
| 'value_head':self.adapter.value_head.state_dict(),'optimizer':self.optimizer.state_dict(), | |
| 'optimizer_steps':self.optimizer_steps,'rank_rng':rank_rng}) | |
| write_json(self.out/'last_checkpoint.json',{'path':str(path),'update':update,'saved':now()}) | |
| if not self.args.validate_only: | |
| keep=set(self.config['eval_updates'])|{0,update,update-1} | |
| for previous in path.parent.glob('update_*.pt'): | |
| if int(previous.stem.split('_')[1]) not in keep:previous.unlink() | |
| dist.barrier() | |
| return path | |
| def restore(self,path,legacy=False): | |
| saved=torch.load(path,map_location='cpu',weights_only=True) | |
| if legacy: | |
| keys=['source_sft','flow_sha256','actor_lr','critic_lr','clip_ratio','gamma','gae_lambda', | |
| 'noise_std','flow_steps','chunk_size','rollout_chunks','minibatch_chunks','max_updates'] | |
| for key in keys: | |
| if saved['config'][key]!=self.config[key]:raise ValueError(f'Source checkpoint differs: {key}') | |
| elif saved['config']!=self.config:raise ValueError('Distributed checkpoint configuration mismatch') | |
| self.adapter.actor.load_state_dict(saved['actor']);self.adapter.value_head.load_state_dict(saved['value_head']) | |
| self.optimizer.load_state_dict(saved['optimizer']) | |
| self.update=saved['update'];self.optimizer_steps=saved['optimizer_steps'] | |
| if legacy: | |
| self.episode_counter=saved['episode_counter'] | |
| if RANK==0: | |
| self.generator.set_state(saved['generator_state']);self.rng.bit_generator.state=saved['numpy_generator_state'] | |
| else: | |
| state=saved['rank_rng'][RANK] | |
| self.episode_counter=state['episode_counter'] | |
| self.generator.set_state(state['generator_state']);self.rng.bit_generator.state=state['numpy_generator_state'] | |
| torch.set_rng_state(state['torch_rng']);torch.cuda.set_rng_state(state['cuda_rng']) | |
| self.adapter.train();del saved;gc.collect() | |
| dist.barrier() | |
| def update_policy(self,rollout,validation=False): | |
| all_advantages=normalized_advantages(rollout._targets[0]) | |
| order=np.random.default_rng(42+self.update).permutation(len(rollout)).tolist() | |
| metrics=[];start=time.monotonic();initial_gaps=[] | |
| self.mark('validation_backward' if validation else 'optimize',chunks=len(rollout)) | |
| with torch.no_grad(): | |
| for i in order[RANK:RANK+3]: | |
| batch=rollout.batch([i],device='cuda') | |
| replay=self.adapter(batch['packed'][0],batch['trajectory']) | |
| initial_gaps.append(float((replay['logprobs']-batch['old_logprobs']).abs().max())) | |
| gap=torch.tensor(max(initial_gaps),device='cuda');dist.all_reduce(gap,op=dist.ReduceOp.MAX) | |
| if gap.item()>1e-3:raise RuntimeError(f'Worker old-policy replay mismatch: {gap.item()}') | |
| for begin in range(0,len(order),128): | |
| global_ids=order[begin:begin+128] | |
| if validation:global_ids=global_ids[:16] | |
| ids=global_ids[RANK::WORLD] | |
| self.optimizer.zero_grad(set_to_none=True) | |
| sums={'loss':0.,'policy_loss':0.,'value_loss':0.,'ratio_mean':0.,'clip_fraction':0.,'joint_kl':0.} | |
| for i in ids: | |
| batch=rollout.batch([i],device='cuda') | |
| replay=self.adapter(batch['packed'][0],batch['trajectory']) | |
| delta=replay['logprobs'].detach()-batch['old_logprobs'] | |
| if not torch.isfinite(delta).all() or delta.abs().max()>60:raise RuntimeError('Excessive/nonfinite joint density drift') | |
| loss,m=chunk_ppo_loss(replay['logprobs'],batch['old_logprobs'],all_advantages[i:i+1].to('cuda'),replay['values'],batch['returns']) | |
| (loss/len(global_ids)).backward() | |
| sums['loss']+=loss.item()/len(global_ids) | |
| for k,v in m.items():sums[k]+=v.item()/len(global_ids) | |
| sums['joint_kl']+=(delta.exp()-1-delta).mean().item()/len(global_ids) | |
| del loss,m,replay,batch | |
| values=torch.tensor(list(sums.values()),device='cuda',dtype=torch.float64) | |
| dist.all_reduce(values,op=dist.ReduceOp.SUM) | |
| sums=dict(zip(sums,values.tolist())) | |
| if sums['joint_kl']>.05 and not validation: | |
| self.optimizer.zero_grad(set_to_none=True) | |
| metrics.append({**sums,'early_stop':True});break | |
| # Every rank contributes its share of the GLOBAL mean loss. SUM gives | |
| # exactly the serial gradient; dividing again would shrink the update. | |
| for p in self.parameters: | |
| present=torch.tensor(int(p.grad is not None),device='cuda') | |
| dist.all_reduce(present,op=dist.ReduceOp.SUM) | |
| if present.item()==0:continue | |
| if p.grad is None:p.grad=torch.zeros_like(p) | |
| dist.all_reduce(p.grad,op=dist.ReduceOp.SUM) | |
| if not torch.isfinite(p.grad).all():raise RuntimeError('Nonfinite synchronized gradient') | |
| norm=torch.nn.utils.clip_grad_norm_(self.parameters,1.,error_if_nonfinite=True) | |
| self.optimizer.step();self.optimizer_steps+=1 | |
| for p in self.parameters: | |
| if not torch.isfinite(p).all():raise RuntimeError('Nonfinite updated parameter') | |
| sums.update(gradient_norm=float(norm),optimizer_steps=self.optimizer_steps) | |
| metrics.append(sums);self.mark('validation_optimizer' if validation else 'optimize',records_processed=min(begin+len(global_ids),len(order)),metrics=sums) | |
| if validation:break | |
| self.optimizer.zero_grad(set_to_none=True) | |
| self.check_replica_parity() | |
| return {'minibatches':metrics,'initial_logprob_gaps':initial_gaps,'seconds':time.monotonic()-start, | |
| 'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9,'world_size':WORLD,'replica_parity':True} | |
| def check_replica_parity(self): | |
| for p in self.parameters: | |
| reference=p.detach().clone() | |
| dist.broadcast(reference,src=0) | |
| if not torch.equal(reference,p):raise RuntimeError('Actor/critic replicas diverged') | |
| def evaluate(self,update,trials=None,label=''): | |
| existing=self.out/f'eval_{update:03d}{label}.json' | |
| trials=self.args.eval_trials if trials is None else trials | |
| if existing.exists():return json.loads(existing.read_text()) | |
| results=[];start=time.monotonic() | |
| self.mark('evaluate',eval_update=update) | |
| for task in range(10): | |
| if task%WORLD!=RANK:continue | |
| env,adapt=self.make_env(task) | |
| try: | |
| for episode in range(trials): | |
| raw,_=env.reset(seed=1000+episode) | |
| gen=torch.Generator(device='cuda').manual_seed(1000+episode) | |
| self.adapter.policy.reset() | |
| success=False;length=0 | |
| while True: | |
| actions,_=self.adapter.predict_action_batch(adapt(raw),processor_bridge=self.bridge,mode='eval',generator=gen) | |
| outcome=execute_environment_chunk(env,actions[0]);raw=outcome['observation'] | |
| length+=int(outcome['executed'].sum());success|=bool(outcome['info'].get('is_success',False)) | |
| if outcome['terminated'] or outcome['truncated']:break | |
| results.append({'task':task,'episode':episode,'seed':1000+episode,'initial_state':episode,'success':success,'length':length}) | |
| self.mark('evaluate',eval_update=update,trials=len(results),successes=sum(r['success'] for r in results)) | |
| finally:env.close() | |
| write_json(self.out/f'eval_shard_{RANK}.json',{'update':update,'label':label,'episodes':results}) | |
| dist.barrier() | |
| if RANK==0: | |
| results=[] | |
| for rank in range(WORLD): | |
| shard=json.loads((self.out/f'eval_shard_{rank}.json').read_text()) | |
| if shard['update']!=update or shard['label']!=label:raise RuntimeError('Stale evaluation shard') | |
| results.extend(shard['episodes']) | |
| results.sort(key=lambda x:(x['task'],x['episode'])) | |
| if len(results)!=trials*10:raise RuntimeError('Incomplete distributed evaluation') | |
| info={'update':update,'n_episodes':len(results),'n_success':sum(r['success'] for r in results), | |
| 'success_rate':sum(r['success'] for r in results)/len(results),'episodes':results, | |
| 'seconds':time.monotonic()-start,'finished':now(),'protocol':f'{trials*10} paired trials', | |
| 'world_size':WORLD} | |
| write_json(existing,info) | |
| dist.barrier() | |
| return json.loads(existing.read_text()) | |
| def validate(self): | |
| self.restore(self.args.source_checkpoint,legacy=True) | |
| initial=self.checkpoint(self.update) | |
| rollout,stats=self.collect(640,validation=True) | |
| if not any(ep['success'] and ep['reward']>0 and ep['terminated'] for ep in stats['episodes']): | |
| raise RuntimeError('Distributed preflight did not observe successful terminal reward') | |
| if not any(ep['length']==280 and ep['truncated'] for ep in stats['episodes']): | |
| raise RuntimeError('Distributed preflight did not observe production truncation') | |
| result=self.update_policy(rollout,validation=True) | |
| post=self.checkpoint(self.update+1) | |
| batch=rollout.batch([0],device='cuda') | |
| with torch.no_grad():expected=self.adapter(batch['packed'][0],batch['trajectory']) | |
| self.restore(post) | |
| with torch.no_grad():restored=self.adapter(batch['packed'][0],batch['trajectory']) | |
| for key in ['values','logprobs']:torch.testing.assert_close(expected[key],restored[key],rtol=0,atol=0) | |
| self.restore(initial) | |
| with torch.no_grad():replay=self.adapter(batch['packed'][0],batch['trajectory']) | |
| torch.testing.assert_close(replay['logprobs'],batch['old_logprobs'],rtol=0,atol=1e-3) | |
| self.check_replica_parity() | |
| if RANK==0: | |
| write_json(self.out/'validation.json',{'finished':now(),'config':self.config,'collection':stats, | |
| 'optimizer_validation':result,'passed':True,'world_size':WORLD, | |
| 'checkpoint_actor_critic_optimizer_roundtrip':True,'replica_parity':True}) | |
| self.mark('validated');dist.barrier() | |
| def train(self): | |
| validation=json.loads(Path(self.args.validation).read_text()) | |
| if not validation['passed'] or validation['config']!=self.config:raise RuntimeError('Matching successful preflight required') | |
| self.mark('starting_extended_training') | |
| last=self.out/'last_checkpoint.json' | |
| if self.args.resume and last.exists(): | |
| self.restore(json.loads(last.read_text())['path']) | |
| elif last.exists():raise RuntimeError('Existing pilot checkpoint; use explicit resume') | |
| else: | |
| self.restore(self.args.source_checkpoint,legacy=True) | |
| self.checkpoint(self.update) | |
| baseline=self.evaluate(0) | |
| if self.update in self.config['eval_updates'] and self.update>0: | |
| current=self.evaluate(self.update) | |
| if baseline['success_rate']-current['success_rate']>=.20: | |
| self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return | |
| for update in range(self.update+1,self.config['max_updates']+1): | |
| self.update=update-1 | |
| if self.args.bootstrap_rollout and update==int(Path(self.args.source_checkpoint).stem.split('_')[1])+1: | |
| state=torch.load(self.args.bootstrap_rollout,map_location='cpu',weights_only=True) | |
| if state['policy_version']!=self.update:raise RuntimeError('Bootstrap rollout policy version differs') | |
| rollout=MolmoChunkRollout.from_state_dict(state);del state | |
| stats={'chunks':len(rollout),'reused_single_gpu_rollout':self.args.bootstrap_rollout} | |
| if len(rollout)!=2048:raise RuntimeError('Expected 2048 bootstrap chunks') | |
| else: | |
| rollout,stats=self.collect() | |
| if RANK==0:write_json(self.out/f'rollout_{update:03d}.json',stats) | |
| result=self.update_policy(rollout) | |
| self.update=update;self.checkpoint(update) | |
| if RANK==0:write_json(self.out/f'update_{update:03d}.json',result) | |
| del rollout;gc.collect();torch.cuda.empty_cache() | |
| if update in self.config['eval_updates']: | |
| current=self.evaluate(update) | |
| if baseline['success_rate']-current['success_rate']>=.20: | |
| self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return | |
| self.evaluate(self.update,trials=50,label='_benchmark500') | |
| self.finish('complete') | |
| def finish(self,status,**fields): | |
| self.mark(status,**fields) | |
| def main(): | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument('--output-dir',required=True) | |
| parser.add_argument('--validate-only',action='store_true') | |
| parser.add_argument('--validation') | |
| parser.add_argument('--source-checkpoint',required=True) | |
| parser.add_argument('--bootstrap-rollout') | |
| parser.add_argument('--resume',action='store_true') | |
| parser.add_argument('--max-updates',type=int,default=150) | |
| parser.add_argument('--eval-every',type=int,default=10) | |
| parser.add_argument('--eval-trials',type=int,default=10) | |
| args=parser.parse_args() | |
| if min(args.max_updates,args.eval_every,args.eval_trials)<=0 or args.eval_trials>50: | |
| parser.error('Positive budgets required; evaluation trials must be <=50/task') | |
| logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s',force=True) | |
| if WORLD!=4:raise RuntimeError('This driver requires exactly four synchronous workers') | |
| torch.cuda.set_device(0) | |
| dist.init_process_group('nccl',timeout=datetime.timedelta(hours=2),device_id=torch.device('cuda',0)) | |
| lock=None | |
| if RANK==0: | |
| lock=(ROOT/'spatial_ppo_launch.lock').open('w') | |
| fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| dist.barrier() | |
| pilot=None | |
| try: | |
| pilot=Pilot(args) | |
| if args.validate_only:pilot.validate() | |
| else:pilot.train() | |
| except BaseException as error: | |
| log.exception('PPO driver failed') | |
| if pilot is not None:pilot.mark('failed',error=repr(error)) | |
| raise | |
| finally: | |
| dist.destroy_process_group() | |
| if __name__=='__main__':main() | |