PPO-run1 / run_spatial_ppo_distributed.py
Riasok's picture
Upload original MolmoAct2 checkpoint, processors and provenance
362f4f0 verified
Raw History Blame Contribute Delete
27.5 kB
"""Synchronous four-GPU Spatial PPO with a fixed global rollout/minibatch budget."""
from __future__ import annotations
import argparse
import datetime
import fcntl
import gc
import hashlib
import json
import logging
import multiprocessing
import os
from pathlib import Path
import random
import time
import traceback
# Each process sees one GPU; EGL retains the physical device index.
RANK=int(os.environ.get('RANK','0'))
WORLD=int(os.environ.get('WORLD_SIZE','1'))
PHYSICAL_GPUS=os.environ.get('MOLMO_GPU_IDS','4,5,6,7').split(',')
os.environ['CUDA_VISIBLE_DEVICES']=PHYSICAL_GPUS[RANK]
os.environ['MUJOCO_EGL_DEVICE_ID']=PHYSICAL_GPUS[RANK]
import torch.distributed as dist
import gymnasium as gym
import numpy as np
import torch
from lerobot.envs.libero import LiberoEnv
from lerobot.envs.utils import preprocess_observation
from lerobot.processor.env_processor import LiberoProcessorStep
from libero.libero import benchmark
from runtime.flow_ppo_pilot_v1 import (
load_sft_flow_adapter, MolmoSingleEnvCollector, MolmoChunkRollout,
execute_environment_chunk, chunk_ppo_loss,
)
ROOT=Path(__file__).resolve().parent
CACHE=Path('/cache/minjaeoh/molmoact2')
SFT=CACHE/'runs/fewshot_sft/checkpoints/001000/pretrained_model'
FLOW=ROOT/'runtime/flow_ppo_pilot_v1.py'
KST=datetime.timezone(datetime.timedelta(hours=9))
log=logging.getLogger('spatial_ppo')
def now(): return datetime.datetime.now(KST).isoformat()
def write_json(path,obj):
path.parent.mkdir(parents=True,exist_ok=True)
tmp=path.with_suffix(path.suffix+'.tmp');tmp.write_text(json.dumps(obj,indent=2)+'\n');tmp.replace(path)
def atomic_save(path,obj):
tmp=path.with_suffix(path.suffix+'.tmp');torch.save(obj,tmp);tmp.replace(path)
def batch_arrays(value):
if isinstance(value,dict):return {k:batch_arrays(v) for k,v in value.items()}
if isinstance(value,np.ndarray):return np.expand_dims(value,0)
return value
def task_quotas(total):
return [total//10+(i<total%10) for i in range(10)]
def normalized_advantages(values):
return (values-values.mean())/values.std(unbiased=False).clamp_min(1e-8)
def environment_worker(connection,task,init_index):
"""Own EGL in a process that never initializes torch distributed."""
os.environ['CUDA_VISIBLE_DEVICES']='5'
os.environ['MUJOCO_EGL_DEVICE_ID']='5'
env=None
try:
suite=benchmark.get_benchmark_dict()['libero_spatial']()
base=LiberoEnv(task_suite=suite,task_id=task,task_suite_name='libero_spatial',
obs_type='pixels_agent_pos',observation_height=256,observation_width=256,
camera_name_mapping={'agentview_image':'image','robot0_eye_in_hand_image':'wrist_image'},
episode_index=init_index,num_steps_wait=50,control_mode='relative',hard_reset=True,control_freq=20)
env=gym.wrappers.TimeLimit(base,max_episode_steps=280)
connection.send(('ok',base.task_description))
while True:
command,payload=connection.recv()
if command=='reset':
kwargs,index=payload
if index is not None:env.unwrapped.init_state_id=index
result=env.reset(**kwargs)
elif command=='step':result=env.step(payload)
elif command=='close':break
else:raise ValueError(f'Unknown environment command: {command}')
connection.send(('ok',result))
except BaseException:
connection.send(('error',traceback.format_exc()))
raise
finally:
if env is not None:env.close()
connection.close()
class EnvironmentProxy:
def __init__(self,task,init_index):
context=multiprocessing.get_context('spawn')
self.connection,child=context.Pipe()
self.process=context.Process(target=environment_worker,args=(child,task,init_index),daemon=True)
self.process.start();child.close()
self.task_description=self.receive();self.init_state_id=None
self.unwrapped=self
def receive(self):
if not self.connection.poll(180):raise RuntimeError('Environment worker response timed out')
status,result=self.connection.recv()
if status!='ok':raise RuntimeError(result)
return result
def reset(self,**kwargs):
self.connection.send(('reset',(kwargs,self.init_state_id)))
return self.receive()
def step(self,action):
self.connection.send(('step',action))
return self.receive()
def close(self):
if self.process.is_alive():
self.connection.send(('close',None));self.process.join(15)
if self.process.is_alive():self.process.terminate();self.process.join(5)
self.connection.close()
class Pilot:
def __init__(self,args):
self.args=args;self.out=Path(args.output_dir);self.out.mkdir(parents=True,exist_ok=True)
self.config={'source_sft':str(SFT),'flow_sha256':hashlib.sha256(FLOW.read_bytes()).hexdigest(),
'execution':'synchronous replicated actor/critic; NCCL gradient sum across four GPUs',
'render_gpu':5,'world_size':WORLD,'physical_gpus':PHYSICAL_GPUS,'source_checkpoint':args.source_checkpoint,
'bootstrap_rollout':args.bootstrap_rollout,
'max_updates':args.max_updates,'rollout_chunks':2048,'minibatch_chunks':128,'microbatch_chunks':1,
'actor_lr':1e-6,'critic_lr':1e-4,'clip_ratio':.2,'value_coefficient':.5,
'gamma':.99,'gae_lambda':.95,'grad_clip':1.0,'epochs':1,'target_joint_kl':.05,
'noise_std':.05,'flow_steps':10,'chunk_size':10,'seed':42,
'eval_updates':sorted(set([0,args.max_updates]+list(range(args.eval_every,args.max_updates+1,args.eval_every)))),'eval_trials_per_task':args.eval_trials,'eval_seed':1000,
'regression_stop_absolute_drop':.20,'horizon':280,'settling_steps':50,
'eval_protocol':f'{args.eval_trials*10} paired trials; final benchmark separately recorded',
'driver_sha256':hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'resume':'Only whole PPO-update boundaries; unfinished updates recollect from the last durable checkpoint'}
existing=self.out/'config.json'
if RANK==0:
if existing.exists() and json.loads(existing.read_text())!=self.config:
raise RuntimeError('Distributed configuration differs from saved run')
write_json(existing,self.config)
dist.barrier()
torch.manual_seed(42);np.random.seed(42);random.seed(42)
torch.backends.cuda.matmul.allow_tf32=False
torch.backends.cudnn.allow_tf32=False
self.rng=np.random.default_rng(42+RANK)
self.generator=torch.Generator(device='cuda').manual_seed(42+RANK)
self.optimizer_steps=0;self.episode_counter=0;self.update=0
self.mark('loading')
self.adapter,self.bridge=load_sft_flow_adapter(str(SFT),noise_std=.05)
self.adapter.to('cuda').train()
groups=[{'params':list(self.adapter.actor.parameters()),'lr':1e-6,'name':'actor'},
{'params':list(self.adapter.value_head.parameters()),'lr':1e-4,'name':'critic'}]
self.parameters=[p for group in groups for p in group['params']]
if len({id(p) for p in self.parameters})!=len(self.parameters):raise RuntimeError('Duplicate optimizer parameter')
assert {id(p) for p in self.parameters}=={id(p) for p in self.adapter.parameters() if p.requires_grad}
self.optimizer=torch.optim.AdamW(groups,betas=(.9,.95),eps=1e-8,weight_decay=0.0)
self.suite=benchmark.get_benchmark_dict()['libero_spatial']()
self.env_processor=LiberoProcessorStep()
self.mark('loaded')
def mark(self,phase,**fields):
self.status={'updated':now(),'pid':os.getpid(),'phase':phase,'update':self.update,
'optimizer_steps':self.optimizer_steps,'gpu':os.environ.get('CUDA_VISIBLE_DEVICES'),**fields}
write_json(self.out/f'status_rank{RANK}.json',{**self.status,'rank':RANK,'world_size':WORLD})
if RANK==0:write_json(self.out/'status.json',{**self.status,'rank':RANK,'world_size':WORLD,'gpus':PHYSICAL_GPUS})
log.info('%s',json.dumps(self.status))
def make_env(self,task,init_index=0):
env=EnvironmentProxy(task,init_index)
def adapt(raw):
p=self.env_processor._process_observation(preprocess_observation(batch_arrays(raw)))
return {'main_images':(p[self.bridge.image_keys[0]]*255).round().to(torch.uint8).permute(0,2,3,1),
'wrist_images':(p[self.bridge.image_keys[1]]*255).round().to(torch.uint8).permute(0,2,3,1),
'states':p['observation.state'],'task_descriptions':[env.task_description]}
return env,adapt
def collect(self,count=2048,validation=False):
rollout=MolmoChunkRollout(capacity=count,policy_version=self.update)
episodes=[];steps=0;started=time.monotonic()
quotas=task_quotas(count)
for task,quota in enumerate(quotas):
if not quota or task%WORLD!=RANK:continue
env,adapt=self.make_env(task,int(self.rng.integers(0,50)))
collector=MolmoSingleEnvCollector(env,self.adapter,self.bridge,adapt,env_id=task,policy_version=self.update)
task_records=0;ep_steps=0;ep_reward=0.;ep_success=False
try:
while task_records<quota:
if collector.needs_reset:
env.unwrapped.init_state_id=int(self.rng.integers(0,50))
collector.reset(seed=100000+RANK*10000000+self.episode_counter)
self.episode_counter+=1;ep_steps=0;ep_reward=0.;ep_success=False
result=collector.collect(rollout,generator=self.generator)
n=int(result['executed'].sum());steps+=n;ep_steps+=n
ep_reward+=float(result['rewards'].sum());ep_success|=bool(result['info'].get('is_success',False))
task_records+=1
if result['terminated'] or result['truncated']:
episodes.append({'task':task,'length':ep_steps,'reward':ep_reward,'success':ep_success,
'terminated':result['terminated'],'truncated':result['truncated']})
log.info('episode %s',json.dumps(episodes[-1]))
if len(rollout)%32==0:self.mark('validation_collect' if validation else 'collect',chunks=len(rollout),target_chunks=count,environment_steps=steps)
finally:env.close()
rollout.finalize(gamma=.99,gae_lambda=.95)
stats={'chunks':len(rollout),'environment_steps':steps,'episodes':episodes,'seconds':time.monotonic()-started,
'storage_tensor_bytes':sum(x.numel()*x.element_size() for r in rollout._records for x in r['packed'].values()),
'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9}
shard=self.out/f'rollout_rank{RANK}.pt'
atomic_save(shard,rollout.state_dict())
write_json(self.out/f'rollout_rank{RANK}.json',stats)
del rollout;gc.collect()
dist.barrier()
merged=MolmoChunkRollout(capacity=count,policy_version=self.update)
all_stats=[]
for rank in range(WORLD):
state=torch.load(self.out/f'rollout_rank{rank}.pt',map_location='cpu',weights_only=True)
if state['policy_version']!=self.update:raise RuntimeError('Stale worker rollout')
# Task streams belong to exactly one rank. Finalize after merging preserves GAE.
merged._records.extend(state['records'])
all_stats.append(json.loads((self.out/f'rollout_rank{rank}.json').read_text()))
del state
if len(merged)!=count:raise RuntimeError(f'Expected {count} global chunks, got {len(merged)}')
merged.finalize(gamma=.99,gae_lambda=.95)
stats={'chunks':count,'environment_steps':sum(x['environment_steps'] for x in all_stats),
'episodes':[ep for x in all_stats for ep in x['episodes']],
'seconds':time.monotonic()-started,'workers':all_stats,
'storage_tensor_bytes':sum(x['storage_tensor_bytes'] for x in all_stats)}
return merged,stats
def checkpoint(self,update):
path=self.out/'checkpoints'/f'update_{update:03d}.pt'
local={'episode_counter':self.episode_counter,'generator_state':self.generator.get_state(),
'numpy_generator_state':self.rng.bit_generator.state,'torch_rng':torch.get_rng_state(),
'cuda_rng':torch.cuda.get_rng_state()}
rank_rng=[None]*WORLD
dist.all_gather_object(rank_rng,local)
if RANK==0:
path.parent.mkdir(exist_ok=True)
atomic_save(path,{'config':self.config,'update':update,'actor':self.adapter.actor.state_dict(),
'value_head':self.adapter.value_head.state_dict(),'optimizer':self.optimizer.state_dict(),
'optimizer_steps':self.optimizer_steps,'rank_rng':rank_rng})
write_json(self.out/'last_checkpoint.json',{'path':str(path),'update':update,'saved':now()})
if not self.args.validate_only:
keep=set(self.config['eval_updates'])|{0,update,update-1}
for previous in path.parent.glob('update_*.pt'):
if int(previous.stem.split('_')[1]) not in keep:previous.unlink()
dist.barrier()
return path
def restore(self,path,legacy=False):
saved=torch.load(path,map_location='cpu',weights_only=True)
if legacy:
keys=['source_sft','flow_sha256','actor_lr','critic_lr','clip_ratio','gamma','gae_lambda',
'noise_std','flow_steps','chunk_size','rollout_chunks','minibatch_chunks','max_updates']
for key in keys:
if saved['config'][key]!=self.config[key]:raise ValueError(f'Source checkpoint differs: {key}')
elif saved['config']!=self.config:raise ValueError('Distributed checkpoint configuration mismatch')
self.adapter.actor.load_state_dict(saved['actor']);self.adapter.value_head.load_state_dict(saved['value_head'])
self.optimizer.load_state_dict(saved['optimizer'])
self.update=saved['update'];self.optimizer_steps=saved['optimizer_steps']
if legacy:
self.episode_counter=saved['episode_counter']
if RANK==0:
self.generator.set_state(saved['generator_state']);self.rng.bit_generator.state=saved['numpy_generator_state']
else:
state=saved['rank_rng'][RANK]
self.episode_counter=state['episode_counter']
self.generator.set_state(state['generator_state']);self.rng.bit_generator.state=state['numpy_generator_state']
torch.set_rng_state(state['torch_rng']);torch.cuda.set_rng_state(state['cuda_rng'])
self.adapter.train();del saved;gc.collect()
dist.barrier()
def update_policy(self,rollout,validation=False):
all_advantages=normalized_advantages(rollout._targets[0])
order=np.random.default_rng(42+self.update).permutation(len(rollout)).tolist()
metrics=[];start=time.monotonic();initial_gaps=[]
self.mark('validation_backward' if validation else 'optimize',chunks=len(rollout))
with torch.no_grad():
for i in order[RANK:RANK+3]:
batch=rollout.batch([i],device='cuda')
replay=self.adapter(batch['packed'][0],batch['trajectory'])
initial_gaps.append(float((replay['logprobs']-batch['old_logprobs']).abs().max()))
gap=torch.tensor(max(initial_gaps),device='cuda');dist.all_reduce(gap,op=dist.ReduceOp.MAX)
if gap.item()>1e-3:raise RuntimeError(f'Worker old-policy replay mismatch: {gap.item()}')
for begin in range(0,len(order),128):
global_ids=order[begin:begin+128]
if validation:global_ids=global_ids[:16]
ids=global_ids[RANK::WORLD]
self.optimizer.zero_grad(set_to_none=True)
sums={'loss':0.,'policy_loss':0.,'value_loss':0.,'ratio_mean':0.,'clip_fraction':0.,'joint_kl':0.}
for i in ids:
batch=rollout.batch([i],device='cuda')
replay=self.adapter(batch['packed'][0],batch['trajectory'])
delta=replay['logprobs'].detach()-batch['old_logprobs']
if not torch.isfinite(delta).all() or delta.abs().max()>60:raise RuntimeError('Excessive/nonfinite joint density drift')
loss,m=chunk_ppo_loss(replay['logprobs'],batch['old_logprobs'],all_advantages[i:i+1].to('cuda'),replay['values'],batch['returns'])
(loss/len(global_ids)).backward()
sums['loss']+=loss.item()/len(global_ids)
for k,v in m.items():sums[k]+=v.item()/len(global_ids)
sums['joint_kl']+=(delta.exp()-1-delta).mean().item()/len(global_ids)
del loss,m,replay,batch
values=torch.tensor(list(sums.values()),device='cuda',dtype=torch.float64)
dist.all_reduce(values,op=dist.ReduceOp.SUM)
sums=dict(zip(sums,values.tolist()))
if sums['joint_kl']>.05 and not validation:
self.optimizer.zero_grad(set_to_none=True)
metrics.append({**sums,'early_stop':True});break
# Every rank contributes its share of the GLOBAL mean loss. SUM gives
# exactly the serial gradient; dividing again would shrink the update.
for p in self.parameters:
present=torch.tensor(int(p.grad is not None),device='cuda')
dist.all_reduce(present,op=dist.ReduceOp.SUM)
if present.item()==0:continue
if p.grad is None:p.grad=torch.zeros_like(p)
dist.all_reduce(p.grad,op=dist.ReduceOp.SUM)
if not torch.isfinite(p.grad).all():raise RuntimeError('Nonfinite synchronized gradient')
norm=torch.nn.utils.clip_grad_norm_(self.parameters,1.,error_if_nonfinite=True)
self.optimizer.step();self.optimizer_steps+=1
for p in self.parameters:
if not torch.isfinite(p).all():raise RuntimeError('Nonfinite updated parameter')
sums.update(gradient_norm=float(norm),optimizer_steps=self.optimizer_steps)
metrics.append(sums);self.mark('validation_optimizer' if validation else 'optimize',records_processed=min(begin+len(global_ids),len(order)),metrics=sums)
if validation:break
self.optimizer.zero_grad(set_to_none=True)
self.check_replica_parity()
return {'minibatches':metrics,'initial_logprob_gaps':initial_gaps,'seconds':time.monotonic()-start,
'peak_gpu_gb':torch.cuda.max_memory_allocated()/1e9,'world_size':WORLD,'replica_parity':True}
@torch.no_grad()
def check_replica_parity(self):
for p in self.parameters:
reference=p.detach().clone()
dist.broadcast(reference,src=0)
if not torch.equal(reference,p):raise RuntimeError('Actor/critic replicas diverged')
@torch.no_grad()
def evaluate(self,update,trials=None,label=''):
existing=self.out/f'eval_{update:03d}{label}.json'
trials=self.args.eval_trials if trials is None else trials
if existing.exists():return json.loads(existing.read_text())
results=[];start=time.monotonic()
self.mark('evaluate',eval_update=update)
for task in range(10):
if task%WORLD!=RANK:continue
env,adapt=self.make_env(task)
try:
for episode in range(trials):
raw,_=env.reset(seed=1000+episode)
gen=torch.Generator(device='cuda').manual_seed(1000+episode)
self.adapter.policy.reset()
success=False;length=0
while True:
actions,_=self.adapter.predict_action_batch(adapt(raw),processor_bridge=self.bridge,mode='eval',generator=gen)
outcome=execute_environment_chunk(env,actions[0]);raw=outcome['observation']
length+=int(outcome['executed'].sum());success|=bool(outcome['info'].get('is_success',False))
if outcome['terminated'] or outcome['truncated']:break
results.append({'task':task,'episode':episode,'seed':1000+episode,'initial_state':episode,'success':success,'length':length})
self.mark('evaluate',eval_update=update,trials=len(results),successes=sum(r['success'] for r in results))
finally:env.close()
write_json(self.out/f'eval_shard_{RANK}.json',{'update':update,'label':label,'episodes':results})
dist.barrier()
if RANK==0:
results=[]
for rank in range(WORLD):
shard=json.loads((self.out/f'eval_shard_{rank}.json').read_text())
if shard['update']!=update or shard['label']!=label:raise RuntimeError('Stale evaluation shard')
results.extend(shard['episodes'])
results.sort(key=lambda x:(x['task'],x['episode']))
if len(results)!=trials*10:raise RuntimeError('Incomplete distributed evaluation')
info={'update':update,'n_episodes':len(results),'n_success':sum(r['success'] for r in results),
'success_rate':sum(r['success'] for r in results)/len(results),'episodes':results,
'seconds':time.monotonic()-start,'finished':now(),'protocol':f'{trials*10} paired trials',
'world_size':WORLD}
write_json(existing,info)
dist.barrier()
return json.loads(existing.read_text())
def validate(self):
self.restore(self.args.source_checkpoint,legacy=True)
initial=self.checkpoint(self.update)
rollout,stats=self.collect(640,validation=True)
if not any(ep['success'] and ep['reward']>0 and ep['terminated'] for ep in stats['episodes']):
raise RuntimeError('Distributed preflight did not observe successful terminal reward')
if not any(ep['length']==280 and ep['truncated'] for ep in stats['episodes']):
raise RuntimeError('Distributed preflight did not observe production truncation')
result=self.update_policy(rollout,validation=True)
post=self.checkpoint(self.update+1)
batch=rollout.batch([0],device='cuda')
with torch.no_grad():expected=self.adapter(batch['packed'][0],batch['trajectory'])
self.restore(post)
with torch.no_grad():restored=self.adapter(batch['packed'][0],batch['trajectory'])
for key in ['values','logprobs']:torch.testing.assert_close(expected[key],restored[key],rtol=0,atol=0)
self.restore(initial)
with torch.no_grad():replay=self.adapter(batch['packed'][0],batch['trajectory'])
torch.testing.assert_close(replay['logprobs'],batch['old_logprobs'],rtol=0,atol=1e-3)
self.check_replica_parity()
if RANK==0:
write_json(self.out/'validation.json',{'finished':now(),'config':self.config,'collection':stats,
'optimizer_validation':result,'passed':True,'world_size':WORLD,
'checkpoint_actor_critic_optimizer_roundtrip':True,'replica_parity':True})
self.mark('validated');dist.barrier()
def train(self):
validation=json.loads(Path(self.args.validation).read_text())
if not validation['passed'] or validation['config']!=self.config:raise RuntimeError('Matching successful preflight required')
self.mark('starting_extended_training')
last=self.out/'last_checkpoint.json'
if self.args.resume and last.exists():
self.restore(json.loads(last.read_text())['path'])
elif last.exists():raise RuntimeError('Existing pilot checkpoint; use explicit resume')
else:
self.restore(self.args.source_checkpoint,legacy=True)
self.checkpoint(self.update)
baseline=self.evaluate(0)
if self.update in self.config['eval_updates'] and self.update>0:
current=self.evaluate(self.update)
if baseline['success_rate']-current['success_rate']>=.20:
self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return
for update in range(self.update+1,self.config['max_updates']+1):
self.update=update-1
if self.args.bootstrap_rollout and update==int(Path(self.args.source_checkpoint).stem.split('_')[1])+1:
state=torch.load(self.args.bootstrap_rollout,map_location='cpu',weights_only=True)
if state['policy_version']!=self.update:raise RuntimeError('Bootstrap rollout policy version differs')
rollout=MolmoChunkRollout.from_state_dict(state);del state
stats={'chunks':len(rollout),'reused_single_gpu_rollout':self.args.bootstrap_rollout}
if len(rollout)!=2048:raise RuntimeError('Expected 2048 bootstrap chunks')
else:
rollout,stats=self.collect()
if RANK==0:write_json(self.out/f'rollout_{update:03d}.json',stats)
result=self.update_policy(rollout)
self.update=update;self.checkpoint(update)
if RANK==0:write_json(self.out/f'update_{update:03d}.json',result)
del rollout;gc.collect();torch.cuda.empty_cache()
if update in self.config['eval_updates']:
current=self.evaluate(update)
if baseline['success_rate']-current['success_rate']>=.20:
self.finish('stopped_on_regression',baseline=baseline['success_rate'],current=current['success_rate']);return
self.evaluate(self.update,trials=50,label='_benchmark500')
self.finish('complete')
def finish(self,status,**fields):
self.mark(status,**fields)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--output-dir',required=True)
parser.add_argument('--validate-only',action='store_true')
parser.add_argument('--validation')
parser.add_argument('--source-checkpoint',required=True)
parser.add_argument('--bootstrap-rollout')
parser.add_argument('--resume',action='store_true')
parser.add_argument('--max-updates',type=int,default=150)
parser.add_argument('--eval-every',type=int,default=10)
parser.add_argument('--eval-trials',type=int,default=10)
args=parser.parse_args()
if min(args.max_updates,args.eval_every,args.eval_trials)<=0 or args.eval_trials>50:
parser.error('Positive budgets required; evaluation trials must be <=50/task')
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s',force=True)
if WORLD!=4:raise RuntimeError('This driver requires exactly four synchronous workers')
torch.cuda.set_device(0)
dist.init_process_group('nccl',timeout=datetime.timedelta(hours=2),device_id=torch.device('cuda',0))
lock=None
if RANK==0:
lock=(ROOT/'spatial_ppo_launch.lock').open('w')
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
dist.barrier()
pilot=None
try:
pilot=Pilot(args)
if args.validate_only:pilot.validate()
else:pilot.train()
except BaseException as error:
log.exception('PPO driver failed')
if pilot is not None:pilot.mark('failed',error=repr(error))
raise
finally:
dist.destroy_process_group()
if __name__=='__main__':main()