Spaces:
Running
Running
Download scripts/train-macro-ppo.py from orcnd/field-orders-ai-lab: direct link, hf CLI and curl.
- Browser
- Download file 3.99 kB
-
https://huggingface.co/spaces/orcnd/field-orders-ai-lab/resolve/main/scripts/train-macro-ppo.py
- Command line
-
hf download hf://spaces/orcnd/field-orders-ai-lab/scripts/train-macro-ppo.py
-
curl -L -o train-macro-ppo.py https://huggingface.co/spaces/orcnd/field-orders-ai-lab/resolve/main/scripts/train-macro-ppo.py
3.99 kB
| import os | |
| os.environ['OPENBLAS_NUM_THREADS']='4';os.environ['OMP_NUM_THREADS']='4' | |
| import argparse,base64,json,subprocess,time,sys | |
| from pathlib import Path | |
| import numpy as np | |
| from lib.numpy_policy import Dense,macro_probabilities,macro_log_prob,macro_ppo_gradient,gae | |
| root=Path(__file__).resolve().parent.parent | |
| parser=argparse.ArgumentParser();parser.add_argument('--updates',type=int,default=24);parser.add_argument('--seed',type=int,default=83641);parser.add_argument('--out',default='.local/battle-v3/macro-ppo');args=parser.parse_args() | |
| out=root/args.out;out.mkdir(parents=True,exist_ok=True);rng=np.random.default_rng(args.seed);base=json.loads((root/'.local/battle-v3/commander-macro-bc.json').read_text());actor=Dense(base);anchor=Dense(base);critic=Dense(sizes=[116,64,32,1],mean=actor.mean,std=actor.std,rng=rng) | |
| n=4;horizon=12;process=subprocess.Popen(['node','scripts/macro-rl-env.mjs',f'envs={n}',f'seed={args.seed+10000}'],cwd=root,stdin=subprocess.PIPE,stdout=subprocess.PIPE,text=True,encoding='utf-8',bufsize=1) | |
| def exchange(d): | |
| process.stdin.write(json.dumps(d,separators=(',',':'))+'\n');process.stdin.flush();r=json.loads(process.stdout.readline()) | |
| if 'error' in r:raise RuntimeError(r['error']) | |
| r['obs']=np.frombuffer(base64.b64decode(r['obs']),dtype='<f4').copy().reshape(n,116);return r | |
| r=exchange({'type':'reset'});report={'algorithm':'hybrid categorical/Gaussian PPO + GAE','seed':args.seed,'macroTransitions':0,'updates':[],'episodes':[],'initialization':'offline macro demonstrations','macroStepSeconds':2.5,'soldier':'frozen soldier-ppo-v2'};started=time.time() | |
| try: | |
| for update in range(args.updates): | |
| xs=[];cs=[];zs=[];ms=[];logs=[];rs=[];vs=[];ds=[] | |
| for step in range(horizon): | |
| x=r['obs'];m=np.array(r['masks']);raw=actor.forward(x);prob=macro_probabilities(raw,m);cat=np.array([[rng.choice(p.shape[1],p=p[i]/p[i].sum()) for p in prob] for i in range(n)]);z=raw[:,36:38]+rng.normal(0,.3,size=(n,2));log=macro_log_prob(raw,cat,z,m) | |
| actions=[{'command':int(c[0]),'formation':int(c[1]),'strategy':int(c[2]),'group':int(c[3]),'task':int(c[4]),'ttl':[2.5,5,10][c[5]],'x':float(30+(np.tanh(q[0])+1)*520),'y':float(47+(np.tanh(q[1])+1)*285.5)} for c,q in zip(cat,z)] | |
| xs.append(x);cs.append(cat);zs.append(z);ms.append(m);logs.append(log);vs.append(critic.forward(x)[:,0]);r=exchange({'type':'step','actions':actions});rs.append(r['rewards']);ds.append(r['dones']);report['episodes']+=r['summaries'] | |
| adv,ret=gae(np.array(rs,dtype='f'),np.array(vs),critic.forward(r['obs'])[:,0],np.array(ds),gamma=.97,lam=.95);x=np.array(xs).reshape(-1,116);cat=np.array(cs).reshape(-1,6);z=np.array(zs).reshape(-1,2);mask=np.array(ms).reshape(-1,9);oldlog=np.array(logs).reshape(-1);a=adv.reshape(-1);a=(a-a.mean())/(a.std()+1e-8);returns=ret.reshape(-1);target=anchor.forward(x);kls=[] | |
| for epoch in range(6): | |
| ids=rng.permutation(len(x));raw,cache=actor.forward(x[ids],True);grad,kl=macro_ppo_gradient(raw,cat[ids],z[ids],mask[ids],oldlog[ids],a[ids]);grad+=.004*(raw-target[ids])/len(ids);actor.update(actor.gradients(cache,grad),1e-4);value,vc=critic.forward(x[ids],True);critic.update(critic.gradients(vc,np.clip(value-returns[ids,None],-5,5)/len(ids)),.001);kls.append(kl) | |
| report['macroTransitions']+=n*horizon;entry={'update':update+1,'macroTransitions':report['macroTransitions'],'reward':float(np.mean(rs)),'KL':float(np.mean(kls)),'seconds':round(time.time()-started,1)};report['updates'].append(entry);report['seconds']=time.time()-started | |
| if(update+1)%8==0 or update+1==args.updates: | |
| actor.export(out/f'commander-ppo-{update+1:03d}.json','commander-macro-ppo-v3',{'algorithm':'PPO','updates':update+1,'macroTransitions':report['macroTransitions'],'seed':args.seed,'gamma':.97,'gaeLambda':.95,'initialBCSamples':2880});(out/'training-report.json').write_text(json.dumps(report,indent=2)) | |
| print(json.dumps(entry),flush=True) | |
| finally: | |
| process.stdin.write('{"type":"close"}\n');process.stdin.flush();process.stdin.close();process.wait(timeout=30) | |