Next-Reward / experiment.py
Lamapi's picture
Release Next-Reward with checkpoint and illustrated evaluation
394a40d verified
Raw History Blame Contribute Delete
4.55 kB
"""Train a model of grid transitions and evaluate policies on disjoint layouts."""
import argparse
import hashlib
import json
import random
from collections import deque
from pathlib import Path
import numpy as np
from next_reward import MOVES, NextReward
ORIGINAL=(".....",".X...",".X...","...X.","....G")
def key(board):
return hashlib.sha256("\n".join(board).encode()).hexdigest()
def generate(rng,n,wall=.20):
cells=rng.choices(".#X",weights=[1-wall-.1,wall,.1],k=n*n)
cells[rng.randrange(n*n)]="G"
return tuple("".join(cells[i:i+n]) for i in range(0,n*n,n))
def advance(b,pos,a):
y,x=pos
r,c=y+MOVES[a][0],x+MOVES[a][1]
if not (0<=r<len(b) and 0<=c<len(b[0])) or b[r][c]=="#": return pos
return r,c
def starts(b):
return [(i,j) for i,row in enumerate(b) for j,ch in enumerate(row) if ch=="."]
def train(seed):
rng=random.Random(seed)
counts=np.zeros((16,4,5),int)
sums=np.zeros(3); freq=np.zeros(3,int)
layouts=set()
for _ in range(650):
b=generate(rng,rng.choice([5,6,7]),rng.uniform(.10,.45))
if key(b)==key(ORIGINAL) or key(b) in layouts:continue
layouts.add(key(b))
places=starts(b)
if not places: continue
for _ in range(100):
p=rng.choice(places)
mask=sum(1<<i for i in range(4) if advance(b,p,i)==p)
a=rng.randrange(4)
q=advance(b,p,a)
counts[mask,a,MOVES.index((q[0]-p[0],q[1]-p[1]))]+=1
event=".GX".index(b[q[0]][q[1]])
sums[event]+=(-.02,1.,-1.)[event]
freq[event]+=1
return NextReward(counts,sums/freq),layouts
def distances(b):
goal=next((i,row.index("G")) for i,row in enumerate(b) if "G" in row)
d={goal:0}; queue=deque([goal])
while queue:
p=queue.popleft()
for a in range(4):
q=advance(b,p,a)
if q not in d and b[q[0]][q[1]]=='.':
d[q]=d[p]+1
queue.append(q)
return goal,d
def play(b,p,policy):
seen=set()
while p not in seen:
seen.add(p)
q=advance(b,p,int(policy[p]))
if b[q[0]][q[1]]=="G":return len(seen)
if b[q[0]][q[1]]=="X":return None
p=q
return None
def evaluate(model,boards):
totals=dict(maps=len(boards),reachable=0,unreachable=0,success=0,shortest=0,greedy_success=0)
for b in boards:
goal,d=distances(b)
pi=model.policy(b); greedy=np.zeros(pi.shape,int)
for p in starts(b):
# One-step Manhattan baseline with access to visible pits and walls.
def score(a):
q=advance(b,p,a)
if q==p or b[q[0]][q[1]]=="X":return float("inf")
return abs(q[0]-goal[0])+abs(q[1]-goal[1])
greedy[p]=min(range(4),key=score)
for p in starts(b):
if p not in d:
totals["unreachable"]+=1;continue
totals["reachable"]+=1
steps=play(b,p,pi)
totals["success"]+=steps is not None
totals["shortest"]+=steps==d[p]
totals["greedy_success"]+=play(b,p,greedy) is not None
return totals
def main(out):
out.mkdir(parents=True,exist_ok=True)
results=[]
trained=[train(seed) for seed in (73,101,149)]
seen=set.union(*(layouts for _,layouts in trained))
seen.add(key(ORIGINAL))
suites={"original":[ORIGINAL]}
rng=random.Random(8901)
for n in (5,8,12):
boards=[]
while len(boards)<40:
board=generate(rng,n)
if key(board) not in seen:
seen.add(key(board));boards.append(board)
suites[str(n)]=boards
for seed,(model,layouts) in zip((73,101,149),trained):
if seed==73:
model.save(out/'model.json')
restored=NextReward.load(out/'model.json')
for boards in suites.values():
for b in boards: np.testing.assert_array_equal(model.policy(b),restored.policy(b))
model=restored
result=dict(seed=seed,train_maps=len(layouts),transitions=int(model.counts.sum()),
suites={k:evaluate(model,v) for k,v in suites.items()})
results.append(result)
print(seed,result["suites"],flush=True)
(out/'evaluation.json').write_text(json.dumps(results,indent=2))
(out/'test_maps.json').write_text(json.dumps(suites,indent=1))
if __name__=="__main__":
parser=argparse.ArgumentParser()
parser.add_argument("--output",type=Path,default=Path("."))
main(parser.parse_args().output)