Download experiment.py from thelamapi/Next-Reward: direct link, hf CLI and curl.
- Browser
- Download file 4.55 kB
-
https://huggingface.co/thelamapi/Next-Reward/resolve/main/experiment.py
- Command line
-
hf download hf://thelamapi/Next-Reward/experiment.py
-
curl -L -o experiment.py https://huggingface.co/thelamapi/Next-Reward/resolve/main/experiment.py
4.55 kB
| """Train a model of grid transitions and evaluate policies on disjoint layouts.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| import random | |
| from collections import deque | |
| from pathlib import Path | |
| import numpy as np | |
| from next_reward import MOVES, NextReward | |
| ORIGINAL=(".....",".X...",".X...","...X.","....G") | |
| def key(board): | |
| return hashlib.sha256("\n".join(board).encode()).hexdigest() | |
| def generate(rng,n,wall=.20): | |
| cells=rng.choices(".#X",weights=[1-wall-.1,wall,.1],k=n*n) | |
| cells[rng.randrange(n*n)]="G" | |
| return tuple("".join(cells[i:i+n]) for i in range(0,n*n,n)) | |
| def advance(b,pos,a): | |
| y,x=pos | |
| r,c=y+MOVES[a][0],x+MOVES[a][1] | |
| if not (0<=r<len(b) and 0<=c<len(b[0])) or b[r][c]=="#": return pos | |
| return r,c | |
| def starts(b): | |
| return [(i,j) for i,row in enumerate(b) for j,ch in enumerate(row) if ch=="."] | |
| def train(seed): | |
| rng=random.Random(seed) | |
| counts=np.zeros((16,4,5),int) | |
| sums=np.zeros(3); freq=np.zeros(3,int) | |
| layouts=set() | |
| for _ in range(650): | |
| b=generate(rng,rng.choice([5,6,7]),rng.uniform(.10,.45)) | |
| if key(b)==key(ORIGINAL) or key(b) in layouts:continue | |
| layouts.add(key(b)) | |
| places=starts(b) | |
| if not places: continue | |
| for _ in range(100): | |
| p=rng.choice(places) | |
| mask=sum(1<<i for i in range(4) if advance(b,p,i)==p) | |
| a=rng.randrange(4) | |
| q=advance(b,p,a) | |
| counts[mask,a,MOVES.index((q[0]-p[0],q[1]-p[1]))]+=1 | |
| event=".GX".index(b[q[0]][q[1]]) | |
| sums[event]+=(-.02,1.,-1.)[event] | |
| freq[event]+=1 | |
| return NextReward(counts,sums/freq),layouts | |
| def distances(b): | |
| goal=next((i,row.index("G")) for i,row in enumerate(b) if "G" in row) | |
| d={goal:0}; queue=deque([goal]) | |
| while queue: | |
| p=queue.popleft() | |
| for a in range(4): | |
| q=advance(b,p,a) | |
| if q not in d and b[q[0]][q[1]]=='.': | |
| d[q]=d[p]+1 | |
| queue.append(q) | |
| return goal,d | |
| def play(b,p,policy): | |
| seen=set() | |
| while p not in seen: | |
| seen.add(p) | |
| q=advance(b,p,int(policy[p])) | |
| if b[q[0]][q[1]]=="G":return len(seen) | |
| if b[q[0]][q[1]]=="X":return None | |
| p=q | |
| return None | |
| def evaluate(model,boards): | |
| totals=dict(maps=len(boards),reachable=0,unreachable=0,success=0,shortest=0,greedy_success=0) | |
| for b in boards: | |
| goal,d=distances(b) | |
| pi=model.policy(b); greedy=np.zeros(pi.shape,int) | |
| for p in starts(b): | |
| # One-step Manhattan baseline with access to visible pits and walls. | |
| def score(a): | |
| q=advance(b,p,a) | |
| if q==p or b[q[0]][q[1]]=="X":return float("inf") | |
| return abs(q[0]-goal[0])+abs(q[1]-goal[1]) | |
| greedy[p]=min(range(4),key=score) | |
| for p in starts(b): | |
| if p not in d: | |
| totals["unreachable"]+=1;continue | |
| totals["reachable"]+=1 | |
| steps=play(b,p,pi) | |
| totals["success"]+=steps is not None | |
| totals["shortest"]+=steps==d[p] | |
| totals["greedy_success"]+=play(b,p,greedy) is not None | |
| return totals | |
| def main(out): | |
| out.mkdir(parents=True,exist_ok=True) | |
| results=[] | |
| trained=[train(seed) for seed in (73,101,149)] | |
| seen=set.union(*(layouts for _,layouts in trained)) | |
| seen.add(key(ORIGINAL)) | |
| suites={"original":[ORIGINAL]} | |
| rng=random.Random(8901) | |
| for n in (5,8,12): | |
| boards=[] | |
| while len(boards)<40: | |
| board=generate(rng,n) | |
| if key(board) not in seen: | |
| seen.add(key(board));boards.append(board) | |
| suites[str(n)]=boards | |
| for seed,(model,layouts) in zip((73,101,149),trained): | |
| if seed==73: | |
| model.save(out/'model.json') | |
| restored=NextReward.load(out/'model.json') | |
| for boards in suites.values(): | |
| for b in boards: np.testing.assert_array_equal(model.policy(b),restored.policy(b)) | |
| model=restored | |
| result=dict(seed=seed,train_maps=len(layouts),transitions=int(model.counts.sum()), | |
| suites={k:evaluate(model,v) for k,v in suites.items()}) | |
| results.append(result) | |
| print(seed,result["suites"],flush=True) | |
| (out/'evaluation.json').write_text(json.dumps(results,indent=2)) | |
| (out/'test_maps.json').write_text(json.dumps(suites,indent=1)) | |
| if __name__=="__main__": | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument("--output",type=Path,default=Path(".")) | |
| main(parser.parse_args().output) | |