File size: 2,506 Bytes
394a40d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Inference for Next Reward. Boards use . (floor), # (wall), X (pit), G (goal)."""
import json
from pathlib import Path
import numpy as np
MOVES = ((1,0),(0,1),(-1,0),(0,-1),(0,0))
ACTIONS = ("down","right","up","left")
def geometry(board):
if not board or not board[0] or any(len(row)!=len(board[0]) for row in board):
raise ValueError("Expected a nonempty rectangular board")
if sum(row.count("G") for row in board)!=1 or any(set(row)-set(".X#G") for row in board):
raise ValueError("Use exactly one G and symbols . # X")
grid=np.array([list(row) for row in board])
h,w=grid.shape
y,x=np.indices((h,w))
dest=[]
mask=np.zeros((h,w),dtype=int)
for i,(dy,dx) in enumerate(MOVES):
ny,nx=y+dy,x+dx
blocked=(ny<0)|(ny>=h)|(nx<0)|(nx>=w)|(grid[ny.clip(0,h-1),nx.clip(0,w-1)]=="#")
if i<4: mask |= blocked.astype(int)<<i
dest.append(np.where(blocked,y*w+x,ny*w+nx))
return grid,mask,np.stack(dest,axis=-1)
class NextReward:
def __init__(self,counts,rewards):
counts=np.asarray(counts,dtype=float)
self.rewards=np.asarray(rewards,dtype=float)
if counts.shape!=(16,4,5) or not np.isfinite(counts).all() or np.any(counts<0) or np.any(counts.sum(-1)==0):
raise ValueError("Missing or invalid transition counts")
if self.rewards.shape!=(3,) or not np.isfinite(self.rewards).all():
raise ValueError("Invalid learned rewards")
self.counts=counts
self.prob=counts/counts.sum(-1,keepdims=True)
@classmethod
def load(cls,path):
d=json.loads(Path(path).read_text())
if d["format"]!="next-reward-v1": raise ValueError("Unsupported model format")
return cls(d["counts"],d["rewards"])
def save(self,path):
Path(path).write_text(json.dumps(dict(format="next-reward-v1",counts=self.counts.astype(int).tolist(),
rewards=self.rewards.tolist()),indent=1))
def policy(self,board):
grid,mask,dest=geometry(board)
reward=np.where(grid=="G",self.rewards[1],np.where(grid=="X",self.rewards[2],self.rewards[0]))
value=np.zeros(grid.shape)
for _ in range(grid.size+1):
outcomes=(reward+0.98*value).ravel()[dest]
q=np.einsum("ijak,ijk->ija",self.prob[mask],outcomes)
updated=np.where(grid==".",q.max(-1),0.)
if np.max(np.abs(updated-value))<1e-12: break
value=updated
return np.where(grid==".",q.argmax(-1),-1)
|