File size: 2,506 Bytes
394a40d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
"""Inference for Next Reward. Boards use . (floor), # (wall), X (pit), G (goal)."""
import json
from pathlib import Path
import numpy as np

MOVES = ((1,0),(0,1),(-1,0),(0,-1),(0,0))
ACTIONS = ("down","right","up","left")

def geometry(board):
    if not board or not board[0] or any(len(row)!=len(board[0]) for row in board):
        raise ValueError("Expected a nonempty rectangular board")
    if sum(row.count("G") for row in board)!=1 or any(set(row)-set(".X#G") for row in board):
        raise ValueError("Use exactly one G and symbols . # X")
    grid=np.array([list(row) for row in board])
    h,w=grid.shape
    y,x=np.indices((h,w))
    dest=[]
    mask=np.zeros((h,w),dtype=int)
    for i,(dy,dx) in enumerate(MOVES):
        ny,nx=y+dy,x+dx
        blocked=(ny<0)|(ny>=h)|(nx<0)|(nx>=w)|(grid[ny.clip(0,h-1),nx.clip(0,w-1)]=="#")
        if i<4: mask |= blocked.astype(int)<<i
        dest.append(np.where(blocked,y*w+x,ny*w+nx))
    return grid,mask,np.stack(dest,axis=-1)

class NextReward:
    def __init__(self,counts,rewards):
        counts=np.asarray(counts,dtype=float)
        self.rewards=np.asarray(rewards,dtype=float)
        if counts.shape!=(16,4,5) or not np.isfinite(counts).all() or np.any(counts<0) or np.any(counts.sum(-1)==0):
            raise ValueError("Missing or invalid transition counts")
        if self.rewards.shape!=(3,) or not np.isfinite(self.rewards).all():
            raise ValueError("Invalid learned rewards")
        self.counts=counts
        self.prob=counts/counts.sum(-1,keepdims=True)

    @classmethod
    def load(cls,path):
        d=json.loads(Path(path).read_text())
        if d["format"]!="next-reward-v1": raise ValueError("Unsupported model format")
        return cls(d["counts"],d["rewards"])

    def save(self,path):
        Path(path).write_text(json.dumps(dict(format="next-reward-v1",counts=self.counts.astype(int).tolist(),
            rewards=self.rewards.tolist()),indent=1))

    def policy(self,board):
        grid,mask,dest=geometry(board)
        reward=np.where(grid=="G",self.rewards[1],np.where(grid=="X",self.rewards[2],self.rewards[0]))
        value=np.zeros(grid.shape)
        for _ in range(grid.size+1):
            outcomes=(reward+0.98*value).ravel()[dest]
            q=np.einsum("ijak,ijk->ija",self.prob[mask],outcomes)
            updated=np.where(grid==".",q.max(-1),0.)
            if np.max(np.abs(updated-value))<1e-12: break
            value=updated
        return np.where(grid==".",q.argmax(-1),-1)