Delete 10_Next_Reward.ipynb
Browse files- 10_Next_Reward.ipynb +0 -88
10_Next_Reward.ipynb
DELETED
|
@@ -1,88 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"nbformat": 4,
|
| 3 |
-
"nbformat_minor": 5,
|
| 4 |
-
"metadata": {
|
| 5 |
-
"kernelspec": {
|
| 6 |
-
"name": "python3",
|
| 7 |
-
"display_name": "Python 3",
|
| 8 |
-
"language": "python"
|
| 9 |
-
}
|
| 10 |
-
},
|
| 11 |
-
"cells": [
|
| 12 |
-
{
|
| 13 |
-
"cell_type": "markdown",
|
| 14 |
-
"metadata": {},
|
| 15 |
-
"id": "c949dc6e",
|
| 16 |
-
"source": [
|
| 17 |
-
"# Next Reward · Eğitim, test ve Hugging Face yayını\n",
|
| 18 |
-
"CPU oturumu yeterli. Son hücreyi çalıştırınca model ve değerlendirme dosyaları özel depoya yüklenir."
|
| 19 |
-
]
|
| 20 |
-
},
|
| 21 |
-
{
|
| 22 |
-
"cell_type": "code",
|
| 23 |
-
"metadata": {},
|
| 24 |
-
"id": "73b262f4",
|
| 25 |
-
"source": [
|
| 26 |
-
"from pathlib import Path\n",
|
| 27 |
-
"import tempfile, subprocess, sys\n",
|
| 28 |
-
"folder=Path(tempfile.mkdtemp(prefix='next-reward-'))\n",
|
| 29 |
-
"assets={'experiment.py': '\"\"\"Train a model of grid transitions and evaluate policies on disjoint layouts.\"\"\"\\nimport argparse\\nimport hashlib\\nimport json\\nimport random\\nfrom collections import deque\\nfrom pathlib import Path\\nimport numpy as np\\nfrom next_reward import MOVES, NextReward\\n\\nORIGINAL=(\".....\",\".X...\",\".X...\",\"...X.\",\"....G\")\\n\\ndef key(board):\\n return hashlib.sha256(\"\\\\n\".join(board).encode()).hexdigest()\\n\\ndef generate(rng,n,wall=.20):\\n cells=rng.choices(\".#X\",weights=[1-wall-.1,wall,.1],k=n*n)\\n cells[rng.randrange(n*n)]=\"G\"\\n return tuple(\"\".join(cells[i:i+n]) for i in range(0,n*n,n))\\n\\ndef advance(b,pos,a):\\n y,x=pos\\n r,c=y+MOVES[a][0],x+MOVES[a][1]\\n if not (0<=r<len(b) and 0<=c<len(b[0])) or b[r][c]==\"#\": return pos\\n return r,c\\n\\ndef starts(b):\\n return [(i,j) for i,row in enumerate(b) for j,ch in enumerate(row) if ch==\".\"]\\n\\ndef train(seed):\\n rng=random.Random(seed)\\n counts=np.zeros((16,4,5),int)\\n sums=np.zeros(3); freq=np.zeros(3,int)\\n layouts=set()\\n for _ in range(650):\\n b=generate(rng,rng.choice([5,6,7]),rng.uniform(.10,.45))\\n if key(b)==key(ORIGINAL) or key(b) in layouts:continue\\n layouts.add(key(b))\\n places=starts(b)\\n if not places: continue\\n for _ in range(100):\\n p=rng.choice(places)\\n mask=sum(1<<i for i in range(4) if advance(b,p,i)==p)\\n a=rng.randrange(4)\\n q=advance(b,p,a)\\n counts[mask,a,MOVES.index((q[0]-p[0],q[1]-p[1]))]+=1\\n event=\".GX\".index(b[q[0]][q[1]])\\n sums[event]+=(-.02,1.,-1.)[event]\\n freq[event]+=1\\n return NextReward(counts,sums/freq),layouts\\n\\ndef distances(b):\\n goal=next((i,row.index(\"G\")) for i,row in enumerate(b) if \"G\" in row)\\n d={goal:0}; queue=deque([goal])\\n while queue:\\n p=queue.popleft()\\n for a in range(4):\\n q=advance(b,p,a)\\n if q not in d and b[q[0]][q[1]]==\\'.\\':\\n d[q]=d[p]+1\\n queue.append(q)\\n return goal,d\\n\\ndef play(b,p,policy):\\n seen=set()\\n while p not in seen:\\n seen.add(p)\\n q=advance(b,p,int(policy[p]))\\n if b[q[0]][q[1]]==\"G\":return len(seen)\\n if b[q[0]][q[1]]==\"X\":return None\\n p=q\\n return None\\n\\ndef evaluate(model,boards):\\n totals=dict(maps=len(boards),reachable=0,unreachable=0,success=0,shortest=0,greedy_success=0)\\n for b in boards:\\n goal,d=distances(b)\\n pi=model.policy(b); greedy=np.zeros(pi.shape,int)\\n for p in starts(b):\\n # One-step Manhattan baseline with access to visible pits and walls.\\n def score(a):\\n q=advance(b,p,a)\\n if q==p or b[q[0]][q[1]]==\"X\":return float(\"inf\")\\n return abs(q[0]-goal[0])+abs(q[1]-goal[1])\\n greedy[p]=min(range(4),key=score)\\n for p in starts(b):\\n if p not in d:\\n totals[\"unreachable\"]+=1;continue\\n totals[\"reachable\"]+=1\\n steps=play(b,p,pi)\\n totals[\"success\"]+=steps is not None\\n totals[\"shortest\"]+=steps==d[p]\\n totals[\"greedy_success\"]+=play(b,p,greedy) is not None\\n return totals\\n\\ndef main(out):\\n out.mkdir(parents=True,exist_ok=True)\\n results=[]\\n trained=[train(seed) for seed in (73,101,149)]\\n seen=set.union(*(layouts for _,layouts in trained))\\n seen.add(key(ORIGINAL))\\n suites={\"original\":[ORIGINAL]}\\n rng=random.Random(8901)\\n for n in (5,8,12):\\n boards=[]\\n while len(boards)<40:\\n board=generate(rng,n)\\n if key(board) not in seen:\\n seen.add(key(board));boards.append(board)\\n suites[str(n)]=boards\\n for seed,(model,layouts) in zip((73,101,149),trained):\\n if seed==73:\\n model.save(out/\\'model.json\\')\\n restored=NextReward.load(out/\\'model.json\\')\\n for boards in suites.values():\\n for b in boards: np.testing.assert_array_equal(model.policy(b),restored.policy(b))\\n model=restored\\n result=dict(seed=seed,train_maps=len(layouts),transitions=int(model.counts.sum()),\\n suites={k:evaluate(model,v) for k,v in suites.items()})\\n results.append(result)\\n print(seed,result[\"suites\"],flush=True)\\n (out/\\'evaluation.json\\').write_text(json.dumps(results,indent=2))\\n (out/\\'test_maps.json\\').write_text(json.dumps(suites,indent=1))\\nif __name__==\"__main__\":\\n parser=argparse.ArgumentParser()\\n parser.add_argument(\"--output\",type=Path,default=Path(\".\"))\\n main(parser.parse_args().output)\\n\\n', 'next_reward.py': '\"\"\"Inference for Next Reward. Boards use . (floor), # (wall), X (pit), G (goal).\"\"\"\\nimport json\\nfrom pathlib import Path\\nimport numpy as np\\n\\nMOVES = ((1,0),(0,1),(-1,0),(0,-1),(0,0))\\nACTIONS = (\"down\",\"right\",\"up\",\"left\")\\n\\ndef geometry(board):\\n if not board or not board[0] or any(len(row)!=len(board[0]) for row in board):\\n raise ValueError(\"Expected a nonempty rectangular board\")\\n if sum(row.count(\"G\") for row in board)!=1 or any(set(row)-set(\".X#G\") for row in board):\\n raise ValueError(\"Use exactly one G and symbols . # X\")\\n grid=np.array([list(row) for row in board])\\n h,w=grid.shape\\n y,x=np.indices((h,w))\\n dest=[]\\n mask=np.zeros((h,w),dtype=int)\\n for i,(dy,dx) in enumerate(MOVES):\\n ny,nx=y+dy,x+dx\\n blocked=(ny<0)|(ny>=h)|(nx<0)|(nx>=w)|(grid[ny.clip(0,h-1),nx.clip(0,w-1)]==\"#\")\\n if i<4: mask |= blocked.astype(int)<<i\\n dest.append(np.where(blocked,y*w+x,ny*w+nx))\\n return grid,mask,np.stack(dest,axis=-1)\\n\\nclass NextReward:\\n def __init__(self,counts,rewards):\\n counts=np.asarray(counts,dtype=float)\\n self.rewards=np.asarray(rewards,dtype=float)\\n if counts.shape!=(16,4,5) or not np.isfinite(counts).all() or np.any(counts<0) or np.any(counts.sum(-1)==0):\\n raise ValueError(\"Missing or invalid transition counts\")\\n if self.rewards.shape!=(3,) or not np.isfinite(self.rewards).all():\\n raise ValueError(\"Invalid learned rewards\")\\n self.counts=counts\\n self.prob=counts/counts.sum(-1,keepdims=True)\\n\\n @classmethod\\n def load(cls,path):\\n d=json.loads(Path(path).read_text())\\n if d[\"format\"]!=\"next-reward-v1\": raise ValueError(\"Unsupported model format\")\\n return cls(d[\"counts\"],d[\"rewards\"])\\n\\n def save(self,path):\\n Path(path).write_text(json.dumps(dict(format=\"next-reward-v1\",counts=self.counts.astype(int).tolist(),\\n rewards=self.rewards.tolist()),indent=1))\\n\\n def policy(self,board):\\n grid,mask,dest=geometry(board)\\n reward=np.where(grid==\"G\",self.rewards[1],np.where(grid==\"X\",self.rewards[2],self.rewards[0]))\\n value=np.zeros(grid.shape)\\n for _ in range(grid.size+1):\\n outcomes=(reward+0.98*value).ravel()[dest]\\n q=np.einsum(\"ijak,ijk->ija\",self.prob[mask],outcomes)\\n updated=np.where(grid==\".\",q.max(-1),0.)\\n if np.max(np.abs(updated-value))<1e-12: break\\n value=updated\\n return np.where(grid==\".\",q.argmax(-1),-1)\\n\\n', 'README.md': '---\\ntags:\\n- reinforcement-learning\\n- model-based-rl\\n- gridworld\\n- numpy\\nlanguage:\\n- en\\n---\\n\\n# Next Reward\\n\\nNext Reward learns a transition table and reward values from simulated grid moves. Given a visible map, it computes a new policy by value iteration. One JSON checkpoint runs on CPU with NumPy.\\n\\n## Try the checkpoint\\n\\nDownload the repository files into one directory and run this Python snippet there.\\n\\n~~~python\\nfrom next_reward import NextReward, ACTIONS\\n\\nboard = (\".....\", \".X...\", \".X...\", \"...X.\", \"....G\")\\nagent = NextReward.load(\"model.json\")\\npolicy = agent.policy(board)\\nprint(ACTIONS[policy[0, 0]])\\n~~~\\n\\nCoordinates are (row, column). Each dot is a valid starting cell. A wall (#) blocks movement. Entering a pit (X) or the goal (G) ends the episode. Policy entries for terminal cells and walls are -1.\\n\\n## What it learns\\n\\nTraining samples random state-action transitions on 5×5, 6×6 and 7×7 procedural maps. The model estimates displacement frequencies for four actions under sixteen neighboring wall patterns and mean rewards for three destination types. Inference uses the map geometry, known terminal types, those learned estimates, and a discount of 0.98. The exported checkpoint uses the prespecified training seed 73.\\n\\n## Held-out evaluation\\n\\nAll three training seeds (73, 101, 149) were evaluated on the same 120 new layouts. Test maps were excluded from all three training sets by content hash. Every reachable floor cell was used as a start. BFS supplied the shortest-path reference during scoring. The greedy baseline picks the safe neighboring cell closest to the goal by Manhattan distance, breaking ties by action order.\\n\\n| Map size | Maps | Reachable starts | Agent reached goal on a shortest path | Greedy reached goal |\\n|---|---:|---:|---:|---:|\\n| 5×5 | 40 | 569 | 569 | 474 |\\n| 8×8 | 40 | 1,417 | 1,417 | 927 |\\n| 12×12 | 40 | 3,373 | 3,373 | 1,555 |\\n\\nCounts are per training seed. All three produced the same scores. Another 1,122 floor cells had no path to the goal and were counted separately. The original demo layout was scored separately at 21/21 for both methods.\\n\\nThis task assumes a fully visible static grid and deterministic motion. The evaluation covers rearranged layouts under the same movement rules. Real-world navigation, changing obstacles, partial observations and stochastic actuators are outside the measured scope.\\n\\n## Reproduce\\n\\nRun `python experiment.py --output run` to train and score three seeds. The command writes a reloadable checkpoint, per-seed metrics and the exact test layouts. It checks that serialization preserves the exported policy on every evaluation board. The Colab notebook includes training, inference and an optional Hugging Face upload cell.\\n\\n', 'requirements.txt': 'numpy>=1.24\\n'}\n",
|
| 30 |
-
"for name,content in assets.items():\n",
|
| 31 |
-
" (folder/name).write_text(content,encoding='utf8')\n",
|
| 32 |
-
"subprocess.run([sys.executable,'experiment.py','--output','.'],cwd=folder,check=True)\n",
|
| 33 |
-
"print(folder)"
|
| 34 |
-
],
|
| 35 |
-
"outputs": [],
|
| 36 |
-
"execution_count": null
|
| 37 |
-
},
|
| 38 |
-
{
|
| 39 |
-
"cell_type": "code",
|
| 40 |
-
"metadata": {},
|
| 41 |
-
"id": "b6eeb3d8",
|
| 42 |
-
"source": [
|
| 43 |
-
"import json\n",
|
| 44 |
-
"sys.path.insert(0,str(folder))\n",
|
| 45 |
-
"from next_reward import NextReward,ACTIONS\n",
|
| 46 |
-
"agent=NextReward.load(folder/'model.json')\n",
|
| 47 |
-
"board=('.....','.X...','.X...','...X.','....G')\n",
|
| 48 |
-
"print(ACTIONS[agent.policy(board)[0,0]])\n",
|
| 49 |
-
"print(json.loads((folder/'evaluation.json').read_text())[0]['suites'])"
|
| 50 |
-
],
|
| 51 |
-
"outputs": [],
|
| 52 |
-
"execution_count": null
|
| 53 |
-
},
|
| 54 |
-
{
|
| 55 |
-
"cell_type": "markdown",
|
| 56 |
-
"metadata": {},
|
| 57 |
-
"id": "360879ee",
|
| 58 |
-
"source": [
|
| 59 |
-
"## İsteğe bağlı yayın\n",
|
| 60 |
-
"Aşağıdaki hücre thelamapi/Next-Lab-Reward deposuna yükler. Yeni depo varsayılan olarak özeldir."
|
| 61 |
-
]
|
| 62 |
-
},
|
| 63 |
-
{
|
| 64 |
-
"cell_type": "code",
|
| 65 |
-
"metadata": {},
|
| 66 |
-
"id": "16ba8331",
|
| 67 |
-
"source": [
|
| 68 |
-
"from getpass import getpass\n",
|
| 69 |
-
"import os\n",
|
| 70 |
-
"from huggingface_hub import HfApi\n",
|
| 71 |
-
"token=os.getenv('HF_TOKEN')\n",
|
| 72 |
-
"if not token:\n",
|
| 73 |
-
" try:\n",
|
| 74 |
-
" from google.colab import userdata\n",
|
| 75 |
-
" token=userdata.get('HF_TOKEN')\n",
|
| 76 |
-
" except Exception:\n",
|
| 77 |
-
" pass\n",
|
| 78 |
-
"api=HfApi(token=token or getpass('HF write token: ').strip())\n",
|
| 79 |
-
"repo='thelamapi/Next-Lab-Reward'\n",
|
| 80 |
-
"api.create_repo(repo_id=repo,private=True,exist_ok=True)\n",
|
| 81 |
-
"commit=api.upload_folder(repo_id=repo,folder_path=str(folder),allow_patterns=list(assets)+['model.json','evaluation.json','test_maps.json'],commit_message='Release Next Reward gridworld model and evaluation')\n",
|
| 82 |
-
"print(commit.commit_url)"
|
| 83 |
-
],
|
| 84 |
-
"outputs": [],
|
| 85 |
-
"execution_count": null
|
| 86 |
-
}
|
| 87 |
-
]
|
| 88 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|