RLMaze / app.py
schrabfab's picture
Upload 2 files
cce4bb7 verified
Raw History Blame Contribute Delete
18.6 kB
"""
GridWorld Warehouse Navigator
Gradio + Gymnasium interactive maze game with RL-style rewards.
"""
import random
from collections import deque
import gradio as gr
import gymnasium as gym
import numpy as np
from gymnasium import spaces
# ── Constants ────────────────────────────────────────────────────────────────
GRID_SIZE = 10
MAX_STEPS = 200
OBS_DENSITY = 0.20
# Actions
UP, RIGHT, DOWN, LEFT = 0, 1, 2, 3
ACTION_DELTAS = {UP: (-1, 0), RIGHT: (0, 1), DOWN: (1, 0), LEFT: (0, -1)}
# RL rewards
R_GOAL = 100.0
R_CLOSER = 1.0
R_FARTHER = -1.5
R_INVALID = -2.0
R_NEW_CELL = 0.5
R_STEP = -0.1
R_TIMEOUT = -10.0
# SVG rendering
CELL = 52
PAD = 6
# ── Gymnasium Environment ─────────────────────────────────────────────────────
class GridWorldEnv(gym.Env):
"""
A simple GridWorld environment.
Observation: [agent_x_norm, agent_y_norm, goal_x_norm, goal_y_norm] (float32, 0-1)
Actions: 0=UP, 1=RIGHT, 2=DOWN, 3=LEFT
"""
metadata = {"render_modes": ["rgb_array"]}
def __init__(self, grid_size: int = GRID_SIZE, obs_density: float = OBS_DENSITY,
max_steps: int = MAX_STEPS):
super().__init__()
self.grid_size = grid_size
self.obs_density = obs_density
self.max_steps = max_steps
self.action_space = spaces.Discrete(4)
self.observation_space = spaces.Box(
low=0.0, high=1.0, shape=(4,), dtype=np.float32
)
# State – initialised by reset()
self.grid: list[list[str]] = []
self.agent_pos: tuple[int, int] = (0, 0)
self.start_pos: tuple[int, int] = (0, 0)
self.goal_pos: tuple[int, int] = (0, 0)
self.steps = 0
self.total_score = 0.0
self.last_reward = 0.0
self.visited: set[tuple[int, int]] = set()
self.goal_reached = False
self.reset()
# ── maze generation ───────────────────────────────────────────────────────
def _bfs_connected(self, grid, start, goal) -> bool:
queue = deque([start])
seen = {start}
while queue:
r, c = queue.popleft()
if (r, c) == goal:
return True
for dr, dc in ACTION_DELTAS.values():
nr, nc = r + dr, c + dc
if 0 <= nr < self.grid_size and 0 <= nc < self.grid_size:
if grid[nr][nc] != 'X' and (nr, nc) not in seen:
seen.add((nr, nc))
queue.append((nr, nc))
return False
def _make_grid(self):
"""Generate a random, solvable grid. Retries up to 200 times."""
for _ in range(200):
g = [['.' for _ in range(self.grid_size)] for _ in range(self.grid_size)]
for r in range(self.grid_size):
for c in range(self.grid_size):
if random.random() < self.obs_density:
g[r][c] = 'X'
empties = [(r, c) for r in range(self.grid_size)
for c in range(self.grid_size) if g[r][c] == '.']
if len(empties) < 2:
continue
start = random.choice(empties)
empties.remove(start)
goal = random.choice(empties)
if self._bfs_connected(g, start, goal):
g[start[0]][start[1]] = 'S'
g[goal[0]][goal[1]] = 'G'
return g, start, goal
# Fallback: empty grid, corners
g = [['.' for _ in range(self.grid_size)] for _ in range(self.grid_size)]
start, goal = (0, 0), (self.grid_size - 1, self.grid_size - 1)
g[start[0]][start[1]] = 'S'
g[goal[0]][goal[1]] = 'G'
return g, start, goal
# ── gym interface ─────────────────────────────────────────────────────────
def reset(self, *, seed=None, options=None):
super().reset(seed=seed)
self.grid, self.start_pos, self.goal_pos = self._make_grid()
self.agent_pos = self.start_pos
self.steps = 0
self.total_score = 0.0
self.last_reward = 0.0
self.visited = {self.agent_pos}
self.goal_reached = False
return self._obs(), {}
def step(self, action: int):
if self.goal_reached or self.steps >= self.max_steps:
return self._obs(), 0.0, True, False, {}
dr, dc = ACTION_DELTAS[action]
nr, nc = self.agent_pos[0] + dr, self.agent_pos[1] + dc
prev_dist = self._manhattan(self.agent_pos, self.goal_pos)
# ── invalid move ──────────────────────────────────────────────────────
if not (0 <= nr < self.grid_size and 0 <= nc < self.grid_size) \
or self.grid[nr][nc] == 'X':
self.steps += 1
reward = R_INVALID
truncated = self.steps >= self.max_steps
if truncated:
reward += R_TIMEOUT
self.last_reward = reward
self.total_score += reward
return self._obs(), reward, False, truncated, {"invalid": True}
# ── valid move ────────────────────────────────────────────────────────
self.agent_pos = (nr, nc)
self.steps += 1
new_dist = self._manhattan(self.agent_pos, self.goal_pos)
if self.agent_pos == self.goal_pos:
reward = R_GOAL
self.goal_reached = True
terminated = True
truncated = False
else:
reward = R_STEP
reward += R_CLOSER if new_dist < prev_dist else R_FARTHER
if self.agent_pos not in self.visited:
reward += R_NEW_CELL
self.visited.add(self.agent_pos)
terminated = False
truncated = self.steps >= self.max_steps
if truncated:
reward += R_TIMEOUT
self.last_reward = reward
self.total_score += reward
return self._obs(), reward, terminated, truncated, {}
# ── helpers ───────────────────────────────────────────────────────────────
def _obs(self) -> np.ndarray:
n = self.grid_size - 1
ax = self.agent_pos[1] / n
ay = self.agent_pos[0] / n
gx = self.goal_pos[1] / n
gy = self.goal_pos[0] / n
return np.array([ax, ay, gx, gy], dtype=np.float32)
def _manhattan(self, a, b) -> int:
return abs(a[0] - b[0]) + abs(a[1] - b[1])
# ── Single shared environment instance ───────────────────────────────────────
env = GridWorldEnv()
# ── SVG grid renderer ─────────────────────────────────────────────────────────
def render_svg(e: GridWorldEnv) -> str:
n = e.grid_size
W = n * CELL + PAD * 2
H = n * CELL + PAD * 2
parts = [
f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" '
f'style="background:#12122a;border-radius:10px;display:block">'
]
# ── cells ─────────────────────────────────────────────────────────────────
for r in range(n):
for c in range(n):
x = PAD + c * CELL
y = PAD + r * CELL
cell = e.grid[r][c]
if cell == 'X':
fill, stroke = '#1e1e38', '#0d0d20'
label, lclr = 'βœ•', '#3a3a5c'
elif cell == 'S':
fill, stroke = '#0d2a4a', '#2196F3'
label, lclr = 'S', '#2196F3'
elif cell == 'G':
fill, stroke = '#0a2a0a', '#4CAF50'
label, lclr = 'G', '#4CAF50'
elif (r, c) in e.visited:
fill, stroke = '#1e1e36', '#16163a'
label, lclr = '', ''
else:
fill, stroke = '#181830', '#12122a'
label, lclr = '', ''
parts.append(
f'<rect x="{x+1}" y="{y+1}" width="{CELL-2}" height="{CELL-2}" '
f'fill="{fill}" stroke="{stroke}" stroke-width="1.5" rx="4"/>'
)
# visited dot
if (r, c) in e.visited and cell not in ('S', 'G', 'X') and (r, c) != e.agent_pos:
cx_ = x + CELL // 2
cy_ = y + CELL // 2
parts.append(f'<circle cx="{cx_}" cy="{cy_}" r="3" fill="#2a2a4e"/>')
if label:
cx_ = x + CELL // 2
cy_ = y + CELL // 2
parts.append(
f'<text x="{cx_}" y="{cy_+5}" text-anchor="middle" '
f'font-family="monospace" font-size="14" font-weight="bold" '
f'fill="{lclr}">{label}</text>'
)
# ── agent ─────────────────────────────────────────────────────────────────
ar, ac = e.agent_pos
ax_ = PAD + ac * CELL + CELL // 2
ay_ = PAD + ar * CELL + CELL // 2
r_ = CELL // 2 - 6
if e.goal_reached:
parts.append(
f'<circle cx="{ax_}" cy="{ay_}" r="{r_}" fill="#FFD700" '
f'stroke="#FFA000" stroke-width="2.5"/>'
)
parts.append(
f'<text x="{ax_}" y="{ay_+6}" text-anchor="middle" '
f'font-size="18" fill="#333">β˜…</text>'
)
else:
parts.append(
f'<circle cx="{ax_}" cy="{ay_}" r="{r_}" fill="#E53935" '
f'stroke="#FF8A80" stroke-width="2"/>'
)
parts.append(
f'<text x="{ax_}" y="{ay_+5}" text-anchor="middle" '
f'font-family="monospace" font-size="13" font-weight="bold" fill="white">A</text>'
)
parts.append('</svg>')
return ''.join(parts)
# ── Scoreboard HTML ───────────────────────────────────────────────────────────
def render_scoreboard(e: GridWorldEnv) -> str:
dist = e._manhattan(e.agent_pos, e.goal_pos)
timeout = e.steps >= e.max_steps and not e.goal_reached
if e.goal_reached:
status_text = 'βœ… GOAL REACHED!'
status_color = '#69F0AE'
elif timeout:
status_text = '⏰ TIME LIMIT'
status_color = '#FF6B6B'
else:
status_text = 'πŸ”„ In progress'
status_color = '#90CAF9'
lr_color = '#69F0AE' if e.last_reward > 0 else ('#FF6B6B' if e.last_reward < 0 else '#aaa')
def row(label, value, color='#e0e0ff'):
return (
f'<div style="margin-bottom:10px">'
f'<div style="font-size:10px;color:#6060a0;text-transform:uppercase;'
f'letter-spacing:.08em">{label}</div>'
f'<div style="font-size:15px;font-weight:bold;color:{color}">{value}</div>'
f'</div>'
)
html = (
'<div style="font-family:\'Courier New\',monospace;background:#12122a;'
'border-radius:10px;padding:18px;color:#e0e0ff;min-width:200px;'
'border:1px solid #2a2a4e">'
'<div style="font-size:15px;font-weight:bold;margin-bottom:14px;color:#90CAF9;'
'border-bottom:1px solid #2a2a4e;padding-bottom:8px">πŸ“Š Scoreboard</div>'
)
html += row('Total Score', f'{e.total_score:+.1f}', '#FFD700')
html += row('Last Reward', f'{e.last_reward:+.2f}', lr_color)
html += row('Steps', f'{e.steps} / {e.max_steps}')
html += row('Agent Position', f'({e.agent_pos[1]}, {e.agent_pos[0]})')
html += row('Goal Position', f'({e.goal_pos[1]}, {e.goal_pos[0]})')
html += row('Manhattan Distance', str(dist), '#90CAF9')
html += row('Status', status_text, status_color)
html += '</div>'
return html
# ── Reward legend HTML ────────────────────────────────────────────────────────
REWARD_LEGEND = """
<div style="font-family:'Courier New',monospace;background:#12122a;border-radius:10px;
padding:14px;color:#e0e0ff;border:1px solid #2a2a4e;margin-top:10px">
<div style="font-size:13px;font-weight:bold;color:#90CAF9;margin-bottom:8px">⚑ Reward Table</div>
<table style="border-collapse:collapse;font-size:12px;width:100%">
<tr><td style="color:#69F0AE;padding:2px 6px">+100</td><td>Reach goal</td></tr>
<tr><td style="color:#69F0AE;padding:2px 6px">+1.0</td><td>Move closer</td></tr>
<tr><td style="color:#69F0AE;padding:2px 6px">+0.5</td><td>Visit new cell</td></tr>
<tr><td style="color:#FF6B6B;padding:2px 6px">-0.1</td><td>Each step</td></tr>
<tr><td style="color:#FF6B6B;padding:2px 6px">-1.5</td><td>Move farther</td></tr>
<tr><td style="color:#FF6B6B;padding:2px 6px">-2.0</td><td>Invalid move</td></tr>
<tr><td style="color:#FF6B6B;padding:2px 6px">-10 </td><td>Timeout</td></tr>
</table>
</div>
"""
LEGEND_BAR = """
<div style="display:flex;gap:14px;padding:10px 6px;font-family:'Courier New',monospace;
font-size:12px;flex-wrap:wrap;align-items:center">
<span><span style="background:#0d2a4a;color:#2196F3;padding:1px 7px;border-radius:3px;
border:1px solid #2196F3">S</span> Start</span>
<span><span style="background:#0a2a0a;color:#4CAF50;padding:1px 7px;border-radius:3px;
border:1px solid #4CAF50">G</span> Goal</span>
<span><span style="background:#E53935;color:white;padding:1px 7px;border-radius:3px">A</span> Agent</span>
<span><span style="background:#1e1e38;color:#3a3a5c;padding:1px 7px;border-radius:3px">βœ•</span> Obstacle</span>
<span><span style="background:#1e1e36;color:#555;padding:1px 7px;border-radius:3px">Β·</span> Visited</span>
</div>
"""
# ── Gradio action handlers ────────────────────────────────────────────────────
def _apply(action: int):
env.step(action)
return render_svg(env), render_scoreboard(env)
def act_up(): return _apply(UP)
def act_right(): return _apply(RIGHT)
def act_down(): return _apply(DOWN)
def act_left(): return _apply(LEFT)
def act_reset():
env.reset()
return render_svg(env), render_scoreboard(env)
# ── Keyboard JS ──────────────────────────────────────────────────────────────
KEYBOARD_JS = """
<script>
(function() {
function findBtn(text) {
return [...document.querySelectorAll('button')].find(
b => b.textContent.trim() === text
);
}
document.addEventListener('keydown', function(e) {
var map = { ArrowUp:'↑', ArrowRight:'β†’', ArrowDown:'↓', ArrowLeft:'←' };
if (!map[e.key]) return;
e.preventDefault();
var btn = findBtn(map[e.key]);
if (btn) btn.click();
});
})();
</script>
"""
# ── Build Gradio UI ───────────────────────────────────────────────────────────
CSS = """
.move-btn { min-width:56px !important; height:52px !important;
font-size:22px !important; padding:0 !important; }
.center-col { display:flex; flex-direction:column; align-items:center; }
footer { display:none !important; }
"""
with gr.Blocks(theme=gr.themes.Base(), css=CSS, title="GridWorld Navigator") as demo:
gr.Markdown("# 🏭 GridWorld Warehouse Navigator")
gr.Markdown(
"Move the red agent **A** to the green goal **G** β€” avoid dark obstacles **βœ•**. "
"Use arrow keys **↑ ↓ ← β†’** or the on-screen buttons."
)
with gr.Row(equal_height=True):
# ── Left: grid + controls ──────────────────────────────────────────────
with gr.Column(scale=3, elem_classes="center-col"):
grid_out = gr.HTML(render_svg(env))
gr.HTML(LEGEND_BAR)
# D-pad layout
with gr.Row():
gr.HTML('<div style="flex:1"></div>')
btn_up = gr.Button("↑", elem_classes="move-btn")
gr.HTML('<div style="flex:1"></div>')
with gr.Row():
btn_left = gr.Button("←", elem_classes="move-btn")
btn_down = gr.Button("↓", elem_classes="move-btn")
btn_right = gr.Button("β†’", elem_classes="move-btn")
btn_reset = gr.Button("πŸ”„ Reset / New Maze", variant="primary", size="lg")
# ── Right: scoreboard + reward table ──────────────────────────────────
with gr.Column(scale=1):
score_out = gr.HTML(render_scoreboard(env))
gr.HTML(REWARD_LEGEND)
# keyboard listener
gr.HTML(KEYBOARD_JS)
# ── wire buttons ──────────────────────────────────────────────────────────
outs = [grid_out, score_out]
btn_up.click(act_up, outputs=outs)
btn_right.click(act_right, outputs=outs)
btn_down.click(act_down, outputs=outs)
btn_left.click(act_left, outputs=outs)
btn_reset.click(act_reset, outputs=outs)
if __name__ == "__main__":
demo.launch()