Upload trained agent

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+replay.mp4 filter=lfs diff=lfs merge=lfs -text
+results.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,82 @@
+---
+library_name: pytorch
+tags:
+- reinforcement-learning
+- atari
+- phoenix
+- ppo
+- custom-implementation
+model-index:
+- name: ppo-ALE/Phoenix-v5
+  results:
+  - task:
+      type: reinforcement-learning
+      name: reinforcement-learning
+    dataset:
+      name: ALE/Phoenix-v5
+      type: ALE/Phoenix-v5
+    metrics:
+    - type: mean_reward
+      value: 4687.19 +/- 974.80
+      name: mean_reward
+---
+# PPO agent playing ALE/Phoenix-v5
+
+A from-scratch PyTorch implementation (PPO) trained for 2,000,000 agent steps (8,000,000 frames).
+
+![results](results.png)
+<video src="replay.mp4" controls></video>
+
+## Evaluation
+| metric | value |
+|---|---|
+| mean score (stochastic policy) | 4687.2 ± 974.8 over 32 episodes |
+| mean score (greedy policy) | 4578.8 |
+| environment | `ALE/Phoenix-v5` (sticky actions 0.25, sign-clipped rewards for training, raw score for evaluation) |
+
+## Model
+* Input: 4 stacked 84x84 grayscale frames.  Network: `impala` CNN trunk (15 conv layers) + 2-layer actor and critic heads; 1,223,129 parameters.
+* Actions: 8 discrete (NOOP, FIRE, RIGHT, LEFT, DOWN, RIGHTFIRE, LEFTFIRE, DOWNFIRE).
+
+## Hyper-parameters
+```json
+{
+ "algo": "ppo",
+ "game": "phoenix",
+ "seed": 0,
+ "tag": "",
+ "total_steps": 2000000,
+ "num_envs": 16,
+ "sticky": 0.25,
+ "noop_max": 30,
+ "max_frames": 108000,
+ "clip_reward": true,
+ "arch": "impala",
+ "hidden": 256,
+ "lr": 0.00025,
+ "lr_anneal": true,
+ "adam_eps": 1e-05,
+ "max_grad_norm": 0.5,
+ "gamma": 0.99,
+ "gae_lambda": 0.95,
+ "n_steps": 128,
+ "ent_coef": 0.01,
+ "vf_coef": 0.5,
+ "norm_adv": true,
+ "clip_eps": 0.1,
+ "ppo_epochs": 4,
+ "minibatches": 4,
+ "clip_vloss": false,
+ "target_kl": null,
+ "baseline": "batch",
+ "max_episode_steps": 4000,
+ "chunk": 512,
+ "eval_every": 300000,
+ "eval_episodes": 8,
+ "ckpt_every": 500000,
+ "print_every_s": 20.0,
+ "device": "auto"
+}
+```
+
+See `load_example.py` for loading. Weights are stored as `model.safetensors`.

config.json

@@ -0,0 +1,37 @@
+{
+ "algo": "ppo",
+ "game": "phoenix",
+ "seed": 0,
+ "tag": "",
+ "total_steps": 2000000,
+ "num_envs": 16,
+ "sticky": 0.25,
+ "noop_max": 30,
+ "max_frames": 108000,
+ "clip_reward": true,
+ "arch": "impala",
+ "hidden": 256,
+ "lr": 0.00025,
+ "lr_anneal": true,
+ "adam_eps": 1e-05,
+ "max_grad_norm": 0.5,
+ "gamma": 0.99,
+ "gae_lambda": 0.95,
+ "n_steps": 128,
+ "ent_coef": 0.01,
+ "vf_coef": 0.5,
+ "norm_adv": true,
+ "clip_eps": 0.1,
+ "ppo_epochs": 4,
+ "minibatches": 4,
+ "clip_vloss": false,
+ "target_kl": null,
+ "baseline": "batch",
+ "max_episode_steps": 4000,
+ "chunk": 512,
+ "eval_every": 300000,
+ "eval_episodes": 8,
+ "ckpt_every": 500000,
+ "print_every_s": 20.0,
+ "device": "auto"
+}

load_example.py

@@ -0,0 +1,11 @@
+# pip install ale-py gymnasium torch safetensors huggingface_hub
+# You also need the phoenix_rl package that trained this model (it defines the network class).
+import json, torch
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+from phoenix_rl.config import Cfg
+from phoenix_rl.nets import ActorCritic
+repo = "<your-username>/ppo-phoenix"
+cfg = Cfg(**json.load(open(hf_hub_download(repo, "config.json"))))
+net = ActorCritic(8, cfg.arch, cfg.hidden); net.load_state_dict(load_file(hf_hub_download(repo, "model.safetensors"))); net.eval()
+# obs: uint8 tensor (B, 4, 84, 84);  logits, value = net(obs);  action = logits.argmax(-1)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40d3f6261058c198a87c25778f4acc915918c6ad2cd185b00e44820d98afb07c
+size 4896076

replay.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a6e8dee4a356f73b656724b75f5ae043c5fb17d4ac6c3e14e3f8b2418d2c28e
+size 768864

results.json

@@ -0,0 +1,8 @@
+{
+ "mean": 4687.1875,
+ "std": 974.8020131512603,
+ "n_episodes": 32,
+ "greedy_mean": 4578.75,
+ "seed": 0,
+ "train_steps": 2000896
+}