metrics: RoboCasa Kitchen 24-task success rate, ckpt 60k + 80k (EMA, seed42av, N_ENVS=1)

metrics/eval_metrics.json

@@ -0,0 +1,317 @@
+{
+ "protocol": {
+  "benchmark": "RoboCasa Kitchen, 24 tasks x 50 episodes (1200 total), env layout 5, N_ENVS=1",
+  "weights": "EVAL_WEIGHTS=ema -- EMA video DiT + EMA action head, loaded by key-swap (843 + 247 tensors, src==dst verified)",
+  "seeds": "WAM_EVAL_ACTION_SEED=42 WAM_EVAL_VIDEO_SEED=42 (seed42av cell)",
+  "budget": "50 episodes/task; dropcur WAM_DITACTION_EXCLUDE_CURRENT=1, matching the training yaml",
+  "code": "branch ctxpool-bmethod @ eb321c8 (pinned eval tree). That commit also carries the attach_ctx_pool fix WITHOUT which no pooled checkpoint loads at eval at all (meta-device + fp32-on-bf16).",
+  "render_bug": "N_ENVS=1 deliberately: at N_ENVS=5 the EGL buffer-swap bug mis-serves 5.9% of render calls vs 0.05% at 1. huiwon's detector scanned the 60k cell: 4/1200 episodes contaminated (0.3%), ALL of them already failures, so the reported number can only move UP by at most 0.33pp.",
+  "measured": "2026-09-13 / 2026-09-14, A100-80GB, kakao"
+ },
+ "robocasa_kitchen": {
+  "60000": {
+   "overall_success_rate": 0.6192,
+   "successes": 743,
+   "episodes": 1200,
+   "tasks_complete": 24,
+   "per_task": {
+    "TurnSinkSpout": {
+     "task_index": 0,
+     "successes": 31,
+     "episodes": 50,
+     "success_rate": 0.62
+    },
+    "TurnOnStove": {
+     "task_index": 1,
+     "successes": 13,
+     "episodes": 50,
+     "success_rate": 0.26
+    },
+    "TurnOnSinkFaucet": {
+     "task_index": 2,
+     "successes": 34,
+     "episodes": 50,
+     "success_rate": 0.68
+    },
+    "TurnOnMicrowave": {
+     "task_index": 3,
+     "successes": 44,
+     "episodes": 50,
+     "success_rate": 0.88
+    },
+    "TurnOffStove": {
+     "task_index": 4,
+     "successes": 10,
+     "episodes": 50,
+     "success_rate": 0.2
+    },
+    "TurnOffSinkFaucet": {
+     "task_index": 5,
+     "successes": 47,
+     "episodes": 50,
+     "success_rate": 0.94
+    },
+    "TurnOffMicrowave": {
+     "task_index": 6,
+     "successes": 50,
+     "episodes": 50,
+     "success_rate": 1.0
+    },
+    "PnPStoveToCounter": {
+     "task_index": 7,
+     "successes": 23,
+     "episodes": 50,
+     "success_rate": 0.46
+    },
+    "PnPSinkToCounter": {
+     "task_index": 8,
+     "successes": 31,
+     "episodes": 50,
+     "success_rate": 0.62
+    },
+    "PnPMicrowaveToCounter": {
+     "task_index": 9,
+     "successes": 19,
+     "episodes": 50,
+     "success_rate": 0.38
+    },
+    "PnPCounterToStove": {
+     "task_index": 10,
+     "successes": 20,
+     "episodes": 50,
+     "success_rate": 0.4
+    },
+    "PnPCounterToSink": {
+     "task_index": 11,
+     "successes": 28,
+     "episodes": 50,
+     "success_rate": 0.56
+    },
+    "PnPCounterToMicrowave": {
+     "task_index": 12,
+     "successes": 16,
+     "episodes": 50,
+     "success_rate": 0.32
+    },
+    "PnPCounterToCab": {
+     "task_index": 13,
+     "successes": 25,
+     "episodes": 50,
+     "success_rate": 0.5
+    },
+    "PnPCabToCounter": {
+     "task_index": 14,
+     "successes": 25,
+     "episodes": 50,
+     "success_rate": 0.5
+    },
+    "OpenSingleDoor": {
+     "task_index": 15,
+     "successes": 33,
+     "episodes": 50,
+     "success_rate": 0.66
+    },
+    "OpenDrawer": {
+     "task_index": 16,
+     "successes": 34,
+     "episodes": 50,
+     "success_rate": 0.68
+    },
+    "OpenDoubleDoor": {
+     "task_index": 17,
+     "successes": 27,
+     "episodes": 50,
+     "success_rate": 0.54
+    },
+    "CoffeeSetupMug": {
+     "task_index": 18,
+     "successes": 18,
+     "episodes": 50,
+     "success_rate": 0.36
+    },
+    "CoffeeServeMug": {
+     "task_index": 19,
+     "successes": 38,
+     "episodes": 50,
+     "success_rate": 0.76
+    },
+    "CoffeePressButton": {
+     "task_index": 20,
+     "successes": 42,
+     "episodes": 50,
+     "success_rate": 0.84
+    },
+    "CloseSingleDoor": {
+     "task_index": 21,
+     "successes": 49,
+     "episodes": 50,
+     "success_rate": 0.98
+    },
+    "CloseDrawer": {
+     "task_index": 22,
+     "successes": 49,
+     "episodes": 50,
+     "success_rate": 0.98
+    },
+    "CloseDoubleDoor": {
+     "task_index": 23,
+     "successes": 37,
+     "episodes": 50,
+     "success_rate": 0.74
+    }
+   }
+  },
+  "80000": {
+   "overall_success_rate": 0.645,
+   "successes": 774,
+   "episodes": 1200,
+   "tasks_complete": 24,
+   "per_task": {
+    "TurnSinkSpout": {
+     "task_index": 0,
+     "successes": 40,
+     "episodes": 50,
+     "success_rate": 0.8
+    },
+    "TurnOnStove": {
+     "task_index": 1,
+     "successes": 13,
+     "episodes": 50,
+     "success_rate": 0.26
+    },
+    "TurnOnSinkFaucet": {
+     "task_index": 2,
+     "successes": 42,
+     "episodes": 50,
+     "success_rate": 0.84
+    },
+    "TurnOnMicrowave": {
+     "task_index": 3,
+     "successes": 45,
+     "episodes": 50,
+     "success_rate": 0.9
+    },
+    "TurnOffStove": {
+     "task_index": 4,
+     "successes": 9,
+     "episodes": 50,
+     "success_rate": 0.18
+    },
+    "TurnOffSinkFaucet": {
+     "task_index": 5,
+     "successes": 48,
+     "episodes": 50,
+     "success_rate": 0.96
+    },
+    "TurnOffMicrowave": {
+     "task_index": 6,
+     "successes": 50,
+     "episodes": 50,
+     "success_rate": 1.0
+    },
+    "PnPStoveToCounter": {
+     "task_index": 7,
+     "successes": 23,
+     "episodes": 50,
+     "success_rate": 0.46
+    },
+    "PnPSinkToCounter": {
+     "task_index": 8,
+     "successes": 31,
+     "episodes": 50,
+     "success_rate": 0.62
+    },
+    "PnPMicrowaveToCounter": {
+     "task_index": 9,
+     "successes": 10,
+     "episodes": 50,
+     "success_rate": 0.2
+    },
+    "PnPCounterToStove": {
+     "task_index": 10,
+     "successes": 27,
+     "episodes": 50,
+     "success_rate": 0.54
+    },
+    "PnPCounterToSink": {
+     "task_index": 11,
+     "successes": 28,
+     "episodes": 50,
+     "success_rate": 0.56
+    },
+    "PnPCounterToMicrowave": {
+     "task_index": 12,
+     "successes": 18,
+     "episodes": 50,
+     "success_rate": 0.36
+    },
+    "PnPCounterToCab": {
+     "task_index": 13,
+     "successes": 23,
+     "episodes": 50,
+     "success_rate": 0.46
+    },
+    "PnPCabToCounter": {
+     "task_index": 14,
+     "successes": 27,
+     "episodes": 50,
+     "success_rate": 0.54
+    },
+    "OpenSingleDoor": {
+     "task_index": 15,
+     "successes": 36,
+     "episodes": 50,
+     "success_rate": 0.72
+    },
+    "OpenDrawer": {
+     "task_index": 16,
+     "successes": 40,
+     "episodes": 50,
+     "success_rate": 0.8
+    },
+    "OpenDoubleDoor": {
+     "task_index": 17,
+     "successes": 32,
+     "episodes": 50,
+     "success_rate": 0.64
+    },
+    "CoffeeSetupMug": {
+     "task_index": 18,
+     "successes": 11,
+     "episodes": 50,
+     "success_rate": 0.22
+    },
+    "CoffeeServeMug": {
+     "task_index": 19,
+     "successes": 34,
+     "episodes": 50,
+     "success_rate": 0.68
+    },
+    "CoffeePressButton": {
+     "task_index": 20,
+     "successes": 46,
+     "episodes": 50,
+     "success_rate": 0.92
+    },
+    "CloseSingleDoor": {
+     "task_index": 21,
+     "successes": 48,
+     "episodes": 50,
+     "success_rate": 0.96
+    },
+    "CloseDrawer": {
+     "task_index": 22,
+     "successes": 50,
+     "episodes": 50,
+     "success_rate": 1.0
+    },
+    "CloseDoubleDoor": {
+     "task_index": 23,
+     "successes": 43,
+     "episodes": 50,
+     "success_rate": 0.86
+    }
+   }
+  }
+ }
+}