Upload folder using huggingface_hub

robocasa/turn_off_sink_faucet/baseline_seed10000/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.52,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.52,
+  "turn_off_sink_faucet/video_10000_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10000/media/turn_off_sink_faucet/tjoome9z.mp4",
+  "turn_off_sink_faucet/video_10001_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10000/media/turn_off_sink_faucet/tfcltkoa.mp4",
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10000/media/turn_off_sink_faucet/8bh9ekuf.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10000/media/turn_off_sink_faucet/bolwj8pa.mp4"
+}

robocasa/turn_off_sink_faucet/baseline_seed10001/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.58,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.58,
+  "turn_off_sink_faucet/video_10001_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10001/media/turn_off_sink_faucet/l6zugavg.mp4",
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10001/media/turn_off_sink_faucet/s1tetsfc.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10001/media/turn_off_sink_faucet/qhni49ba.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10001/media/turn_off_sink_faucet/hhzvooix.mp4"
+}

robocasa/turn_off_sink_faucet/baseline_seed10002/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.58,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.58,
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10002/media/turn_off_sink_faucet/cfle2d3d.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10002/media/turn_off_sink_faucet/wm08fc27.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10002/media/turn_off_sink_faucet/03m3ip8j.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10002/media/turn_off_sink_faucet/4igjx69a.mp4"
+}

robocasa/turn_off_sink_faucet/baseline_seed10003/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.46,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.46,
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10003/media/turn_off_sink_faucet/qt47bmvy.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10003/media/turn_off_sink_faucet/3ubtjar2.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10003/media/turn_off_sink_faucet/ulxdwupd.mp4",
+  "turn_off_sink_faucet/video_10006_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10003/media/turn_off_sink_faucet/j2voa8iq.mp4"
+}

robocasa/turn_off_sink_faucet/baseline_seed10004/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.48,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.48,
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10004/media/turn_off_sink_faucet/1hqj9ezd.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10004/media/turn_off_sink_faucet/lir5oqnt.mp4",
+  "turn_off_sink_faucet/video_10006_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10004/media/turn_off_sink_faucet/tsq4ruxy.mp4",
+  "turn_off_sink_faucet/video_10007_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10004/media/turn_off_sink_faucet/g96xwu0j.mp4"
+}

robocasa/turn_off_sink_faucet/bon_n8_seed10000/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.54,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.54,
+  "turn_off_sink_faucet/video_10000_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10000/media/turn_off_sink_faucet/1m4uh0f4.mp4",
+  "turn_off_sink_faucet/video_10001_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10000/media/turn_off_sink_faucet/j7oaey4j.mp4",
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10000/media/turn_off_sink_faucet/0hlsh6v7.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10000/media/turn_off_sink_faucet/e4vw4vuq.mp4"
+}

robocasa/turn_off_sink_faucet/bon_n8_seed10001/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.58,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.58,
+  "turn_off_sink_faucet/video_10001_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10001/media/turn_off_sink_faucet/s7ronezc.mp4",
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10001/media/turn_off_sink_faucet/gt977plu.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10001/media/turn_off_sink_faucet/7gpj9iou.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10001/media/turn_off_sink_faucet/be5mi7a1.mp4"
+}

robocasa/turn_off_sink_faucet/bon_n8_seed10002/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.64,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.64,
+  "turn_off_sink_faucet/video_10002_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10002/media/turn_off_sink_faucet/smluism8.mp4",
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10002/media/turn_off_sink_faucet/aixvrsjc.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10002/media/turn_off_sink_faucet/rgtmon9b.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10002/media/turn_off_sink_faucet/q1ssycmw.mp4"
+}

robocasa/turn_off_sink_faucet/bon_n8_seed10003/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.6,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.6,
+  "turn_off_sink_faucet/video_10003_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10003/media/turn_off_sink_faucet/mswy7vqg.mp4",
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10003/media/turn_off_sink_faucet/wl81rp7d.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10003/media/turn_off_sink_faucet/24u3uk4t.mp4",
+  "turn_off_sink_faucet/video_10006_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10003/media/turn_off_sink_faucet/9pdrltty.mp4"
+}

robocasa/turn_off_sink_faucet/bon_n8_seed10004/eval_log.json

@@ -0,0 +1,11 @@
+{
+  "checkpoint": "/workspace/oat/my_models/robocasa_turn_off_sink_faucet_topk_ep0500_sr0.580.ckpt",
+  "mean_success_rate_mean": 0.44,
+  "mean_tokens_used_mean": 8.0,
+  "num_exp": 1,
+  "turn_off_sink_faucet/mean_success_rate_mean": 0.44,
+  "turn_off_sink_faucet/video_10004_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10004/media/turn_off_sink_faucet/x4e7bv8l.mp4",
+  "turn_off_sink_faucet/video_10005_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10004/media/turn_off_sink_faucet/q1skn86j.mp4",
+  "turn_off_sink_faucet/video_10006_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10004/media/turn_off_sink_faucet/9jsouygy.mp4",
+  "turn_off_sink_faucet/video_10007_0": "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10004/media/turn_off_sink_faucet/k21ci3z6.mp4"
+}

robocasa/turn_off_sink_faucet/summary_literal5.json

@@ -0,0 +1,64 @@
+{
+  "protocol": {
+    "layout": "literal_5_seeds",
+    "seeds": [
+      10000,
+      10001,
+      10002,
+      10003,
+      10004
+    ],
+    "n_test": 50,
+    "n_exp_per_seed": 1,
+    "aggregation": "mean over seeds; primary uncertainty SEM; also SD",
+    "delta": "mean(method)-mean(baseline); SEM_delta=sqrt(SEM_m^2+SEM_b^2) unpaired/conservative (paired SEM of diffs optional later)"
+  },
+  "methods": {
+    "baseline": {
+      "per_seed_sr": {
+        "10000": 0.52,
+        "10001": 0.58,
+        "10002": 0.58,
+        "10003": 0.46,
+        "10004": 0.48
+      },
+      "mean": 0.524,
+      "sd": 0.055497747702046414,
+      "sem": 0.024819347291981705,
+      "n_seeds": 5,
+      "eval_logs": [
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10000/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10001/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10002/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10003/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/baseline_seed10004/eval_log.json"
+      ]
+    },
+    "bon_n8": {
+      "per_seed_sr": {
+        "10000": 0.54,
+        "10001": 0.58,
+        "10002": 0.64,
+        "10003": 0.6,
+        "10004": 0.44
+      },
+      "mean": 0.56,
+      "sd": 0.07615773105863909,
+      "sem": 0.0340587727318528,
+      "n_seeds": 5,
+      "eval_logs": [
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10000/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10001/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10002/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10003/eval_log.json",
+        "/workspace/oat/output/eval/matched_s10000/robocasa/turn_off_sink_faucet/bon_n8_seed10004/eval_log.json"
+      ]
+    }
+  },
+  "deltas": {
+    "delta_bon_n8_minus_baseline": {
+      "delta": 0.03600000000000003,
+      "sem_delta": 0.04214261501141095
+    }
+  }
+}