Add Blockwise-OAT baseline artifacts (eval/ar_eval_log.json)
Browse files- eval/ar_eval_log.json +71 -0
eval/ar_eval_log.json
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_mean": 0.7933333333333333,
|
| 3 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_std": 0.030550504633038912,
|
| 4 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_stderr": 0.017638342073763927,
|
| 5 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/video_1002_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/c9yklo2q.mp4",
|
| 6 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/video_1012_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/h2w769rv.mp4",
|
| 7 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_mean": 0.9666666666666667,
|
| 8 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_std": 0.023094010767585053,
|
| 9 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_stderr": 0.013333333333333346,
|
| 10 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/video_1003_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/xjxd5y28.mp4",
|
| 11 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/video_1013_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/3y82tmm9.mp4",
|
| 12 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_mean": 0.7533333333333333,
|
| 13 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_std": 0.023094010767585053,
|
| 14 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_stderr": 0.013333333333333346,
|
| 15 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/video_1009_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/ps7zftfh.mp4",
|
| 16 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/video_1019_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/qil2l178.mp4",
|
| 17 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_mean": 0.5333333333333333,
|
| 18 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_std": 0.08326663997864531,
|
| 19 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_stderr": 0.04807401700618653,
|
| 20 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/video_1008_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/2wvze99v.mp4",
|
| 21 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/video_1018_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/kptlyktd.mp4",
|
| 22 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_mean": 0.3866666666666667,
|
| 23 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_std": 0.09865765724632494,
|
| 24 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_stderr": 0.056960024968783544,
|
| 25 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/video_1007_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/itk4yt28.mp4",
|
| 26 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/video_1017_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/khq36t8p.mp4",
|
| 27 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_mean": 0.4466666666666667,
|
| 28 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_std": 0.09865765724632497,
|
| 29 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_stderr": 0.05696002496878356,
|
| 30 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/video_1000_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/6w4b63wj.mp4",
|
| 31 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/video_1010_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/z42ynf4w.mp4",
|
| 32 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_mean": 0.47333333333333333,
|
| 33 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_std": 0.06110100926607789,
|
| 34 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_stderr": 0.03527668414752789,
|
| 35 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/video_1001_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/cv82otz4.mp4",
|
| 36 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/video_1011_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/4qsgsxba.mp4",
|
| 37 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_mean": 0.4533333333333333,
|
| 38 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_std": 0.05033222956847166,
|
| 39 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_stderr": 0.029059326290271154,
|
| 40 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/video_1004_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/n9gg2bor.mp4",
|
| 41 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/video_1014_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/vhre78cl.mp4",
|
| 42 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_mean": 0.42,
|
| 43 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_std": 0.034641016151377546,
|
| 44 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_stderr": 0.02,
|
| 45 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/video_1006_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/22xn3mim.mp4",
|
| 46 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/video_1016_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/zg00pqzi.mp4",
|
| 47 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_mean": 0.6466666666666666,
|
| 48 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_std": 0.02309401076758505,
|
| 49 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_stderr": 0.013333333333333345,
|
| 50 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/video_1005_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/bp2m64ub.mp4",
|
| 51 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/video_1015_0": "output/eval/blockwise_original_oat/confirm_pt50/ar/media/STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/5cgymmjv.mp4",
|
| 52 |
+
"blockwise_prefix_len": null,
|
| 53 |
+
"blockwise_refine_iters": null,
|
| 54 |
+
"blockwise_tail_checkpoint": null,
|
| 55 |
+
"blockwise_tail_param_ratio": null,
|
| 56 |
+
"checkpoint": "output/baselines/original_oat/hf/policy_ep-0250_sr-0.596.ckpt",
|
| 57 |
+
"episodes_per_task_approx": 50.0,
|
| 58 |
+
"mean_success_rate_mean": 0.5873333333333334,
|
| 59 |
+
"mean_success_rate_std": 0.0030550504633038962,
|
| 60 |
+
"mean_success_rate_stderr": 0.0017638342073763955,
|
| 61 |
+
"n_test": 500,
|
| 62 |
+
"n_test_per_task": 50,
|
| 63 |
+
"num_exp": 3,
|
| 64 |
+
"num_tasks": 10,
|
| 65 |
+
"seed_stride": 500,
|
| 66 |
+
"task_suite": "libero10",
|
| 67 |
+
"test_start_seed": 1000,
|
| 68 |
+
"tokenizer_checkpoint": "/workspace/OAT-RoboMimic-Fine-tune/BLT-OAT/output/baselines/original_oat/hf/tokenizer_ep-0950_mse-0.002.ckpt",
|
| 69 |
+
"tokenizer_overridden": true,
|
| 70 |
+
"use_blockwise": false
|
| 71 |
+
}
|