Add Blockwise-OAT baseline artifacts (eval/blockwise_eval_log.json)
Browse files- eval/blockwise_eval_log.json +71 -0
eval/blockwise_eval_log.json
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_mean": 0.6999999999999998,
|
| 3 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_std": 0.03464101615137752,
|
| 4 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/mean_success_rate_stderr": 0.019999999999999987,
|
| 5 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/video_1002_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/l7ci2e5q.mp4",
|
| 6 |
+
"KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/video_1012_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE3_turn_on_the_stove_and_put_the_moka_pot_on_it/md08cmd5.mp4",
|
| 7 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_mean": 0.9333333333333332,
|
| 8 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_std": 0.05033222956847165,
|
| 9 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/mean_success_rate_stderr": 0.02905932629027115,
|
| 10 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/video_1003_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/9ieopqf7.mp4",
|
| 11 |
+
"KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/video_1013_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it/feqrnqgg.mp4",
|
| 12 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_mean": 0.7133333333333334,
|
| 13 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_std": 0.07571877794400365,
|
| 14 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/mean_success_rate_stderr": 0.04371625682868001,
|
| 15 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/video_1009_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/ihjpxaga.mp4",
|
| 16 |
+
"KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/video_1019_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it/22i41kk7.mp4",
|
| 17 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_mean": 0.45999999999999996,
|
| 18 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_std": 0.04000000000000001,
|
| 19 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/mean_success_rate_stderr": 0.023094010767585035,
|
| 20 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/video_1008_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/fkdjjej2.mp4",
|
| 21 |
+
"KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/video_1018_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/KITCHEN_SCENE8_put_both_moka_pots_on_the_stove/5fx2a5vr.mp4",
|
| 22 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_mean": 0.2866666666666667,
|
| 23 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_std": 0.023094010767585018,
|
| 24 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/mean_success_rate_stderr": 0.013333333333333327,
|
| 25 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/video_1007_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/sveusxq8.mp4",
|
| 26 |
+
"LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/video_1017_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE1_put_both_the_alphabet_soup_and_the_cream_cheese_box_in_the_basket/nqgvpdhg.mp4",
|
| 27 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_mean": 0.41333333333333333,
|
| 28 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_std": 0.11015141094572205,
|
| 29 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/mean_success_rate_stderr": 0.06359594676112972,
|
| 30 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/video_1000_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/rrc4m051.mp4",
|
| 31 |
+
"LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/video_1010_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE2_put_both_the_alphabet_soup_and_the_tomato_sauce_in_the_basket/hl69ytgh.mp4",
|
| 32 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_mean": 0.38666666666666666,
|
| 33 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_std": 0.11015141094572203,
|
| 34 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/mean_success_rate_stderr": 0.06359594676112972,
|
| 35 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/video_1001_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/kniazpyd.mp4",
|
| 36 |
+
"LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/video_1011_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE2_put_both_the_cream_cheese_box_and_the_butter_in_the_basket/z4jfjas9.mp4",
|
| 37 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_mean": 0.45999999999999996,
|
| 38 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_std": 0.05291502622129183,
|
| 39 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/mean_success_rate_stderr": 0.030550504633038947,
|
| 40 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/video_1004_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/x791xjln.mp4",
|
| 41 |
+
"LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/video_1014_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE5_put_the_white_mug_on_the_left_plate_and_put_the_yellow_and_white_mug_on_the_right_plate/b8v6ihen.mp4",
|
| 42 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_mean": 0.26666666666666666,
|
| 43 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_std": 0.07023769168568493,
|
| 44 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/mean_success_rate_stderr": 0.04055175020198814,
|
| 45 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/video_1006_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/6bkrodkh.mp4",
|
| 46 |
+
"LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/video_1016_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/LIVING_ROOM_SCENE6_put_the_white_mug_on_the_plate_and_put_the_chocolate_pudding_to_the_right_of_the_plate/8qniowkt.mp4",
|
| 47 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_mean": 0.6133333333333334,
|
| 48 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_std": 0.06429100507328636,
|
| 49 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/mean_success_rate_stderr": 0.03711842908553348,
|
| 50 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/video_1005_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/nr82yetg.mp4",
|
| 51 |
+
"STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/video_1015_0": "output/eval/blockwise_original_oat/confirm_pt50/blockwise_p4_r1/media/STUDY_SCENE1_pick_up_the_book_and_place_it_in_the_back_compartment_of_the_caddy/g9dlb9r8.mp4",
|
| 52 |
+
"blockwise_prefix_len": 4,
|
| 53 |
+
"blockwise_refine_iters": 1,
|
| 54 |
+
"blockwise_tail_checkpoint": "/workspace/OAT-RoboMimic-Fine-tune/BLT-OAT/output/blockwise/original_oat_tail_p4_r1.pt",
|
| 55 |
+
"blockwise_tail_param_ratio": 0.9008816185088926,
|
| 56 |
+
"checkpoint": "output/baselines/original_oat/hf/policy_ep-0250_sr-0.596.ckpt",
|
| 57 |
+
"episodes_per_task_approx": 50.0,
|
| 58 |
+
"mean_success_rate_mean": 0.5233333333333333,
|
| 59 |
+
"mean_success_rate_std": 0.018036999011291594,
|
| 60 |
+
"mean_success_rate_stderr": 0.010413666234542216,
|
| 61 |
+
"n_test": 500,
|
| 62 |
+
"n_test_per_task": 50,
|
| 63 |
+
"num_exp": 3,
|
| 64 |
+
"num_tasks": 10,
|
| 65 |
+
"seed_stride": 500,
|
| 66 |
+
"task_suite": "libero10",
|
| 67 |
+
"test_start_seed": 1000,
|
| 68 |
+
"tokenizer_checkpoint": "/workspace/OAT-RoboMimic-Fine-tune/BLT-OAT/output/baselines/original_oat/hf/tokenizer_ep-0950_mse-0.002.ckpt",
|
| 69 |
+
"tokenizer_overridden": true,
|
| 70 |
+
"use_blockwise": true
|
| 71 |
+
}
|