Upload verified evaluation/full-suite-manifest.json
Browse files
evaluation/full-suite-manifest.json
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"baseline": {
|
| 3 |
+
"checkpoint": 55,
|
| 4 |
+
"episodes": 400,
|
| 5 |
+
"mean_partial_score": 0.39425,
|
| 6 |
+
"source_url": "https://app.notion.com/p/40ded7607a4081f48d77c47f8549784c",
|
| 7 |
+
"successes": 122
|
| 8 |
+
},
|
| 9 |
+
"checkpoint": 1000,
|
| 10 |
+
"decision_rule": "Finish all400 checkpoint1000 Precision episodes, then Memory, Generalization, Long Horizon and Open regardless of success rate or partial score; checkpoint950 cancelled by user.",
|
| 11 |
+
"execution_sources": {
|
| 12 |
+
"generalization.sbatch": "d6fc7abb1b6170704fa60e4ca75bb574952253316f59ca4ffc85d83bfdeff6ca",
|
| 13 |
+
"long_horizon.sbatch": "bdf112f24eb2fa5c85567731046544277cd209a561f50d65f54a07bb03a2a7e0",
|
| 14 |
+
"memory.sbatch": "efc073d50f80bb115c41cc9b8988c1afab1010e6c047f330cd71f12d8c3b1175",
|
| 15 |
+
"open.sbatch": "5dfd7afdcf8f4b5b97c5fd688b980f58bd61a1a4101a37c825005b55f2ed051c",
|
| 16 |
+
"precision_handoff.py": "9a783d94112f12542c6715aac55f802e66d392367c1c72cd8f9aa93f65374378",
|
| 17 |
+
"prepare_followup.py": "c2e457fdea0f9ef3a33f5c233ea019aea31ff89fb2982a5032cc4cadac2d01ac",
|
| 18 |
+
"suite_common.py": "157988d7033ea528b2dc9200359a4edfbb41e5d2f692da6487108669abf11b1f",
|
| 19 |
+
"suite_driver.py": "d4fc61af349bd499eddd5d061675405dedede1cda918765b1954d2ec85a55d8e",
|
| 20 |
+
"test_followup.py": "106e26b1be0059be2444054628000e1dee482ba315adec2ce0b9a46ba371941e"
|
| 21 |
+
},
|
| 22 |
+
"id": "c62f853a713ce9200730a1b937a619358a9c146f2d914ddddfe8988b27ae47d6",
|
| 23 |
+
"input_hashes": {
|
| 24 |
+
"inputs/authorization.json": "23aba269d4c3f71508be5fdc44e45fbfda7c358fef1ac4b761690f25cbbace45",
|
| 25 |
+
"inputs/generalization.json": "de5a880e127039c689082262619791f26eb66ae5ed1fe929b8308f93a818f6b5",
|
| 26 |
+
"inputs/long_horizon.json": "26a9c66ed3082a59ccbb11c1c43d5bc2ad0123ab66a3ffd27750e834c9de2d20",
|
| 27 |
+
"inputs/memory.json": "b961cfc82459c2a394922903cc6d275aaf647d8cb4561ebafc7dc259bc73928b",
|
| 28 |
+
"inputs/open.json": "3daaf3e86b2f7dacad8fbe063efc23aa06d805a5c50a60cac347395fec87f4a7",
|
| 29 |
+
"inputs/unconditional-authorization.json": "a33af04a144b2d59a15c8fe4a74fd93a6b5b94ab7539316abc09b553226bf43a"
|
| 30 |
+
},
|
| 31 |
+
"performance_condition": "none",
|
| 32 |
+
"precision_protocol": "266df4b5d9b8eac16145600a8c4f9223d15e03285d61473c3bbd8c43b694f211",
|
| 33 |
+
"precision_root": "/home/praveen/experiments/vpp2-recovery1000-950-precision-20261103",
|
| 34 |
+
"precision_tasks": [
|
| 35 |
+
"build_tower",
|
| 36 |
+
"insert_tubes",
|
| 37 |
+
"deposit_coin",
|
| 38 |
+
"fasten_screws",
|
| 39 |
+
"insert_key",
|
| 40 |
+
"play_Xylophone",
|
| 41 |
+
"plug_in_charger",
|
| 42 |
+
"pour_balls_into_vase"
|
| 43 |
+
],
|
| 44 |
+
"replacements": [],
|
| 45 |
+
"revision_of": "f5491d41dd847cc86f52a9f9b8d7ffc9b8aa9642a595372aa20d51f5026b534f",
|
| 46 |
+
"schema": "checkpoint1000-unconditional-followup-v2",
|
| 47 |
+
"selection": "original",
|
| 48 |
+
"suite_order": [
|
| 49 |
+
"memory",
|
| 50 |
+
"generalization",
|
| 51 |
+
"long_horizon",
|
| 52 |
+
"open"
|
| 53 |
+
],
|
| 54 |
+
"suites": {
|
| 55 |
+
"generalization": {
|
| 56 |
+
"episodes_file": "inputs/generalization.json",
|
| 57 |
+
"expected": 600,
|
| 58 |
+
"name": "generalization",
|
| 59 |
+
"overrides": {},
|
| 60 |
+
"reporting_holds": {},
|
| 61 |
+
"runtime": {
|
| 62 |
+
"protocol": "4d705e28dc3ee7b401ea3f851cfa2b205981a5009d65cc01f3bd467489d50ab6",
|
| 63 |
+
"root": "/data/praveen/vpp2-recovery1000-followup-20261104/generalization/runtime",
|
| 64 |
+
"update": 1000
|
| 65 |
+
},
|
| 66 |
+
"task_order": [
|
| 67 |
+
"stack_bowls",
|
| 68 |
+
"push_T",
|
| 69 |
+
"pack_objects_into_box",
|
| 70 |
+
"fold_clothes",
|
| 71 |
+
"hang_mugs",
|
| 72 |
+
"sweep_blocks",
|
| 73 |
+
"pour_liquid_into_cup",
|
| 74 |
+
"make_toast",
|
| 75 |
+
"arrange_largest_number",
|
| 76 |
+
"sort_nesting_dolls_by_size",
|
| 77 |
+
"store_laptop_and_headphones",
|
| 78 |
+
"stack_blocks"
|
| 79 |
+
]
|
| 80 |
+
},
|
| 81 |
+
"long_horizon": {
|
| 82 |
+
"episodes_file": "inputs/long_horizon.json",
|
| 83 |
+
"expected": 400,
|
| 84 |
+
"name": "long_horizon",
|
| 85 |
+
"overrides": {},
|
| 86 |
+
"reporting_holds": {},
|
| 87 |
+
"runtime": {
|
| 88 |
+
"protocol": "c2867b1bb38b88d54cbe92103a5cf6d2020c3d44f67c89412182265c4bac8256",
|
| 89 |
+
"root": "/data/praveen/vpp2-recovery1000-followup-20261104/long_horizon/runtime",
|
| 90 |
+
"update": 1000
|
| 91 |
+
},
|
| 92 |
+
"task_order": [
|
| 93 |
+
"put_bottles_into_dustbin",
|
| 94 |
+
"fill_pen_holder",
|
| 95 |
+
"classify_objects",
|
| 96 |
+
"play_tic_tac_toe",
|
| 97 |
+
"fill_egg_holder",
|
| 98 |
+
"organize_table",
|
| 99 |
+
"make_kong",
|
| 100 |
+
"play_stacking_toy"
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
"memory": {
|
| 104 |
+
"episodes_file": "inputs/memory.json",
|
| 105 |
+
"expected": 300,
|
| 106 |
+
"name": "memory",
|
| 107 |
+
"overrides": {},
|
| 108 |
+
"reporting_holds": {},
|
| 109 |
+
"runtime": {
|
| 110 |
+
"protocol": "35ef508554aee3ce8b89ddc3826fb6eda60bd022db79d6d91a60f6f37e7014f8",
|
| 111 |
+
"root": "/data/praveen/vpp2-recovery1000-followup-20261104/memory/runtime",
|
| 112 |
+
"update": 1000
|
| 113 |
+
},
|
| 114 |
+
"task_order": [
|
| 115 |
+
"cover_blocks",
|
| 116 |
+
"match_and_pick_from_conveyor",
|
| 117 |
+
"swap_blocks",
|
| 118 |
+
"swap_T",
|
| 119 |
+
"press_by_number",
|
| 120 |
+
"imitate_sorting_sequence"
|
| 121 |
+
]
|
| 122 |
+
},
|
| 123 |
+
"open": {
|
| 124 |
+
"episodes_file": "inputs/open.json",
|
| 125 |
+
"expected": 400,
|
| 126 |
+
"name": "open",
|
| 127 |
+
"overrides": {},
|
| 128 |
+
"reporting_holds": {},
|
| 129 |
+
"runtime": {
|
| 130 |
+
"protocol": "92558f946ce17044692364bcb3515da3925bcf3c646266c8dc9ce1f38771384e",
|
| 131 |
+
"root": "/data/praveen/vpp2-recovery1000-followup-20261104/open/runtime",
|
| 132 |
+
"update": 1000
|
| 133 |
+
},
|
| 134 |
+
"task_order": [
|
| 135 |
+
"align_blocks",
|
| 136 |
+
"general_pickup",
|
| 137 |
+
"stack_blocks_by_language",
|
| 138 |
+
"solve_equation",
|
| 139 |
+
"classify_objects_by_language",
|
| 140 |
+
"pick_from_conveyor_by_image",
|
| 141 |
+
"store_tools_in_toolbox",
|
| 142 |
+
"pour_by_language"
|
| 143 |
+
]
|
| 144 |
+
}
|
| 145 |
+
}
|
| 146 |
+
}
|