| { |
| "evaluation_type": "offline paired-action metrics", |
| "timestamp": "2026-04-17 17:27:07", |
| "num_records": { |
| "train": 240, |
| "val": 80, |
| "test": 80 |
| }, |
| "models": { |
| "identity": { |
| "train_metrics": { |
| "mean_break_clean_l2": 0.7432431578636169, |
| "mean_break_expert_l2": 1.4091085195541382, |
| "mean_clean_mse": 0.3958700895309448, |
| "mean_control_clean_l2": 0.7472800016403198, |
| "mean_control_expert_l2": 1.4053751230239868, |
| "mean_control_mse": 0.37037795782089233, |
| "break_gt_control_rate_clean_ref": 0.5458333333333333, |
| "break_gt_control_rate_expert_ref": 0.4875, |
| "mean_clean_ref_gap": -0.004036815837025642, |
| "mean_expert_ref_gap": 0.003733340883627534, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.543735682964325, |
| "mean_break_expert_l2": 1.5344077348709106, |
| "mean_clean_mse": 0.45629745721817017, |
| "mean_control_clean_l2": 0.6087575554847717, |
| "mean_control_expert_l2": 1.5666840076446533, |
| "mean_control_mse": 0.42383497953414917, |
| "break_gt_control_rate_clean_ref": 0.4666666666666667, |
| "break_gt_control_rate_expert_ref": 0.49166666666666664, |
| "mean_clean_ref_gap": -0.06502188742160797, |
| "mean_expert_ref_gap": -0.032276351004838943 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.9427506923675537, |
| "mean_break_expert_l2": 1.2838094234466553, |
| "mean_clean_mse": 0.3354426920413971, |
| "mean_control_clean_l2": 0.8858024477958679, |
| "mean_control_expert_l2": 1.2440663576126099, |
| "mean_control_mse": 0.3169209063053131, |
| "break_gt_control_rate_clean_ref": 0.625, |
| "break_gt_control_rate_expert_ref": 0.48333333333333334, |
| "mean_clean_ref_gap": 0.056948255747556686, |
| "mean_expert_ref_gap": 0.039743032306432724 |
| } |
| }, |
| "num_records": 240 |
| }, |
| "val_metrics": { |
| "mean_break_clean_l2": 0.6865259408950806, |
| "mean_break_expert_l2": 1.3638770580291748, |
| "mean_clean_mse": 0.35426950454711914, |
| "mean_control_clean_l2": 0.6174260377883911, |
| "mean_control_expert_l2": 1.4026581048965454, |
| "mean_control_mse": 0.3633658289909363, |
| "break_gt_control_rate_clean_ref": 0.575, |
| "break_gt_control_rate_expert_ref": 0.4375, |
| "mean_clean_ref_gap": 0.06909994781017303, |
| "mean_expert_ref_gap": -0.03878090903162956, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.6781854629516602, |
| "mean_break_expert_l2": 1.4673945903778076, |
| "mean_clean_mse": 0.43167275190353394, |
| "mean_control_clean_l2": 0.5327732563018799, |
| "mean_control_expert_l2": 1.5085182189941406, |
| "mean_control_mse": 0.3970142602920532, |
| "break_gt_control_rate_clean_ref": 0.575, |
| "break_gt_control_rate_expert_ref": 0.375, |
| "mean_clean_ref_gap": 0.14541222155094147, |
| "mean_expert_ref_gap": -0.041123662143945694 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.694866418838501, |
| "mean_break_expert_l2": 1.2603596448898315, |
| "mean_clean_mse": 0.27686622738838196, |
| "mean_control_clean_l2": 0.7020787596702576, |
| "mean_control_expert_l2": 1.2967978715896606, |
| "mean_control_mse": 0.32971736788749695, |
| "break_gt_control_rate_clean_ref": 0.575, |
| "break_gt_control_rate_expert_ref": 0.5, |
| "mean_clean_ref_gap": -0.007212340831756592, |
| "mean_expert_ref_gap": -0.036438144743442535 |
| } |
| }, |
| "num_records": 80 |
| }, |
| "test_metrics": { |
| "mean_break_clean_l2": 0.6800279021263123, |
| "mean_break_expert_l2": 1.6663868427276611, |
| "mean_clean_mse": 0.39530259370803833, |
| "mean_control_clean_l2": 0.7465093731880188, |
| "mean_control_expert_l2": 1.351813554763794, |
| "mean_control_mse": 0.3437272906303406, |
| "break_gt_control_rate_clean_ref": 0.5125, |
| "break_gt_control_rate_expert_ref": 0.5625, |
| "mean_clean_ref_gap": -0.06648143380880356, |
| "mean_expert_ref_gap": 0.31457358598709106, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.4548130929470062, |
| "mean_break_expert_l2": 1.7907543182373047, |
| "mean_clean_mse": 0.4607141613960266, |
| "mean_control_clean_l2": 0.6214543581008911, |
| "mean_control_expert_l2": 1.5308020114898682, |
| "mean_control_mse": 0.40925103425979614, |
| "break_gt_control_rate_clean_ref": 0.4, |
| "break_gt_control_rate_expert_ref": 0.55, |
| "mean_clean_ref_gap": -0.1666412204504013, |
| "mean_expert_ref_gap": 0.25995248556137085 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.9052427411079407, |
| "mean_break_expert_l2": 1.5420197248458862, |
| "mean_clean_mse": 0.32989105582237244, |
| "mean_control_clean_l2": 0.8715643882751465, |
| "mean_control_expert_l2": 1.1728250980377197, |
| "mean_control_mse": 0.27820348739624023, |
| "break_gt_control_rate_clean_ref": 0.625, |
| "break_gt_control_rate_expert_ref": 0.575, |
| "mean_clean_ref_gap": 0.033678364008665085, |
| "mean_expert_ref_gap": 0.3691946864128113 |
| } |
| }, |
| "num_records": 80 |
| } |
| }, |
| "action_only_adapter": { |
| "best_val_loss_total": 1.5956528186798096, |
| "train_metrics": { |
| "mean_break_clean_l2": 0.36038321256637573, |
| "mean_break_expert_l2": 1.1645487546920776, |
| "mean_clean_mse": 0.20006108283996582, |
| "mean_control_clean_l2": 0.2829788327217102, |
| "mean_control_expert_l2": 1.1402603387832642, |
| "mean_control_mse": 0.19075056910514832, |
| "break_gt_control_rate_clean_ref": 0.5875, |
| "break_gt_control_rate_expert_ref": 0.5291666666666667, |
| "mean_clean_ref_gap": 0.07740439474582672, |
| "mean_expert_ref_gap": 0.024288399145007133, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.25087252259254456, |
| "mean_break_expert_l2": 1.198777675628662, |
| "mean_clean_mse": 0.21315599977970123, |
| "mean_control_clean_l2": 0.22465568780899048, |
| "mean_control_expert_l2": 1.1870900392532349, |
| "mean_control_mse": 0.20506463944911957, |
| "break_gt_control_rate_clean_ref": 0.5083333333333333, |
| "break_gt_control_rate_expert_ref": 0.55, |
| "mean_clean_ref_gap": 0.026216844096779823, |
| "mean_expert_ref_gap": 0.011687582358717918 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.4698939025402069, |
| "mean_break_expert_l2": 1.1303197145462036, |
| "mean_clean_mse": 0.1869661659002304, |
| "mean_control_clean_l2": 0.34130197763442993, |
| "mean_control_expert_l2": 1.0934306383132935, |
| "mean_control_mse": 0.17643649876117706, |
| "break_gt_control_rate_clean_ref": 0.6666666666666666, |
| "break_gt_control_rate_expert_ref": 0.5083333333333333, |
| "mean_clean_ref_gap": 0.12859195470809937, |
| "mean_expert_ref_gap": 0.0368892140686512 |
| } |
| }, |
| "num_records": 240 |
| }, |
| "val_metrics": { |
| "mean_break_clean_l2": 0.34965047240257263, |
| "mean_break_expert_l2": 1.1548038721084595, |
| "mean_clean_mse": 0.1989673674106598, |
| "mean_control_clean_l2": 0.2915192246437073, |
| "mean_control_expert_l2": 1.1498476266860962, |
| "mean_control_mse": 0.19426663219928741, |
| "break_gt_control_rate_clean_ref": 0.6125, |
| "break_gt_control_rate_expert_ref": 0.55, |
| "mean_clean_ref_gap": 0.05813126638531685, |
| "mean_expert_ref_gap": 0.004956287331879139, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.2960914373397827, |
| "mean_break_expert_l2": 1.1769672632217407, |
| "mean_clean_mse": 0.2139633595943451, |
| "mean_control_clean_l2": 0.262840211391449, |
| "mean_control_expert_l2": 1.1790968179702759, |
| "mean_control_mse": 0.2026643306016922, |
| "break_gt_control_rate_clean_ref": 0.575, |
| "break_gt_control_rate_expert_ref": 0.5, |
| "mean_clean_ref_gap": 0.03325119987130165, |
| "mean_expert_ref_gap": -0.002129593398422003 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.4032095968723297, |
| "mean_break_expert_l2": 1.1326402425765991, |
| "mean_clean_mse": 0.1839713752269745, |
| "mean_control_clean_l2": 0.3201982378959656, |
| "mean_control_expert_l2": 1.1205980777740479, |
| "mean_control_mse": 0.18586888909339905, |
| "break_gt_control_rate_clean_ref": 0.65, |
| "break_gt_control_rate_expert_ref": 0.6, |
| "mean_clean_ref_gap": 0.08301133662462234, |
| "mean_expert_ref_gap": 0.012042166665196419 |
| } |
| }, |
| "num_records": 80 |
| }, |
| "test_metrics": { |
| "mean_break_clean_l2": 0.3397909700870514, |
| "mean_break_expert_l2": 1.2354549169540405, |
| "mean_clean_mse": 0.21382936835289001, |
| "mean_control_clean_l2": 0.3245377540588379, |
| "mean_control_expert_l2": 1.1614665985107422, |
| "mean_control_mse": 0.19767430424690247, |
| "break_gt_control_rate_clean_ref": 0.5375, |
| "break_gt_control_rate_expert_ref": 0.5125, |
| "mean_clean_ref_gap": 0.015253203921020031, |
| "mean_expert_ref_gap": 0.07398830354213715, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 0.23053522408008575, |
| "mean_break_expert_l2": 1.2631430625915527, |
| "mean_clean_mse": 0.22174766659736633, |
| "mean_control_clean_l2": 0.25600486993789673, |
| "mean_control_expert_l2": 1.2138757705688477, |
| "mean_control_mse": 0.21398238837718964, |
| "break_gt_control_rate_clean_ref": 0.45, |
| "break_gt_control_rate_expert_ref": 0.475, |
| "mean_clean_ref_gap": -0.025469651445746422, |
| "mean_expert_ref_gap": 0.04926721006631851 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 0.44904670119285583, |
| "mean_break_expert_l2": 1.2077667713165283, |
| "mean_clean_mse": 0.2059110701084137, |
| "mean_control_clean_l2": 0.39307063817977905, |
| "mean_control_expert_l2": 1.1090574264526367, |
| "mean_control_mse": 0.1813661754131317, |
| "break_gt_control_rate_clean_ref": 0.625, |
| "break_gt_control_rate_expert_ref": 0.55, |
| "mean_clean_ref_gap": 0.05597605183720589, |
| "mean_expert_ref_gap": 0.09870939701795578 |
| } |
| }, |
| "num_records": 80 |
| } |
| }, |
| "text_action_adapter": { |
| "best_val_loss_total": 0.36617913842201233, |
| "train_metrics": { |
| "mean_break_clean_l2": 1.9662588834762573, |
| "mean_break_expert_l2": 2.3562726974487305, |
| "mean_clean_mse": 0.1834717094898224, |
| "mean_control_clean_l2": 0.2672695815563202, |
| "mean_control_expert_l2": 1.0719788074493408, |
| "mean_control_mse": 0.1756630837917328, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.6989891529083252, |
| "mean_expert_ref_gap": 1.2842938899993896, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 1.7676277160644531, |
| "mean_break_expert_l2": 2.259204864501953, |
| "mean_clean_mse": 0.2054469883441925, |
| "mean_control_clean_l2": 0.2036994993686676, |
| "mean_control_expert_l2": 1.1436399221420288, |
| "mean_control_mse": 0.19497980177402496, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.5639280080795288, |
| "mean_expert_ref_gap": 1.1155647039413452 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 2.1648900508880615, |
| "mean_break_expert_l2": 2.4533402919769287, |
| "mean_clean_mse": 0.16149643063545227, |
| "mean_control_clean_l2": 0.3308396339416504, |
| "mean_control_expert_l2": 1.0003175735473633, |
| "mean_control_mse": 0.1563463807106018, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.8340502977371216, |
| "mean_expert_ref_gap": 1.453023076057434 |
| } |
| }, |
| "num_records": 240 |
| }, |
| "val_metrics": { |
| "mean_break_clean_l2": 1.9402227401733398, |
| "mean_break_expert_l2": 2.3244988918304443, |
| "mean_clean_mse": 0.168838232755661, |
| "mean_control_clean_l2": 0.23467329144477844, |
| "mean_control_expert_l2": 1.0611259937286377, |
| "mean_control_mse": 0.17241854965686798, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.7055492401123047, |
| "mean_expert_ref_gap": 1.2633731365203857, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 1.78369140625, |
| "mean_break_expert_l2": 2.2142577171325684, |
| "mean_clean_mse": 0.19796983897686005, |
| "mean_control_clean_l2": 0.19418850541114807, |
| "mean_control_expert_l2": 1.1197407245635986, |
| "mean_control_mse": 0.18957313895225525, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.5895028114318848, |
| "mean_expert_ref_gap": 1.0945169925689697 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 2.0967538356781006, |
| "mean_break_expert_l2": 2.4347403049468994, |
| "mean_clean_mse": 0.13970664143562317, |
| "mean_control_clean_l2": 0.2751580476760864, |
| "mean_control_expert_l2": 1.0025112628936768, |
| "mean_control_mse": 0.15526394546031952, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.8215957880020142, |
| "mean_expert_ref_gap": 1.4322290420532227 |
| } |
| }, |
| "num_records": 80 |
| }, |
| "test_metrics": { |
| "mean_break_clean_l2": 1.9752228260040283, |
| "mean_break_expert_l2": 2.467482089996338, |
| "mean_clean_mse": 0.18618619441986084, |
| "mean_control_clean_l2": 0.273476779460907, |
| "mean_control_expert_l2": 1.0540869235992432, |
| "mean_control_mse": 0.1706114113330841, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.7017459869384766, |
| "mean_expert_ref_gap": 1.4133949279785156, |
| "by_case": { |
| "break_target_object": { |
| "mean_break_clean_l2": 1.7627042531967163, |
| "mean_break_expert_l2": 2.334259033203125, |
| "mean_clean_mse": 0.21386170387268066, |
| "mean_control_clean_l2": 0.2173435240983963, |
| "mean_control_expert_l2": 1.1480493545532227, |
| "mean_control_mse": 0.19736817479133606, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.5453606843948364, |
| "mean_expert_ref_gap": 1.1862096786499023 |
| }, |
| "break_executability_condition": { |
| "mean_break_clean_l2": 2.18774151802063, |
| "mean_break_expert_l2": 2.6007046699523926, |
| "mean_clean_mse": 0.1585106998682022, |
| "mean_control_clean_l2": 0.3296099901199341, |
| "mean_control_expert_l2": 0.9601246118545532, |
| "mean_control_mse": 0.14385461807250977, |
| "break_gt_control_rate_clean_ref": 1.0, |
| "break_gt_control_rate_expert_ref": 1.0, |
| "mean_clean_ref_gap": 1.8581314086914062, |
| "mean_expert_ref_gap": 1.640580415725708 |
| } |
| }, |
| "num_records": 80 |
| } |
| } |
| } |
| } |
|
|
