diff --git a/.gitattributes b/.gitattributes index 992acf30b7f282135486970a323877bb3b3bf8e8..0d6ea1b18d0e633a0171630b445dcb8e99337ded 100644 --- a/.gitattributes +++ b/.gitattributes @@ -875,3 +875,43 @@ _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_left.png filter=lfs diff=lfs merge=lfs -text _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_right.png filter=lfs diff=lfs merge=lfs -text _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text +results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/case.json new file mode 100644 index 0000000000000000000000000000000000000000..e42b56cc3859703440cc93d19cc64a389ff32d4d --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/case.json @@ -0,0 +1,164 @@ +{ + "task": 0, + "episode": "motherboard_0909_episode_000.pt", + "category": "none", + "index": 1134, + "t_start": 1134, + "start_seconds": 75.6, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.015625, + "long_future_contact_fraction": 0.016666666666666666, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "first_contact_tick": 125, + "long_label": "none_at_start", + "output_subdir": "00_motherboard_0909_episode_000_none", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f05fb6b9f2f1c73d1025683d3236a9c6113362af --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:714dff21d089ab0704ddfbd44d1973286728628fe074079c2fa46da4a93b2f06 +size 7552869 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/motherboard_0909_episode_000_t01134_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/motherboard_0909_episode_000_t01134_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a577a31d0f271c87d1b3b07f0485a91424d1fb6 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/motherboard_0909_episode_000_t01134_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a1d930a777e0fa90a5fcb5dda9fc83df647db7cc8f407c4ff095f71c9258630 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..563fef1ae696906b8b1ae1e0e2eb5b29fadfb963 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 1134, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 1134, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 28.118625935497626, + "ssim": 0.8981439036327015, + "lpips": 0.02593047171831131 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 33.22567826043501, + "ssim": 0.9705809567376241, + "lpips": 0.028415463864803314 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 16.614699992395067, + "ssim": 0.6578296037120219, + "lpips": 0.13281607627868652 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 48.21878437631539, + "ssim": 0.9935058084502003, + "lpips": 0.004225203301757574 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 44.9079643326986, + "ssim": 0.98389798483938, + "lpips": 0.023684388026595116 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 1142, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.875, + "history_latent_mse": 0.07125979661941528, + "psnr": 22.299567988857767, + "ssim": 0.7740196461835518, + "lpips": 0.08631207048892975 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.4609375, + "history_latent_mse": 0.07626854628324509, + "psnr": 27.307863321953803, + "ssim": 0.9027393856884827, + "lpips": 0.07224798202514648 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8515625, + "history_latent_mse": 0.4535127580165863, + "psnr": 11.077167592783326, + "ssim": 0.35698008339236, + "lpips": 0.3966830372810364 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.693359375, + "history_latent_mse": 0.00990868080407381, + "psnr": 45.312933483226345, + "ssim": 0.9895434031357497, + "lpips": 0.016883008182048798 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4931640625, + "history_latent_mse": 0.03798486664891243, + "psnr": 42.18905422712518, + "ssim": 0.9758137255932309, + "lpips": 0.049356162548065186 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 1150, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.1796875, + "history_latent_mse": 0.1578950434923172, + "psnr": 20.496099549491138, + "ssim": 0.7374106287444411, + "lpips": 0.1451627016067505 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.3125, + "history_latent_mse": 0.16328312456607819, + "psnr": 24.916684340802206, + "ssim": 0.8430255329021106, + "lpips": 0.126552015542984 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.90625, + "history_latent_mse": 0.802646815776825, + "psnr": 10.756574512710799, + "ssim": 0.3366269340047899, + "lpips": 0.45835721492767334 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.92529296875, + "history_latent_mse": 0.021422749385237694, + "psnr": 43.282370651116224, + "ssim": 0.9847204751128106, + "lpips": 0.03882313519716263 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.390625, + "history_latent_mse": 0.05414335057139397, + "psnr": 41.48125187958886, + "ssim": 0.9737438472038825, + "lpips": 0.07059751451015472 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 1158, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.09375, + "history_latent_mse": 0.186897411942482, + "psnr": 20.132890376872112, + "ssim": 0.7151798327633927, + "lpips": 0.15986736118793488 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.6640625, + "history_latent_mse": 0.2769032120704651, + "psnr": 22.686464420994497, + "ssim": 0.7762327048393174, + "lpips": 0.15843285620212555 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.046875, + "history_latent_mse": 0.7685828804969788, + "psnr": 11.316598704771296, + "ssim": 0.27996114215063156, + "lpips": 0.46487122774124146 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.947509765625, + "history_latent_mse": 0.031877338886260986, + "psnr": 41.39235837791431, + "ssim": 0.9798823696250478, + "lpips": 0.0765223354101181 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.28125, + "history_latent_mse": 0.05412241816520691, + "psnr": 40.054445265286745, + "ssim": 0.9706170962246483, + "lpips": 0.09510093927383423 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 1166, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.4375, + "history_latent_mse": 0.19426828622817993, + "psnr": 19.521938358523744, + "ssim": 0.711563055300819, + "lpips": 0.17140625417232513 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.9921875, + "history_latent_mse": 0.3184545934200287, + "psnr": 21.595329401906657, + "ssim": 0.7292829632491972, + "lpips": 0.19176793098449707 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.8025609850883484, + "psnr": 11.085940117735175, + "ssim": 0.26395832446885137, + "lpips": 0.4824531674385071 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.044921875, + "history_latent_mse": 0.04302438721060753, + "psnr": 39.914173602008624, + "ssim": 0.9754849562365518, + "lpips": 0.11436552554368973 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.2421875, + "history_latent_mse": 0.07244016975164413, + "psnr": 38.92377956111862, + "ssim": 0.9673908140162797, + "lpips": 0.11861604452133179 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 1174, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.96875, + "history_latent_mse": 0.21044723689556122, + "psnr": 19.172171786625192, + "ssim": 0.704517872746727, + "lpips": 0.18479205667972565 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.82421875, + "history_latent_mse": 0.4116292893886566, + "psnr": 22.70440798830937, + "ssim": 0.7446463412912648, + "lpips": 0.19933201372623444 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.109375, + "history_latent_mse": 0.7731020450592041, + "psnr": 10.901734091153113, + "ssim": 0.250261526565162, + "lpips": 0.5134169459342957 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.259765625, + "history_latent_mse": 0.051759108901023865, + "psnr": 39.16722800912323, + "ssim": 0.9711905790082329, + "lpips": 0.14375180006027222 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.66015625, + "history_latent_mse": 0.0699012503027916, + "psnr": 38.23995041546551, + "ssim": 0.9635273440027555, + "lpips": 0.13030634820461273 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 1182, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.421875, + "history_latent_mse": 0.22228819131851196, + "psnr": 19.06247131517149, + "ssim": 0.6924016519226703, + "lpips": 0.18284600973129272 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.93505859375, + "history_latent_mse": 0.37236669659614563, + "psnr": 22.312370155604597, + "ssim": 0.7331450899010764, + "lpips": 0.21978585422039032 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.921875, + "history_latent_mse": 0.8304977416992188, + "psnr": 10.793835822812479, + "ssim": 0.24135036171360116, + "lpips": 0.4925817549228668 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.2421875, + "history_latent_mse": 0.0620768703520298, + "psnr": 38.341802320897685, + "ssim": 0.9668993105821726, + "lpips": 0.17329636216163635 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.1953125, + "history_latent_mse": 0.09515998512506485, + "psnr": 37.828723643381004, + "ssim": 0.9592576191761607, + "lpips": 0.14613281190395355 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1190, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.140625, + "history_latent_mse": 0.2338160276412964, + "psnr": 19.07268901263889, + "ssim": 0.6884140759189079, + "lpips": 0.1924017369747162 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.0859375, + "history_latent_mse": 0.37667128443717957, + "psnr": 22.223930212587973, + "ssim": 0.7211460010588575, + "lpips": 0.23736724257469177 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.11328125, + "history_latent_mse": 0.9431016445159912, + "psnr": 11.30551157083791, + "ssim": 0.24893031947198674, + "lpips": 0.4797683358192444 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.31640625, + "history_latent_mse": 0.07115178555250168, + "psnr": 37.64922164261528, + "ssim": 0.9626026930802947, + "lpips": 0.20532503724098206 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.525390625, + "history_latent_mse": 0.09037545323371887, + "psnr": 36.95520132192448, + "ssim": 0.9554855120856747, + "lpips": 0.17302285134792328 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1198, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.46875, + "history_latent_mse": 0.24051082134246826, + "psnr": 19.06611351386221, + "ssim": 0.6793518227863704, + "lpips": 0.2010258138179779 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.75, + "history_latent_mse": 0.3760389983654022, + "psnr": 21.29236028691256, + "ssim": 0.6912351388757134, + "lpips": 0.26064425706863403 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.9166877865791321, + "psnr": 11.075601469624822, + "ssim": 0.2391405727778968, + "lpips": 0.5041201114654541 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.58203125, + "history_latent_mse": 0.08001843839883804, + "psnr": 36.95966657540948, + "ssim": 0.9588211571345229, + "lpips": 0.22562216222286224 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.05859375, + "history_latent_mse": 0.10932174324989319, + "psnr": 36.3994679137689, + "ssim": 0.9515778298865583, + "lpips": 0.1953134536743164 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1206, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.25, + "history_latent_mse": 0.2604706883430481, + "psnr": 18.503925060285766, + "ssim": 0.644975085064724, + "lpips": 0.24730649590492249 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.498046875, + "history_latent_mse": 0.4385768175125122, + "psnr": 19.918945132686176, + "ssim": 0.6337497940723535, + "lpips": 0.2764352560043335 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.125, + "history_latent_mse": 0.8151993751525879, + "psnr": 10.891914773891202, + "ssim": 0.23601986912850992, + "lpips": 0.5140556693077087 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.509765625, + "history_latent_mse": 0.0883679911494255, + "psnr": 36.5672491510434, + "ssim": 0.9549003553892959, + "lpips": 0.24738405644893646 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.95703125, + "history_latent_mse": 0.10421737283468246, + "psnr": 36.06402608273454, + "ssim": 0.9495501773313848, + "lpips": 0.19591185450553894 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 1214, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.8671875, + "history_latent_mse": 0.2957853674888611, + "psnr": 18.30719393323559, + "ssim": 0.6242453652896285, + "lpips": 0.26864248514175415 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.515625, + "history_latent_mse": 0.4766908586025238, + "psnr": 19.296219314166898, + "ssim": 0.6084653542439014, + "lpips": 0.29249680042266846 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.1171875, + "history_latent_mse": 0.823030948638916, + "psnr": 9.929037641527676, + "ssim": 0.23528381378321522, + "lpips": 0.5044862627983093 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.4716796875, + "history_latent_mse": 0.09646136313676834, + "psnr": 36.09376945101387, + "ssim": 0.9505532080139466, + "lpips": 0.2651330232620239 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.25, + "history_latent_mse": 0.12446943670511246, + "psnr": 35.77057445272215, + "ssim": 0.9483529378923702, + "lpips": 0.1994127333164215 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1222, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3203125, + "history_latent_mse": 0.30306458473205566, + "psnr": 17.93564056800799, + "ssim": 0.6129561302921299, + "lpips": 0.2883601784706116 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.0625, + "history_latent_mse": 0.48860955238342285, + "psnr": 18.96807842006445, + "ssim": 0.5910876772669825, + "lpips": 0.31704819202423096 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.703125, + "history_latent_mse": 0.9537672400474548, + "psnr": 9.410330656003293, + "ssim": 0.2376455665181072, + "lpips": 0.5271283388137817 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.58984375, + "history_latent_mse": 0.10451682657003403, + "psnr": 35.790241161950455, + "ssim": 0.9467691672545042, + "lpips": 0.2830126881599426 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.0078125, + "history_latent_mse": 0.1126982644200325, + "psnr": 35.091368951824094, + "ssim": 0.9443291247814589, + "lpips": 0.2325611114501953 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1230, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.46875, + "history_latent_mse": 0.31999102234840393, + "psnr": 17.928656525741033, + "ssim": 0.614931015609635, + "lpips": 0.2899765074253082 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.99609375, + "history_latent_mse": 0.5375950932502747, + "psnr": 18.303168251730394, + "ssim": 0.5682538687196317, + "lpips": 0.3424307405948639 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.046875, + "history_latent_mse": 1.0547698736190796, + "psnr": 9.388592113479417, + "ssim": 0.23906650579267608, + "lpips": 0.5055872201919556 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.69921875, + "history_latent_mse": 0.11771353334188461, + "psnr": 35.4045823403573, + "ssim": 0.9431462540202011, + "lpips": 0.29494112730026245 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.671875, + "history_latent_mse": 0.14025308191776276, + "psnr": 34.770440023358226, + "ssim": 0.942512951717249, + "lpips": 0.22735056281089783 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1238, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.5390625, + "history_latent_mse": 0.3158722519874573, + "psnr": 17.91332162646378, + "ssim": 0.6223806892138105, + "lpips": 0.2958202362060547 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.703125, + "history_latent_mse": 0.5239625573158264, + "psnr": 17.56523429353433, + "ssim": 0.5341453904969209, + "lpips": 0.3766480088233948 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5078125, + "history_latent_mse": 0.934349775314331, + "psnr": 9.723535208620142, + "ssim": 0.23142451541698372, + "lpips": 0.513656735420227 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.88671875, + "history_latent_mse": 0.12004297971725464, + "psnr": 34.77172152401946, + "ssim": 0.9387000663377864, + "lpips": 0.3137485980987549 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.19921875, + "history_latent_mse": 0.13056305050849915, + "psnr": 34.15736555453252, + "ssim": 0.9386077918616212, + "lpips": 0.23681756854057312 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1246, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.51171875, + "history_latent_mse": 0.2952965199947357, + "psnr": 17.643402573338243, + "ssim": 0.6265155324322179, + "lpips": 0.2974652051925659 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.6053735017776489, + "psnr": 16.593013323686236, + "ssim": 0.5137983982273475, + "lpips": 0.4190220236778259 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.0, + "history_latent_mse": 0.9508810043334961, + "psnr": 9.878230801841482, + "ssim": 0.19101576200356277, + "lpips": 0.5357831716537476 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.9375, + "history_latent_mse": 0.12828531861305237, + "psnr": 34.28512883129351, + "ssim": 0.9350560243901624, + "lpips": 0.3242204487323761 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.458984375, + "history_latent_mse": 0.13128380477428436, + "psnr": 33.080836199969895, + "ssim": 0.9308731355698692, + "lpips": 0.2695121169090271 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67baac83973ce66f5fcfcb57bb158f42de73e6f7 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf61e43e0a6412de10a31f00ef4aa6dfcfa4a9cdf8b2aae2d144b20c6e01d0f +size 918193 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..31dc8120f985cc5dee2d37735b9267cfeb70dd88 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.08200626075267792, + "wrist/video/latent_future_mse": 0.08200626075267792, + "video/psnr": 28.118625935497626, + "wrist/video/psnr": 28.118625935497626, + "video/ssim": 0.8981439036327015, + "wrist/video/ssim": 0.8981439036327015, + "video/lpips": 0.02593047171831131, + "wrist/video/lpips": 0.02593047171831131, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.08269955962896347, + "wrist/wrist_left/latent_future_mse": 0.08269955962896347, + "wrist_left/psnr": 33.22567826043501, + "wrist/wrist_left/psnr": 33.22567826043501, + "wrist_left/ssim": 0.9705809567376241, + "wrist/wrist_left/ssim": 0.9705809567376241, + "wrist_left/lpips": 0.028415463864803314, + "wrist/wrist_left/lpips": 0.028415463864803314, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.47750023007392883, + "wrist/wrist_right/latent_future_mse": 0.47750023007392883, + "wrist_right/psnr": 16.614699992395067, + "wrist/wrist_right/psnr": 16.614699992395067, + "wrist_right/ssim": 0.6578296037120219, + "wrist/wrist_right/ssim": 0.6578296037120219, + "wrist_right/lpips": 0.13281607627868652, + "wrist/wrist_right/lpips": 0.13281607627868652, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.007856979966163635, + "wrist/tactile_left/latent_future_mse": 0.007856979966163635, + "tactile_left/psnr": 48.21878437631539, + "wrist/tactile_left/psnr": 48.21878437631539, + "tactile_left/ssim": 0.9935058084502003, + "wrist/tactile_left/ssim": 0.9935058084502003, + "tactile_left/lpips": 0.004225203301757574, + "wrist/tactile_left/lpips": 0.004225203301757574, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.03919291868805885, + "wrist/tactile_right/latent_future_mse": 0.03919291868805885, + "tactile_right/psnr": 44.9079643326986, + "wrist/tactile_right/psnr": 44.9079643326986, + "tactile_right/ssim": 0.98389798483938, + "wrist/tactile_right/ssim": 0.98389798483938, + "tactile_right/lpips": 0.023684388026595116, + "wrist/tactile_right/lpips": 0.023684388026595116, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 1134, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08200626075267792, + "psnr": 28.118625935497626, + "ssim": 0.8981439036327015, + "lpips": 0.02593047171831131 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08269955962896347, + "psnr": 33.22567826043501, + "ssim": 0.9705809567376241, + "lpips": 0.028415463864803314 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.47750023007392883, + "psnr": 16.614699992395067, + "ssim": 0.6578296037120219, + "lpips": 0.13281607627868652 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.007856979966163635, + "psnr": 48.21878437631539, + "ssim": 0.9935058084502003, + "lpips": 0.004225203301757574 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.03919291868805885, + "psnr": 44.9079643326986, + "ssim": 0.98389798483938, + "lpips": 0.023684388026595116 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/case.json new file mode 100644 index 0000000000000000000000000000000000000000..954769cdb6ed72fcfa6168fe000b59a30ae5a76b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/case.json @@ -0,0 +1,168 @@ +{ + "task": 1, + "episode": "motherboard_0909_episode_000.pt", + "category": "onset", + "index": 1825, + "t_start": 1825, + "start_seconds": 121.66666666666667, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.7265625, + "long_future_contact_fraction": 0.775, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "01_motherboard_0909_episode_000_onset", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc38adb88a9fd99ba5cca46109e9ef262572c2cf --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70df5a834c3c49340462c55c679b91256fe24fc5365d57fd4813ce8b2de6cc69 +size 9526143 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/motherboard_0909_episode_000_t01825_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/motherboard_0909_episode_000_t01825_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cf924f7ab949e5879aa934ad3aac2a7360bd2a3 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/motherboard_0909_episode_000_t01825_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6734064a9940d0d8856e6db42f2085b0011cd908057f98627b4dc5da31497560 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..e029db9990b1940c6877e7d594dc7b096678afd4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 1825, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 1825, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 23.628176260948578, + "ssim": 0.8456452647494677, + "lpips": 0.0570581778883934 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 16.974798261240593, + "ssim": 0.430608030325784, + "lpips": 0.1560155153274536 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 11.3508814597468, + "ssim": 0.42637402380382783, + "lpips": 0.37309491634368896 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 37.21835685193686, + "ssim": 0.9798802064902503, + "lpips": 0.055090613663196564 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.144626811115348, + "ssim": 0.9719350201675224, + "lpips": 0.10720653086900711 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 1833, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.453125, + "history_latent_mse": 0.11924435943365097, + "psnr": 21.38447871325007, + "ssim": 0.7454026098413906, + "lpips": 0.09989136457443237 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.703125, + "history_latent_mse": 0.6149580478668213, + "psnr": 15.709534166775507, + "ssim": 0.3481757982898162, + "lpips": 0.23345258831977844 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.75, + "history_latent_mse": 0.8365908265113831, + "psnr": 10.51854516089019, + "ssim": 0.39563176970008596, + "lpips": 0.4264390170574188 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.1328125, + "history_latent_mse": 0.07114292681217194, + "psnr": 27.08653562731253, + "ssim": 0.9231090405566171, + "lpips": 0.26742565631866455 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.8671875, + "history_latent_mse": 0.102386474609375, + "psnr": 26.352180183774756, + "ssim": 0.9428494174808446, + "lpips": 0.20953956246376038 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 1841, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.03125, + "history_latent_mse": 0.15389490127563477, + "psnr": 20.58760675670795, + "ssim": 0.7152439390162091, + "lpips": 0.13344064354896545 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.32421875, + "history_latent_mse": 0.5376542210578918, + "psnr": 14.86372024979256, + "ssim": 0.31986041049769304, + "lpips": 0.3564751148223877 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 0.8208544254302979, + "psnr": 9.399308772525803, + "ssim": 0.379937250363267, + "lpips": 0.4914754331111908 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.9609375, + "history_latent_mse": 0.18323363363742828, + "psnr": 27.37818445738302, + "ssim": 0.9299320902182369, + "lpips": 0.25641900300979614 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.9951171875, + "history_latent_mse": 0.13819794356822968, + "psnr": 26.38334682961372, + "ssim": 0.9408343045795153, + "lpips": 0.21969859302043915 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 1849, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.68359375, + "history_latent_mse": 0.19861239194869995, + "psnr": 20.758812785152756, + "ssim": 0.6975106537333856, + "lpips": 0.1406661719083786 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5, + "history_latent_mse": 0.8406947255134583, + "psnr": 12.81062295552455, + "ssim": 0.2740668538115486, + "lpips": 0.45320695638656616 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.66796875, + "history_latent_mse": 0.9032319188117981, + "psnr": 12.964569428178333, + "ssim": 0.44145546590248474, + "lpips": 0.46219319105148315 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.640625, + "history_latent_mse": 0.18205487728118896, + "psnr": 27.98484835314955, + "ssim": 0.9292138306951223, + "lpips": 0.2694128751754761 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.44140625, + "history_latent_mse": 0.14041712880134583, + "psnr": 41.67464345589414, + "ssim": 0.9743010559371356, + "lpips": 0.08515998721122742 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 1857, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8359375, + "history_latent_mse": 0.21089641749858856, + "psnr": 20.12968986430085, + "ssim": 0.6729452297464844, + "lpips": 0.19104285538196564 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.78125, + "history_latent_mse": 0.9549667835235596, + "psnr": 12.550545095641612, + "ssim": 0.42146506309265463, + "lpips": 0.4507025480270386 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8984375, + "history_latent_mse": 0.7145854830741882, + "psnr": 11.401634918412771, + "ssim": 0.28026669964421147, + "lpips": 0.6360324621200562 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8828125, + "history_latent_mse": 0.16469396650791168, + "psnr": 38.39359132029856, + "ssim": 0.9650768356833449, + "lpips": 0.1758243292570114 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.859375, + "history_latent_mse": 0.06372665613889694, + "psnr": 39.74709760258228, + "ssim": 0.9697240396062548, + "lpips": 0.11890912055969238 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 1865, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.703125, + "history_latent_mse": 0.26115942001342773, + "psnr": 19.3840588315676, + "ssim": 0.6533332812739424, + "lpips": 0.22177474200725555 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.52734375, + "history_latent_mse": 0.7400798201560974, + "psnr": 11.641789232035197, + "ssim": 0.35105336418918753, + "lpips": 0.49183470010757446 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5703125, + "history_latent_mse": 1.0202034711837769, + "psnr": 10.671679805235513, + "ssim": 0.2757385503072361, + "lpips": 0.7221993207931519 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.2265625, + "history_latent_mse": 0.092548668384552, + "psnr": 38.86592262744519, + "ssim": 0.9639488963676137, + "lpips": 0.1817716360092163 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.01171875, + "history_latent_mse": 0.08237791061401367, + "psnr": 28.937859597528977, + "ssim": 0.9130735440172923, + "lpips": 0.3372594118118286 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 1873, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.234375, + "history_latent_mse": 0.2853606045246124, + "psnr": 19.327087853647637, + "ssim": 0.6527707724798322, + "lpips": 0.23417320847511292 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.5546875, + "history_latent_mse": 0.9626197814941406, + "psnr": 10.949636084625007, + "ssim": 0.29039771588789137, + "lpips": 0.5295237302780151 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 1.1418527364730835, + "psnr": 9.80005694351729, + "ssim": 0.27183607856813097, + "lpips": 0.7450927495956421 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6171875, + "history_latent_mse": 0.08214738219976425, + "psnr": 38.044049077303185, + "ssim": 0.9611224301634556, + "lpips": 0.19384586811065674 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.6328125, + "history_latent_mse": 0.2798396050930023, + "psnr": 29.630710620583017, + "ssim": 0.9067128979478171, + "lpips": 0.3555738925933838 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1881, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9765625, + "history_latent_mse": 0.3022192120552063, + "psnr": 18.474017428693067, + "ssim": 0.6382623648501708, + "lpips": 0.2747034728527069 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9296875, + "history_latent_mse": 0.9861699938774109, + "psnr": 9.804466590576641, + "ssim": 0.239695646269107, + "lpips": 0.5884726047515869 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.171875, + "history_latent_mse": 1.1945067644119263, + "psnr": 9.817503092548431, + "ssim": 0.2559563394471839, + "lpips": 0.7234587669372559 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.546875, + "history_latent_mse": 0.08684799820184708, + "psnr": 37.02358126070998, + "ssim": 0.9571071971857603, + "lpips": 0.21416965126991272 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.890625, + "history_latent_mse": 0.3355695307254791, + "psnr": 30.56725607826592, + "ssim": 0.9054872014132207, + "lpips": 0.3531824052333832 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1889, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.53125, + "history_latent_mse": 0.3008727729320526, + "psnr": 18.334771306226575, + "ssim": 0.6482847651536765, + "lpips": 0.28134870529174805 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.23046875, + "history_latent_mse": 1.0673683881759644, + "psnr": 10.502831838929554, + "ssim": 0.3037068785100905, + "lpips": 0.5442860722541809 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 1.1673964262008667, + "psnr": 9.045239052512038, + "ssim": 0.18471718775414525, + "lpips": 0.6944596767425537 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.7578125, + "history_latent_mse": 0.09910959005355835, + "psnr": 36.619810000881145, + "ssim": 0.9556470391864594, + "lpips": 0.2256094217300415 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.5546875, + "history_latent_mse": 0.2062147855758667, + "psnr": 36.28021961420545, + "ssim": 0.959326061857189, + "lpips": 0.17003518342971802 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1897, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.765625, + "history_latent_mse": 0.30186331272125244, + "psnr": 18.03807525162402, + "ssim": 0.6454518931632223, + "lpips": 0.28907760977745056 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.875, + "history_latent_mse": 0.8890538811683655, + "psnr": 10.39684811802661, + "ssim": 0.29498948139481107, + "lpips": 0.5650955438613892 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.7734375, + "history_latent_mse": 1.1140291690826416, + "psnr": 9.289455988436062, + "ssim": 0.1557308833748376, + "lpips": 0.7086127996444702 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6171875, + "history_latent_mse": 0.09790438413619995, + "psnr": 36.211948050450346, + "ssim": 0.9528436610809317, + "lpips": 0.23762783408164978 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.359375, + "history_latent_mse": 0.14760760962963104, + "psnr": 34.08932658177453, + "ssim": 0.9368645450533417, + "lpips": 0.21939662098884583 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 1905, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.25, + "history_latent_mse": 0.28883758187294006, + "psnr": 18.00619140224066, + "ssim": 0.645133041821521, + "lpips": 0.29140937328338623 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.53125, + "history_latent_mse": 1.0083221197128296, + "psnr": 10.23226909703255, + "ssim": 0.28376860883500077, + "lpips": 0.5770002603530884 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.765625, + "history_latent_mse": 1.3987174034118652, + "psnr": 9.058174410777262, + "ssim": 0.14708021609148147, + "lpips": 0.7367289066314697 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6865234375, + "history_latent_mse": 0.10319709032773972, + "psnr": 35.50569664133912, + "ssim": 0.9492254278084, + "lpips": 0.24971535801887512 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.9677734375, + "history_latent_mse": 0.23123686015605927, + "psnr": 30.87939622028728, + "ssim": 0.8752212486062663, + "lpips": 0.30587390065193176 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1913, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.140625, + "history_latent_mse": 0.2750135064125061, + "psnr": 18.022429446139647, + "ssim": 0.6455458783856715, + "lpips": 0.29137641191482544 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.76171875, + "history_latent_mse": 1.0009616613388062, + "psnr": 10.371446021281326, + "ssim": 0.3005361599410692, + "lpips": 0.5711427330970764 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.84375, + "history_latent_mse": 1.2773358821868896, + "psnr": 9.281254755236965, + "ssim": 0.1496813331936439, + "lpips": 0.7296895980834961 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.650390625, + "history_latent_mse": 0.10978487133979797, + "psnr": 35.15160195860626, + "ssim": 0.9460204977305288, + "lpips": 0.2653445899486542 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.27734375, + "history_latent_mse": 0.26345571875572205, + "psnr": 31.984257250549504, + "ssim": 0.8987391876954239, + "lpips": 0.2942860722541809 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1921, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.42138671875, + "history_latent_mse": 0.2782556712627411, + "psnr": 18.08716396623567, + "ssim": 0.6525679444946482, + "lpips": 0.2933219075202942 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.75, + "history_latent_mse": 0.9220336079597473, + "psnr": 10.313384160235174, + "ssim": 0.30653793556181264, + "lpips": 0.5786222219467163 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.890625, + "history_latent_mse": 1.3143733739852905, + "psnr": 9.654781771154955, + "ssim": 0.16233785932343134, + "lpips": 0.7144385576248169 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.875, + "history_latent_mse": 0.11926188319921494, + "psnr": 34.73191641558897, + "ssim": 0.9422505534304545, + "lpips": 0.2749377191066742 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.84375, + "history_latent_mse": 0.22833473980426788, + "psnr": 33.316754080222914, + "ssim": 0.9276727551495014, + "lpips": 0.24914956092834473 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1929, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.892578125, + "history_latent_mse": 0.27144190669059753, + "psnr": 18.12713456455959, + "ssim": 0.6514087079592135, + "lpips": 0.2898755371570587 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0703125, + "history_latent_mse": 0.9486039280891418, + "psnr": 10.831998972595065, + "ssim": 0.34260404403467953, + "lpips": 0.5509961843490601 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.59375, + "history_latent_mse": 1.3222182989120483, + "psnr": 9.676479397509294, + "ssim": 0.17591857036204672, + "lpips": 0.6677525043487549 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.84716796875, + "history_latent_mse": 0.12162520736455917, + "psnr": 34.26179215064562, + "ssim": 0.9379269060301725, + "lpips": 0.2951134443283081 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.45703125, + "history_latent_mse": 0.17624182999134064, + "psnr": 35.574963847934846, + "ssim": 0.9519700138394485, + "lpips": 0.17888721823692322 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1937, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 7.125, + "history_latent_mse": 0.28930458426475525, + "psnr": 17.933937036292257, + "ssim": 0.6501697360484575, + "lpips": 0.2973931133747101 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9375, + "history_latent_mse": 0.951725959777832, + "psnr": 10.521384358361725, + "ssim": 0.3264058407296058, + "lpips": 0.5590139031410217 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.671875, + "history_latent_mse": 1.1301602125167847, + "psnr": 9.693420192053367, + "ssim": 0.17648837550633995, + "lpips": 0.7731425762176514 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.125, + "history_latent_mse": 0.12921133637428284, + "psnr": 34.00654352401008, + "ssim": 0.9354487104185354, + "lpips": 0.3045352101325989 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.75390625, + "history_latent_mse": 0.10979020595550537, + "psnr": 31.363891055101227, + "ssim": 0.9245440910377623, + "lpips": 0.280612051486969 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1dd89d645ee1bf6f5c6706d02bb8ccf212265348 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a04544d9b48f7e181852631ba6543ac50e3f6e94c3b26ac4fbcaaeadcd3df1c +size 1265979 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..95bafacfc41bf9d38008e434a021ddd000d8f396 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.15089699625968933, + "wrist/video/latent_future_mse": 0.15089699625968933, + "video/psnr": 23.628176260948578, + "wrist/video/psnr": 23.628176260948578, + "video/ssim": 0.8456452647494677, + "wrist/video/ssim": 0.8456452647494677, + "video/lpips": 0.0570581778883934, + "wrist/video/lpips": 0.0570581778883934, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.6843722462654114, + "wrist/wrist_left/latent_future_mse": 0.6843722462654114, + "wrist_left/psnr": 16.974798261240593, + "wrist/wrist_left/psnr": 16.974798261240593, + "wrist_left/ssim": 0.430608030325784, + "wrist/wrist_left/ssim": 0.430608030325784, + "wrist_left/lpips": 0.1560155153274536, + "wrist/wrist_left/lpips": 0.1560155153274536, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.9355753064155579, + "wrist/wrist_right/latent_future_mse": 0.9355753064155579, + "wrist_right/psnr": 11.3508814597468, + "wrist/wrist_right/psnr": 11.3508814597468, + "wrist_right/ssim": 0.42637402380382783, + "wrist/wrist_right/ssim": 0.42637402380382783, + "wrist_right/lpips": 0.37309491634368896, + "wrist/wrist_right/lpips": 0.37309491634368896, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.07507113367319107, + "wrist/tactile_left/latent_future_mse": 0.07507113367319107, + "tactile_left/psnr": 37.21835685193686, + "wrist/tactile_left/psnr": 37.21835685193686, + "tactile_left/ssim": 0.9798802064902503, + "wrist/tactile_left/ssim": 0.9798802064902503, + "tactile_left/lpips": 0.055090613663196564, + "wrist/tactile_left/lpips": 0.055090613663196564, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.1042751669883728, + "wrist/tactile_right/latent_future_mse": 0.1042751669883728, + "tactile_right/psnr": 31.144626811115348, + "wrist/tactile_right/psnr": 31.144626811115348, + "tactile_right/ssim": 0.9719350201675224, + "wrist/tactile_right/ssim": 0.9719350201675224, + "tactile_right/lpips": 0.10720653086900711, + "wrist/tactile_right/lpips": 0.10720653086900711, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 1825, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.15089699625968933, + "psnr": 23.628176260948578, + "ssim": 0.8456452647494677, + "lpips": 0.0570581778883934 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.6843722462654114, + "psnr": 16.974798261240593, + "ssim": 0.430608030325784, + "lpips": 0.1560155153274536 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.9355753064155579, + "psnr": 11.3508814597468, + "ssim": 0.42637402380382783, + "lpips": 0.37309491634368896 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.07507113367319107, + "psnr": 37.21835685193686, + "ssim": 0.9798802064902503, + "lpips": 0.055090613663196564 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.1042751669883728, + "psnr": 31.144626811115348, + "ssim": 0.9719350201675224, + "lpips": 0.10720653086900711 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/case.json new file mode 100644 index 0000000000000000000000000000000000000000..4948e622e90d3aedebacaa8d4c66b2d6710e439e --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/case.json @@ -0,0 +1,168 @@ +{ + "task": 2, + "episode": "motherboard_0909_episode_000.pt", + "category": "release", + "index": 3072, + "t_start": 3072, + "start_seconds": 204.8, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.7734375, + "long_future_contact_fraction": 0.7583333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "02_motherboard_0909_episode_000_release", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca1a1bf864533381fb37fb1a16c55a85261a16e4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98b09b60768a3011884e2aebcc363698237271e882e07b3c939b581f319eb01 +size 9025700 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/motherboard_0909_episode_000_t03072_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/motherboard_0909_episode_000_t03072_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c8bb7c397ade764fae769d606707a60527f868d --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/motherboard_0909_episode_000_t03072_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd08ce704e39dbcdb41edd46f6924ee2482860068255be037fd1c48f49a699bd +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..da75b364b0617cc0c1aaf64dbd1a79eec02460a4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 3072, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 3072, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 29.266572537636293, + "ssim": 0.9162150885896057, + "lpips": 0.03114306926727295 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 15.72964153701189, + "ssim": 0.5237543840369523, + "lpips": 0.18139484524726868 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.913819208591171, + "ssim": 0.5439860906598222, + "lpips": 0.30286651849746704 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 42.44182842336647, + "ssim": 0.9840405275886075, + "lpips": 0.014103841967880726 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 43.15277773614905, + "ssim": 0.9819352677548217, + "lpips": 0.04016111418604851 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 3080, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.658203125, + "history_latent_mse": 0.0524832159280777, + "psnr": 26.480900577635992, + "ssim": 0.8568048789627838, + "lpips": 0.0566643625497818 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.59375, + "history_latent_mse": 0.5854699611663818, + "psnr": 14.425112517205427, + "ssim": 0.44195591168688314, + "lpips": 0.2938387393951416 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.6425979137420654, + "psnr": 11.877670430829415, + "ssim": 0.4240810047598062, + "lpips": 0.4234185218811035 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.53125, + "history_latent_mse": 0.03738224878907204, + "psnr": 37.894477381030505, + "ssim": 0.9701083073248673, + "lpips": 0.06112324446439743 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.41015625, + "history_latent_mse": 0.05036762356758118, + "psnr": 42.507885252321444, + "ssim": 0.9766029615344074, + "lpips": 0.055503640323877335 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 3088, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.1484375, + "history_latent_mse": 0.07951003313064575, + "psnr": 24.564863578668753, + "ssim": 0.7881806175917664, + "lpips": 0.08079169690608978 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.296875, + "history_latent_mse": 0.6849780082702637, + "psnr": 12.90153598569784, + "ssim": 0.38549987983287787, + "lpips": 0.36882656812667847 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5859375, + "history_latent_mse": 0.7878640294075012, + "psnr": 11.660039349859384, + "ssim": 0.38050448241676854, + "lpips": 0.5146379470825195 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.48828125, + "history_latent_mse": 0.06754805892705917, + "psnr": 34.91500774183322, + "ssim": 0.9589869880474067, + "lpips": 0.14406634867191315 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.48828125, + "history_latent_mse": 0.055852070450782776, + "psnr": 35.54515781236948, + "ssim": 0.9663785536995418, + "lpips": 0.1137726902961731 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 3096, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.890625, + "history_latent_mse": 0.12277263402938843, + "psnr": 22.94232235508399, + "ssim": 0.729770033078336, + "lpips": 0.10369352996349335 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.3984375, + "history_latent_mse": 0.690626323223114, + "psnr": 13.911253496880104, + "ssim": 0.4345450204978216, + "lpips": 0.34035584330558777 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.8453990817070007, + "psnr": 10.980996235036901, + "ssim": 0.3287601789194513, + "lpips": 0.5500631332397461 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.72265625, + "history_latent_mse": 0.09262625128030777, + "psnr": 33.46081177972756, + "ssim": 0.9531914052591324, + "lpips": 0.20742003619670868 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.34375, + "history_latent_mse": 0.11502578109502792, + "psnr": 31.539306887337162, + "ssim": 0.9472127857497132, + "lpips": 0.203257218003273 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 3104, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.9296875, + "history_latent_mse": 0.16665054857730865, + "psnr": 22.04007984596984, + "ssim": 0.6941891443404591, + "lpips": 0.124331995844841 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.7421875, + "history_latent_mse": 0.6611791253089905, + "psnr": 14.176640399708514, + "ssim": 0.4220857849868887, + "lpips": 0.3271068036556244 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.640625, + "history_latent_mse": 0.8994737267494202, + "psnr": 10.491407354591216, + "ssim": 0.3351243739596609, + "lpips": 0.5160884857177734 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.7421875, + "history_latent_mse": 0.11128786951303482, + "psnr": 34.6155133245452, + "ssim": 0.9577253908695167, + "lpips": 0.21651971340179443 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.1015625, + "history_latent_mse": 0.11482458561658859, + "psnr": 31.327422280966797, + "ssim": 0.942726219103224, + "lpips": 0.21856579184532166 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 3112, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.352783203125, + "history_latent_mse": 0.18367813527584076, + "psnr": 21.127098852448007, + "ssim": 0.6933516038223533, + "lpips": 0.17727592587471008 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.5, + "history_latent_mse": 0.5865779519081116, + "psnr": 12.512415844269501, + "ssim": 0.3622295575154916, + "lpips": 0.40887606143951416 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 0.9623259902000427, + "psnr": 9.850495173981773, + "ssim": 0.3399974859369542, + "lpips": 0.48278534412384033 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.390625, + "history_latent_mse": 0.11763932555913925, + "psnr": 34.978677140953806, + "ssim": 0.9573239829106417, + "lpips": 0.22706487774848938 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.70703125, + "history_latent_mse": 0.13144007325172424, + "psnr": 33.42194081729072, + "ssim": 0.9504479238586145, + "lpips": 0.209794282913208 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 3120, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6640625, + "history_latent_mse": 0.23482775688171387, + "psnr": 19.742974716246493, + "ssim": 0.6568032426650807, + "lpips": 0.24461683630943298 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9609375, + "history_latent_mse": 0.7560710310935974, + "psnr": 11.087393192067854, + "ssim": 0.30024546702065225, + "lpips": 0.4932715892791748 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5, + "history_latent_mse": 0.9448109269142151, + "psnr": 10.017077900899105, + "ssim": 0.32922143118481717, + "lpips": 0.49438393115997314 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.60546875, + "history_latent_mse": 0.10446634143590927, + "psnr": 35.094482493360594, + "ssim": 0.9545911342300593, + "lpips": 0.2322356104850769 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.37109375, + "history_latent_mse": 0.12086028605699539, + "psnr": 34.39604525779137, + "ssim": 0.9508120283033794, + "lpips": 0.20816770195960999 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 3128, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0234375, + "history_latent_mse": 0.26398757100105286, + "psnr": 19.490515236262617, + "ssim": 0.6357417017243336, + "lpips": 0.2562229037284851 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.6015625, + "history_latent_mse": 0.9454730153083801, + "psnr": 11.381691494621517, + "ssim": 0.317029280474494, + "lpips": 0.46522700786590576 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7890625, + "history_latent_mse": 0.9471634030342102, + "psnr": 9.323565723535017, + "ssim": 0.31195205039034823, + "lpips": 0.5405911803245544 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.15771484375, + "history_latent_mse": 0.10136154294013977, + "psnr": 34.641060180658116, + "ssim": 0.9482866741629074, + "lpips": 0.260652095079422 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.40234375, + "history_latent_mse": 0.1180509701371193, + "psnr": 36.045222383582995, + "ssim": 0.9531250000497351, + "lpips": 0.2066667228937149 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 3136, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.953125, + "history_latent_mse": 0.2903461456298828, + "psnr": 19.49002682443959, + "ssim": 0.6248671674031994, + "lpips": 0.23285439610481262 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 0.8181669116020203, + "psnr": 11.839906286502913, + "ssim": 0.34480539758716977, + "lpips": 0.4296375513076782 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8125, + "history_latent_mse": 0.9962184429168701, + "psnr": 8.001149714870516, + "ssim": 0.26536818740541224, + "lpips": 0.597160816192627 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.546875, + "history_latent_mse": 0.12413499504327774, + "psnr": 32.79141311437882, + "ssim": 0.9389920308518498, + "lpips": 0.3098233938217163 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.171875, + "history_latent_mse": 0.1330520361661911, + "psnr": 36.726209603006545, + "ssim": 0.9541922844243047, + "lpips": 0.20737692713737488 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 3144, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6788330078125, + "history_latent_mse": 0.27717462182044983, + "psnr": 19.398272521696065, + "ssim": 0.6096253384874204, + "lpips": 0.22750045359134674 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.75, + "history_latent_mse": 0.8745848536491394, + "psnr": 10.611273957111177, + "ssim": 0.32374390172224476, + "lpips": 0.4980604350566864 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.8125, + "history_latent_mse": 1.1955797672271729, + "psnr": 6.962309834326984, + "ssim": 0.2628506639940223, + "lpips": 0.6147164106369019 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.9453125, + "history_latent_mse": 0.14682048559188843, + "psnr": 30.486920618259962, + "ssim": 0.9299237964500228, + "lpips": 0.33560019731521606 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.609375, + "history_latent_mse": 0.1349182277917862, + "psnr": 36.70657183525083, + "ssim": 0.953313381429965, + "lpips": 0.1944001019001007 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 3152, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.15234375, + "history_latent_mse": 0.299398273229599, + "psnr": 19.78143673961738, + "ssim": 0.6238783178037695, + "lpips": 0.21857289969921112 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9296875, + "history_latent_mse": 0.8995222449302673, + "psnr": 12.112046005977927, + "ssim": 0.31583914984852396, + "lpips": 0.4641914665699005 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.421875, + "history_latent_mse": 1.4405235052108765, + "psnr": 8.447949295659507, + "ssim": 0.2894142125139586, + "lpips": 0.616000771522522 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.146484375, + "history_latent_mse": 0.18555539846420288, + "psnr": 35.50311870711727, + "ssim": 0.9480858985537217, + "lpips": 0.25570857524871826 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.3828125, + "history_latent_mse": 0.11988797038793564, + "psnr": 37.18997158102523, + "ssim": 0.9534102759059571, + "lpips": 0.17949232459068298 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 3160, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.5, + "history_latent_mse": 0.26305127143859863, + "psnr": 19.199608709573234, + "ssim": 0.611331433193068, + "lpips": 0.24953411519527435 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.888671875, + "history_latent_mse": 0.7852938771247864, + "psnr": 11.572953367318476, + "ssim": 0.3238332913964056, + "lpips": 0.4967406392097473 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.1484375, + "history_latent_mse": 1.145728588104248, + "psnr": 7.744809140730141, + "ssim": 0.2059366484145767, + "lpips": 0.6874375343322754 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.99609375, + "history_latent_mse": 0.1139211654663086, + "psnr": 35.27091367991169, + "ssim": 0.9441998356097521, + "lpips": 0.27397745847702026 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.0625, + "history_latent_mse": 0.10503560304641724, + "psnr": 36.1373293969658, + "ssim": 0.9487875146287383, + "lpips": 0.20599575340747833 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 3168, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3203125, + "history_latent_mse": 0.3029901683330536, + "psnr": 18.914070770327754, + "ssim": 0.6005699051833406, + "lpips": 0.26203030347824097 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.890625, + "history_latent_mse": 0.8589582443237305, + "psnr": 11.431910071869627, + "ssim": 0.32711357091987503, + "lpips": 0.5092601776123047 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.34375, + "history_latent_mse": 1.2575098276138306, + "psnr": 7.438335581507751, + "ssim": 0.20773836807506874, + "lpips": 0.7033286094665527 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.0, + "history_latent_mse": 0.11988425999879837, + "psnr": 35.07300876696385, + "ssim": 0.9421615157874426, + "lpips": 0.2858327329158783 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.54296875, + "history_latent_mse": 0.1247773990035057, + "psnr": 31.765331780163567, + "ssim": 0.9338935658553225, + "lpips": 0.32855433225631714 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 3176, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8671875, + "history_latent_mse": 0.29801949858665466, + "psnr": 17.743571328599067, + "ssim": 0.5835674410820922, + "lpips": 0.32023608684539795 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.2421875, + "history_latent_mse": 0.880891740322113, + "psnr": 11.860265498249019, + "ssim": 0.3625592751989118, + "lpips": 0.4984472990036011 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.3125, + "history_latent_mse": 1.3354878425598145, + "psnr": 6.901163704143754, + "ssim": 0.25132595270990854, + "lpips": 0.7489310503005981 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.81640625, + "history_latent_mse": 0.12284906953573227, + "psnr": 34.826902851967375, + "ssim": 0.9396243239896089, + "lpips": 0.29613763093948364 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.65625, + "history_latent_mse": 0.18616968393325806, + "psnr": 32.35612792799206, + "ssim": 0.9313905496953596, + "lpips": 0.32109880447387695 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 3184, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.2265625, + "history_latent_mse": 0.3595130741596222, + "psnr": 17.001402687593565, + "ssim": 0.5801625562166594, + "lpips": 0.35640954971313477 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.46875, + "history_latent_mse": 0.8259716033935547, + "psnr": 11.853545381287065, + "ssim": 0.37933143170024974, + "lpips": 0.5165786147117615 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.015625, + "history_latent_mse": 1.6166692972183228, + "psnr": 6.541689035094549, + "ssim": 0.24061577418572283, + "lpips": 0.7551593780517578 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.796875, + "history_latent_mse": 0.1284082978963852, + "psnr": 34.565221408096896, + "ssim": 0.9368704220705981, + "lpips": 0.30383628606796265 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0625, + "history_latent_mse": 0.17708100378513336, + "psnr": 34.23701526944256, + "ssim": 0.9404904003820407, + "lpips": 0.26351332664489746 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..005d1b621352aa758423fc74a6de0c521d8b9a86 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ac3d49cdc986fd14efc80d5e8963944437f8c445b59f78c452075b47fee9ab +size 1015121 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..6be22a9d2707a7428bf162fa493c567a1fc31437 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.07159221917390823, + "wrist/video/latent_future_mse": 0.07159221917390823, + "video/psnr": 29.266572537636293, + "wrist/video/psnr": 29.266572537636293, + "video/ssim": 0.9162150885896057, + "wrist/video/ssim": 0.9162150885896057, + "video/lpips": 0.03114306926727295, + "wrist/video/lpips": 0.03114306926727295, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.658707857131958, + "wrist/wrist_left/latent_future_mse": 0.658707857131958, + "wrist_left/psnr": 15.72964153701189, + "wrist/wrist_left/psnr": 15.72964153701189, + "wrist_left/ssim": 0.5237543840369523, + "wrist/wrist_left/ssim": 0.5237543840369523, + "wrist_left/lpips": 0.18139484524726868, + "wrist/wrist_left/lpips": 0.18139484524726868, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.7194229960441589, + "wrist/wrist_right/latent_future_mse": 0.7194229960441589, + "wrist_right/psnr": 13.913819208591171, + "wrist/wrist_right/psnr": 13.913819208591171, + "wrist_right/ssim": 0.5439860906598222, + "wrist/wrist_right/ssim": 0.5439860906598222, + "wrist_right/lpips": 0.30286651849746704, + "wrist/wrist_right/lpips": 0.30286651849746704, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.044789332896471024, + "wrist/tactile_left/latent_future_mse": 0.044789332896471024, + "tactile_left/psnr": 42.44182842336647, + "wrist/tactile_left/psnr": 42.44182842336647, + "tactile_left/ssim": 0.9840405275886075, + "wrist/tactile_left/ssim": 0.9840405275886075, + "tactile_left/lpips": 0.014103841967880726, + "wrist/tactile_left/lpips": 0.014103841967880726, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.05245906487107277, + "wrist/tactile_right/latent_future_mse": 0.05245906487107277, + "tactile_right/psnr": 43.15277773614905, + "wrist/tactile_right/psnr": 43.15277773614905, + "tactile_right/ssim": 0.9819352677548217, + "wrist/tactile_right/ssim": 0.9819352677548217, + "tactile_right/lpips": 0.04016111418604851, + "wrist/tactile_right/lpips": 0.04016111418604851, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 3072, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.07159221917390823, + "psnr": 29.266572537636293, + "ssim": 0.9162150885896057, + "lpips": 0.03114306926727295 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.658707857131958, + "psnr": 15.72964153701189, + "ssim": 0.5237543840369523, + "lpips": 0.18139484524726868 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.7194229960441589, + "psnr": 13.913819208591171, + "ssim": 0.5439860906598222, + "lpips": 0.30286651849746704 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.044789332896471024, + "psnr": 42.44182842336647, + "ssim": 0.9840405275886075, + "lpips": 0.014103841967880726 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.05245906487107277, + "psnr": 43.15277773614905, + "ssim": 0.9819352677548217, + "lpips": 0.04016111418604851 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/case.json new file mode 100644 index 0000000000000000000000000000000000000000..1d72c10d571f1fc97cf7a86a91df6b7fcb6e278d --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/case.json @@ -0,0 +1,164 @@ +{ + "task": 3, + "episode": "motherboard_0909_episode_000.pt", + "category": "full", + "index": 2531, + "t_start": 2531, + "start_seconds": 168.73333333333332, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.9296875, + "long_future_contact_fraction": 0.925, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "03_motherboard_0909_episode_000_full", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50c55bece5c4aaf5806b25c9937a2a4a618432f4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905ecc7502ec5a7ac205070796140eb1548ce6678a8464ca206854983c7b55cb +size 9207150 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/motherboard_0909_episode_000_t02531_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/motherboard_0909_episode_000_t02531_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c877846732b7ad042ef4e106f01ecfde40f4b3e --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/motherboard_0909_episode_000_t02531_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6246e71358c7c6d85690bab70bab37942573d0a7e02e9e73f01c34a8ea20a8 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..bd360b5a07b9335e995a056e6f147ade1c477632 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 2531, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 2531, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 26.63426720435477, + "ssim": 0.8708790760218011, + "lpips": 0.046272218227386475 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 17.216493839691783, + "ssim": 0.5951449385403667, + "lpips": 0.17227669060230255 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 17.487450501330382, + "ssim": 0.5787619882113139, + "lpips": 0.23368918895721436 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 35.386292833088994, + "ssim": 0.9682620496470609, + "lpips": 0.0447380356490612 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 36.69995933064942, + "ssim": 0.9682634386253878, + "lpips": 0.08494263142347336 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 2539, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8251953125, + "history_latent_mse": 0.08070540428161621, + "psnr": 23.159437122001776, + "ssim": 0.7823098292669202, + "lpips": 0.10296832025051117 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.515625, + "history_latent_mse": 0.39089512825012207, + "psnr": 14.673834614553565, + "ssim": 0.4183632750673575, + "lpips": 0.3741599917411804 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8359375, + "history_latent_mse": 0.527142345905304, + "psnr": 14.254749471245711, + "ssim": 0.4171200134046572, + "lpips": 0.4276435971260071 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.78125, + "history_latent_mse": 0.06735280156135559, + "psnr": 28.736440877463714, + "ssim": 0.9332238196872995, + "lpips": 0.19336062669754028 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.609375, + "history_latent_mse": 0.0874987244606018, + "psnr": 37.26545928086202, + "ssim": 0.9663097214724079, + "lpips": 0.1253584921360016 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 2547, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.39453125, + "history_latent_mse": 0.15959617495536804, + "psnr": 20.62850763959014, + "ssim": 0.7172446505481008, + "lpips": 0.17381739616394043 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.953125, + "history_latent_mse": 0.6985142827033997, + "psnr": 12.82548513288514, + "ssim": 0.30840404812611005, + "lpips": 0.5225712060928345 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.46875, + "history_latent_mse": 0.8651549220085144, + "psnr": 12.95824588599153, + "ssim": 0.36013512019582494, + "lpips": 0.5288517475128174 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.3671875, + "history_latent_mse": 0.15326480567455292, + "psnr": 29.3778627177778, + "ssim": 0.9308608957824773, + "lpips": 0.2518291175365448 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7109375, + "history_latent_mse": 0.09487878531217575, + "psnr": 34.76537606447303, + "ssim": 0.9603698907407732, + "lpips": 0.1629692018032074 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 2555, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.03125, + "history_latent_mse": 0.24894385039806366, + "psnr": 17.443917565051784, + "ssim": 0.6693827035691184, + "lpips": 0.3044857978820801 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.265625, + "history_latent_mse": 0.9179932475090027, + "psnr": 10.550608387552627, + "ssim": 0.2635874614567964, + "lpips": 0.6275637149810791 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.1015625, + "history_latent_mse": 0.9075899124145508, + "psnr": 12.775938497835632, + "ssim": 0.32981508164611417, + "lpips": 0.5660387277603149 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.984375, + "history_latent_mse": 0.14413142204284668, + "psnr": 31.98423471974735, + "ssim": 0.9414440889533799, + "lpips": 0.23925256729125977 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.25, + "history_latent_mse": 0.11077144742012024, + "psnr": 33.64269211697651, + "ssim": 0.9556408978265329, + "lpips": 0.19989131391048431 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 2563, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.0, + "history_latent_mse": 0.39638951420783997, + "psnr": 14.73174274772068, + "ssim": 0.585983706585437, + "lpips": 0.415120005607605 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8521728515625, + "history_latent_mse": 1.1219066381454468, + "psnr": 8.928059073629477, + "ssim": 0.224150783567039, + "lpips": 0.7316471338272095 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.12890625, + "history_latent_mse": 1.0076806545257568, + "psnr": 11.184012805939638, + "ssim": 0.2882455510490546, + "lpips": 0.6224209070205688 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.0078125, + "history_latent_mse": 0.11010429263114929, + "psnr": 33.260507171447415, + "ssim": 0.9461954082057156, + "lpips": 0.2397955358028412 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.189453125, + "history_latent_mse": 0.1103661060333252, + "psnr": 33.35748976734123, + "ssim": 0.9512292889887657, + "lpips": 0.21728402376174927 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 2571, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.46875, + "history_latent_mse": 0.4892379939556122, + "psnr": 14.334460869912744, + "ssim": 0.5790020853021631, + "lpips": 0.41715461015701294 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9609375, + "history_latent_mse": 1.3448820114135742, + "psnr": 8.799930353471497, + "ssim": 0.21221004452292824, + "lpips": 0.716631293296814 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.43359375, + "history_latent_mse": 1.133210301399231, + "psnr": 11.191955502330352, + "ssim": 0.28797609721676753, + "lpips": 0.6440451145172119 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4609375, + "history_latent_mse": 0.12806642055511475, + "psnr": 33.859292154893545, + "ssim": 0.9467556135771815, + "lpips": 0.24504689872264862 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.59375, + "history_latent_mse": 0.1392369419336319, + "psnr": 32.49246806577232, + "ssim": 0.9465398813390566, + "lpips": 0.23140281438827515 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 2579, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.46875, + "history_latent_mse": 0.4454944431781769, + "psnr": 14.302113527236903, + "ssim": 0.5739056927373726, + "lpips": 0.4254215955734253 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.96875, + "history_latent_mse": 1.3893589973449707, + "psnr": 8.653165509360154, + "ssim": 0.22678990052173326, + "lpips": 0.7852842807769775 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.1640625, + "history_latent_mse": 1.2213624715805054, + "psnr": 10.658522710345189, + "ssim": 0.2944790973066083, + "lpips": 0.6900660991668701 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.39453125, + "history_latent_mse": 0.14844247698783875, + "psnr": 34.90857239958195, + "ssim": 0.9470455298839704, + "lpips": 0.25731635093688965 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7578125, + "history_latent_mse": 0.170336052775383, + "psnr": 31.603033853314226, + "ssim": 0.9423910748516358, + "lpips": 0.2559289336204529 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 2587, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.25, + "history_latent_mse": 0.5087829232215881, + "psnr": 14.300809302476104, + "ssim": 0.5767790701594095, + "lpips": 0.41574013233184814 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.96875, + "history_latent_mse": 1.5244522094726562, + "psnr": 8.586503810292381, + "ssim": 0.20867409667261005, + "lpips": 0.7698798179626465 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9375, + "history_latent_mse": 1.2617968320846558, + "psnr": 10.188565542458578, + "ssim": 0.25988805688630157, + "lpips": 0.6863692998886108 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.55859375, + "history_latent_mse": 0.1374787837266922, + "psnr": 34.29122747192379, + "ssim": 0.938551057802476, + "lpips": 0.29119420051574707 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.44921875, + "history_latent_mse": 0.1452258825302124, + "psnr": 31.989948746552052, + "ssim": 0.9404435053929187, + "lpips": 0.25835925340652466 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 2595, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.84375, + "history_latent_mse": 0.5014035701751709, + "psnr": 14.053325979340862, + "ssim": 0.5619428936991968, + "lpips": 0.4426591396331787 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5703125, + "history_latent_mse": 1.4670085906982422, + "psnr": 8.45139375971582, + "ssim": 0.21053082769187503, + "lpips": 0.7753310203552246 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.71875, + "history_latent_mse": 1.214146375656128, + "psnr": 8.064663961919155, + "ssim": 0.216773069440242, + "lpips": 0.8136317729949951 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.0625, + "history_latent_mse": 0.1508377343416214, + "psnr": 34.87256996632177, + "ssim": 0.939595877735945, + "lpips": 0.2883746027946472 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.16015625, + "history_latent_mse": 0.16494834423065186, + "psnr": 33.6269204922963, + "ssim": 0.9425436603443749, + "lpips": 0.2461204081773758 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 2603, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.87109375, + "history_latent_mse": 0.5396252870559692, + "psnr": 13.939093041392185, + "ssim": 0.5519655360276372, + "lpips": 0.4710853695869446 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9765625, + "history_latent_mse": 1.5143388509750366, + "psnr": 8.205375822657874, + "ssim": 0.2038023523271112, + "lpips": 0.7657009363174438 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.25, + "history_latent_mse": 1.5066324472427368, + "psnr": 8.40044824459646, + "ssim": 0.21608138567351615, + "lpips": 0.8176727890968323 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.65625, + "history_latent_mse": 0.13566608726978302, + "psnr": 34.98019656406381, + "ssim": 0.9396104027733518, + "lpips": 0.29628050327301025 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0703125, + "history_latent_mse": 0.144988015294075, + "psnr": 33.08963139480401, + "ssim": 0.9410875123563129, + "lpips": 0.258383572101593 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 2611, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.15625, + "history_latent_mse": 0.5618721842765808, + "psnr": 14.022926307752998, + "ssim": 0.5597091502249111, + "lpips": 0.4689983129501343 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.125, + "history_latent_mse": 1.482651710510254, + "psnr": 9.247492241476783, + "ssim": 0.3291559204751646, + "lpips": 0.6797118782997131 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 1.5147126913070679, + "psnr": 9.54024800688658, + "ssim": 0.28481042074729657, + "lpips": 0.7541435956954956 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.7109375, + "history_latent_mse": 0.1551610678434372, + "psnr": 35.599234237202694, + "ssim": 0.9427807967268611, + "lpips": 0.2821560204029083 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.765625, + "history_latent_mse": 0.1496855467557907, + "psnr": 35.99517843138076, + "ssim": 0.9486212645952063, + "lpips": 0.21314403414726257 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 2619, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.44140625, + "history_latent_mse": 0.5302004814147949, + "psnr": 14.880947695835854, + "ssim": 0.57533410681392, + "lpips": 0.45671528577804565 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.296875, + "history_latent_mse": 1.2381802797317505, + "psnr": 10.703357894145372, + "ssim": 0.41701729843814095, + "lpips": 0.5924857258796692 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 1.3446159362792969, + "psnr": 8.821429504322131, + "ssim": 0.19334200080630412, + "lpips": 0.814721405506134 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3515625, + "history_latent_mse": 0.12978319823741913, + "psnr": 35.598542324647646, + "ssim": 0.9397215758602377, + "lpips": 0.2898758053779602 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7109375, + "history_latent_mse": 0.15500086545944214, + "psnr": 34.32492532341694, + "ssim": 0.9427065419641114, + "lpips": 0.25140976905822754 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 2627, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3125, + "history_latent_mse": 0.47130608558654785, + "psnr": 14.606693310270844, + "ssim": 0.5531711847126617, + "lpips": 0.4825262427330017 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.046875, + "history_latent_mse": 1.0705623626708984, + "psnr": 12.064152210124384, + "ssim": 0.516445524010195, + "lpips": 0.5754836797714233 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.78125, + "history_latent_mse": 1.6478544473648071, + "psnr": 9.411113216018585, + "ssim": 0.1698900599853368, + "lpips": 0.8225197792053223 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.88671875, + "history_latent_mse": 0.1231420636177063, + "psnr": 35.32644705073464, + "ssim": 0.9375313809980473, + "lpips": 0.3004121780395508 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.640625, + "history_latent_mse": 0.19039489328861237, + "psnr": 30.3456437967443, + "ssim": 0.9275757916035403, + "lpips": 0.33385026454925537 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 2635, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3203125, + "history_latent_mse": 0.5852118730545044, + "psnr": 14.079720521704612, + "ssim": 0.5082219844715823, + "lpips": 0.5390398502349854 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8359375, + "history_latent_mse": 0.8839182257652283, + "psnr": 12.705496023461402, + "ssim": 0.5040707284931212, + "lpips": 0.5906945466995239 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4765625, + "history_latent_mse": 1.4683102369308472, + "psnr": 9.855829945014992, + "ssim": 0.17281228171489707, + "lpips": 0.746903121471405 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.056884765625, + "history_latent_mse": 0.12720677256584167, + "psnr": 35.04194541254678, + "ssim": 0.9346703476639409, + "lpips": 0.3099118173122406 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.8828125, + "history_latent_mse": 0.20562660694122314, + "psnr": 29.746095590532956, + "ssim": 0.9229609742765656, + "lpips": 0.3514562249183655 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 2643, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.71875, + "history_latent_mse": 0.6232137084007263, + "psnr": 14.082933236823468, + "ssim": 0.5156893884621685, + "lpips": 0.5388348698616028 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.140625, + "history_latent_mse": 0.8454455733299255, + "psnr": 12.986729481331821, + "ssim": 0.5114012672230022, + "lpips": 0.5878117084503174 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.94921875, + "history_latent_mse": 1.3163434267044067, + "psnr": 9.973529345020598, + "ssim": 0.17466833704868473, + "lpips": 0.684074878692627 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.978729248046875, + "history_latent_mse": 0.13266299664974213, + "psnr": 34.50713700039933, + "ssim": 0.9316547931630519, + "lpips": 0.3206460475921631 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7578125, + "history_latent_mse": 0.22606706619262695, + "psnr": 31.080840879215664, + "ssim": 0.9219519881025942, + "lpips": 0.3415656089782715 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b00b47da12d17ad444a5aed816b5147797f88392 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd7e7a72f29030b8926dc1564b3a4ce93c6e574ba8b187b6b275d95707acc66 +size 1201037 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..70b9520fffe6f545d8ffc3325761325b5b4814ca --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.0986940786242485, + "wrist/video/latent_future_mse": 0.0986940786242485, + "video/psnr": 26.63426720435477, + "wrist/video/psnr": 26.63426720435477, + "video/ssim": 0.8708790760218011, + "wrist/video/ssim": 0.8708790760218011, + "video/lpips": 0.046272218227386475, + "wrist/video/lpips": 0.046272218227386475, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.4402367174625397, + "wrist/wrist_left/latent_future_mse": 0.4402367174625397, + "wrist_left/psnr": 17.216493839691783, + "wrist/wrist_left/psnr": 17.216493839691783, + "wrist_left/ssim": 0.5951449385403667, + "wrist/wrist_left/ssim": 0.5951449385403667, + "wrist_left/lpips": 0.17227669060230255, + "wrist/wrist_left/lpips": 0.17227669060230255, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.599057137966156, + "wrist/wrist_right/latent_future_mse": 0.599057137966156, + "wrist_right/psnr": 17.487450501330382, + "wrist/wrist_right/psnr": 17.487450501330382, + "wrist_right/ssim": 0.5787619882113139, + "wrist/wrist_right/ssim": 0.5787619882113139, + "wrist_right/lpips": 0.23368918895721436, + "wrist/wrist_right/lpips": 0.23368918895721436, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.0710873007774353, + "wrist/tactile_left/latent_future_mse": 0.0710873007774353, + "tactile_left/psnr": 35.386292833088994, + "wrist/tactile_left/psnr": 35.386292833088994, + "tactile_left/ssim": 0.9682620496470609, + "wrist/tactile_left/ssim": 0.9682620496470609, + "tactile_left/lpips": 0.0447380356490612, + "wrist/tactile_left/lpips": 0.0447380356490612, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.09564441442489624, + "wrist/tactile_right/latent_future_mse": 0.09564441442489624, + "tactile_right/psnr": 36.69995933064942, + "wrist/tactile_right/psnr": 36.69995933064942, + "tactile_right/ssim": 0.9682634386253878, + "wrist/tactile_right/ssim": 0.9682634386253878, + "tactile_right/lpips": 0.08494263142347336, + "wrist/tactile_right/lpips": 0.08494263142347336, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_000.pt", + "t_start": 2531, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.0986940786242485, + "psnr": 26.63426720435477, + "ssim": 0.8708790760218011, + "lpips": 0.046272218227386475 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.4402367174625397, + "psnr": 17.216493839691783, + "ssim": 0.5951449385403667, + "lpips": 0.17227669060230255 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.599057137966156, + "psnr": 17.487450501330382, + "ssim": 0.5787619882113139, + "lpips": 0.23368918895721436 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.0710873007774353, + "psnr": 35.386292833088994, + "ssim": 0.9682620496470609, + "lpips": 0.0447380356490612 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.09564441442489624, + "psnr": 36.69995933064942, + "ssim": 0.9682634386253878, + "lpips": 0.08494263142347336 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/case.json new file mode 100644 index 0000000000000000000000000000000000000000..eec7151ad8aeb38ac20e291686f18a3eeb9edfc6 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/case.json @@ -0,0 +1,164 @@ +{ + "task": 4, + "episode": "motherboard_0909_episode_001.pt", + "category": "none", + "index": 3319, + "t_start": 3, + "start_seconds": 0.2, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.0, + "long_future_contact_fraction": 0.0, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": null, + "long_label": "none_throughout", + "output_subdir": "04_motherboard_0909_episode_001_none", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e880b3e72ba66b3e9ff6806d3a0583001316f653 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8ab5fc9e5a51129d7d014edfa4eb5efc9d2b055993ddff6c3ff768511e85aba +size 4678993 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/motherboard_0909_episode_001_t00003_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/motherboard_0909_episode_001_t00003_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..081367fd032a7e44972b9f0502311bd0db107876 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/motherboard_0909_episode_001_t00003_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b783f20cd087d6d9acf43772992bc5d2b7b17dc38c31e8ee9bafb1b4f2b79aec +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..6c8b6ae746222755a5a337716ff05d9be180aa77 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 3, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 3, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 34.98245723682948, + "ssim": 0.9629138931467476, + "lpips": 0.010414157062768936 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 39.17661504835171, + "ssim": 0.9774012756239003, + "lpips": 0.014707624912261963 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 41.112583412445105, + "ssim": 0.980698017129837, + "lpips": 0.011336026713252068 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.88916107867081, + "ssim": 0.992888085249837, + "lpips": 0.005414151586592197 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 50.09711170465344, + "ssim": 0.9943909986189488, + "lpips": 0.004877166822552681 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 11, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 0.80078125, + "history_latent_mse": 0.016177501529455185, + "psnr": 32.65360172062122, + "ssim": 0.9451018438629752, + "lpips": 0.018051061779260635 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.51953125, + "history_latent_mse": 0.0688566341996193, + "psnr": 34.70874410652181, + "ssim": 0.9550191848141394, + "lpips": 0.030690759420394897 + }, + "wrist_right": { + "history_latent_max_abs_error": 1.609375, + "history_latent_mse": 0.051370058208703995, + "psnr": 36.70862851801243, + "ssim": 0.9665328940070996, + "lpips": 0.021000880748033524 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.7998046875, + "history_latent_mse": 0.01607528142631054, + "psnr": 43.30408058688558, + "ssim": 0.9866618062365831, + "lpips": 0.02096220850944519 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.78125, + "history_latent_mse": 0.01136635523289442, + "psnr": 45.19819940703411, + "ssim": 0.9887515562169498, + "lpips": 0.029524262994527817 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 19, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 0.859375, + "history_latent_mse": 0.024617746472358704, + "psnr": 30.657814377587233, + "ssim": 0.9261530313214832, + "lpips": 0.026675928384065628 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.75, + "history_latent_mse": 0.1057824194431305, + "psnr": 31.89597433160625, + "ssim": 0.9414306908418408, + "lpips": 0.052067264914512634 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.13671875, + "history_latent_mse": 0.07896627485752106, + "psnr": 33.360781933264356, + "ssim": 0.9495707114799177, + "lpips": 0.03309304267168045 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.2509765625, + "history_latent_mse": 0.03207814693450928, + "psnr": 41.132266877457944, + "ssim": 0.9818701938980837, + "lpips": 0.0422329306602478 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.115234375, + "history_latent_mse": 0.03048039972782135, + "psnr": 42.74665725794399, + "ssim": 0.9834887554005778, + "lpips": 0.05530240386724472 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 27, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.09375, + "history_latent_mse": 0.03539456054568291, + "psnr": 29.217249573879712, + "ssim": 0.9059627175180314, + "lpips": 0.035778772085905075 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.51171875, + "history_latent_mse": 0.12455450743436813, + "psnr": 29.53919078110118, + "ssim": 0.929478994425008, + "lpips": 0.06983967125415802 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.3984375, + "history_latent_mse": 0.09827543050050735, + "psnr": 31.221406923166764, + "ssim": 0.9340979552251343, + "lpips": 0.04688713699579239 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.41015625, + "history_latent_mse": 0.041825417429208755, + "psnr": 39.6108188738143, + "ssim": 0.9777722940812191, + "lpips": 0.07898698002099991 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.083984375, + "history_latent_mse": 0.03997945412993431, + "psnr": 41.088363148294775, + "ssim": 0.9800138270826642, + "lpips": 0.07595539093017578 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 35, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.453125, + "history_latent_mse": 0.04712721332907677, + "psnr": 28.073694923796836, + "ssim": 0.8865141116325409, + "lpips": 0.047943443059921265 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.078125, + "history_latent_mse": 0.14618082344532013, + "psnr": 27.72459402619125, + "ssim": 0.9197344237354016, + "lpips": 0.0823066234588623 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.5234375, + "history_latent_mse": 0.11575943976640701, + "psnr": 29.226118852301507, + "ssim": 0.9202291093930883, + "lpips": 0.06048545986413956 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.326171875, + "history_latent_mse": 0.050554513931274414, + "psnr": 38.41573621709636, + "ssim": 0.9740903761437776, + "lpips": 0.11617135256528854 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.28125, + "history_latent_mse": 0.048963647335767746, + "psnr": 39.43631928449023, + "ssim": 0.9763008183608433, + "lpips": 0.09465165436267853 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 43, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.609375, + "history_latent_mse": 0.06006389856338501, + "psnr": 27.02987894339884, + "ssim": 0.866084420760076, + "lpips": 0.05735699087381363 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.98828125, + "history_latent_mse": 0.1538659930229187, + "psnr": 25.91892593024735, + "ssim": 0.8915482906987001, + "lpips": 0.13015079498291016 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.5859375, + "history_latent_mse": 0.1415880173444748, + "psnr": 27.698779252768805, + "ssim": 0.9043232175014985, + "lpips": 0.07837487757205963 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.37060546875, + "history_latent_mse": 0.0584084577858448, + "psnr": 37.7379841467458, + "ssim": 0.970306988952025, + "lpips": 0.14769673347473145 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.19140625, + "history_latent_mse": 0.057037342339754105, + "psnr": 38.661940769686765, + "ssim": 0.9726057034187406, + "lpips": 0.11783362179994583 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 51, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.08984375, + "history_latent_mse": 0.07282394915819168, + "psnr": 26.190869255919587, + "ssim": 0.8458709432522004, + "lpips": 0.06871116906404495 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.6796875, + "history_latent_mse": 0.22041599452495575, + "psnr": 24.700992186275258, + "ssim": 0.8639734152728547, + "lpips": 0.16663044691085815 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.19921875, + "history_latent_mse": 0.15733292698860168, + "psnr": 26.35364796082934, + "ssim": 0.8889362924344137, + "lpips": 0.09919115900993347 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.458984375, + "history_latent_mse": 0.06807977706193924, + "psnr": 36.85889801614486, + "ssim": 0.9663484443781485, + "lpips": 0.1788940131664276 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.484619140625, + "history_latent_mse": 0.06784757971763611, + "psnr": 38.00342742716552, + "ssim": 0.9689376271909127, + "lpips": 0.1367427408695221 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 59, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.84765625, + "history_latent_mse": 0.08585706353187561, + "psnr": 25.4728454101052, + "ssim": 0.8278488497254033, + "lpips": 0.08031848073005676 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.9453125, + "history_latent_mse": 0.2418694943189621, + "psnr": 22.741613755245133, + "ssim": 0.7890121080437303, + "lpips": 0.24681398272514343 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.06640625, + "history_latent_mse": 0.18104489147663116, + "psnr": 25.48509429605201, + "ssim": 0.863099005353641, + "lpips": 0.11643271893262863 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.423828125, + "history_latent_mse": 0.07726151496171951, + "psnr": 36.215608122520976, + "ssim": 0.9625128813389783, + "lpips": 0.20342519879341125 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.40625, + "history_latent_mse": 0.07355325669050217, + "psnr": 37.22816225778982, + "ssim": 0.9651628246134882, + "lpips": 0.15830203890800476 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 67, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.10546875, + "history_latent_mse": 0.09972519427537918, + "psnr": 24.810640359296006, + "ssim": 0.8092628495916824, + "lpips": 0.0904068723320961 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.546875, + "history_latent_mse": 0.28614574670791626, + "psnr": 21.360640624845622, + "ssim": 0.7470997269319821, + "lpips": 0.28648388385772705 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.71875, + "history_latent_mse": 0.2093622088432312, + "psnr": 24.62544191727146, + "ssim": 0.8383657738887982, + "lpips": 0.13321352005004883 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.44921875, + "history_latent_mse": 0.08496720343828201, + "psnr": 35.345682259461014, + "ssim": 0.9586259166662039, + "lpips": 0.23060844838619232 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.493408203125, + "history_latent_mse": 0.0822221040725708, + "psnr": 36.436543148097485, + "ssim": 0.9616665582813301, + "lpips": 0.17387701570987701 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 75, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.3828125, + "history_latent_mse": 0.11592298746109009, + "psnr": 24.177785101261165, + "ssim": 0.7940892053250914, + "lpips": 0.09998476505279541 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.7578125, + "history_latent_mse": 0.3163505792617798, + "psnr": 20.864276155296217, + "ssim": 0.7295377803498351, + "lpips": 0.3027108311653137 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.46435546875, + "history_latent_mse": 0.24694335460662842, + "psnr": 23.87788299042554, + "ssim": 0.8193955600194242, + "lpips": 0.15296679735183716 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.541015625, + "history_latent_mse": 0.09362727403640747, + "psnr": 34.78741039492542, + "ssim": 0.9547197357495681, + "lpips": 0.25133681297302246 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.525390625, + "history_latent_mse": 0.08905778080224991, + "psnr": 35.892239294070905, + "ssim": 0.9580291924617752, + "lpips": 0.19143548607826233 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 83, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.515625, + "history_latent_mse": 0.12416509538888931, + "psnr": 23.771543933786234, + "ssim": 0.7821418086178215, + "lpips": 0.10931544005870819 + }, + "wrist_left": { + "history_latent_max_abs_error": 2.8984375, + "history_latent_mse": 0.32416948676109314, + "psnr": 20.65019768223056, + "ssim": 0.7196238203856296, + "lpips": 0.3087509870529175 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.71875, + "history_latent_mse": 0.2711307406425476, + "psnr": 23.208677977308643, + "ssim": 0.8031497119100225, + "lpips": 0.1665535718202591 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.548828125, + "history_latent_mse": 0.10238610953092575, + "psnr": 34.228455996369156, + "ssim": 0.950545647103486, + "lpips": 0.2692863941192627 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.56640625, + "history_latent_mse": 0.09811829775571823, + "psnr": 35.527784126835904, + "ssim": 0.9543523143871042, + "lpips": 0.20773538947105408 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 91, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.625, + "history_latent_mse": 0.1330309510231018, + "psnr": 23.2575131675048, + "ssim": 0.7703655601357615, + "lpips": 0.11754289269447327 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.478515625, + "history_latent_mse": 0.3368115723133087, + "psnr": 20.464269335672892, + "ssim": 0.7121922812898331, + "lpips": 0.3153577446937561 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.625, + "history_latent_mse": 0.2756876051425934, + "psnr": 22.60791053573388, + "ssim": 0.7901893278748782, + "lpips": 0.17656481266021729 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.630859375, + "history_latent_mse": 0.11201604455709457, + "psnr": 33.760866912896724, + "ssim": 0.9461824642387036, + "lpips": 0.28509899973869324 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.71875, + "history_latent_mse": 0.10582425445318222, + "psnr": 35.03517923183785, + "ssim": 0.9508115855691162, + "lpips": 0.22077180445194244 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 99, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.8515625, + "history_latent_mse": 0.14295947551727295, + "psnr": 22.767795813713203, + "ssim": 0.7567105433703114, + "lpips": 0.12600049376487732 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.6171875, + "history_latent_mse": 0.33440685272216797, + "psnr": 20.289214883884632, + "ssim": 0.7069400382975393, + "lpips": 0.32493889331817627 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.7578125, + "history_latent_mse": 0.2922215163707733, + "psnr": 22.042752144313628, + "ssim": 0.7740129932497972, + "lpips": 0.19255070388317108 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6728515625, + "history_latent_mse": 0.1231573224067688, + "psnr": 33.411619844836665, + "ssim": 0.9415519517741281, + "lpips": 0.2940673232078552 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.83203125, + "history_latent_mse": 0.11262484639883041, + "psnr": 34.4821552377769, + "ssim": 0.9460351792023809, + "lpips": 0.2379671335220337 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 107, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.1015625, + "history_latent_mse": 0.15253527462482452, + "psnr": 22.41080653955966, + "ssim": 0.7440511207261107, + "lpips": 0.13436497747898102 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.140625, + "history_latent_mse": 0.3626815378665924, + "psnr": 20.165464811224048, + "ssim": 0.6996546904626508, + "lpips": 0.32388269901275635 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.658935546875, + "history_latent_mse": 0.304993599653244, + "psnr": 21.357920729699682, + "ssim": 0.7567205235876333, + "lpips": 0.20732323825359344 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.8359375, + "history_latent_mse": 0.12264937162399292, + "psnr": 32.91540619581552, + "ssim": 0.9368707716007975, + "lpips": 0.3127726912498474 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.978515625, + "history_latent_mse": 0.11911370605230331, + "psnr": 33.81251976800809, + "ssim": 0.9419957173487334, + "lpips": 0.25386711955070496 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 115, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.2265625, + "history_latent_mse": 0.16395385563373566, + "psnr": 21.352153566226427, + "ssim": 0.7121591690641549, + "lpips": 0.1606350988149643 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.08984375, + "history_latent_mse": 0.3456915318965912, + "psnr": 20.105780881761074, + "ssim": 0.6982235168499099, + "lpips": 0.3303559422492981 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.734375, + "history_latent_mse": 0.3107913136482239, + "psnr": 20.490728589145085, + "ssim": 0.7300526933251967, + "lpips": 0.24053047597408295 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.033203125, + "history_latent_mse": 0.131675586104393, + "psnr": 32.407308936669715, + "ssim": 0.9330809563942799, + "lpips": 0.32732903957366943 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.2421875, + "history_latent_mse": 0.12655149400234222, + "psnr": 33.08481006743399, + "ssim": 0.938295244543474, + "lpips": 0.2669263780117035 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f08bc4dcfdc7813b7464dfddf0871fa4b36ee4b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b1e62289dfad8a71829f168aa8dcc8d3b4e52d2244868cc24c33f0dddcc078 +size 578075 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..bd6c5a19db525fb855c8de49396c5980cae0aaee --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.03628881648182869, + "wrist/video/latent_future_mse": 0.03628881648182869, + "video/psnr": 34.98245723682948, + "wrist/video/psnr": 34.98245723682948, + "video/ssim": 0.9629138931467476, + "wrist/video/ssim": 0.9629138931467476, + "video/lpips": 0.010414157062768936, + "wrist/video/lpips": 0.010414157062768936, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.0688975527882576, + "wrist/wrist_left/latent_future_mse": 0.0688975527882576, + "wrist_left/psnr": 39.17661504835171, + "wrist/wrist_left/psnr": 39.17661504835171, + "wrist_left/ssim": 0.9774012756239003, + "wrist/wrist_left/ssim": 0.9774012756239003, + "wrist_left/lpips": 0.014707624912261963, + "wrist/wrist_left/lpips": 0.014707624912261963, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.05638621747493744, + "wrist/wrist_right/latent_future_mse": 0.05638621747493744, + "wrist_right/psnr": 41.112583412445105, + "wrist/wrist_right/psnr": 41.112583412445105, + "wrist_right/ssim": 0.980698017129837, + "wrist/wrist_right/ssim": 0.980698017129837, + "wrist_right/lpips": 0.011336026713252068, + "wrist/wrist_right/lpips": 0.011336026713252068, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.013817056082189083, + "wrist/tactile_left/latent_future_mse": 0.013817056082189083, + "tactile_left/psnr": 47.88916107867081, + "wrist/tactile_left/psnr": 47.88916107867081, + "tactile_left/ssim": 0.992888085249837, + "wrist/tactile_left/ssim": 0.992888085249837, + "tactile_left/lpips": 0.005414151586592197, + "wrist/tactile_left/lpips": 0.005414151586592197, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.00848926417529583, + "wrist/tactile_right/latent_future_mse": 0.00848926417529583, + "tactile_right/psnr": 50.09711170465344, + "wrist/tactile_right/psnr": 50.09711170465344, + "tactile_right/ssim": 0.9943909986189488, + "wrist/tactile_right/ssim": 0.9943909986189488, + "tactile_right/lpips": 0.004877166822552681, + "wrist/tactile_right/lpips": 0.004877166822552681, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 3, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.03628881648182869, + "psnr": 34.98245723682948, + "ssim": 0.9629138931467476, + "lpips": 0.010414157062768936 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.0688975527882576, + "psnr": 39.17661504835171, + "ssim": 0.9774012756239003, + "lpips": 0.014707624912261963 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.05638621747493744, + "psnr": 41.112583412445105, + "ssim": 0.980698017129837, + "lpips": 0.011336026713252068 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.013817056082189083, + "psnr": 47.88916107867081, + "ssim": 0.992888085249837, + "lpips": 0.005414151586592197 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.00848926417529583, + "psnr": 50.09711170465344, + "ssim": 0.9943909986189488, + "lpips": 0.004877166822552681 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/case.json new file mode 100644 index 0000000000000000000000000000000000000000..6ed402153b52cbdc147dfb3417dc53803615dcac --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/case.json @@ -0,0 +1,168 @@ +{ + "task": 5, + "episode": "motherboard_0909_episode_001.pt", + "category": "onset", + "index": 4059, + "t_start": 760, + "start_seconds": 50.666666666666664, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6953125, + "long_future_contact_fraction": 0.7416666666666667, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "05_motherboard_0909_episode_001_onset", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad4a6017df21ac3af4255a0786ffc04a830dbfdb --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efd25e4c8d5dc4a3ca2022e9125977e0dda87ddfda30496c0256577693bc7072 +size 9794147 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/motherboard_0909_episode_001_t00760_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/motherboard_0909_episode_001_t00760_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..409b15ac95fda59e16ddf408e0ec131ca29ba23e --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/motherboard_0909_episode_001_t00760_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb1f433278c24e0a40f64c711ce45ff1da495a2a7f2d9f685fb24702cc2f2b7 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..3cddd40023b3f23c1120a093b440b300eca409a9 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 760, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 760, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 23.97182098554074, + "ssim": 0.8125829526693039, + "lpips": 0.07511236518621445 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.948032842082776, + "ssim": 0.3378560447266604, + "lpips": 0.47823843359947205 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 15.361403592715266, + "ssim": 0.5478371830024589, + "lpips": 0.3370887041091919 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 38.742343650877785, + "ssim": 0.9835824167841523, + "lpips": 0.041552357375621796 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 46.984432773655556, + "ssim": 0.9918726462661673, + "lpips": 0.013785661198198795 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 768, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.828125, + "history_latent_mse": 0.16472817957401276, + "psnr": 20.773689788720986, + "ssim": 0.702239032431079, + "lpips": 0.161298930644989 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.749755859375, + "history_latent_mse": 0.8072547912597656, + "psnr": 12.957019796188575, + "ssim": 0.33489133909746094, + "lpips": 0.49681586027145386 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.0, + "history_latent_mse": 0.5608716607093811, + "psnr": 13.357670605569608, + "ssim": 0.4937799383731884, + "lpips": 0.3790629208087921 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.64453125, + "history_latent_mse": 0.06477810442447662, + "psnr": 41.98333673446485, + "ssim": 0.981422978192573, + "lpips": 0.036314770579338074 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.2216796875, + "history_latent_mse": 0.025710830464959145, + "psnr": 44.02224942484943, + "ssim": 0.9845782055239337, + "lpips": 0.0436430424451828 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 776, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0390625, + "history_latent_mse": 0.22434532642364502, + "psnr": 21.411604121735902, + "ssim": 0.7241759497755744, + "lpips": 0.1670190840959549 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0703125, + "history_latent_mse": 0.7591099739074707, + "psnr": 13.382551202141775, + "ssim": 0.2598375119677453, + "lpips": 0.5377963185310364 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.75, + "history_latent_mse": 0.6611424088478088, + "psnr": 12.686662470364169, + "ssim": 0.4378278147681438, + "lpips": 0.4320305585861206 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.667236328125, + "history_latent_mse": 0.053133945912122726, + "psnr": 29.017501133794287, + "ssim": 0.9454906762667659, + "lpips": 0.19702446460723877 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.048828125, + "history_latent_mse": 0.035533007234334946, + "psnr": 43.24735895355716, + "ssim": 0.9815280666806907, + "lpips": 0.06259869039058685 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 784, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.625, + "history_latent_mse": 0.18156300485134125, + "psnr": 19.813532405047198, + "ssim": 0.6956227564452014, + "lpips": 0.24994304776191711 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.640625, + "history_latent_mse": 0.7792689204216003, + "psnr": 10.840011900492755, + "ssim": 0.2175243299314709, + "lpips": 0.5774753093719482 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.625, + "history_latent_mse": 0.710976779460907, + "psnr": 12.708692467527909, + "ssim": 0.46053409200751866, + "lpips": 0.47454896569252014 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.546875, + "history_latent_mse": 0.15830831229686737, + "psnr": 29.84664517899878, + "ssim": 0.953936999401324, + "lpips": 0.19601863622665405 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.177734375, + "history_latent_mse": 0.04353208467364311, + "psnr": 42.08288312723045, + "ssim": 0.9780972762830105, + "lpips": 0.08162233233451843 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 792, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.640625, + "history_latent_mse": 0.25846076011657715, + "psnr": 19.261874864915804, + "ssim": 0.6692308307134727, + "lpips": 0.2730814516544342 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.453125, + "history_latent_mse": 0.8949176669120789, + "psnr": 9.92673343964626, + "ssim": 0.4358654626353847, + "lpips": 0.5115290880203247 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8984375, + "history_latent_mse": 0.7173665165901184, + "psnr": 9.293844716863788, + "ssim": 0.23060218251391312, + "lpips": 0.5908918380737305 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.4140625, + "history_latent_mse": 0.15500132739543915, + "psnr": 39.90042011569596, + "ssim": 0.9705844551989551, + "lpips": 0.14060094952583313 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.287109375, + "history_latent_mse": 0.05181732773780823, + "psnr": 40.93387182630329, + "ssim": 0.9750033902078178, + "lpips": 0.09806308150291443 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 800, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 0.2749159634113312, + "psnr": 19.2806336321409, + "ssim": 0.6734096140701664, + "lpips": 0.25466710329055786 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1328125, + "history_latent_mse": 0.9686599373817444, + "psnr": 10.464444482360339, + "ssim": 0.4894071927984617, + "lpips": 0.5146087408065796 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.3828125, + "history_latent_mse": 1.1812862157821655, + "psnr": 8.351546856531277, + "ssim": 0.17845092278843294, + "lpips": 0.6624883413314819 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.47265625, + "history_latent_mse": 0.06376153975725174, + "psnr": 39.330011846542305, + "ssim": 0.9668452695236465, + "lpips": 0.162667915225029 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.2783203125, + "history_latent_mse": 0.05738365277647972, + "psnr": 33.65080702739904, + "ssim": 0.9520888476662168, + "lpips": 0.2124367356300354 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 808, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.63330078125, + "history_latent_mse": 0.2533222436904907, + "psnr": 19.185930506613264, + "ssim": 0.6614148823732806, + "lpips": 0.28340551257133484 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.21875, + "history_latent_mse": 1.0155400037765503, + "psnr": 11.5018379751468, + "ssim": 0.4703773290802423, + "lpips": 0.5402319431304932 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.578125, + "history_latent_mse": 1.164634108543396, + "psnr": 8.047975165782235, + "ssim": 0.17040814911619756, + "lpips": 0.6948565244674683 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.580078125, + "history_latent_mse": 0.0728769525885582, + "psnr": 38.43696015877503, + "ssim": 0.9633539665800088, + "lpips": 0.18397042155265808 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.5390625, + "history_latent_mse": 0.15528421103954315, + "psnr": 30.482760363891877, + "ssim": 0.9316210718878324, + "lpips": 0.29477837681770325 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 816, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.16796875, + "history_latent_mse": 0.2984682321548462, + "psnr": 18.461471730632585, + "ssim": 0.6432727285054268, + "lpips": 0.3496913015842438 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.109375, + "history_latent_mse": 0.872347891330719, + "psnr": 10.769764364051005, + "ssim": 0.3029838169425373, + "lpips": 0.6494001150131226 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 1.3400568962097168, + "psnr": 7.740216071696815, + "ssim": 0.16791392220948567, + "lpips": 0.7081996202468872 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.5224609375, + "history_latent_mse": 0.08166665583848953, + "psnr": 37.54734938192997, + "ssim": 0.9595712611877427, + "lpips": 0.20131734013557434 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.8125, + "history_latent_mse": 0.16986556351184845, + "psnr": 29.36502662780875, + "ssim": 0.9217778719544037, + "lpips": 0.3236025273799896 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 824, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.828125, + "history_latent_mse": 0.2938782274723053, + "psnr": 18.5235528992905, + "ssim": 0.6452774613458347, + "lpips": 0.34617170691490173 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.6640625, + "history_latent_mse": 1.1267484426498413, + "psnr": 10.979310461236569, + "ssim": 0.24758757208566562, + "lpips": 0.721572995185852 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.578125, + "history_latent_mse": 1.5104466676712036, + "psnr": 7.804949675021238, + "ssim": 0.15574708281079835, + "lpips": 0.7038141489028931 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.8984375, + "history_latent_mse": 0.10589944571256638, + "psnr": 33.30858080413044, + "ssim": 0.9326610898617956, + "lpips": 0.2946099638938904 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0390625, + "history_latent_mse": 0.17906594276428223, + "psnr": 29.386979883751515, + "ssim": 0.9234396478463931, + "lpips": 0.31560975313186646 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 832, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.6484375, + "history_latent_mse": 0.29150208830833435, + "psnr": 18.801432301325395, + "ssim": 0.6654753952910855, + "lpips": 0.3038299083709717 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.6875, + "history_latent_mse": 1.4942198991775513, + "psnr": 7.578415182211341, + "ssim": 0.15813444732251838, + "lpips": 0.7655599117279053 + }, + "wrist_right": { + "history_latent_max_abs_error": 7.2734375, + "history_latent_mse": 1.5599931478500366, + "psnr": 8.69173064918552, + "ssim": 0.14549984755213147, + "lpips": 0.6497141718864441 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.7578125, + "history_latent_mse": 0.22963522374629974, + "psnr": 34.0053449152069, + "ssim": 0.9345576496650239, + "lpips": 0.3088558316230774 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.10546875, + "history_latent_mse": 0.1841275691986084, + "psnr": 30.267839570359605, + "ssim": 0.9280124234953808, + "lpips": 0.30279654264450073 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 840, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.3664903938770294, + "psnr": 18.4862279240429, + "ssim": 0.6609155744072859, + "lpips": 0.2633398771286011 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.109375, + "history_latent_mse": 1.6551098823547363, + "psnr": 7.660092129532145, + "ssim": 0.2063683648179046, + "lpips": 0.7334326505661011 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.6875, + "history_latent_mse": 1.2903972864151, + "psnr": 9.634885592697719, + "ssim": 0.21263329836904776, + "lpips": 0.6240497827529907 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.625, + "history_latent_mse": 0.14765022695064545, + "psnr": 33.23548299191736, + "ssim": 0.9318444574210466, + "lpips": 0.30578720569610596 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.171875, + "history_latent_mse": 0.17726582288742065, + "psnr": 37.76077328802814, + "ssim": 0.9590818585007767, + "lpips": 0.18256664276123047 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 848, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.2265625, + "history_latent_mse": 0.3046681582927704, + "psnr": 18.709655100884643, + "ssim": 0.650614385218887, + "lpips": 0.234390988945961 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.203125, + "history_latent_mse": 1.516667366027832, + "psnr": 8.81480209739876, + "ssim": 0.2781303169195639, + "lpips": 0.6655868291854858 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.75, + "history_latent_mse": 1.1167012453079224, + "psnr": 10.98071122293361, + "ssim": 0.2703117738891817, + "lpips": 0.5898345112800598 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.890625, + "history_latent_mse": 0.1776418834924698, + "psnr": 35.887101881085634, + "ssim": 0.9509251859429928, + "lpips": 0.2477835714817047 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.6748046875, + "history_latent_mse": 0.09520227462053299, + "psnr": 35.238608750384515, + "ssim": 0.9507887189950697, + "lpips": 0.22817659378051758 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 856, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3046875, + "history_latent_mse": 0.27612948417663574, + "psnr": 18.590045602181135, + "ssim": 0.652070265922557, + "lpips": 0.23579567670822144 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.6875, + "history_latent_mse": 1.2576309442520142, + "psnr": 9.062799913712063, + "ssim": 0.27118083845238455, + "lpips": 0.6443774104118347 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.640625, + "history_latent_mse": 1.0636368989944458, + "psnr": 10.750034044458946, + "ssim": 0.3228667446565132, + "lpips": 0.5970298051834106 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.02734375, + "history_latent_mse": 0.13855737447738647, + "psnr": 35.77824613720884, + "ssim": 0.9494368221539021, + "lpips": 0.25469911098480225 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.5859375, + "history_latent_mse": 0.14308881759643555, + "psnr": 32.26957627504644, + "ssim": 0.9360240426541555, + "lpips": 0.314947247505188 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 864, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.2701523005962372, + "psnr": 18.504458378406493, + "ssim": 0.6583827715134457, + "lpips": 0.25816187262535095 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.4921875, + "history_latent_mse": 1.231392741203308, + "psnr": 8.901461708555818, + "ssim": 0.25798425067663955, + "lpips": 0.6264577507972717 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.2578125, + "history_latent_mse": 0.9444217681884766, + "psnr": 9.285897667881601, + "ssim": 0.28747021138644513, + "lpips": 0.6289656758308411 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6640625, + "history_latent_mse": 0.11717787384986877, + "psnr": 35.4269862753644, + "ssim": 0.9457311159669539, + "lpips": 0.27127423882484436 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.71875, + "history_latent_mse": 0.2028433233499527, + "psnr": 31.57250570206226, + "ssim": 0.9245300013350644, + "lpips": 0.3654765486717224 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 872, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.8125, + "history_latent_mse": 0.2978976368904114, + "psnr": 17.773418873175515, + "ssim": 0.6404226633787303, + "lpips": 0.3210606575012207 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.28125, + "history_latent_mse": 1.4542804956436157, + "psnr": 8.89524846520872, + "ssim": 0.27624061718745085, + "lpips": 0.6418129801750183 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.453125, + "history_latent_mse": 1.1367557048797607, + "psnr": 8.078108348028705, + "ssim": 0.2556396720105939, + "lpips": 0.670830488204956 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.7421875, + "history_latent_mse": 0.12477728724479675, + "psnr": 35.10459493823618, + "ssim": 0.9424944582689578, + "lpips": 0.28546005487442017 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.8359375, + "history_latent_mse": 0.2300664782524109, + "psnr": 34.46117796678137, + "ssim": 0.9412823137088673, + "lpips": 0.2880580425262451 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6fda7f0dc28220a74458fa979c2a3cc72f58ab3b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8508f7a4e233120bcd928e8e2f95d80204cb6f88a0ea3665cb15fc30cb0ba4b3 +size 1316592 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..fee6417b4ab4ba7744f513a28b9a3d7d6682f895 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.19504790008068085, + "wrist/video/latent_future_mse": 0.19504790008068085, + "video/psnr": 23.97182098554074, + "wrist/video/psnr": 23.97182098554074, + "video/ssim": 0.8125829526693039, + "wrist/video/ssim": 0.8125829526693039, + "video/lpips": 0.07511236518621445, + "wrist/video/lpips": 0.07511236518621445, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.9129447340965271, + "wrist/wrist_left/latent_future_mse": 0.9129447340965271, + "wrist_left/psnr": 13.948032842082776, + "wrist/wrist_left/psnr": 13.948032842082776, + "wrist_left/ssim": 0.3378560447266604, + "wrist/wrist_left/ssim": 0.3378560447266604, + "wrist_left/lpips": 0.47823843359947205, + "wrist/wrist_left/lpips": 0.47823843359947205, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.6459246277809143, + "wrist/wrist_right/latent_future_mse": 0.6459246277809143, + "wrist_right/psnr": 15.361403592715266, + "wrist/wrist_right/psnr": 15.361403592715266, + "wrist_right/ssim": 0.5478371830024589, + "wrist/wrist_right/ssim": 0.5478371830024589, + "wrist_right/lpips": 0.3370887041091919, + "wrist/wrist_right/lpips": 0.3370887041091919, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.08018997311592102, + "wrist/tactile_left/latent_future_mse": 0.08018997311592102, + "tactile_left/psnr": 38.742343650877785, + "wrist/tactile_left/psnr": 38.742343650877785, + "tactile_left/ssim": 0.9835824167841523, + "wrist/tactile_left/ssim": 0.9835824167841523, + "tactile_left/lpips": 0.041552357375621796, + "wrist/tactile_left/lpips": 0.041552357375621796, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.022527383640408516, + "wrist/tactile_right/latent_future_mse": 0.022527383640408516, + "tactile_right/psnr": 46.984432773655556, + "wrist/tactile_right/psnr": 46.984432773655556, + "tactile_right/ssim": 0.9918726462661673, + "wrist/tactile_right/ssim": 0.9918726462661673, + "tactile_right/lpips": 0.013785661198198795, + "wrist/tactile_right/lpips": 0.013785661198198795, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 760, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.19504790008068085, + "psnr": 23.97182098554074, + "ssim": 0.8125829526693039, + "lpips": 0.07511236518621445 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.9129447340965271, + "psnr": 13.948032842082776, + "ssim": 0.3378560447266604, + "lpips": 0.47823843359947205 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.6459246277809143, + "psnr": 15.361403592715266, + "ssim": 0.5478371830024589, + "lpips": 0.3370887041091919 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08018997311592102, + "psnr": 38.742343650877785, + "ssim": 0.9835824167841523, + "lpips": 0.041552357375621796 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.022527383640408516, + "psnr": 46.984432773655556, + "ssim": 0.9918726462661673, + "lpips": 0.013785661198198795 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/case.json new file mode 100644 index 0000000000000000000000000000000000000000..8eda86988a5222555e06c8bcc1a0bef56a5ae780 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/case.json @@ -0,0 +1,168 @@ +{ + "task": 6, + "episode": "motherboard_0909_episode_001.pt", + "category": "release", + "index": 6112, + "t_start": 2829, + "start_seconds": 188.6, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6953125, + "long_future_contact_fraction": 0.675, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "06_motherboard_0909_episode_001_release", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..350bf3fdd62f45410ef8c1c335e8d61563257882 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20c9fd5f94287c73f2541a8d89687770ae361b845b90fee22b4e046caf86e266 +size 8171715 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/motherboard_0909_episode_001_t02829_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/motherboard_0909_episode_001_t02829_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb56b5bee3c63334d1a49cf8212aea54f1213fd3 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/motherboard_0909_episode_001_t02829_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:405160612d929df4ed208ad723babcf18f25e656f3202d9757a0c8a6e3bd8a03 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..d579aed6e1da06c6e9d5ed55b06296299ac1c6e8 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 2829, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 2829, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 25.756199167378753, + "ssim": 0.835266090630362, + "lpips": 0.05219259858131409 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 9.795115082654975, + "ssim": 0.3088627337972464, + "lpips": 0.5185638070106506 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 21.236576841817993, + "ssim": 0.833024592249541, + "lpips": 0.11528465896844864 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 37.94824790738401, + "ssim": 0.9632617663474357, + "lpips": 0.08573136478662491 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.52862412781565, + "ssim": 0.9924801051994103, + "lpips": 0.013260400854051113 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 2837, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.515625, + "history_latent_mse": 0.12307489663362503, + "psnr": 23.72925858408616, + "ssim": 0.7761899281313098, + "lpips": 0.07246850430965424 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.65625, + "history_latent_mse": 1.2207019329071045, + "psnr": 8.630960419020989, + "ssim": 0.20055540644528894, + "lpips": 0.6255948543548584 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.84375, + "history_latent_mse": 0.31101498007774353, + "psnr": 19.73899343110421, + "ssim": 0.7430886145307394, + "lpips": 0.15666377544403076 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.93359375, + "history_latent_mse": 0.09051945060491562, + "psnr": 37.439019622510806, + "ssim": 0.9635076161667111, + "lpips": 0.12076577544212341 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.159423828125, + "history_latent_mse": 0.024197274819016457, + "psnr": 44.48048027060615, + "ssim": 0.9852560510154088, + "lpips": 0.04280985891819 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 2845, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.35546875, + "history_latent_mse": 0.14202453196048737, + "psnr": 23.43751343681503, + "ssim": 0.7771850841248189, + "lpips": 0.07657304406166077 + }, + "wrist_left": { + "history_latent_max_abs_error": 7.609375, + "history_latent_mse": 1.2571264505386353, + "psnr": 9.566423281198027, + "ssim": 0.15526217428360162, + "lpips": 0.6325905323028564 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.90625, + "history_latent_mse": 0.4207431375980377, + "psnr": 17.3419850995055, + "ssim": 0.6332227410823139, + "lpips": 0.2565159797668457 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.828125, + "history_latent_mse": 0.09976337105035782, + "psnr": 35.387877081236866, + "ssim": 0.9514584495842969, + "lpips": 0.18000397086143494 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.9580078125, + "history_latent_mse": 0.032493047416210175, + "psnr": 43.35396284367206, + "ssim": 0.9817948579709243, + "lpips": 0.06435484439134598 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 2853, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.48046875, + "history_latent_mse": 0.13837173581123352, + "psnr": 23.269153282176926, + "ssim": 0.7713606991305408, + "lpips": 0.08427079766988754 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.3515625, + "history_latent_mse": 1.243075966835022, + "psnr": 9.803793547610336, + "ssim": 0.15565791287500588, + "lpips": 0.6375032663345337 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 0.5453107357025146, + "psnr": 15.749417675969656, + "ssim": 0.5873199043996946, + "lpips": 0.3279944062232971 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.701171875, + "history_latent_mse": 0.12913911044597626, + "psnr": 30.941367570492837, + "ssim": 0.8596609505029358, + "lpips": 0.29118359088897705 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.150146484375, + "history_latent_mse": 0.04036927968263626, + "psnr": 42.60904908641229, + "ssim": 0.978562387292111, + "lpips": 0.08306442201137543 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 2861, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.484375, + "history_latent_mse": 0.15298156440258026, + "psnr": 22.931096522383424, + "ssim": 0.7530826154687198, + "lpips": 0.0916934460401535 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.34375, + "history_latent_mse": 1.1576999425888062, + "psnr": 9.602714612953354, + "ssim": 0.12942120882511385, + "lpips": 0.6547681093215942 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8203125, + "history_latent_mse": 0.5596463680267334, + "psnr": 15.209122319573844, + "ssim": 0.5545493435156822, + "lpips": 0.35842907428741455 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.390625, + "history_latent_mse": 0.2723928987979889, + "psnr": 29.160411673850604, + "ssim": 0.7936044125988544, + "lpips": 0.3303782641887665 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.1348876953125, + "history_latent_mse": 0.048756178468465805, + "psnr": 41.11739088054814, + "ssim": 0.9749281700328197, + "lpips": 0.10401212424039841 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 2869, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.4609375, + "history_latent_mse": 0.16085605323314667, + "psnr": 22.353856441894724, + "ssim": 0.7333473397406705, + "lpips": 0.11020157486200333 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.859375, + "history_latent_mse": 1.2130208015441895, + "psnr": 9.827940443273292, + "ssim": 0.120363783339094, + "lpips": 0.6839362978935242 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7021484375, + "history_latent_mse": 0.5513512492179871, + "psnr": 14.426776491518376, + "ssim": 0.5260012032368425, + "lpips": 0.40512168407440186 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.60546875, + "history_latent_mse": 0.3037930428981781, + "psnr": 29.655984399543414, + "ssim": 0.8017136302310763, + "lpips": 0.3270813524723053 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.400390625, + "history_latent_mse": 0.05523292347788811, + "psnr": 40.29634074004274, + "ssim": 0.9717637859116596, + "lpips": 0.12469758838415146 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 2877, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.78125, + "history_latent_mse": 0.1902128905057907, + "psnr": 19.4039788749634, + "ssim": 0.6855616918287396, + "lpips": 0.2106633484363556 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.7734375, + "history_latent_mse": 1.1584187746047974, + "psnr": 10.287976859647348, + "ssim": 0.16378251630219276, + "lpips": 0.7025153636932373 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7265625, + "history_latent_mse": 0.6200301051139832, + "psnr": 12.910240263466672, + "ssim": 0.4701324883426373, + "lpips": 0.48982176184654236 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.4921875, + "history_latent_mse": 0.3263384997844696, + "psnr": 29.934611188531743, + "ssim": 0.8139848020527061, + "lpips": 0.33004093170166016 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.466552734375, + "history_latent_mse": 0.06368327885866165, + "psnr": 39.24159367842562, + "ssim": 0.9684283080090664, + "lpips": 0.13924658298492432 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 2885, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.703125, + "history_latent_mse": 0.30108970403671265, + "psnr": 17.764057073614584, + "ssim": 0.6620262777791566, + "lpips": 0.2986719608306885 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.78125, + "history_latent_mse": 1.2482184171676636, + "psnr": 10.03961572574659, + "ssim": 0.18423219634043292, + "lpips": 0.704594612121582 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.859375, + "history_latent_mse": 0.8195632100105286, + "psnr": 11.468674621173065, + "ssim": 0.42334443091871987, + "lpips": 0.5219416618347168 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.125, + "history_latent_mse": 0.3108655512332916, + "psnr": 31.394821750293097, + "ssim": 0.875660407564387, + "lpips": 0.29375094175338745 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.36328125, + "history_latent_mse": 0.06977532804012299, + "psnr": 38.478987549231675, + "ssim": 0.9648117161060317, + "lpips": 0.1598256230354309 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 2893, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.78515625, + "history_latent_mse": 0.3189527690410614, + "psnr": 17.48441877434705, + "ssim": 0.6254595757164626, + "lpips": 0.3275079131126404 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9140625, + "history_latent_mse": 1.1481460332870483, + "psnr": 9.389247640325724, + "ssim": 0.1767917168409729, + "lpips": 0.6874648332595825 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6875, + "history_latent_mse": 0.8640062808990479, + "psnr": 11.339433764633501, + "ssim": 0.42499702225197766, + "lpips": 0.5273013710975647 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.20703125, + "history_latent_mse": 0.3068641722202301, + "psnr": 33.69861295645229, + "ssim": 0.9492698384083883, + "lpips": 0.24424457550048828 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4892578125, + "history_latent_mse": 0.07717099785804749, + "psnr": 37.740329840455, + "ssim": 0.9618718538699986, + "lpips": 0.17238739132881165 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 2901, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.34765625, + "history_latent_mse": 0.3440483808517456, + "psnr": 16.933706100900928, + "ssim": 0.6001300345545135, + "lpips": 0.3501172959804535 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.5859375, + "history_latent_mse": 1.1391674280166626, + "psnr": 10.52694284469202, + "ssim": 0.20861702685694775, + "lpips": 0.6568442583084106 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.0234375, + "history_latent_mse": 0.8517677783966064, + "psnr": 11.265056864826166, + "ssim": 0.4087446973819553, + "lpips": 0.5313889980316162 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.1015625, + "history_latent_mse": 0.13151998817920685, + "psnr": 30.829325047674693, + "ssim": 0.9280656745489044, + "lpips": 0.31298744678497314 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.48779296875, + "history_latent_mse": 0.08445736765861511, + "psnr": 37.40243412292781, + "ssim": 0.9589167978689157, + "lpips": 0.1800118386745453 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 2909, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.96875, + "history_latent_mse": 0.37753233313560486, + "psnr": 18.03464233729001, + "ssim": 0.6383288619058014, + "lpips": 0.28571292757987976 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.65625, + "history_latent_mse": 1.238336443901062, + "psnr": 10.779121594349029, + "ssim": 0.2390221962666336, + "lpips": 0.654449462890625 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.375, + "history_latent_mse": 0.8353622555732727, + "psnr": 11.476342685836116, + "ssim": 0.42685966672093406, + "lpips": 0.5434567928314209 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.71875, + "history_latent_mse": 0.22170914709568024, + "psnr": 30.056813774227344, + "ssim": 0.9215089027334331, + "lpips": 0.3433040380477905 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7998046875, + "history_latent_mse": 0.09339123964309692, + "psnr": 37.12693345464745, + "ssim": 0.9559216184310354, + "lpips": 0.18842431902885437 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 2917, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.484375, + "history_latent_mse": 0.32711321115493774, + "psnr": 19.61492843296683, + "ssim": 0.6587162664755082, + "lpips": 0.23245057463645935 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.703125, + "history_latent_mse": 1.164857029914856, + "psnr": 9.9483687954496, + "ssim": 0.2461362383961195, + "lpips": 0.6712664365768433 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9375, + "history_latent_mse": 0.9033206105232239, + "psnr": 12.222441214228827, + "ssim": 0.4488215279057117, + "lpips": 0.5212984085083008 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.234375, + "history_latent_mse": 0.2137620896100998, + "psnr": 30.74562702996214, + "ssim": 0.9239795544912861, + "lpips": 0.34028100967407227 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.720458984375, + "history_latent_mse": 0.09752737730741501, + "psnr": 36.76620970072229, + "ssim": 0.9528357943842705, + "lpips": 0.2019694447517395 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 2925, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.21875, + "history_latent_mse": 0.3174107074737549, + "psnr": 17.630640348833893, + "ssim": 0.6352566776951215, + "lpips": 0.3112691640853882 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.875, + "history_latent_mse": 1.1844148635864258, + "psnr": 9.926584509785014, + "ssim": 0.2263229233906025, + "lpips": 0.6708772778511047 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.2890625, + "history_latent_mse": 0.8612427711486816, + "psnr": 11.288683097340076, + "ssim": 0.39453047331131896, + "lpips": 0.5532624125480652 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.26953125, + "history_latent_mse": 0.23575620353221893, + "psnr": 31.900122410023602, + "ssim": 0.9339039367245892, + "lpips": 0.3118135333061218 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.818359375, + "history_latent_mse": 0.10420220345258713, + "psnr": 36.18968641226627, + "ssim": 0.9493757037877266, + "lpips": 0.2170557975769043 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 2933, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6982421875, + "history_latent_mse": 0.3512437045574188, + "psnr": 17.38577659661196, + "ssim": 0.6256218854209, + "lpips": 0.33145877718925476 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.8837890625, + "history_latent_mse": 1.0546153783798218, + "psnr": 9.186732640274805, + "ssim": 0.22645646590277574, + "lpips": 0.7241584062576294 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.828125, + "history_latent_mse": 0.9743571877479553, + "psnr": 11.340736695156577, + "ssim": 0.3807836173228526, + "lpips": 0.5694361925125122 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.517578125, + "history_latent_mse": 0.20394068956375122, + "psnr": 32.77276578471606, + "ssim": 0.9385675745388503, + "lpips": 0.29230186343193054 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.87109375, + "history_latent_mse": 0.11122607439756393, + "psnr": 35.49436248095677, + "ssim": 0.9463017571242149, + "lpips": 0.23368960618972778 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 2941, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8125, + "history_latent_mse": 0.31654423475265503, + "psnr": 17.264726033259684, + "ssim": 0.6227012440019195, + "lpips": 0.34232401847839355 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.65625, + "history_latent_mse": 1.1066244840621948, + "psnr": 9.659802896710838, + "ssim": 0.2412243192254273, + "lpips": 0.7216579914093018 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.46875, + "history_latent_mse": 0.8346479535102844, + "psnr": 11.25499090369395, + "ssim": 0.3682623444970198, + "lpips": 0.5757245421409607 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.1630859375, + "history_latent_mse": 0.14695937931537628, + "psnr": 28.03061389141117, + "ssim": 0.890513159166643, + "lpips": 0.40389591455459595 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.921875, + "history_latent_mse": 0.11754439026117325, + "psnr": 34.854910955501516, + "ssim": 0.9429673473778937, + "lpips": 0.24597874283790588 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6649d00180e90d9b7ef596df20f336574e3217f0 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:621c80cb39cecb67d779dc27dd75fa929c05edab9e00529a6c8ebe921ffab4b0 +size 1013997 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..d2d380745e2ec1ef368fb31095e771e3af196fad --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.14344173669815063, + "wrist/video/latent_future_mse": 0.14344173669815063, + "video/psnr": 25.756199167378753, + "wrist/video/psnr": 25.756199167378753, + "video/ssim": 0.835266090630362, + "wrist/video/ssim": 0.835266090630362, + "video/lpips": 0.05219259858131409, + "wrist/video/lpips": 0.05219259858131409, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 1.364241600036621, + "wrist/wrist_left/latent_future_mse": 1.364241600036621, + "wrist_left/psnr": 9.795115082654975, + "wrist/wrist_left/psnr": 9.795115082654975, + "wrist_left/ssim": 0.3088627337972464, + "wrist/wrist_left/ssim": 0.3088627337972464, + "wrist_left/lpips": 0.5185638070106506, + "wrist/wrist_left/lpips": 0.5185638070106506, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.3349973261356354, + "wrist/wrist_right/latent_future_mse": 0.3349973261356354, + "wrist_right/psnr": 21.236576841817993, + "wrist/wrist_right/psnr": 21.236576841817993, + "wrist_right/ssim": 0.833024592249541, + "wrist/wrist_right/ssim": 0.833024592249541, + "wrist_right/lpips": 0.11528465896844864, + "wrist/wrist_right/lpips": 0.11528465896844864, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.13295195996761322, + "wrist/tactile_left/latent_future_mse": 0.13295195996761322, + "tactile_left/psnr": 37.94824790738401, + "wrist/tactile_left/psnr": 37.94824790738401, + "tactile_left/ssim": 0.9632617663474357, + "wrist/tactile_left/ssim": 0.9632617663474357, + "tactile_left/lpips": 0.08573136478662491, + "wrist/tactile_left/lpips": 0.08573136478662491, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.020939955487847328, + "wrist/tactile_right/latent_future_mse": 0.020939955487847328, + "tactile_right/psnr": 47.52862412781565, + "wrist/tactile_right/psnr": 47.52862412781565, + "tactile_right/ssim": 0.9924801051994103, + "wrist/tactile_right/ssim": 0.9924801051994103, + "tactile_right/lpips": 0.013260400854051113, + "wrist/tactile_right/lpips": 0.013260400854051113, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 2829, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.14344173669815063, + "psnr": 25.756199167378753, + "ssim": 0.835266090630362, + "lpips": 0.05219259858131409 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 1.364241600036621, + "psnr": 9.795115082654975, + "ssim": 0.3088627337972464, + "lpips": 0.5185638070106506 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.3349973261356354, + "psnr": 21.236576841817993, + "ssim": 0.833024592249541, + "lpips": 0.11528465896844864 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.13295195996761322, + "psnr": 37.94824790738401, + "ssim": 0.9632617663474357, + "lpips": 0.08573136478662491 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.020939955487847328, + "psnr": 47.52862412781565, + "ssim": 0.9924801051994103, + "lpips": 0.013260400854051113 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/case.json new file mode 100644 index 0000000000000000000000000000000000000000..66285065a5464dcef0298af1d71c35ae5fa048ef --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/case.json @@ -0,0 +1,164 @@ +{ + "task": 7, + "episode": "motherboard_0909_episode_001.pt", + "category": "full", + "index": 3783, + "t_start": 467, + "start_seconds": 31.133333333333333, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 1.0, + "long_future_contact_fraction": 1.0, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "07_motherboard_0909_episode_001_full", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c252bf0136b06dd745c3f80f0dbdc05d265fb7f0 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93eaa759e50d10c4c859f7e4b04f373b1f797ea4b595417b605b47efdc90d739 +size 9918900 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/motherboard_0909_episode_001_t00467_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/motherboard_0909_episode_001_t00467_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..80c5569d902c8389069743f44f3e1ec378b8f62b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/motherboard_0909_episode_001_t00467_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59797154e4b06242f84e50ad885f189b8e67b5fda3fc98cc8f6d24afd6a3e83f +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..56035a1e5832e9362d13863080c4281174a9c3ae --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 467, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 467, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.79918631869247, + "ssim": 0.9405153807669218, + "lpips": 0.02165227383375168 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.23797058027921, + "ssim": 0.37099834081172656, + "lpips": 0.26520758867263794 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 23.549866051629607, + "ssim": 0.8096416508700895, + "lpips": 0.03447118028998375 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.6589653665956, + "ssim": 0.9321616081257167, + "lpips": 0.05463609844446182 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 48.496738263424476, + "ssim": 0.9923670189592797, + "lpips": 0.01329369843006134 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 475, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.501953125, + "history_latent_mse": 0.038583289831876755, + "psnr": 26.31516018778616, + "ssim": 0.861799448383671, + "lpips": 0.05737560987472534 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.09375, + "history_latent_mse": 0.7361770272254944, + "psnr": 12.79601016778255, + "ssim": 0.30969395159162827, + "lpips": 0.31969374418258667 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.40234375, + "history_latent_mse": 0.28392481803894043, + "psnr": 14.486079482442706, + "ssim": 0.4949890674921087, + "lpips": 0.3516920804977417 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.953125, + "history_latent_mse": 0.14671771228313446, + "psnr": 29.277967452739585, + "ssim": 0.8990907521873879, + "lpips": 0.0953884869813919 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.0908203125, + "history_latent_mse": 0.022759249433875084, + "psnr": 45.55136709167517, + "ssim": 0.9852842482069954, + "lpips": 0.046231709420681 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 483, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.5234375, + "history_latent_mse": 0.09620427340269089, + "psnr": 24.007064138568246, + "ssim": 0.804829790183206, + "lpips": 0.09571818262338638 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.921875, + "history_latent_mse": 0.5916845798492432, + "psnr": 12.967128597131323, + "ssim": 0.29712932782732093, + "lpips": 0.33074498176574707 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 0.6580809354782104, + "psnr": 11.356134943324816, + "ssim": 0.3446265954153085, + "lpips": 0.5659664869308472 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.7109375, + "history_latent_mse": 0.15363924205303192, + "psnr": 28.8090069389898, + "ssim": 0.9088995408238202, + "lpips": 0.1651235818862915 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.00390625, + "history_latent_mse": 0.03241310268640518, + "psnr": 31.994262208566965, + "ssim": 0.9564679607191622, + "lpips": 0.14736680686473846 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 491, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1796875, + "history_latent_mse": 0.13702355325222015, + "psnr": 23.108039635288865, + "ssim": 0.7755334281797419, + "lpips": 0.1123148649930954 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.0546875, + "history_latent_mse": 0.6756782531738281, + "psnr": 12.793753746870264, + "ssim": 0.2742819901556849, + "lpips": 0.3706228733062744 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.421875, + "history_latent_mse": 1.033644199371338, + "psnr": 9.760602571694516, + "ssim": 0.27222759723965817, + "lpips": 0.673262357711792 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.40625, + "history_latent_mse": 0.17140686511993408, + "psnr": 27.893361026706845, + "ssim": 0.8995547627416888, + "lpips": 0.22723011672496796 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.3125, + "history_latent_mse": 0.16317220032215118, + "psnr": 25.44779896955425, + "ssim": 0.9001322201876891, + "lpips": 0.3060959577560425 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 499, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9140625, + "history_latent_mse": 0.145286425948143, + "psnr": 22.452700552466467, + "ssim": 0.7564218286071277, + "lpips": 0.13158687949180603 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9453125, + "history_latent_mse": 0.6630589365959167, + "psnr": 12.735603953299917, + "ssim": 0.25915939732947124, + "lpips": 0.3853752017021179 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 1.3365473747253418, + "psnr": 9.575725774411907, + "ssim": 0.24839668410395532, + "lpips": 0.6950333118438721 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.5, + "history_latent_mse": 0.18892385065555573, + "psnr": 28.950394937487612, + "ssim": 0.9064372387162206, + "lpips": 0.22515344619750977 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.078125, + "history_latent_mse": 0.194100484251976, + "psnr": 25.976016470264103, + "ssim": 0.9031468628933291, + "lpips": 0.2997230589389801 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 507, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.46875, + "history_latent_mse": 0.17265097796916962, + "psnr": 22.12153378459734, + "ssim": 0.7399131665768094, + "lpips": 0.13239380717277527 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.328125, + "history_latent_mse": 0.7654544711112976, + "psnr": 13.19588603190276, + "ssim": 0.245385362142353, + "lpips": 0.42265164852142334 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4296875, + "history_latent_mse": 1.4771820306777954, + "psnr": 9.868379911496417, + "ssim": 0.24579138735088846, + "lpips": 0.6728012561798096 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.39453125, + "history_latent_mse": 0.18204034864902496, + "psnr": 33.071920351725154, + "ssim": 0.9478811417088681, + "lpips": 0.23996196687221527 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.859375, + "history_latent_mse": 0.23697365820407867, + "psnr": 31.78723903249546, + "ssim": 0.9332352943099247, + "lpips": 0.22054944932460785 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 515, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.546875, + "history_latent_mse": 0.1732872575521469, + "psnr": 22.092048251786164, + "ssim": 0.7341361098893324, + "lpips": 0.13828861713409424 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0390625, + "history_latent_mse": 0.9025480151176453, + "psnr": 12.89017545825965, + "ssim": 0.22243866644091775, + "lpips": 0.44017040729522705 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.3125, + "history_latent_mse": 1.3514305353164673, + "psnr": 9.99361304746172, + "ssim": 0.24867001090078328, + "lpips": 0.6729084849357605 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.611328125, + "history_latent_mse": 0.10685992240905762, + "psnr": 35.2135256327645, + "ssim": 0.9519153977284582, + "lpips": 0.22909444570541382 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.4609375, + "history_latent_mse": 0.13948722183704376, + "psnr": 33.91254432022666, + "ssim": 0.9436976898652631, + "lpips": 0.20800288021564484 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 523, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.78125, + "history_latent_mse": 0.18281030654907227, + "psnr": 21.616150418851504, + "ssim": 0.7130694839616778, + "lpips": 0.14592614769935608 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.41015625, + "history_latent_mse": 0.7243251204490662, + "psnr": 12.147014829832607, + "ssim": 0.21480263566876462, + "lpips": 0.4695311486721039 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.625, + "history_latent_mse": 1.3498749732971191, + "psnr": 9.99459628709759, + "ssim": 0.24844907579690578, + "lpips": 0.6677268743515015 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.887847900390625, + "history_latent_mse": 0.11193692684173584, + "psnr": 30.417405280121386, + "ssim": 0.9348383820622825, + "lpips": 0.28978782892227173 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.8828125, + "history_latent_mse": 0.13924987614154816, + "psnr": 28.32221857469206, + "ssim": 0.915345191641968, + "lpips": 0.27507686614990234 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 531, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.0244140625, + "history_latent_mse": 0.20070268213748932, + "psnr": 21.595208796680776, + "ssim": 0.7095022237653092, + "lpips": 0.1485729068517685 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.703125, + "history_latent_mse": 0.7811102867126465, + "psnr": 11.488227607426406, + "ssim": 0.18940519903417638, + "lpips": 0.5430371761322021 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.921875, + "history_latent_mse": 1.2838374376296997, + "psnr": 9.861292711297281, + "ssim": 0.23835221022442032, + "lpips": 0.6952875256538391 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.460693359375, + "history_latent_mse": 0.13836847245693207, + "psnr": 30.704273615064643, + "ssim": 0.9334615105258581, + "lpips": 0.30334582924842834 + }, + "tactile_right": { + "history_latent_max_abs_error": 4.203125, + "history_latent_mse": 0.1873796433210373, + "psnr": 27.747420059560728, + "ssim": 0.9104643220980273, + "lpips": 0.28804826736450195 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 539, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.25, + "history_latent_mse": 0.20865654945373535, + "psnr": 21.243586781750817, + "ssim": 0.705088544593211, + "lpips": 0.1626243144273758 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 0.9288724064826965, + "psnr": 11.002118071262242, + "ssim": 0.17393266477691682, + "lpips": 0.553251326084137 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.703125, + "history_latent_mse": 1.4197317361831665, + "psnr": 9.752625886859974, + "ssim": 0.22054168103310604, + "lpips": 0.7114371061325073 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.857421875, + "history_latent_mse": 0.13340142369270325, + "psnr": 28.882137735029243, + "ssim": 0.9217843835359045, + "lpips": 0.33387595415115356 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.40625, + "history_latent_mse": 0.20258593559265137, + "psnr": 26.002112080153758, + "ssim": 0.8940531755965547, + "lpips": 0.31770050525665283 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 547, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.265625, + "history_latent_mse": 0.21568655967712402, + "psnr": 20.71393640340554, + "ssim": 0.6971538289891093, + "lpips": 0.1866607815027237 + }, + "wrist_left": { + "history_latent_max_abs_error": 7.296875, + "history_latent_mse": 1.0170633792877197, + "psnr": 10.583965772619102, + "ssim": 0.15645569639115398, + "lpips": 0.5840693712234497 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.8046875, + "history_latent_mse": 1.4844478368759155, + "psnr": 9.332335345733306, + "ssim": 0.21095143258539678, + "lpips": 0.7388973236083984 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8046875, + "history_latent_mse": 0.1654040813446045, + "psnr": 28.466022300380352, + "ssim": 0.9156529455911936, + "lpips": 0.3493713438510895 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.92578125, + "history_latent_mse": 0.2870751619338989, + "psnr": 25.738675338292985, + "ssim": 0.8864533827947134, + "lpips": 0.32033681869506836 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 555, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9140625, + "history_latent_mse": 0.23213285207748413, + "psnr": 18.418336245687666, + "ssim": 0.635391789287816, + "lpips": 0.2802475094795227 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.984375, + "history_latent_mse": 1.0951322317123413, + "psnr": 10.24336163809393, + "ssim": 0.14385405945986823, + "lpips": 0.6002342700958252 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.984375, + "history_latent_mse": 1.4927717447280884, + "psnr": 9.567484244372084, + "ssim": 0.22314620079496134, + "lpips": 0.7474483251571655 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.6875, + "history_latent_mse": 0.15566276013851166, + "psnr": 28.895847407514093, + "ssim": 0.9176875766239901, + "lpips": 0.35283321142196655 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.47265625, + "history_latent_mse": 0.19630908966064453, + "psnr": 26.98852873065458, + "ssim": 0.902563127788592, + "lpips": 0.31430381536483765 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 563, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.671875, + "history_latent_mse": 0.343946248292923, + "psnr": 18.98158296300633, + "ssim": 0.6381849619906427, + "lpips": 0.25852543115615845 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.375, + "history_latent_mse": 1.0976516008377075, + "psnr": 10.950662272898612, + "ssim": 0.17369673466403074, + "lpips": 0.6137601137161255 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.375, + "history_latent_mse": 1.4897398948669434, + "psnr": 10.226112161393718, + "ssim": 0.23871828101060796, + "lpips": 0.7227326035499573 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8046875, + "history_latent_mse": 0.1755307912826538, + "psnr": 29.078565942808055, + "ssim": 0.9191124151339493, + "lpips": 0.35054081678390503 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0390625, + "history_latent_mse": 0.24020564556121826, + "psnr": 29.978019082581746, + "ssim": 0.9147158204701133, + "lpips": 0.285137414932251 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 571, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.3203125, + "history_latent_mse": 0.28363218903541565, + "psnr": 20.317212784766937, + "ssim": 0.658754892678991, + "lpips": 0.21950367093086243 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8515625, + "history_latent_mse": 1.0092530250549316, + "psnr": 10.922258985847233, + "ssim": 0.1989175263684142, + "lpips": 0.601204514503479 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 1.3066974878311157, + "psnr": 8.987844266268471, + "ssim": 0.17943477307697042, + "lpips": 0.6839389801025391 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.646484375, + "history_latent_mse": 0.17088592052459717, + "psnr": 30.91518111331434, + "ssim": 0.9290990091322813, + "lpips": 0.3286210298538208 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.1171875, + "history_latent_mse": 0.18112920224666595, + "psnr": 31.55793212763819, + "ssim": 0.9209195112747569, + "lpips": 0.2706839442253113 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 579, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.763671875, + "history_latent_mse": 0.271883100271225, + "psnr": 20.167900945191292, + "ssim": 0.6652234890645061, + "lpips": 0.22179129719734192 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 1.0457733869552612, + "psnr": 11.323604293783236, + "ssim": 0.32578410018551085, + "lpips": 0.5162110328674316 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.53125, + "history_latent_mse": 1.2707844972610474, + "psnr": 10.552685403594666, + "ssim": 0.2423959484764282, + "lpips": 0.6761786937713623 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4951171875, + "history_latent_mse": 0.15843868255615234, + "psnr": 33.30651622399341, + "ssim": 0.9356264583221097, + "lpips": 0.307800829410553 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.55078125, + "history_latent_mse": 0.17068034410476685, + "psnr": 29.34550644971781, + "ssim": 0.9071809634667074, + "lpips": 0.30786028504371643 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..db47ed933889e83ea79fe49ec4b5436cc69e12bc --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9deb97f83e7c373fb9360dc7f09e979f8283181f94cbf73a00f06690495d0f71 +size 1115818 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..d19310574c209cfbfd750aa5b11a488e24af1fc2 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.06166359409689903, + "wrist/video/latent_future_mse": 0.06166359409689903, + "video/psnr": 31.79918631869247, + "wrist/video/psnr": 31.79918631869247, + "video/ssim": 0.9405153807669218, + "wrist/video/ssim": 0.9405153807669218, + "video/lpips": 0.02165227383375168, + "wrist/video/lpips": 0.02165227383375168, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.7917399406433105, + "wrist/wrist_left/latent_future_mse": 0.7917399406433105, + "wrist_left/psnr": 13.23797058027921, + "wrist/wrist_left/psnr": 13.23797058027921, + "wrist_left/ssim": 0.37099834081172656, + "wrist/wrist_left/ssim": 0.37099834081172656, + "wrist_left/lpips": 0.26520758867263794, + "wrist/wrist_left/lpips": 0.26520758867263794, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.2945031225681305, + "wrist/wrist_right/latent_future_mse": 0.2945031225681305, + "wrist_right/psnr": 23.549866051629607, + "wrist/wrist_right/psnr": 23.549866051629607, + "wrist_right/ssim": 0.8096416508700895, + "wrist/wrist_right/ssim": 0.8096416508700895, + "wrist_right/lpips": 0.03447118028998375, + "wrist/wrist_right/lpips": 0.03447118028998375, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.18197719752788544, + "wrist/tactile_left/latent_future_mse": 0.18197719752788544, + "tactile_left/psnr": 31.6589653665956, + "wrist/tactile_left/psnr": 31.6589653665956, + "tactile_left/ssim": 0.9321616081257167, + "wrist/tactile_left/ssim": 0.9321616081257167, + "tactile_left/lpips": 0.05463609844446182, + "wrist/tactile_left/lpips": 0.05463609844446182, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.02023722417652607, + "wrist/tactile_right/latent_future_mse": 0.02023722417652607, + "tactile_right/psnr": 48.496738263424476, + "wrist/tactile_right/psnr": 48.496738263424476, + "tactile_right/ssim": 0.9923670189592797, + "wrist/tactile_right/ssim": 0.9923670189592797, + "tactile_right/lpips": 0.01329369843006134, + "wrist/tactile_right/lpips": 0.01329369843006134, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_001.pt", + "t_start": 467, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.06166359409689903, + "psnr": 31.79918631869247, + "ssim": 0.9405153807669218, + "lpips": 0.02165227383375168 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.7917399406433105, + "psnr": 13.23797058027921, + "ssim": 0.37099834081172656, + "lpips": 0.26520758867263794 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.2945031225681305, + "psnr": 23.549866051629607, + "ssim": 0.8096416508700895, + "lpips": 0.03447118028998375 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.18197719752788544, + "psnr": 31.6589653665956, + "ssim": 0.9321616081257167, + "lpips": 0.05463609844446182 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.02023722417652607, + "psnr": 48.496738263424476, + "ssim": 0.9923670189592797, + "lpips": 0.01329369843006134 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/case.json new file mode 100644 index 0000000000000000000000000000000000000000..f8107ab85ad75c7a4c3d125138713335b3087f9a --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/case.json @@ -0,0 +1,164 @@ +{ + "task": 8, + "episode": "motherboard_0909_episode_002.pt", + "category": "none", + "index": 9178, + "t_start": 2600, + "start_seconds": 173.33333333333334, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.296875, + "long_future_contact_fraction": 0.31666666666666665, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 48, + "long_label": "none_at_start", + "output_subdir": "08_motherboard_0909_episode_002_none", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..47e8c70a8f8c845747d566609139310418918346 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca4254b6286794bec0b754baf66ab90c4ce3a68596fbec4ee8920265bf14d89f +size 9150387 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/motherboard_0909_episode_002_t02600_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/motherboard_0909_episode_002_t02600_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce61409693afbe1ed73f840498dc9e6c352e305f --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/motherboard_0909_episode_002_t02600_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d52e45940b4ebaf20ee2df6c07c77bda834e2a28938ce51cba4b4be3f933e02 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..0b0b01b8e632c114c58d55afc49a9378c93615c6 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 2600, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 2600, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 28.321348685265328, + "ssim": 0.8925943476889411, + "lpips": 0.03279535472393036 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.091017709087751, + "ssim": 0.39492816861257507, + "lpips": 0.36116477847099304 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 24.853965446816723, + "ssim": 0.8352598886662768, + "lpips": 0.05153008550405502 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 41.18403310522096, + "ssim": 0.9832525334876951, + "lpips": 0.02327561192214489 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 49.90268913328755, + "ssim": 0.9945657508029224, + "lpips": 0.00416005402803421 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 2608, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.662109375, + "history_latent_mse": 0.06252210587263107, + "psnr": 25.056016261777113, + "ssim": 0.8123191934707812, + "lpips": 0.057601410895586014 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.515625, + "history_latent_mse": 0.7873194813728333, + "psnr": 11.406908536674328, + "ssim": 0.3106318255839948, + "lpips": 0.4997127056121826 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.3320673406124115, + "psnr": 19.62137571764551, + "ssim": 0.726484534351762, + "lpips": 0.1329290121793747 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4403076171875, + "history_latent_mse": 0.06022274121642113, + "psnr": 38.459951842443104, + "ssim": 0.9723180501756127, + "lpips": 0.06336117535829544 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.4609375, + "history_latent_mse": 0.008299936540424824, + "psnr": 45.533771048248056, + "ssim": 0.9875471176546815, + "lpips": 0.035431042313575745 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 2616, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.05859375, + "history_latent_mse": 0.11882045120000839, + "psnr": 22.777301870575716, + "ssim": 0.7625873975756212, + "lpips": 0.09908400475978851 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.765625, + "history_latent_mse": 0.9561381936073303, + "psnr": 10.090516032597188, + "ssim": 0.28333893403866456, + "lpips": 0.5684067010879517 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.83984375, + "history_latent_mse": 0.39902830123901367, + "psnr": 17.950308826066184, + "ssim": 0.6595385340897124, + "lpips": 0.19629105925559998 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.359375, + "history_latent_mse": 0.0908391997218132, + "psnr": 39.71983207556724, + "ssim": 0.9715247745489334, + "lpips": 0.08903728425502777 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.052734375, + "history_latent_mse": 0.03488529473543167, + "psnr": 43.69875048377645, + "ssim": 0.980795143065914, + "lpips": 0.06430771946907043 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 2624, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 6.109375, + "history_latent_mse": 0.1622547060251236, + "psnr": 22.587426926663753, + "ssim": 0.7415447832645492, + "lpips": 0.10635249316692352 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0703125, + "history_latent_mse": 0.9970862865447998, + "psnr": 12.287590301582593, + "ssim": 0.3357961506702284, + "lpips": 0.5827891826629639 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.265625, + "history_latent_mse": 0.4102928638458252, + "psnr": 19.071931842089064, + "ssim": 0.6544825947995186, + "lpips": 0.1864711344242096 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.5087890625, + "history_latent_mse": 0.058560580015182495, + "psnr": 36.06358519657531, + "ssim": 0.9637347616469758, + "lpips": 0.16175225377082825 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.228515625, + "history_latent_mse": 0.04572538658976555, + "psnr": 42.00621258311188, + "ssim": 0.9773745783268488, + "lpips": 0.0855625569820404 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 2632, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.05078125, + "history_latent_mse": 0.18039853870868683, + "psnr": 21.753673762658984, + "ssim": 0.7130341559401685, + "lpips": 0.13442102074623108 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.86328125, + "history_latent_mse": 0.9431846141815186, + "psnr": 9.773000344667825, + "ssim": 0.2835231577152118, + "lpips": 0.6378713846206665 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.4333278238773346, + "psnr": 17.722643049774817, + "ssim": 0.6163580165175663, + "lpips": 0.24650982022285461 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.671875, + "history_latent_mse": 0.18574345111846924, + "psnr": 36.15449144276414, + "ssim": 0.963757722367279, + "lpips": 0.19141283631324768 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.38720703125, + "history_latent_mse": 0.05357510223984718, + "psnr": 40.118525690753074, + "ssim": 0.9739715186053735, + "lpips": 0.10445328801870346 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 2640, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.2509765625, + "history_latent_mse": 0.19860166311264038, + "psnr": 21.217578624354125, + "ssim": 0.685711070643809, + "lpips": 0.15549303591251373 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.671875, + "history_latent_mse": 1.1483262777328491, + "psnr": 9.19808092128992, + "ssim": 0.2847081718282937, + "lpips": 0.7072978019714355 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.5390625, + "history_latent_mse": 0.5245361924171448, + "psnr": 15.883731688812112, + "ssim": 0.5679545891218587, + "lpips": 0.29940831661224365 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.50390625, + "history_latent_mse": 0.1280093640089035, + "psnr": 29.43723227504732, + "ssim": 0.9216788099191737, + "lpips": 0.3343045711517334 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4609375, + "history_latent_mse": 0.05936182662844658, + "psnr": 39.25766468850473, + "ssim": 0.9709061692084675, + "lpips": 0.12409501522779465 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 2648, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7890625, + "history_latent_mse": 0.21204258501529694, + "psnr": 19.38512090289568, + "ssim": 0.647481144288772, + "lpips": 0.24343755841255188 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.625, + "history_latent_mse": 1.2810019254684448, + "psnr": 10.724007446367128, + "ssim": 0.37692082236320723, + "lpips": 0.6167332530021667 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.0, + "history_latent_mse": 0.5713801383972168, + "psnr": 13.182250447540136, + "ssim": 0.48251647281536403, + "lpips": 0.3992757201194763 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.4140625, + "history_latent_mse": 0.21851295232772827, + "psnr": 28.923116852765077, + "ssim": 0.9338280959417321, + "lpips": 0.2944086790084839 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.51171875, + "history_latent_mse": 0.06805455684661865, + "psnr": 38.936124598317065, + "ssim": 0.9678917557435509, + "lpips": 0.13932904601097107 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 2656, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.34442138671875, + "history_latent_mse": 0.26767948269844055, + "psnr": 17.850766554823444, + "ssim": 0.6270236553845159, + "lpips": 0.32330143451690674 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.953125, + "history_latent_mse": 1.0375231504440308, + "psnr": 8.679739857755905, + "ssim": 0.2618732132541619, + "lpips": 0.6813249588012695 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7109375, + "history_latent_mse": 0.7066742777824402, + "psnr": 11.88957933552193, + "ssim": 0.31704826020309596, + "lpips": 0.5538946390151978 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.453125, + "history_latent_mse": 0.18985457718372345, + "psnr": 35.45865590197093, + "ssim": 0.9567743337301984, + "lpips": 0.21679648756980896 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.5703125, + "history_latent_mse": 0.07213427871465683, + "psnr": 38.567830712927716, + "ssim": 0.9644343489232449, + "lpips": 0.1560472548007965 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 2664, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9921875, + "history_latent_mse": 0.3341314494609833, + "psnr": 17.407541849765245, + "ssim": 0.6343979439478783, + "lpips": 0.3289176821708679 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.453125, + "history_latent_mse": 1.2485345602035522, + "psnr": 8.554713734845123, + "ssim": 0.1957293511165109, + "lpips": 0.6916732788085938 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.59375, + "history_latent_mse": 0.8979034423828125, + "psnr": 10.312818019301638, + "ssim": 0.24112920075081407, + "lpips": 0.6276059746742249 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.681396484375, + "history_latent_mse": 0.09610829502344131, + "psnr": 34.867304702965505, + "ssim": 0.9533889254212877, + "lpips": 0.230188250541687 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.5546875, + "history_latent_mse": 0.079752117395401, + "psnr": 35.94042802665755, + "ssim": 0.9604605083296149, + "lpips": 0.168034628033638 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 2672, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3828125, + "history_latent_mse": 0.34064212441444397, + "psnr": 16.663569615394472, + "ssim": 0.6241672982289495, + "lpips": 0.3368901312351227 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9296875, + "history_latent_mse": 1.384458065032959, + "psnr": 8.811951967188609, + "ssim": 0.18888183094606517, + "lpips": 0.7000510692596436 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 1.0975481271743774, + "psnr": 12.15373035377345, + "ssim": 0.24137120067775886, + "lpips": 0.6109684705734253 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.845703125, + "history_latent_mse": 0.10271593183279037, + "psnr": 34.530815974779735, + "ssim": 0.9501715940077164, + "lpips": 0.24294838309288025 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.765625, + "history_latent_mse": 0.13484476506710052, + "psnr": 35.876780672593355, + "ssim": 0.957471398816784, + "lpips": 0.17721398174762726 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 2680, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.90625, + "history_latent_mse": 0.3561190366744995, + "psnr": 16.684784114129513, + "ssim": 0.6306784087934554, + "lpips": 0.35553526878356934 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.03125, + "history_latent_mse": 1.3755773305892944, + "psnr": 8.054303074347155, + "ssim": 0.16430248536919334, + "lpips": 0.7394726276397705 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 1.061068058013916, + "psnr": 11.046967362360014, + "ssim": 0.22449183984726706, + "lpips": 0.6940229535102844 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.892578125, + "history_latent_mse": 0.10966425389051437, + "psnr": 34.07966210811851, + "ssim": 0.9466016925846936, + "lpips": 0.2573740482330322 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.1484375, + "history_latent_mse": 0.1359272450208664, + "psnr": 36.767371455100964, + "ssim": 0.9582651810530349, + "lpips": 0.19474193453788757 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 2688, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8203125, + "history_latent_mse": 0.33579695224761963, + "psnr": 16.621437447278232, + "ssim": 0.6254457710508501, + "lpips": 0.3688615560531616 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.765625, + "history_latent_mse": 1.5805974006652832, + "psnr": 8.256314438561288, + "ssim": 0.195002381675719, + "lpips": 0.7547802329063416 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.57421875, + "history_latent_mse": 1.1915475130081177, + "psnr": 10.287032931084381, + "ssim": 0.18327342522379264, + "lpips": 0.6871622800827026 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.86962890625, + "history_latent_mse": 0.11436286568641663, + "psnr": 33.819331877927844, + "ssim": 0.9434306080679358, + "lpips": 0.27428892254829407 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.36328125, + "history_latent_mse": 0.1414056271314621, + "psnr": 33.88551138492296, + "ssim": 0.9461849065059094, + "lpips": 0.2807251811027527 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 2696, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.328125, + "history_latent_mse": 0.37477412819862366, + "psnr": 16.23899544821048, + "ssim": 0.6225962414283619, + "lpips": 0.37277951836586 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9609375, + "history_latent_mse": 1.351096510887146, + "psnr": 9.647018817138559, + "ssim": 0.2592694553633376, + "lpips": 0.7417600154876709 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.796875, + "history_latent_mse": 1.2963247299194336, + "psnr": 9.076900625682342, + "ssim": 0.19557995003314144, + "lpips": 0.6635410189628601 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.798828125, + "history_latent_mse": 0.12003646045923233, + "psnr": 33.48648257527331, + "ssim": 0.9411911736631288, + "lpips": 0.2868015766143799 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.9765625, + "history_latent_mse": 0.21027012169361115, + "psnr": 36.73372945117108, + "ssim": 0.9546098113056423, + "lpips": 0.21039295196533203 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 2704, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.4453125, + "history_latent_mse": 0.39689913392066956, + "psnr": 15.864726007356378, + "ssim": 0.5792514153645281, + "lpips": 0.37460142374038696 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.46875, + "history_latent_mse": 1.260903000831604, + "psnr": 11.271858707517946, + "ssim": 0.24830740938935164, + "lpips": 0.7026363611221313 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.09375, + "history_latent_mse": 1.1504552364349365, + "psnr": 9.276851296956032, + "ssim": 0.2744532632741905, + "lpips": 0.7161211967468262 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3515625, + "history_latent_mse": 0.1408364176750183, + "psnr": 33.89313188868849, + "ssim": 0.9429844506805167, + "lpips": 0.28668516874313354 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.57421875, + "history_latent_mse": 0.1607610583305359, + "psnr": 36.374560456918644, + "ssim": 0.9519853444771056, + "lpips": 0.20218856632709503 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 2712, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1796875, + "history_latent_mse": 0.4117502272129059, + "psnr": 15.921618564221482, + "ssim": 0.5727381165508021, + "lpips": 0.3835420608520508 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.3125, + "history_latent_mse": 1.1318460702896118, + "psnr": 11.181688953291422, + "ssim": 0.2328037139364576, + "lpips": 0.7135410904884338 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.3203125, + "history_latent_mse": 1.203708529472351, + "psnr": 10.088051017735856, + "ssim": 0.17728977331456708, + "lpips": 0.6910430192947388 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.109375, + "history_latent_mse": 0.14456360042095184, + "psnr": 34.00514072968324, + "ssim": 0.9419275898337744, + "lpips": 0.2840314209461212 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.1015625, + "history_latent_mse": 0.11299251765012741, + "psnr": 35.79170432103377, + "ssim": 0.9494982427214975, + "lpips": 0.21527722477912903 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e36179b46b6b75abe564fc3038da0ae016fec780 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e8702c0320d72212d791db11c0119940b53e9604e06cbc8f3281996bb9c4ae +size 1046877 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..8b6566355e88774b50cc7369ab1d424d4fb78726 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.08749524503946304, + "wrist/video/latent_future_mse": 0.08749524503946304, + "video/psnr": 28.321348685265328, + "wrist/video/psnr": 28.321348685265328, + "video/ssim": 0.8925943476889411, + "wrist/video/ssim": 0.8925943476889411, + "video/lpips": 0.03279535472393036, + "wrist/video/lpips": 0.03279535472393036, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.9110854268074036, + "wrist/wrist_left/latent_future_mse": 0.9110854268074036, + "wrist_left/psnr": 13.091017709087751, + "wrist/wrist_left/psnr": 13.091017709087751, + "wrist_left/ssim": 0.39492816861257507, + "wrist/wrist_left/ssim": 0.39492816861257507, + "wrist_left/lpips": 0.36116477847099304, + "wrist/wrist_left/lpips": 0.36116477847099304, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.3618867099285126, + "wrist/wrist_right/latent_future_mse": 0.3618867099285126, + "wrist_right/psnr": 24.853965446816723, + "wrist/wrist_right/psnr": 24.853965446816723, + "wrist_right/ssim": 0.8352598886662768, + "wrist/wrist_right/ssim": 0.8352598886662768, + "wrist_right/lpips": 0.05153008550405502, + "wrist/wrist_right/lpips": 0.05153008550405502, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.06186702847480774, + "wrist/tactile_left/latent_future_mse": 0.06186702847480774, + "tactile_left/psnr": 41.18403310522096, + "wrist/tactile_left/psnr": 41.18403310522096, + "tactile_left/ssim": 0.9832525334876951, + "wrist/tactile_left/ssim": 0.9832525334876951, + "tactile_left/lpips": 0.02327561192214489, + "wrist/tactile_left/lpips": 0.02327561192214489, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.006720454897731543, + "wrist/tactile_right/latent_future_mse": 0.006720454897731543, + "tactile_right/psnr": 49.90268913328755, + "wrist/tactile_right/psnr": 49.90268913328755, + "tactile_right/ssim": 0.9945657508029224, + "wrist/tactile_right/ssim": 0.9945657508029224, + "tactile_right/lpips": 0.00416005402803421, + "wrist/tactile_right/lpips": 0.00416005402803421, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 2600, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08749524503946304, + "psnr": 28.321348685265328, + "ssim": 0.8925943476889411, + "lpips": 0.03279535472393036 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.9110854268074036, + "psnr": 13.091017709087751, + "ssim": 0.39492816861257507, + "lpips": 0.36116477847099304 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.3618867099285126, + "psnr": 24.853965446816723, + "ssim": 0.8352598886662768, + "lpips": 0.05153008550405502 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.06186702847480774, + "psnr": 41.18403310522096, + "ssim": 0.9832525334876951, + "lpips": 0.02327561192214489 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.006720454897731543, + "psnr": 49.90268913328755, + "ssim": 0.9945657508029224, + "lpips": 0.00416005402803421 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/case.json new file mode 100644 index 0000000000000000000000000000000000000000..0bc428524175107ed76d6b07dbc083a56276ed6a --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/case.json @@ -0,0 +1,168 @@ +{ + "task": 9, + "episode": "motherboard_0909_episode_002.pt", + "category": "onset", + "index": 8502, + "t_start": 1924, + "start_seconds": 128.26666666666668, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6796875, + "long_future_contact_fraction": 0.725, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "09_motherboard_0909_episode_002_onset", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0bf02ef56287c34d911ec7ea9c764ebea35cd4e2 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f102a3702919a87515cd4783e3dc22b40e071ef17a8a1b7ca4748685bb9b7104 +size 9322604 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/motherboard_0909_episode_002_t01924_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/motherboard_0909_episode_002_t01924_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..deb97e4af716876118c3d5739daf2c0ee34c7965 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/motherboard_0909_episode_002_t01924_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebfe87703dd3565a486bce9427b1413e3f14d28c7dd7480a2b43dd4dcc95d50 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..95d86a1344aa0ba0e45628223a2f18af82475001 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1924, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 1924, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 28.494541010181763, + "ssim": 0.9001987403420323, + "lpips": 0.03234739601612091 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.719734189252797, + "ssim": 0.39231427728512597, + "lpips": 0.4066063165664673 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 19.574960276103393, + "ssim": 0.7210154792460803, + "lpips": 0.12814699113368988 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 34.35383595629083, + "ssim": 0.9602559699144639, + "lpips": 0.1290508508682251 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.977118380330225, + "ssim": 0.9926154016976252, + "lpips": 0.010815073736011982 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 1932, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.0625, + "history_latent_mse": 0.06520099192857742, + "psnr": 23.962714844254002, + "ssim": 0.7955335001621708, + "lpips": 0.07955881208181381 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.73046875, + "history_latent_mse": 0.9224420189857483, + "psnr": 13.11207931438436, + "ssim": 0.31669028506194224, + "lpips": 0.46471405029296875 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.7421875, + "history_latent_mse": 0.3062453866004944, + "psnr": 16.549868698848517, + "ssim": 0.6050576354589219, + "lpips": 0.2253406047821045 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.03125, + "history_latent_mse": 0.1582791954278946, + "psnr": 29.25471035808464, + "ssim": 0.9123434732290632, + "lpips": 0.2983797788619995 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.9140625, + "history_latent_mse": 0.018215050920844078, + "psnr": 45.18957114905576, + "ssim": 0.9862938676135624, + "lpips": 0.0343681275844574 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 1940, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.14325185120105743, + "psnr": 22.007912939260272, + "ssim": 0.7510801322104907, + "lpips": 0.13696688413619995 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.143310546875, + "history_latent_mse": 0.9529007077217102, + "psnr": 12.550407116256853, + "ssim": 0.31079316608887714, + "lpips": 0.5635617971420288 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.4453125, + "history_latent_mse": 0.45667243003845215, + "psnr": 15.07651766997017, + "ssim": 0.530489776652124, + "lpips": 0.31246334314346313 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.4765625, + "history_latent_mse": 0.20291654765605927, + "psnr": 28.690421475263285, + "ssim": 0.9209439012702334, + "lpips": 0.3097030222415924 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4140625, + "history_latent_mse": 0.031021064147353172, + "psnr": 43.96852113024764, + "ssim": 0.9816996548485378, + "lpips": 0.05633924901485443 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 1948, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6640625, + "history_latent_mse": 0.1608167141675949, + "psnr": 21.522421300757234, + "ssim": 0.7454080639731002, + "lpips": 0.16455137729644775 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.3671875, + "history_latent_mse": 1.0201411247253418, + "psnr": 10.606711979168896, + "ssim": 0.24972901758778723, + "lpips": 0.6569910049438477 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.650390625, + "history_latent_mse": 0.5518054366111755, + "psnr": 14.4885184375101, + "ssim": 0.5008093578507609, + "lpips": 0.3659552335739136 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.421875, + "history_latent_mse": 0.27974823117256165, + "psnr": 29.903851291108623, + "ssim": 0.9039081659081674, + "lpips": 0.36039865016937256 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.2578125, + "history_latent_mse": 0.03979746624827385, + "psnr": 42.86087280475893, + "ssim": 0.9786900442223285, + "lpips": 0.07667769491672516 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 1956, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.41796875, + "history_latent_mse": 0.18667316436767578, + "psnr": 21.06825143904291, + "ssim": 0.730761179065969, + "lpips": 0.1948888897895813 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.34375, + "history_latent_mse": 1.1364026069641113, + "psnr": 10.918035488498703, + "ssim": 0.24905196239439087, + "lpips": 0.6491527557373047 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.15625, + "history_latent_mse": 0.5982366800308228, + "psnr": 13.276005240833681, + "ssim": 0.4364785162215612, + "lpips": 0.3999193608760834 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.75, + "history_latent_mse": 0.2444906085729599, + "psnr": 30.710338048930662, + "ssim": 0.9178403050311218, + "lpips": 0.36276960372924805 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.28515625, + "history_latent_mse": 0.04743664339184761, + "psnr": 41.414489092750905, + "ssim": 0.9750807832475844, + "lpips": 0.10054214298725128 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 1964, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.375, + "history_latent_mse": 0.1877610832452774, + "psnr": 21.37905688942452, + "ssim": 0.7361981815909024, + "lpips": 0.1788175106048584 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.34375, + "history_latent_mse": 1.1143640279769897, + "psnr": 10.566867273257259, + "ssim": 0.22394047927233213, + "lpips": 0.6912099123001099 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9609375, + "history_latent_mse": 0.7550022602081299, + "psnr": 12.033795558349423, + "ssim": 0.3766393907826975, + "lpips": 0.4243532121181488 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.80078125, + "history_latent_mse": 0.2573679983615875, + "psnr": 30.516190868078503, + "ssim": 0.9227086366406068, + "lpips": 0.32929158210754395 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.31396484375, + "history_latent_mse": 0.054774705320596695, + "psnr": 40.55559135891868, + "ssim": 0.9707607769429034, + "lpips": 0.12342441082000732 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 1972, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.5625, + "history_latent_mse": 0.18440251052379608, + "psnr": 21.448322140419762, + "ssim": 0.7382690503837144, + "lpips": 0.1823701411485672 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.875, + "history_latent_mse": 1.2171566486358643, + "psnr": 10.980842194600859, + "ssim": 0.22798951970539083, + "lpips": 0.6890844702720642 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5703125, + "history_latent_mse": 0.7618011832237244, + "psnr": 11.933428672112338, + "ssim": 0.3656485308585671, + "lpips": 0.43608272075653076 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.89453125, + "history_latent_mse": 0.21533836424350739, + "psnr": 32.67891514097806, + "ssim": 0.9342791349245015, + "lpips": 0.2992141544818878 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7421875, + "history_latent_mse": 0.07232380658388138, + "psnr": 39.9556330312211, + "ssim": 0.9665107315373773, + "lpips": 0.1405923068523407 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1980, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.28125, + "history_latent_mse": 0.1935638189315796, + "psnr": 21.534857150350057, + "ssim": 0.7337740793651191, + "lpips": 0.18501484394073486 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5703125, + "history_latent_mse": 1.136147379875183, + "psnr": 9.385626633760824, + "ssim": 0.251585685163361, + "lpips": 0.6597579717636108 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.234375, + "history_latent_mse": 0.7846826910972595, + "psnr": 11.657608313385378, + "ssim": 0.35648983168469894, + "lpips": 0.44906455278396606 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3984375, + "history_latent_mse": 0.1672704964876175, + "psnr": 35.14580685908464, + "ssim": 0.9541458467079559, + "lpips": 0.23374763131141663 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.490234375, + "history_latent_mse": 0.0723375678062439, + "psnr": 39.13720256579195, + "ssim": 0.9633387434686357, + "lpips": 0.15701869130134583 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1988, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.21875, + "history_latent_mse": 0.20017315447330475, + "psnr": 21.225795799481567, + "ssim": 0.7178684592525852, + "lpips": 0.19729235768318176 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.234375, + "history_latent_mse": 1.0399466753005981, + "psnr": 10.19977398914601, + "ssim": 0.2568572261000118, + "lpips": 0.6993142366409302 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.0078125, + "history_latent_mse": 0.7483503818511963, + "psnr": 11.274434308756307, + "ssim": 0.33643291934566805, + "lpips": 0.481344074010849 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.904296875, + "history_latent_mse": 0.09804151207208633, + "psnr": 30.064202907460835, + "ssim": 0.9294466595947244, + "lpips": 0.3304336965084076 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.783447265625, + "history_latent_mse": 0.07933502644300461, + "psnr": 38.36410765935703, + "ssim": 0.9617357605981217, + "lpips": 0.16658277809619904 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1996, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7783203125, + "history_latent_mse": 0.19483067095279694, + "psnr": 20.796022332181117, + "ssim": 0.6970291095706409, + "lpips": 0.21846133470535278 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.59375, + "history_latent_mse": 1.2884445190429688, + "psnr": 8.5345732240311, + "ssim": 0.21812167249144476, + "lpips": 0.6573114395141602 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4609375, + "history_latent_mse": 0.8637526631355286, + "psnr": 11.588394907596989, + "ssim": 0.31874253038654854, + "lpips": 0.504014253616333 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.640625, + "history_latent_mse": 0.22836430370807648, + "psnr": 35.0312364006038, + "ssim": 0.9559114866201077, + "lpips": 0.22564885020256042 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.91015625, + "history_latent_mse": 0.08338288217782974, + "psnr": 38.22339231314945, + "ssim": 0.9604909657537561, + "lpips": 0.17869964241981506 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 2004, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3515625, + "history_latent_mse": 0.24682515859603882, + "psnr": 20.68731671356072, + "ssim": 0.6880218784464165, + "lpips": 0.22452279925346375 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 1.1261769533157349, + "psnr": 9.777049921622824, + "ssim": 0.20396026410943335, + "lpips": 0.6839240789413452 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.125, + "history_latent_mse": 0.8158014416694641, + "psnr": 11.56899181983272, + "ssim": 0.3151796872871819, + "lpips": 0.5092281103134155 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.828125, + "history_latent_mse": 0.10102817416191101, + "psnr": 32.076169922046354, + "ssim": 0.9284150223207893, + "lpips": 0.3021513819694519 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.875, + "history_latent_mse": 0.0885261595249176, + "psnr": 37.64130693404797, + "ssim": 0.9577040628330352, + "lpips": 0.19019970297813416 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 2012, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 0.22780950367450714, + "psnr": 20.613084215338773, + "ssim": 0.6772894945048399, + "lpips": 0.21608597040176392 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.2421875, + "history_latent_mse": 1.3088301420211792, + "psnr": 9.6812342875172, + "ssim": 0.21068451317170456, + "lpips": 0.6885558366775513 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.703125, + "history_latent_mse": 0.8222996592521667, + "psnr": 11.429433970296365, + "ssim": 0.3199808251895996, + "lpips": 0.519562840461731 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.4140625, + "history_latent_mse": 0.18147200345993042, + "psnr": 33.123136442536094, + "ssim": 0.9371801318928683, + "lpips": 0.2971862256526947 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.97607421875, + "history_latent_mse": 0.09453967958688736, + "psnr": 37.20866620117477, + "ssim": 0.9543945016076231, + "lpips": 0.200607031583786 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 2020, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.5703125, + "history_latent_mse": 0.2750646471977234, + "psnr": 18.990116484982877, + "ssim": 0.6493413329170643, + "lpips": 0.25676825642585754 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0859375, + "history_latent_mse": 1.3223687410354614, + "psnr": 9.587705824669097, + "ssim": 0.18726753872755003, + "lpips": 0.7437233924865723 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.21875, + "history_latent_mse": 0.9825031161308289, + "psnr": 13.301059756459503, + "ssim": 0.3883079032053941, + "lpips": 0.4515150785446167 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.9140625, + "history_latent_mse": 0.16300739347934723, + "psnr": 32.84466091104288, + "ssim": 0.9356234975965597, + "lpips": 0.30726495385169983 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.9296875, + "history_latent_mse": 0.10136353224515915, + "psnr": 36.916005775350065, + "ssim": 0.9510610566425788, + "lpips": 0.21472862362861633 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 2028, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8828125, + "history_latent_mse": 0.34441712498664856, + "psnr": 19.23638701356055, + "ssim": 0.6630829797752018, + "lpips": 0.2497263103723526 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.796875, + "history_latent_mse": 1.4184972047805786, + "psnr": 8.66237412689767, + "ssim": 0.19531306004221402, + "lpips": 0.7483354806900024 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.7814427018165588, + "psnr": 13.167783760965111, + "ssim": 0.3732238680224133, + "lpips": 0.4610782861709595 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.171875, + "history_latent_mse": 0.1639244705438614, + "psnr": 33.81895884774033, + "ssim": 0.9452764275433627, + "lpips": 0.26778459548950195 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.876953125, + "history_latent_mse": 0.10765393823385239, + "psnr": 36.53016635271656, + "ssim": 0.9481246723582624, + "lpips": 0.2317875325679779 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 2036, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6328125, + "history_latent_mse": 0.26435256004333496, + "psnr": 19.115425274813276, + "ssim": 0.6711596677243196, + "lpips": 0.25779545307159424 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 1.3323416709899902, + "psnr": 8.952848489627005, + "ssim": 0.24056197890778758, + "lpips": 0.709993302822113 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.3515625, + "history_latent_mse": 0.7246000170707703, + "psnr": 12.913372381177746, + "ssim": 0.29606079760711046, + "lpips": 0.5317023992538452 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.0, + "history_latent_mse": 0.1420317143201828, + "psnr": 33.528295301648484, + "ssim": 0.943721259135271, + "lpips": 0.2747670114040375 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.9296875, + "history_latent_mse": 0.11418911069631577, + "psnr": 32.94200159856021, + "ssim": 0.9351752249215898, + "lpips": 0.28808432817459106 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c4b95140bbdc557dec2dbae8c4cbe3b3066300d --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7710366802a4177d54ae02fd15080e022a1cccc2ee65a9f963aaae1bb605f813 +size 1131834 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..73af7f7cec8ad5204a4a0f48ce13098bc8574931 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.08656862378120422, + "wrist/video/latent_future_mse": 0.08656862378120422, + "video/psnr": 28.494541010181763, + "wrist/video/psnr": 28.494541010181763, + "video/ssim": 0.9001987403420323, + "wrist/video/ssim": 0.9001987403420323, + "video/lpips": 0.03234739601612091, + "wrist/video/lpips": 0.03234739601612091, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 1.04190993309021, + "wrist/wrist_left/latent_future_mse": 1.04190993309021, + "wrist_left/psnr": 13.719734189252797, + "wrist/wrist_left/psnr": 13.719734189252797, + "wrist_left/ssim": 0.39231427728512597, + "wrist/wrist_left/ssim": 0.39231427728512597, + "wrist_left/lpips": 0.4066063165664673, + "wrist/wrist_left/lpips": 0.4066063165664673, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.34950700402259827, + "wrist/wrist_right/latent_future_mse": 0.34950700402259827, + "wrist_right/psnr": 19.574960276103393, + "wrist/wrist_right/psnr": 19.574960276103393, + "wrist_right/ssim": 0.7210154792460803, + "wrist/wrist_right/ssim": 0.7210154792460803, + "wrist_right/lpips": 0.12814699113368988, + "wrist/wrist_right/lpips": 0.12814699113368988, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.1699931025505066, + "wrist/tactile_left/latent_future_mse": 0.1699931025505066, + "tactile_left/psnr": 34.35383595629083, + "wrist/tactile_left/psnr": 34.35383595629083, + "tactile_left/ssim": 0.9602559699144639, + "wrist/tactile_left/ssim": 0.9602559699144639, + "tactile_left/lpips": 0.1290508508682251, + "wrist/tactile_left/lpips": 0.1290508508682251, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.016070818528532982, + "wrist/tactile_right/latent_future_mse": 0.016070818528532982, + "tactile_right/psnr": 47.977118380330225, + "wrist/tactile_right/psnr": 47.977118380330225, + "tactile_right/ssim": 0.9926154016976252, + "wrist/tactile_right/ssim": 0.9926154016976252, + "tactile_right/lpips": 0.010815073736011982, + "wrist/tactile_right/lpips": 0.010815073736011982, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1924, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08656862378120422, + "psnr": 28.494541010181763, + "ssim": 0.9001987403420323, + "lpips": 0.03234739601612091 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 1.04190993309021, + "psnr": 13.719734189252797, + "ssim": 0.39231427728512597, + "lpips": 0.4066063165664673 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.34950700402259827, + "psnr": 19.574960276103393, + "ssim": 0.7210154792460803, + "lpips": 0.12814699113368988 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.1699931025505066, + "psnr": 34.35383595629083, + "ssim": 0.9602559699144639, + "lpips": 0.1290508508682251 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.016070818528532982, + "psnr": 47.977118380330225, + "ssim": 0.9926154016976252, + "lpips": 0.010815073736011982 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/case.json new file mode 100644 index 0000000000000000000000000000000000000000..e98aed03c522ad513c5dc32108ffa285cb62a3f8 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/case.json @@ -0,0 +1,168 @@ +{ + "task": 10, + "episode": "motherboard_0909_episode_002.pt", + "category": "release", + "index": 7670, + "t_start": 1051, + "start_seconds": 70.06666666666666, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.703125, + "long_future_contact_fraction": 0.6833333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "10_motherboard_0909_episode_002_release", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b021d4fe3fa0a3d19e9afb72c5c1ba32ecf299a --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3c4e82e9158ce3dea7b9859b765b1625ec602bee30dc91028ea8e8796f3580 +size 9288729 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/motherboard_0909_episode_002_t01051_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/motherboard_0909_episode_002_t01051_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..975a08bff6949acf1afe967d519173824068990c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/motherboard_0909_episode_002_t01051_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9cb70b84ba2c9cdc1160f3e249220d547f0c2b58de6ea2c55564e3687469344 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..532b9238371c3600d173e95ab80f096f4d4c802f --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1051, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 1051, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 21.70579439364692, + "ssim": 0.7487737666248115, + "lpips": 0.10364481061697006 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.917306289905477, + "ssim": 0.5433917112992307, + "lpips": 0.35166752338409424 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.314255328985132, + "ssim": 0.37254428174028753, + "lpips": 0.4034028649330139 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.43455620722216, + "ssim": 0.9914911279915801, + "lpips": 0.010060389526188374 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 36.012908386890686, + "ssim": 0.9796672683912191, + "lpips": 0.052117638289928436 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 1059, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8828125, + "history_latent_mse": 0.19145523011684418, + "psnr": 20.572965858812648, + "ssim": 0.710970001159997, + "lpips": 0.138225257396698 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.638671875, + "history_latent_mse": 0.6423529982566833, + "psnr": 13.516367860837518, + "ssim": 0.5525543557787381, + "lpips": 0.3600078225135803 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.5, + "history_latent_mse": 0.7439696192741394, + "psnr": 12.87117840521136, + "ssim": 0.3367273544819314, + "lpips": 0.5313436388969421 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.9921875, + "history_latent_mse": 0.0201504398137331, + "psnr": 44.50269361192944, + "ssim": 0.9846149984730617, + "lpips": 0.03763904049992561 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.03125, + "history_latent_mse": 0.04826243594288826, + "psnr": 30.303925800461357, + "ssim": 0.937742394313432, + "lpips": 0.2274796962738037 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 1067, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.109375, + "history_latent_mse": 0.1893804669380188, + "psnr": 19.73173062672208, + "ssim": 0.6978167933239217, + "lpips": 0.16436676681041718 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.890625, + "history_latent_mse": 0.5532927513122559, + "psnr": 15.73658988659312, + "ssim": 0.6174121886373041, + "lpips": 0.3321990370750427 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.546875, + "history_latent_mse": 0.9186837077140808, + "psnr": 12.148963803249115, + "ssim": 0.28423534811685625, + "lpips": 0.5839634537696838 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.130859375, + "history_latent_mse": 0.03373406454920769, + "psnr": 43.145439407607704, + "ssim": 0.9803783078941283, + "lpips": 0.06457595527172089 + }, + "tactile_right": { + "history_latent_max_abs_error": 4.3828125, + "history_latent_mse": 0.22072608768939972, + "psnr": 29.914465995315, + "ssim": 0.9306001888299158, + "lpips": 0.25166040658950806 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 1075, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3046875, + "history_latent_mse": 0.22361153364181519, + "psnr": 19.374525783289418, + "ssim": 0.6851771163650544, + "lpips": 0.17686377465724945 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.6484375, + "history_latent_mse": 0.5482866764068604, + "psnr": 14.40580781829546, + "ssim": 0.5981672183039262, + "lpips": 0.417755126953125 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4453125, + "history_latent_mse": 0.959981381893158, + "psnr": 11.946151598533774, + "ssim": 0.29195145923274685, + "lpips": 0.625076413154602 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.1953125, + "history_latent_mse": 0.04408858343958855, + "psnr": 41.654705340817756, + "ssim": 0.9758497228080242, + "lpips": 0.1073160320520401 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7265625, + "history_latent_mse": 0.20004482567310333, + "psnr": 33.71182250280812, + "ssim": 0.9599501022043431, + "lpips": 0.1516217291355133 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 1083, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.62109375, + "history_latent_mse": 0.2404736876487732, + "psnr": 19.475909006449978, + "ssim": 0.6823104580030748, + "lpips": 0.17493289709091187 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1171875, + "history_latent_mse": 0.6291408538818359, + "psnr": 14.15158631857005, + "ssim": 0.5675767759606053, + "lpips": 0.4301455318927765 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.375, + "history_latent_mse": 1.077132225036621, + "psnr": 10.9288623603438, + "ssim": 0.22423328286897456, + "lpips": 0.6920446157455444 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.25390625, + "history_latent_mse": 0.056828904896974564, + "psnr": 40.2010693527814, + "ssim": 0.9716287300700102, + "lpips": 0.14263105392456055 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.92138671875, + "history_latent_mse": 0.15522928535938263, + "psnr": 28.187288412448925, + "ssim": 0.934892255494917, + "lpips": 0.25090524554252625 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 1091, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.11328125, + "history_latent_mse": 0.24178510904312134, + "psnr": 19.209573452520676, + "ssim": 0.6769324211521658, + "lpips": 0.1960592120885849 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9453125, + "history_latent_mse": 0.6872124671936035, + "psnr": 14.445496554230253, + "ssim": 0.5882989433316614, + "lpips": 0.4050613343715668 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.765625, + "history_latent_mse": 1.0994254350662231, + "psnr": 10.980361695140832, + "ssim": 0.2196325108875354, + "lpips": 0.701492190361023 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.462890625, + "history_latent_mse": 0.0622323714196682, + "psnr": 39.62354092623315, + "ssim": 0.9680320534859208, + "lpips": 0.16904953122138977 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.98828125, + "history_latent_mse": 0.21067774295806885, + "psnr": 30.012834834987384, + "ssim": 0.9449673722399576, + "lpips": 0.2240869402885437 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 1099, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.939453125, + "history_latent_mse": 0.2516324818134308, + "psnr": 19.071649646988128, + "ssim": 0.6703239225414992, + "lpips": 0.20538562536239624 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.5390625, + "history_latent_mse": 0.6985146999359131, + "psnr": 14.622519949143385, + "ssim": 0.5739531280498738, + "lpips": 0.42581769824028015 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.328125, + "history_latent_mse": 1.036558985710144, + "psnr": 10.553404090866286, + "ssim": 0.23859935969085383, + "lpips": 0.5888689756393433 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.603515625, + "history_latent_mse": 0.07063637673854828, + "psnr": 38.56938063331064, + "ssim": 0.9645202875428234, + "lpips": 0.19006922841072083 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0703125, + "history_latent_mse": 0.194483682513237, + "psnr": 37.23709835515632, + "ssim": 0.963597694697253, + "lpips": 0.1264275163412094 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1107, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.021484375, + "history_latent_mse": 0.26287326216697693, + "psnr": 18.982588465613897, + "ssim": 0.6634089748307322, + "lpips": 0.19958168268203735 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.661600649356842, + "psnr": 13.960439155694505, + "ssim": 0.5586244280798092, + "lpips": 0.4401297867298126 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.59375, + "history_latent_mse": 1.0009546279907227, + "psnr": 11.76144062459371, + "ssim": 0.22865287520523342, + "lpips": 0.6127713322639465 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.578125, + "history_latent_mse": 0.07832357287406921, + "psnr": 37.890044979282834, + "ssim": 0.960876534073699, + "lpips": 0.20986396074295044 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.642578125, + "history_latent_mse": 0.0916355550289154, + "psnr": 29.202562223496162, + "ssim": 0.9325116218164355, + "lpips": 0.24368831515312195 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1115, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.0703125, + "history_latent_mse": 0.2985070049762726, + "psnr": 18.87063162129064, + "ssim": 0.664236822840331, + "lpips": 0.2084948867559433 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.0703125, + "history_latent_mse": 0.6914920210838318, + "psnr": 13.684291218434876, + "ssim": 0.5511398357238233, + "lpips": 0.42790305614471436 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.609375, + "history_latent_mse": 1.038418173789978, + "psnr": 11.582783219065377, + "ssim": 0.2525654462014324, + "lpips": 0.6294841766357422 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.583984375, + "history_latent_mse": 0.08475509285926819, + "psnr": 37.01388024426811, + "ssim": 0.9573513226899314, + "lpips": 0.23101270198822021 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.4453125, + "history_latent_mse": 0.2567020356655121, + "psnr": 33.99360554178683, + "ssim": 0.9551734319336236, + "lpips": 0.1837346851825714 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1123, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.203125, + "history_latent_mse": 0.2939448654651642, + "psnr": 18.17715412128393, + "ssim": 0.6510999195775277, + "lpips": 0.24761679768562317 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.859375, + "history_latent_mse": 0.7940015196800232, + "psnr": 13.909742194821629, + "ssim": 0.5414315159294236, + "lpips": 0.41529518365859985 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.171875, + "history_latent_mse": 1.0120949745178223, + "psnr": 11.004080226784389, + "ssim": 0.19297985969515907, + "lpips": 0.6677682399749756 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.640625, + "history_latent_mse": 0.09185979515314102, + "psnr": 36.52787650534421, + "ssim": 0.9541311011975695, + "lpips": 0.24924236536026 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.03125, + "history_latent_mse": 0.1408538818359375, + "psnr": 27.38385031609992, + "ssim": 0.9199516030655873, + "lpips": 0.3315187394618988 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 1131, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 6.609375, + "history_latent_mse": 0.33626723289489746, + "psnr": 17.971667748756243, + "ssim": 0.6480372524538058, + "lpips": 0.2607712149620056 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5, + "history_latent_mse": 0.7673904299736023, + "psnr": 13.733720293853844, + "ssim": 0.5259447535693419, + "lpips": 0.4164723753929138 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.9609375, + "history_latent_mse": 1.1325275897979736, + "psnr": 10.921450632268485, + "ssim": 0.19377471510416372, + "lpips": 0.671328067779541 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.76953125, + "history_latent_mse": 0.09825915843248367, + "psnr": 35.877050659005796, + "ssim": 0.9503052848534203, + "lpips": 0.2662605047225952 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.7578125, + "history_latent_mse": 0.32508307695388794, + "psnr": 29.788843006738034, + "ssim": 0.9407832789743318, + "lpips": 0.24086053669452667 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1139, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.59375, + "history_latent_mse": 0.29531893134117126, + "psnr": 17.77112384425608, + "ssim": 0.6499401568888821, + "lpips": 0.26503533124923706 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.578125, + "history_latent_mse": 0.7237811088562012, + "psnr": 13.401690261033306, + "ssim": 0.4921055013412148, + "lpips": 0.4435722231864929 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.21875, + "history_latent_mse": 1.1076585054397583, + "psnr": 10.126125775408607, + "ssim": 0.16500421965220174, + "lpips": 0.7014805674552917 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.802734375, + "history_latent_mse": 0.10527119785547256, + "psnr": 35.280069151859365, + "ssim": 0.9466248703981919, + "lpips": 0.28250986337661743 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0, + "history_latent_mse": 0.20544449985027313, + "psnr": 28.076028289688303, + "ssim": 0.9296757592003483, + "lpips": 0.28135526180267334 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1147, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 0.3254244923591614, + "psnr": 16.808664499155658, + "ssim": 0.6269425248241043, + "lpips": 0.3469650149345398 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.4140625, + "history_latent_mse": 0.8004812598228455, + "psnr": 13.26579252525252, + "ssim": 0.48157496756453916, + "lpips": 0.4702306389808655 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.09375, + "history_latent_mse": 1.2286310195922852, + "psnr": 9.979600377413261, + "ssim": 0.20980980649133824, + "lpips": 0.6633951663970947 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.80078125, + "history_latent_mse": 0.1117587685585022, + "psnr": 34.9679222100804, + "ssim": 0.9441989627708849, + "lpips": 0.29147660732269287 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.3359375, + "history_latent_mse": 0.2052658349275589, + "psnr": 31.823824717653455, + "ssim": 0.9447659806502587, + "lpips": 0.20875215530395508 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1155, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.6484375, + "history_latent_mse": 0.3552737534046173, + "psnr": 16.388355078163702, + "ssim": 0.6194681701792746, + "lpips": 0.35402369499206543 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.953125, + "history_latent_mse": 0.8575640320777893, + "psnr": 14.10659877999648, + "ssim": 0.5291990688029247, + "lpips": 0.42841964960098267 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.1640625, + "history_latent_mse": 1.1044225692749023, + "psnr": 10.597968359631658, + "ssim": 0.14089922213148623, + "lpips": 0.632637619972229 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.69921875, + "history_latent_mse": 0.11596766114234924, + "psnr": 34.65374926697558, + "ssim": 0.9408117761481378, + "lpips": 0.30404824018478394 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.984375, + "history_latent_mse": 0.1677505224943161, + "psnr": 32.79951979331682, + "ssim": 0.94448020870965, + "lpips": 0.20812612771987915 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1163, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 6.203125, + "history_latent_mse": 0.34517601132392883, + "psnr": 16.65622415363219, + "ssim": 0.6268693077859788, + "lpips": 0.3555598258972168 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.08203125, + "history_latent_mse": 0.6607910990715027, + "psnr": 13.429481880538866, + "ssim": 0.4835571203646227, + "lpips": 0.4265593886375427 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 1.1640411615371704, + "psnr": 9.996974705102184, + "ssim": 0.16085541408413406, + "lpips": 0.6675382852554321 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.947265625, + "history_latent_mse": 0.12347192317247391, + "psnr": 34.27660548306107, + "ssim": 0.9378913384228709, + "lpips": 0.31526148319244385 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.845703125, + "history_latent_mse": 0.17712484300136566, + "psnr": 29.40471329257469, + "ssim": 0.9227097631859089, + "lpips": 0.2877015769481659 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0012760eba407c54df65959435d95234129f0a67 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3064aac9408b05710f4e20bc9fc264cbd000e683940bde6c22a0ff6491348419 +size 1264307 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..6c45efc1b429c37c0380c9d46bb8d41f469a6f71 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.22004568576812744, + "wrist/video/latent_future_mse": 0.22004568576812744, + "video/psnr": 21.70579439364692, + "wrist/video/psnr": 21.70579439364692, + "video/ssim": 0.7487737666248115, + "wrist/video/ssim": 0.7487737666248115, + "video/lpips": 0.10364481061697006, + "wrist/video/lpips": 0.10364481061697006, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.7275433540344238, + "wrist/wrist_left/latent_future_mse": 0.7275433540344238, + "wrist_left/psnr": 13.917306289905477, + "wrist/wrist_left/psnr": 13.917306289905477, + "wrist_left/ssim": 0.5433917112992307, + "wrist/wrist_left/ssim": 0.5433917112992307, + "wrist_left/lpips": 0.35166752338409424, + "wrist/wrist_left/lpips": 0.35166752338409424, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.8210799694061279, + "wrist/wrist_right/latent_future_mse": 0.8210799694061279, + "wrist_right/psnr": 13.314255328985132, + "wrist/wrist_right/psnr": 13.314255328985132, + "wrist_right/ssim": 0.37254428174028753, + "wrist/wrist_right/ssim": 0.37254428174028753, + "wrist_right/lpips": 0.4034028649330139, + "wrist/wrist_right/lpips": 0.4034028649330139, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.019636327400803566, + "wrist/tactile_left/latent_future_mse": 0.019636327400803566, + "tactile_left/psnr": 47.43455620722216, + "wrist/tactile_left/psnr": 47.43455620722216, + "tactile_left/ssim": 0.9914911279915801, + "wrist/tactile_left/ssim": 0.9914911279915801, + "tactile_left/lpips": 0.010060389526188374, + "wrist/tactile_left/lpips": 0.010060389526188374, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.057538047432899475, + "wrist/tactile_right/latent_future_mse": 0.057538047432899475, + "tactile_right/psnr": 36.012908386890686, + "wrist/tactile_right/psnr": 36.012908386890686, + "tactile_right/ssim": 0.9796672683912191, + "wrist/tactile_right/ssim": 0.9796672683912191, + "tactile_right/lpips": 0.052117638289928436, + "wrist/tactile_right/lpips": 0.052117638289928436, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1051, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.22004568576812744, + "psnr": 21.70579439364692, + "ssim": 0.7487737666248115, + "lpips": 0.10364481061697006 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.7275433540344238, + "psnr": 13.917306289905477, + "ssim": 0.5433917112992307, + "lpips": 0.35166752338409424 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.8210799694061279, + "psnr": 13.314255328985132, + "ssim": 0.37254428174028753, + "lpips": 0.4034028649330139 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.019636327400803566, + "psnr": 47.43455620722216, + "ssim": 0.9914911279915801, + "lpips": 0.010060389526188374 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.057538047432899475, + "psnr": 36.012908386890686, + "ssim": 0.9796672683912191, + "lpips": 0.052117638289928436 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/case.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba8913f33f306e6460740e5ccb01413ef995eea --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/case.json @@ -0,0 +1,164 @@ +{ + "task": 11, + "episode": "motherboard_0909_episode_002.pt", + "category": "full", + "index": 8366, + "t_start": 1788, + "start_seconds": 119.2, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.9609375, + "long_future_contact_fraction": 0.9583333333333334, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "11_motherboard_0909_episode_002_full", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a697622d7690374751b679a0abccea86db6aa229 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec7a6202ab1cb7b1bb0f7bcc3d9bde9cef856fbb55315a2a99d51cd786874771 +size 9281196 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/motherboard_0909_episode_002_t01788_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/motherboard_0909_episode_002_t01788_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6a2e3b86b2a3e7576c8811baeb6d8e61cf8de1f --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/motherboard_0909_episode_002_t01788_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db27538dc183ca7c6fe9e0623b62c3d00690574bf6c02eb9d5e1ec221bfd8be5 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..a93fd55e2b53f1e0621e17049273f1ddff45acd8 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1788, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 1788, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 26.928243022387694, + "ssim": 0.881112507024052, + "lpips": 0.05308378487825394 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 12.208527476765113, + "ssim": 0.39174826497395737, + "lpips": 0.3744303584098816 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 22.595010078670054, + "ssim": 0.8586842305789523, + "lpips": 0.041494183242321014 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.512762453697384, + "ssim": 0.9928329123046913, + "lpips": 0.004961402155458927 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 45.991363331281576, + "ssim": 0.9881811009308914, + "lpips": 0.008251955732703209 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 1796, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.4765625, + "history_latent_mse": 0.0918087363243103, + "psnr": 25.53296551381355, + "ssim": 0.8361636991618869, + "lpips": 0.07748467475175858 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.3984375, + "history_latent_mse": 0.6965205073356628, + "psnr": 11.077813078166908, + "ssim": 0.23444340022056476, + "lpips": 0.5269680023193359 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.1539808064699173, + "psnr": 18.312944734980835, + "ssim": 0.6452345341329491, + "lpips": 0.12518271803855896 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.91015625, + "history_latent_mse": 0.014102048240602016, + "psnr": 31.578289026207997, + "ssim": 0.9389098976164713, + "lpips": 0.19002629816532135 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.21875, + "history_latent_mse": 0.022463515400886536, + "psnr": 33.62367768314635, + "ssim": 0.9603040979089217, + "lpips": 0.09362610429525375 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 1804, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.9296875, + "history_latent_mse": 0.10316038876771927, + "psnr": 23.11109863168658, + "ssim": 0.7473614438444791, + "lpips": 0.11294395476579666 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 0.9288347363471985, + "psnr": 10.920764312212006, + "ssim": 0.23272118643271855, + "lpips": 0.546252429485321 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.34375, + "history_latent_mse": 0.4133683741092682, + "psnr": 13.137852778819994, + "ssim": 0.40249982277185575, + "lpips": 0.37948375940322876 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.99609375, + "history_latent_mse": 0.14419682323932648, + "psnr": 30.350317000640345, + "ssim": 0.9169482796501762, + "lpips": 0.2532126009464264 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0546875, + "history_latent_mse": 0.10360180586576462, + "psnr": 31.03675456812948, + "ssim": 0.9441321426810716, + "lpips": 0.1521255373954773 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 1812, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.34765625, + "history_latent_mse": 0.16476544737815857, + "psnr": 21.573121446444787, + "ssim": 0.6993413185948636, + "lpips": 0.14580155909061432 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9375, + "history_latent_mse": 0.9424673914909363, + "psnr": 10.33575855940352, + "ssim": 0.20880074628149228, + "lpips": 0.5629078149795532 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.75, + "history_latent_mse": 0.8577510714530945, + "psnr": 11.282864711090472, + "ssim": 0.3064799089772862, + "lpips": 0.5411309003829956 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.87890625, + "history_latent_mse": 0.14211508631706238, + "psnr": 31.116486068430866, + "ssim": 0.9253226429369942, + "lpips": 0.25740134716033936 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.609375, + "history_latent_mse": 0.1428862363100052, + "psnr": 31.0769479629996, + "ssim": 0.9454793779833907, + "lpips": 0.1827661246061325 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 1820, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.138671875, + "history_latent_mse": 0.18939103186130524, + "psnr": 21.629391729366212, + "ssim": 0.7012938699855326, + "lpips": 0.15014734864234924 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.84375, + "history_latent_mse": 1.0669331550598145, + "psnr": 10.331316921555974, + "ssim": 0.199128064450759, + "lpips": 0.5630877614021301 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.828125, + "history_latent_mse": 0.9589233994483948, + "psnr": 10.991531053462893, + "ssim": 0.28216960140474057, + "lpips": 0.5574256181716919 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.453125, + "history_latent_mse": 0.1279725283384323, + "psnr": 30.34818102630645, + "ssim": 0.9196318295858037, + "lpips": 0.2910916209220886 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.61328125, + "history_latent_mse": 0.14413614571094513, + "psnr": 31.094203453587703, + "ssim": 0.9439208241247241, + "lpips": 0.19289442896842957 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 1828, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.763671875, + "history_latent_mse": 0.17105866968631744, + "psnr": 21.57272565825709, + "ssim": 0.6994133144205159, + "lpips": 0.1578795611858368 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0, + "history_latent_mse": 0.9195531010627747, + "psnr": 10.758511278308019, + "ssim": 0.21308471502053983, + "lpips": 0.545006513595581 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.46875, + "history_latent_mse": 1.0360995531082153, + "psnr": 11.034660414012304, + "ssim": 0.2794656180203185, + "lpips": 0.5605291724205017 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.921875, + "history_latent_mse": 0.1314925253391266, + "psnr": 32.19489050964277, + "ssim": 0.9310760196734856, + "lpips": 0.2836173474788666 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.1484375, + "history_latent_mse": 0.13723798096179962, + "psnr": 32.46928613742293, + "ssim": 0.9492871966639902, + "lpips": 0.1956261396408081 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 1836, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.21875, + "history_latent_mse": 0.19323168694972992, + "psnr": 21.247040928469648, + "ssim": 0.6917045654818198, + "lpips": 0.16357895731925964 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.0390625, + "history_latent_mse": 0.9569007754325867, + "psnr": 10.866246228435411, + "ssim": 0.22561552004719443, + "lpips": 0.545732855796814 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.234375, + "history_latent_mse": 0.9405404925346375, + "psnr": 10.550392693384683, + "ssim": 0.2743421370436383, + "lpips": 0.5485435724258423 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.5703125, + "history_latent_mse": 0.13657374680042267, + "psnr": 36.030567503970616, + "ssim": 0.9491558446309601, + "lpips": 0.25062796473503113 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.32421875, + "history_latent_mse": 0.1291276216506958, + "psnr": 37.26584706399295, + "ssim": 0.9607607024650588, + "lpips": 0.16582827270030975 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1844, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.046875, + "history_latent_mse": 0.19538849592208862, + "psnr": 20.701350503761347, + "ssim": 0.6809853863053097, + "lpips": 0.1869572103023529 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.0234375, + "history_latent_mse": 0.8590214848518372, + "psnr": 10.775255262974104, + "ssim": 0.22598459287856915, + "lpips": 0.552519679069519 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.859375, + "history_latent_mse": 0.9360775351524353, + "psnr": 10.413576908355886, + "ssim": 0.27925696066199546, + "lpips": 0.5553408861160278 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.6953125, + "history_latent_mse": 0.10964235663414001, + "psnr": 34.894089122992895, + "ssim": 0.9422146957365237, + "lpips": 0.27247172594070435 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.13671875, + "history_latent_mse": 0.09357333928346634, + "psnr": 36.59057267041453, + "ssim": 0.9587506886179369, + "lpips": 0.1707456409931183 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1852, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7421875, + "history_latent_mse": 0.2376762181520462, + "psnr": 19.502281243571222, + "ssim": 0.647725689348525, + "lpips": 0.24902325868606567 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.96875, + "history_latent_mse": 1.1334117650985718, + "psnr": 9.94902207610876, + "ssim": 0.2184455509987499, + "lpips": 0.5784121751785278 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.171875, + "history_latent_mse": 0.9209620952606201, + "psnr": 10.135895286184196, + "ssim": 0.27295193848614707, + "lpips": 0.587500810623169 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8203125, + "history_latent_mse": 0.12299001961946487, + "psnr": 33.02402294584954, + "ssim": 0.9363020462299589, + "lpips": 0.314297616481781 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.3125, + "history_latent_mse": 0.10561762005090714, + "psnr": 37.07582000563181, + "ssim": 0.9606397399141551, + "lpips": 0.14273275434970856 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1860, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.37890625, + "history_latent_mse": 0.26228025555610657, + "psnr": 19.397140579954563, + "ssim": 0.6464943230389761, + "lpips": 0.2448817640542984 + }, + "wrist_left": { + "history_latent_max_abs_error": 7.890625, + "history_latent_mse": 1.201985239982605, + "psnr": 9.635909970383054, + "ssim": 0.22651905949480738, + "lpips": 0.5834014415740967 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.171875, + "history_latent_mse": 0.9925606846809387, + "psnr": 10.10041321500374, + "ssim": 0.2602426550609839, + "lpips": 0.5694988965988159 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.7421875, + "history_latent_mse": 0.1448144167661667, + "psnr": 35.166227151395155, + "ssim": 0.9493974897099434, + "lpips": 0.2630700469017029 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.046875, + "history_latent_mse": 0.09717970341444016, + "psnr": 36.78157155789092, + "ssim": 0.958394604488773, + "lpips": 0.1534227728843689 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 1868, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.40625, + "history_latent_mse": 0.25645875930786133, + "psnr": 19.176027670522814, + "ssim": 0.6439339912479388, + "lpips": 0.24566134810447693 + }, + "wrist_left": { + "history_latent_max_abs_error": 7.078125, + "history_latent_mse": 1.016283631324768, + "psnr": 10.177372977076281, + "ssim": 0.2951535933594328, + "lpips": 0.5810104608535767 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.171875, + "history_latent_mse": 1.1478701829910278, + "psnr": 10.111849108464826, + "ssim": 0.24107693648105152, + "lpips": 0.5813837051391602 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.73046875, + "history_latent_mse": 0.12569254636764526, + "psnr": 36.43659492659581, + "ssim": 0.95246556084568, + "lpips": 0.24293774366378784 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.046875, + "history_latent_mse": 0.09934564679861069, + "psnr": 29.136082936867112, + "ssim": 0.9307495163074104, + "lpips": 0.30521470308303833 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1876, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.1015625, + "history_latent_mse": 0.2695702612400055, + "psnr": 19.05787620159687, + "ssim": 0.6446027616894197, + "lpips": 0.25260162353515625 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9453125, + "history_latent_mse": 1.0066733360290527, + "psnr": 11.606488505806084, + "ssim": 0.276271158204692, + "lpips": 0.554871678352356 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.484375, + "history_latent_mse": 1.0568008422851562, + "psnr": 10.226097597830492, + "ssim": 0.2448470976384448, + "lpips": 0.5806096792221069 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.72265625, + "history_latent_mse": 0.10276634246110916, + "psnr": 36.14781083653512, + "ssim": 0.9495700974749584, + "lpips": 0.2588042914867401 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.37890625, + "history_latent_mse": 0.16980814933776855, + "psnr": 27.46846758458638, + "ssim": 0.9175294941383092, + "lpips": 0.34467166662216187 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1884, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.56640625, + "history_latent_mse": 0.25259941816329956, + "psnr": 19.054002614916207, + "ssim": 0.6455146786162944, + "lpips": 0.25710251927375793 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.390625, + "history_latent_mse": 0.8799088597297668, + "psnr": 11.87364208311794, + "ssim": 0.26873274819491416, + "lpips": 0.535346269607544 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9140625, + "history_latent_mse": 1.0394325256347656, + "psnr": 10.223993025855112, + "ssim": 0.26245761787431776, + "lpips": 0.5796068906784058 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.740234375, + "history_latent_mse": 0.1104775071144104, + "psnr": 35.881144907603264, + "ssim": 0.9467266279070495, + "lpips": 0.27555280923843384 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.625, + "history_latent_mse": 0.17654617130756378, + "psnr": 29.384019975101282, + "ssim": 0.9274400788865472, + "lpips": 0.33017122745513916 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1892, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.32421875, + "history_latent_mse": 0.2831057012081146, + "psnr": 18.737960537256996, + "ssim": 0.637121906395242, + "lpips": 0.2718476951122284 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.21875, + "history_latent_mse": 1.0344409942626953, + "psnr": 11.491233767081413, + "ssim": 0.2719118436774063, + "lpips": 0.5543545484542847 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4765625, + "history_latent_mse": 0.9855496287345886, + "psnr": 10.275915204963377, + "ssim": 0.2672660701528783, + "lpips": 0.6014553904533386 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.771484375, + "history_latent_mse": 0.11773419380187988, + "psnr": 35.40854056063655, + "ssim": 0.9432590692522995, + "lpips": 0.29080629348754883 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.5, + "history_latent_mse": 0.18845005333423615, + "psnr": 30.245651813856153, + "ssim": 0.930084804874755, + "lpips": 0.32582947611808777 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1900, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.828125, + "history_latent_mse": 0.28142765164375305, + "psnr": 18.806752118457474, + "ssim": 0.63274690120223, + "lpips": 0.25915613770484924 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.2578125, + "history_latent_mse": 1.0919095277786255, + "psnr": 11.746133279645681, + "ssim": 0.2780723824408689, + "lpips": 0.5329008102416992 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 1.1187289953231812, + "psnr": 10.376637718760566, + "ssim": 0.267316216746343, + "lpips": 0.6130403280258179 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.83154296875, + "history_latent_mse": 0.12422064691781998, + "psnr": 34.878494580680425, + "ssim": 0.9407055981357528, + "lpips": 0.2986179292201996 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.7734375, + "history_latent_mse": 0.20956473052501678, + "psnr": 31.06561100482899, + "ssim": 0.9332759353234585, + "lpips": 0.3037983775138855 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9393a07d53769938c442a50bf95a3cdeb035c806 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffcd187ae810827326c042f7609113d5ec63c8344ec4736eb740c3a159e067f4 +size 1095977 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdafae05374506560afaf7a7514de787dd2b7ac --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.11844224482774734, + "wrist/video/latent_future_mse": 0.11844224482774734, + "video/psnr": 26.928243022387694, + "wrist/video/psnr": 26.928243022387694, + "video/ssim": 0.881112507024052, + "wrist/video/ssim": 0.881112507024052, + "video/lpips": 0.05308378487825394, + "wrist/video/lpips": 0.05308378487825394, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.7735612988471985, + "wrist/wrist_left/latent_future_mse": 0.7735612988471985, + "wrist_left/psnr": 12.208527476765113, + "wrist/wrist_left/psnr": 12.208527476765113, + "wrist_left/ssim": 0.39174826497395737, + "wrist/wrist_left/ssim": 0.39174826497395737, + "wrist_left/lpips": 0.3744303584098816, + "wrist/wrist_left/lpips": 0.3744303584098816, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.17983035743236542, + "wrist/wrist_right/latent_future_mse": 0.17983035743236542, + "wrist_right/psnr": 22.595010078670054, + "wrist/wrist_right/psnr": 22.595010078670054, + "wrist_right/ssim": 0.8586842305789523, + "wrist/wrist_right/ssim": 0.8586842305789523, + "wrist_right/lpips": 0.041494183242321014, + "wrist/wrist_right/lpips": 0.041494183242321014, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.013836540281772614, + "wrist/tactile_left/latent_future_mse": 0.013836540281772614, + "tactile_left/psnr": 47.512762453697384, + "wrist/tactile_left/psnr": 47.512762453697384, + "tactile_left/ssim": 0.9928329123046913, + "wrist/tactile_left/ssim": 0.9928329123046913, + "tactile_left/lpips": 0.004961402155458927, + "wrist/tactile_left/lpips": 0.004961402155458927, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.02346600405871868, + "wrist/tactile_right/latent_future_mse": 0.02346600405871868, + "tactile_right/psnr": 45.991363331281576, + "wrist/tactile_right/psnr": 45.991363331281576, + "tactile_right/ssim": 0.9881811009308914, + "wrist/tactile_right/ssim": 0.9881811009308914, + "tactile_right/lpips": 0.008251955732703209, + "wrist/tactile_right/lpips": 0.008251955732703209, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0909_episode_002.pt", + "t_start": 1788, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.11844224482774734, + "psnr": 26.928243022387694, + "ssim": 0.881112507024052, + "lpips": 0.05308378487825394 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.7735612988471985, + "psnr": 12.208527476765113, + "ssim": 0.39174826497395737, + "lpips": 0.3744303584098816 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.17983035743236542, + "psnr": 22.595010078670054, + "ssim": 0.8586842305789523, + "lpips": 0.041494183242321014 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.013836540281772614, + "psnr": 47.512762453697384, + "ssim": 0.9928329123046913, + "lpips": 0.004961402155458927 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.02346600405871868, + "psnr": 45.991363331281576, + "ssim": 0.9881811009308914, + "lpips": 0.008251955732703209 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/case.json new file mode 100644 index 0000000000000000000000000000000000000000..7f628108391acd53b87eb1c87b9dfa47323fb19f --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/case.json @@ -0,0 +1,164 @@ +{ + "task": 12, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "none", + "index": 10300, + "t_start": 913, + "start_seconds": 60.86666666666667, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.2734375, + "long_future_contact_fraction": 0.2916666666666667, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "first_contact_tick": 18, + "long_label": "none_at_start", + "output_subdir": "12_motherboard_0911_episode_008_seg00_none", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6836828ead9118f0cd5cff093c88fa616be7eb0c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c572aaca6a46182dceb9129903b0180b2b12999a1d9323879ddf27aa6720bd1 +size 9389084 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/motherboard_0911_episode_008_seg00_t00913_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/motherboard_0911_episode_008_seg00_t00913_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4d4de03baaf987d09422af43d6fe1a1f2298fef --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/motherboard_0911_episode_008_seg00_t00913_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52b94cf3bb1b30c5e3a3a3ca3203b64eac71ad66282fb4bb5a22575a1081a701 +size 1978961 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..6706f3a2a3526cc6fcdf56da5846782a23735a15 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 913, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 913, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 26.99092708910586, + "ssim": 0.892603898996384, + "lpips": 0.04418816789984703 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 19.967447428081677, + "ssim": 0.7395254602707335, + "lpips": 0.12541130185127258 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 16.27012417437985, + "ssim": 0.4244755159640749, + "lpips": 0.3176184892654419 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 50.84456346979567, + "ssim": 0.995048933439145, + "lpips": 0.003001157660037279 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 51.86527554175336, + "ssim": 0.9950497071316373, + "lpips": 0.004702226258814335 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 921, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.8203125, + "history_latent_mse": 0.07646084576845169, + "psnr": 25.01756221081484, + "ssim": 0.8435543897452739, + "lpips": 0.0576213076710701 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.015625, + "history_latent_mse": 0.2769981920719147, + "psnr": 17.714688280664802, + "ssim": 0.6945667609230457, + "lpips": 0.16515487432479858 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.455078125, + "history_latent_mse": 0.45598044991493225, + "psnr": 15.392194585499512, + "ssim": 0.37045252292305964, + "lpips": 0.4226338267326355 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.51953125, + "history_latent_mse": 0.00895365048199892, + "psnr": 47.90721691085927, + "ssim": 0.9921589553536125, + "lpips": 0.009723201394081116 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.9248046875, + "history_latent_mse": 0.011988862417638302, + "psnr": 42.253242120456434, + "ssim": 0.9833294971189872, + "lpips": 0.05198349431157112 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 929, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.2734375, + "history_latent_mse": 0.10423722863197327, + "psnr": 24.351993985251642, + "ssim": 0.8276117530441486, + "lpips": 0.06680246442556381 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.625, + "history_latent_mse": 0.2950395345687866, + "psnr": 16.760954282776414, + "ssim": 0.6644298500618788, + "lpips": 0.18947187066078186 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.8359375, + "history_latent_mse": 0.6448513865470886, + "psnr": 15.212422403575463, + "ssim": 0.37810868609196635, + "lpips": 0.43738850951194763 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.78375244140625, + "history_latent_mse": 0.018043795600533485, + "psnr": 45.675202599994044, + "ssim": 0.9889886726761326, + "lpips": 0.023789089173078537 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.09375, + "history_latent_mse": 0.07073543965816498, + "psnr": 41.61573607207409, + "ssim": 0.98335233788305, + "lpips": 0.02869277447462082 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 937, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.421875, + "history_latent_mse": 0.11082541942596436, + "psnr": 24.125836394783867, + "ssim": 0.8232870762475601, + "lpips": 0.07111717760562897 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.3515625, + "history_latent_mse": 0.33607006072998047, + "psnr": 16.330336878026543, + "ssim": 0.6535598288060325, + "lpips": 0.20491836965084076 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.03125, + "history_latent_mse": 0.6031566262245178, + "psnr": 13.636011058658344, + "ssim": 0.31200518092529406, + "lpips": 0.4759405851364136 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.9393768310546875, + "history_latent_mse": 0.026178449392318726, + "psnr": 44.246911188449936, + "ssim": 0.9858470706495508, + "lpips": 0.03947924077510834 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.58984375, + "history_latent_mse": 0.04587920382618904, + "psnr": 40.21219755173624, + "ssim": 0.9827947366995263, + "lpips": 0.032592352479696274 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 945, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.287109375, + "history_latent_mse": 0.11618608236312866, + "psnr": 24.31984667333338, + "ssim": 0.827706696627391, + "lpips": 0.06999114155769348 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.4609375, + "history_latent_mse": 0.3500951826572418, + "psnr": 16.00713184908757, + "ssim": 0.6446075308035992, + "lpips": 0.2133217602968216 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.4375, + "history_latent_mse": 0.749478816986084, + "psnr": 14.23194649698849, + "ssim": 0.32984866241834776, + "lpips": 0.4689670503139496 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.0859375, + "history_latent_mse": 0.03299197182059288, + "psnr": 42.890322343850684, + "ssim": 0.9832668046397932, + "lpips": 0.05685801059007645 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.357421875, + "history_latent_mse": 0.04650048911571503, + "psnr": 39.61035146404888, + "ssim": 0.9812900659121875, + "lpips": 0.048351116478443146 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 953, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.6640625, + "history_latent_mse": 0.11743078380823135, + "psnr": 23.414015645467487, + "ssim": 0.8025714772560983, + "lpips": 0.0795525535941124 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.9375, + "history_latent_mse": 0.378527969121933, + "psnr": 14.206556580933452, + "ssim": 0.5807955274752888, + "lpips": 0.32298046350479126 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.53125, + "history_latent_mse": 0.7865776419639587, + "psnr": 13.988356098986577, + "ssim": 0.3425369157230629, + "lpips": 0.4726126492023468 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.0625, + "history_latent_mse": 0.03797493502497673, + "psnr": 41.64523439350093, + "ssim": 0.980282681656488, + "lpips": 0.07978159189224243 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.0078125, + "history_latent_mse": 0.05513649061322212, + "psnr": 38.813499040654065, + "ssim": 0.980690612644287, + "lpips": 0.05265729874372482 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 961, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8359375, + "history_latent_mse": 0.13883501291275024, + "psnr": 22.850826550696492, + "ssim": 0.7863342428216948, + "lpips": 0.0928582102060318 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.359375, + "history_latent_mse": 0.5720059275627136, + "psnr": 12.844647146281524, + "ssim": 0.5207070999489407, + "lpips": 0.4237939119338989 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.1171875, + "history_latent_mse": 0.702068030834198, + "psnr": 14.070435924430742, + "ssim": 0.3153879918623502, + "lpips": 0.4552322030067444 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.1328125, + "history_latent_mse": 0.0432741604745388, + "psnr": 40.319852568854486, + "ssim": 0.9771850628717577, + "lpips": 0.10037104785442352 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.744140625, + "history_latent_mse": 0.05675557255744934, + "psnr": 37.63059083038138, + "ssim": 0.9784068008887516, + "lpips": 0.0711846649646759 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 969, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1875, + "history_latent_mse": 0.16040246188640594, + "psnr": 22.512241933192204, + "ssim": 0.7752873227184919, + "lpips": 0.11398039758205414 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.375, + "history_latent_mse": 0.6628901362419128, + "psnr": 13.782476872646008, + "ssim": 0.5419167140262641, + "lpips": 0.40555429458618164 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.46875, + "history_latent_mse": 0.6690407395362854, + "psnr": 14.294843009499694, + "ssim": 0.3580972900003256, + "lpips": 0.4684538245201111 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.23046875, + "history_latent_mse": 0.05124777555465698, + "psnr": 39.249427587400504, + "ssim": 0.973711453445447, + "lpips": 0.1243937760591507 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.634765625, + "history_latent_mse": 0.06495579332113266, + "psnr": 36.62462774437505, + "ssim": 0.9741425265277927, + "lpips": 0.11501732468605042 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 977, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.181640625, + "history_latent_mse": 0.1515711396932602, + "psnr": 22.262815877639653, + "ssim": 0.7600415677959116, + "lpips": 0.12999463081359863 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.4921875, + "history_latent_mse": 0.6058850288391113, + "psnr": 14.431131041410872, + "ssim": 0.5895743099032873, + "lpips": 0.36152568459510803 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.09375, + "history_latent_mse": 0.6347067952156067, + "psnr": 15.700905835861628, + "ssim": 0.38414367383464326, + "lpips": 0.4543244242668152 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.171875, + "history_latent_mse": 0.05765466019511223, + "psnr": 38.29434643340507, + "ssim": 0.9709464493696001, + "lpips": 0.1446119248867035 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.6279296875, + "history_latent_mse": 0.11126349121332169, + "psnr": 36.06853478207791, + "ssim": 0.9743961158264545, + "lpips": 0.10393696278333664 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 985, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.20703125, + "history_latent_mse": 0.18848787248134613, + "psnr": 20.776967741259284, + "ssim": 0.7180156910696066, + "lpips": 0.16729271411895752 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.5625, + "history_latent_mse": 0.5720319151878357, + "psnr": 14.295236296644841, + "ssim": 0.6066075572908771, + "lpips": 0.31601765751838684 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.15625, + "history_latent_mse": 0.6303689479827881, + "psnr": 13.296476584406413, + "ssim": 0.330754484031884, + "lpips": 0.48200833797454834 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.33984375, + "history_latent_mse": 0.06409744173288345, + "psnr": 37.02870779550671, + "ssim": 0.966893369058752, + "lpips": 0.16241252422332764 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.09765625, + "history_latent_mse": 0.08801428228616714, + "psnr": 34.95608254809681, + "ssim": 0.9719626284084035, + "lpips": 0.11206506192684174 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 993, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.890625, + "history_latent_mse": 0.2208760380744934, + "psnr": 20.57648454575418, + "ssim": 0.7159371558777368, + "lpips": 0.16061249375343323 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9765625, + "history_latent_mse": 0.47219619154930115, + "psnr": 14.201905995725264, + "ssim": 0.6074695967643287, + "lpips": 0.30098873376846313 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.4296875, + "history_latent_mse": 0.7978346347808838, + "psnr": 12.640470876776588, + "ssim": 0.29129608668496465, + "lpips": 0.5169528126716614 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.59375, + "history_latent_mse": 0.07278597354888916, + "psnr": 36.56144428305092, + "ssim": 0.9634300038784684, + "lpips": 0.1783471554517746 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.232421875, + "history_latent_mse": 0.08413520455360413, + "psnr": 34.1398860633587, + "ssim": 0.9689921005991062, + "lpips": 0.1398734450340271 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1001, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.21875, + "history_latent_mse": 0.2179524153470993, + "psnr": 20.449199642854552, + "ssim": 0.6999760463685344, + "lpips": 0.17533859610557556 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.3125, + "history_latent_mse": 0.4655846655368805, + "psnr": 13.944563379030638, + "ssim": 0.5942142723405965, + "lpips": 0.3168647885322571 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.46875, + "history_latent_mse": 1.0294398069381714, + "psnr": 11.734180379620657, + "ssim": 0.34415606831690415, + "lpips": 0.5154658555984497 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.40625, + "history_latent_mse": 0.07924490422010422, + "psnr": 36.121269104660996, + "ssim": 0.9604081679300178, + "lpips": 0.1975746750831604 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.78125, + "history_latent_mse": 0.09440696239471436, + "psnr": 33.72876523693314, + "ssim": 0.9664259669889894, + "lpips": 0.15368282794952393 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1009, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.890625, + "history_latent_mse": 0.22975289821624756, + "psnr": 20.196217042554245, + "ssim": 0.6878887111333466, + "lpips": 0.18638533353805542 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.4765625, + "history_latent_mse": 0.4945486783981323, + "psnr": 12.91706264600326, + "ssim": 0.5486050225415919, + "lpips": 0.3515714406967163 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.0, + "history_latent_mse": 0.7808933258056641, + "psnr": 10.805328943842088, + "ssim": 0.34732342476342615, + "lpips": 0.5364969968795776 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.5859375, + "history_latent_mse": 0.085682213306427, + "psnr": 35.60081330189967, + "ssim": 0.9570769225255955, + "lpips": 0.21200713515281677 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.640625, + "history_latent_mse": 0.09576988220214844, + "psnr": 33.22834319946371, + "ssim": 0.9637001272037085, + "lpips": 0.16537395119667053 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1017, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7734375, + "history_latent_mse": 0.24446357786655426, + "psnr": 20.141072479912353, + "ssim": 0.6817173994780263, + "lpips": 0.1915043145418167 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.7109375, + "history_latent_mse": 0.5851922035217285, + "psnr": 12.67676588643856, + "ssim": 0.5273051427767307, + "lpips": 0.3822683095932007 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.466796875, + "history_latent_mse": 0.908463716506958, + "psnr": 11.087691332406578, + "ssim": 0.3114287675518306, + "lpips": 0.5371555089950562 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.841796875, + "history_latent_mse": 0.09181178361177444, + "psnr": 35.15105135299968, + "ssim": 0.9532959688763978, + "lpips": 0.22573015093803406 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.375, + "history_latent_mse": 0.11484029144048691, + "psnr": 32.82465655989018, + "ssim": 0.9606983759934069, + "lpips": 0.19110579788684845 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1025, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.984375, + "history_latent_mse": 0.24917598068714142, + "psnr": 20.088983435834745, + "ssim": 0.678561061188111, + "lpips": 0.19595912098884583 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.875, + "history_latent_mse": 0.5775278806686401, + "psnr": 12.654883929483143, + "ssim": 0.513559189485009, + "lpips": 0.4167664647102356 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.671875, + "history_latent_mse": 0.8694343566894531, + "psnr": 11.108630331529161, + "ssim": 0.38797563745972274, + "lpips": 0.5627725124359131 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.953125, + "history_latent_mse": 0.09838924556970596, + "psnr": 34.427566071470366, + "ssim": 0.9502583113484713, + "lpips": 0.2376347780227661 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.328125, + "history_latent_mse": 0.12842737138271332, + "psnr": 32.05363364609272, + "ssim": 0.95782636919141, + "lpips": 0.20850145816802979 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57c35747655ea8ae8ebc775285c69e4bbeee1dcb --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38492ca16c3edba8ee39bd83ff2df1773bec8108c4cc373e6f042a63fa7bc30e +size 1344225 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..0648d262870de66033a8c9b69321c8f0451e89c8 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.08359571546316147, + "wrist/video/latent_future_mse": 0.08359571546316147, + "video/psnr": 26.99092708910586, + "wrist/video/psnr": 26.99092708910586, + "video/ssim": 0.892603898996384, + "wrist/video/ssim": 0.892603898996384, + "video/lpips": 0.04418816789984703, + "wrist/video/lpips": 0.04418816789984703, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.3021610677242279, + "wrist/wrist_left/latent_future_mse": 0.3021610677242279, + "wrist_left/psnr": 19.967447428081677, + "wrist/wrist_left/psnr": 19.967447428081677, + "wrist_left/ssim": 0.7395254602707335, + "wrist/wrist_left/ssim": 0.7395254602707335, + "wrist_left/lpips": 0.12541130185127258, + "wrist/wrist_left/lpips": 0.12541130185127258, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.4905147850513458, + "wrist/wrist_right/latent_future_mse": 0.4905147850513458, + "wrist_right/psnr": 16.27012417437985, + "wrist/wrist_right/psnr": 16.27012417437985, + "wrist_right/ssim": 0.4244755159640749, + "wrist/wrist_right/ssim": 0.4244755159640749, + "wrist_right/lpips": 0.3176184892654419, + "wrist/wrist_right/lpips": 0.3176184892654419, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.005700790788978338, + "wrist/tactile_left/latent_future_mse": 0.005700790788978338, + "tactile_left/psnr": 50.84456346979567, + "wrist/tactile_left/psnr": 50.84456346979567, + "tactile_left/ssim": 0.995048933439145, + "wrist/tactile_left/ssim": 0.995048933439145, + "tactile_left/lpips": 0.003001157660037279, + "wrist/tactile_left/lpips": 0.003001157660037279, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.008183596655726433, + "wrist/tactile_right/latent_future_mse": 0.008183596655726433, + "tactile_right/psnr": 51.86527554175336, + "wrist/tactile_right/psnr": 51.86527554175336, + "tactile_right/ssim": 0.9950497071316373, + "wrist/tactile_right/ssim": 0.9950497071316373, + "tactile_right/lpips": 0.004702226258814335, + "wrist/tactile_right/lpips": 0.004702226258814335, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 913, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08359571546316147, + "psnr": 26.99092708910586, + "ssim": 0.892603898996384, + "lpips": 0.04418816789984703 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.3021610677242279, + "psnr": 19.967447428081677, + "ssim": 0.7395254602707335, + "lpips": 0.12541130185127258 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.4905147850513458, + "psnr": 16.27012417437985, + "ssim": 0.4244755159640749, + "lpips": 0.3176184892654419 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.005700790788978338, + "psnr": 50.84456346979567, + "ssim": 0.995048933439145, + "lpips": 0.003001157660037279 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.008183596655726433, + "psnr": 51.86527554175336, + "ssim": 0.9950497071316373, + "lpips": 0.004702226258814335 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/case.json new file mode 100644 index 0000000000000000000000000000000000000000..c14f46e917b6d722a7cd355bc63621c6dabfec5d --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/case.json @@ -0,0 +1,168 @@ +{ + "task": 13, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "onset", + "index": 10113, + "t_start": 726, + "start_seconds": 48.4, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.515625, + "long_future_contact_fraction": 0.55, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "13_motherboard_0911_episode_008_seg00_onset", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..043eb6d8b941aa50854eaca56bd66619bbef1c49 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102676c1064310dec4d724dd1732539b08d211795824b829de2be856d3562dea +size 10207214 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/motherboard_0911_episode_008_seg00_t00726_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/motherboard_0911_episode_008_seg00_t00726_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..02cc4a004e304f3f29a87d366bfe77636c13dcf4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/motherboard_0911_episode_008_seg00_t00726_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da45b6a58d6fc9b5b4b97b6c65d0599c81cb7361b065ff6f28e7073e385740e7 +size 1978961 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..7136791a5e601515cc3f0ad9960c2d11c6345ab6 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 726, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 726, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.12251940351486, + "ssim": 0.9337358848760544, + "lpips": 0.01897820457816124 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 14.960669145784786, + "ssim": 0.2846841099277542, + "lpips": 0.2619835138320923 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 20.09410116535415, + "ssim": 0.8357987898903909, + "lpips": 0.04304193705320358 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 35.40484588348979, + "ssim": 0.969891564480726, + "lpips": 0.08059585094451904 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 50.36403959779542, + "ssim": 0.994872124209794, + "lpips": 0.0034496979787945747 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 734, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.57568359375, + "history_latent_mse": 0.03760721907019615, + "psnr": 23.893714886883238, + "ssim": 0.8204769662834801, + "lpips": 0.0678124874830246 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.6875, + "history_latent_mse": 0.5689957737922668, + "psnr": 14.341637634218337, + "ssim": 0.24915379676881239, + "lpips": 0.3974441587924957 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.9375, + "history_latent_mse": 0.14124061167240143, + "psnr": 17.519113712303266, + "ssim": 0.7172091828567272, + "lpips": 0.13472780585289001 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4921875, + "history_latent_mse": 0.09695478528738022, + "psnr": 33.27214415699099, + "ssim": 0.9526542351382581, + "lpips": 0.12748709321022034 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.146240234375, + "history_latent_mse": 0.01764599233865738, + "psnr": 47.24276785108471, + "ssim": 0.9889017867263115, + "lpips": 0.012656893581151962 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 742, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0546875, + "history_latent_mse": 0.11081355810165405, + "psnr": 22.63612309683717, + "ssim": 0.7901186999619326, + "lpips": 0.09322889149188995 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1640625, + "history_latent_mse": 0.6307730078697205, + "psnr": 14.280175484967979, + "ssim": 0.2744324121255549, + "lpips": 0.4070354104042053 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.078125, + "history_latent_mse": 0.27529409527778625, + "psnr": 16.97304030386692, + "ssim": 0.7138549632461786, + "lpips": 0.17569315433502197 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3671875, + "history_latent_mse": 0.11368855834007263, + "psnr": 43.21925572477065, + "ssim": 0.9811123251221517, + "lpips": 0.03611776977777481 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.228515625, + "history_latent_mse": 0.03470168262720108, + "psnr": 45.1791614513502, + "ssim": 0.9861869600429845, + "lpips": 0.02558419480919838 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 750, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.92578125, + "history_latent_mse": 0.13601671159267426, + "psnr": 22.05971078951039, + "ssim": 0.7626783969614743, + "lpips": 0.10541737079620361 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.55908203125, + "history_latent_mse": 0.6276031136512756, + "psnr": 13.75804952005857, + "ssim": 0.2541055708978527, + "lpips": 0.4276825785636902 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6015625, + "history_latent_mse": 0.3330307900905609, + "psnr": 16.673336523401417, + "ssim": 0.6896521628295224, + "lpips": 0.19613346457481384 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.51904296875, + "history_latent_mse": 0.044429581612348557, + "psnr": 41.962922922034494, + "ssim": 0.9790788111326458, + "lpips": 0.051196467131376266 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.2265625, + "history_latent_mse": 0.040319643914699554, + "psnr": 43.35139995686138, + "ssim": 0.9839300879716837, + "lpips": 0.04063522815704346 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 758, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.078125, + "history_latent_mse": 0.17241863906383514, + "psnr": 21.599119722163643, + "ssim": 0.7416996916797329, + "lpips": 0.12093204259872437 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.76953125, + "history_latent_mse": 0.7026738524436951, + "psnr": 13.201588739617765, + "ssim": 0.23548611555331614, + "lpips": 0.4988209307193756 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.125, + "history_latent_mse": 0.3836073577404022, + "psnr": 16.822332017443795, + "ssim": 0.6836966362950094, + "lpips": 0.19169461727142334 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.49609375, + "history_latent_mse": 0.05780808627605438, + "psnr": 37.071262986013906, + "ssim": 0.963236214952448, + "lpips": 0.12417158484458923 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4609375, + "history_latent_mse": 0.048107992857694626, + "psnr": 41.97531252979356, + "ssim": 0.9816100574974771, + "lpips": 0.05867469310760498 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 766, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.25, + "history_latent_mse": 0.18308089673519135, + "psnr": 21.701937189176217, + "ssim": 0.7295050769022435, + "lpips": 0.1226889044046402 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.958984375, + "history_latent_mse": 0.8715968728065491, + "psnr": 12.929700866780948, + "ssim": 0.25636685565857736, + "lpips": 0.5107643604278564 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.140625, + "history_latent_mse": 0.3829778730869293, + "psnr": 16.930155348004437, + "ssim": 0.6901412586731993, + "lpips": 0.160008043050766 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.578125, + "history_latent_mse": 0.0981186032295227, + "psnr": 35.47155319719403, + "ssim": 0.9645085583362654, + "lpips": 0.12689337134361267 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.2823486328125, + "history_latent_mse": 0.05279931426048279, + "psnr": 40.586778512701756, + "ssim": 0.9791341387743809, + "lpips": 0.07825347781181335 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 774, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.234375, + "history_latent_mse": 0.18893440067768097, + "psnr": 21.24170618239504, + "ssim": 0.711314867122613, + "lpips": 0.13207340240478516 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.698242723941803, + "psnr": 11.811697918193204, + "ssim": 0.2612625235783076, + "lpips": 0.528882622718811 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.3984375, + "history_latent_mse": 0.3264119029045105, + "psnr": 17.0335348079962, + "ssim": 0.6925020675273938, + "lpips": 0.16078999638557434 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4296875, + "history_latent_mse": 0.07996673136949539, + "psnr": 38.6430794875599, + "ssim": 0.970977645089591, + "lpips": 0.11217324435710907 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.32666015625, + "history_latent_mse": 0.06137746945023537, + "psnr": 39.28282976625612, + "ssim": 0.9761892817574638, + "lpips": 0.10036586225032806 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 782, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.20328204333782196, + "psnr": 21.351827826385673, + "ssim": 0.7188142074131576, + "lpips": 0.12764237821102142 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.40625, + "history_latent_mse": 0.8319578766822815, + "psnr": 12.58098685804525, + "ssim": 0.2645766112475998, + "lpips": 0.4589235484600067 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.0234375, + "history_latent_mse": 0.3068038523197174, + "psnr": 17.10987577314262, + "ssim": 0.6888259629306013, + "lpips": 0.17081405222415924 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.86328125, + "history_latent_mse": 0.08944788575172424, + "psnr": 37.40659562461981, + "ssim": 0.9689403484000166, + "lpips": 0.1252223253250122 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.46875, + "history_latent_mse": 0.07023967057466507, + "psnr": 38.56612262378166, + "ssim": 0.973364054223161, + "lpips": 0.12144952267408371 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 790, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.34375, + "history_latent_mse": 0.19459551572799683, + "psnr": 21.093687152010435, + "ssim": 0.7158933769536946, + "lpips": 0.13801309466362 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.494140625, + "history_latent_mse": 0.6656257510185242, + "psnr": 13.122499964516333, + "ssim": 0.3140757483708532, + "lpips": 0.4398747384548187 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.078125, + "history_latent_mse": 0.32052841782569885, + "psnr": 17.38459452742008, + "ssim": 0.6984630170806051, + "lpips": 0.1983262449502945 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.83984375, + "history_latent_mse": 0.09700548648834229, + "psnr": 34.384937749752574, + "ssim": 0.9537460615350397, + "lpips": 0.18549248576164246 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.58984375, + "history_latent_mse": 0.07978542894124985, + "psnr": 37.899442837828786, + "ssim": 0.9707114671701258, + "lpips": 0.13966064155101776 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 798, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.21875, + "history_latent_mse": 0.20765471458435059, + "psnr": 20.776779349311635, + "ssim": 0.7035742959183195, + "lpips": 0.15400518476963043 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8359375, + "history_latent_mse": 0.6910180449485779, + "psnr": 11.66623223252412, + "ssim": 0.2481787402204949, + "lpips": 0.49910280108451843 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9921875, + "history_latent_mse": 0.41015055775642395, + "psnr": 16.922603293881814, + "ssim": 0.691880590727197, + "lpips": 0.21014201641082764 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.98828125, + "history_latent_mse": 0.16122955083847046, + "psnr": 29.475905295540095, + "ssim": 0.939482911551425, + "lpips": 0.23121075332164764 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.6328125, + "history_latent_mse": 0.08597981929779053, + "psnr": 36.61531897125691, + "ssim": 0.9676520748291607, + "lpips": 0.16189944744110107 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 806, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.84375, + "history_latent_mse": 0.2088438868522644, + "psnr": 20.72822311234703, + "ssim": 0.6985618651473766, + "lpips": 0.17882874608039856 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1875, + "history_latent_mse": 0.7156643867492676, + "psnr": 12.238759436370179, + "ssim": 0.26981907947176986, + "lpips": 0.45253047347068787 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.390625, + "history_latent_mse": 0.3325895369052887, + "psnr": 17.592426742309737, + "ssim": 0.6946618345756057, + "lpips": 0.2124924659729004 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.57763671875, + "history_latent_mse": 0.17286773025989532, + "psnr": 30.502955866331515, + "ssim": 0.9432009798965779, + "lpips": 0.21931427717208862 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.859375, + "history_latent_mse": 0.10088381916284561, + "psnr": 35.82062702439509, + "ssim": 0.9643192405922869, + "lpips": 0.17499464750289917 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 814, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.8125, + "history_latent_mse": 0.2216993123292923, + "psnr": 20.350852675296792, + "ssim": 0.6888530738036251, + "lpips": 0.2099272459745407 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.84375, + "history_latent_mse": 0.6536296606063843, + "psnr": 13.461568030957828, + "ssim": 0.27631269731522007, + "lpips": 0.5012498497962952 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.83984375, + "history_latent_mse": 0.3575328290462494, + "psnr": 17.171541072668163, + "ssim": 0.6768563089524153, + "lpips": 0.2280459702014923 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.984375, + "history_latent_mse": 0.17930568754673004, + "psnr": 34.66132783907756, + "ssim": 0.9547357976113436, + "lpips": 0.1772235780954361 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.6875, + "history_latent_mse": 0.10364401340484619, + "psnr": 35.229163053268884, + "ssim": 0.9608430777452817, + "lpips": 0.19408352673053741 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 822, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.5703125, + "history_latent_mse": 0.24216747283935547, + "psnr": 20.178965039581307, + "ssim": 0.6864502101343676, + "lpips": 0.21314242482185364 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.75390625, + "history_latent_mse": 0.6824352145195007, + "psnr": 12.493652117919257, + "ssim": 0.2913693905556206, + "lpips": 0.5319913625717163 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.9765625, + "history_latent_mse": 0.3928472101688385, + "psnr": 16.915957653933887, + "ssim": 0.6567060642809783, + "lpips": 0.2566368281841278 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.96875, + "history_latent_mse": 0.13239778578281403, + "psnr": 30.93265261900427, + "ssim": 0.9345873097842627, + "lpips": 0.22871273756027222 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8203125, + "history_latent_mse": 0.11200395971536636, + "psnr": 34.86447018057213, + "ssim": 0.9575596301415626, + "lpips": 0.21290689706802368 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 830, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.94921875, + "history_latent_mse": 0.25137296319007874, + "psnr": 19.302938245882224, + "ssim": 0.6687457864353149, + "lpips": 0.24991613626480103 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.15625, + "history_latent_mse": 0.7480480074882507, + "psnr": 12.488311434550186, + "ssim": 0.3363838083884773, + "lpips": 0.5150729417800903 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.796875, + "history_latent_mse": 0.3936816155910492, + "psnr": 16.56283740591634, + "ssim": 0.6328158748903817, + "lpips": 0.2922106981277466 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.78125, + "history_latent_mse": 0.14990125596523285, + "psnr": 33.49171245430712, + "ssim": 0.9508083455937664, + "lpips": 0.19189786911010742 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.9609375, + "history_latent_mse": 0.12196389585733414, + "psnr": 34.505811871119995, + "ssim": 0.9545588853796998, + "lpips": 0.2272951900959015 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 838, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.015625, + "history_latent_mse": 0.28004178404808044, + "psnr": 19.12518571336588, + "ssim": 0.6570700000077103, + "lpips": 0.26303911209106445 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.9375, + "history_latent_mse": 0.7726134657859802, + "psnr": 13.207228748675082, + "ssim": 0.28917647980682076, + "lpips": 0.5030600428581238 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.609375, + "history_latent_mse": 0.41162291169166565, + "psnr": 16.34060330671803, + "ssim": 0.6260150324533811, + "lpips": 0.3063967227935791 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.4140625, + "history_latent_mse": 0.12827010452747345, + "psnr": 31.472302254783738, + "ssim": 0.9408625909479255, + "lpips": 0.20679588615894318 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.9375, + "history_latent_mse": 0.13053548336029053, + "psnr": 33.967873858095025, + "ssim": 0.9513377386134616, + "lpips": 0.24304315447807312 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1c0e71e6ebab507414887411bd9c275394460bda --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3d4516aad830f4759984db41a57066d7168f55512376938f93372e749be3966 +size 1310995 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..354085b5bd3c0cc7e0fd65f3909eab2e9dd3c991 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.04399647191166878, + "wrist/video/latent_future_mse": 0.04399647191166878, + "video/psnr": 31.12251940351486, + "wrist/video/psnr": 31.12251940351486, + "video/ssim": 0.9337358848760544, + "wrist/video/ssim": 0.9337358848760544, + "video/lpips": 0.01897820457816124, + "wrist/video/lpips": 0.01897820457816124, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.6297520995140076, + "wrist/wrist_left/latent_future_mse": 0.6297520995140076, + "wrist_left/psnr": 14.960669145784786, + "wrist/wrist_left/psnr": 14.960669145784786, + "wrist_left/ssim": 0.2846841099277542, + "wrist/wrist_left/ssim": 0.2846841099277542, + "wrist_left/lpips": 0.2619835138320923, + "wrist/wrist_left/lpips": 0.2619835138320923, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.1451612263917923, + "wrist/wrist_right/latent_future_mse": 0.1451612263917923, + "wrist_right/psnr": 20.09410116535415, + "wrist/wrist_right/psnr": 20.09410116535415, + "wrist_right/ssim": 0.8357987898903909, + "wrist/wrist_right/ssim": 0.8357987898903909, + "wrist_right/lpips": 0.04304193705320358, + "wrist/wrist_right/lpips": 0.04304193705320358, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.10318439453840256, + "wrist/tactile_left/latent_future_mse": 0.10318439453840256, + "tactile_left/psnr": 35.40484588348979, + "wrist/tactile_left/psnr": 35.40484588348979, + "tactile_left/ssim": 0.969891564480726, + "wrist/tactile_left/ssim": 0.969891564480726, + "tactile_left/lpips": 0.08059585094451904, + "wrist/tactile_left/lpips": 0.08059585094451904, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.014547972939908504, + "wrist/tactile_right/latent_future_mse": 0.014547972939908504, + "tactile_right/psnr": 50.36403959779542, + "wrist/tactile_right/psnr": 50.36403959779542, + "tactile_right/ssim": 0.994872124209794, + "wrist/tactile_right/ssim": 0.994872124209794, + "tactile_right/lpips": 0.0034496979787945747, + "wrist/tactile_right/lpips": 0.0034496979787945747, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 726, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.04399647191166878, + "psnr": 31.12251940351486, + "ssim": 0.9337358848760544, + "lpips": 0.01897820457816124 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.6297520995140076, + "psnr": 14.960669145784786, + "ssim": 0.2846841099277542, + "lpips": 0.2619835138320923 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.1451612263917923, + "psnr": 20.09410116535415, + "ssim": 0.8357987898903909, + "lpips": 0.04304193705320358 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.10318439453840256, + "psnr": 35.40484588348979, + "ssim": 0.969891564480726, + "lpips": 0.08059585094451904 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.014547972939908504, + "psnr": 50.36403959779542, + "ssim": 0.994872124209794, + "lpips": 0.0034496979787945747 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/case.json new file mode 100644 index 0000000000000000000000000000000000000000..1075f0ee1ba1a96ef66d8b794cbe2bbff1e0f20c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/case.json @@ -0,0 +1,168 @@ +{ + "task": 14, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "release", + "index": 9633, + "t_start": 246, + "start_seconds": 16.4, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6796875, + "long_future_contact_fraction": 0.6583333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "14_motherboard_0911_episode_008_seg00_release", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78074ed77f4ebdc9ede8bae8e611267ae93e4977 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41647cc6b41399a11c42245f7e508bfcab5370dbf3176f70b3565a81b6536fbf +size 10427563 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/motherboard_0911_episode_008_seg00_t00246_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/motherboard_0911_episode_008_seg00_t00246_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..53cd8eaa477efa9575b3e4b37be9240cf03f0892 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/motherboard_0911_episode_008_seg00_t00246_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1bb8b8bf7ed035febbf0713dffb0a59a7d43888cdaee83e1a38b572e816336 +size 1978961 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..72f8a5b58f54ccc8288e828b5a22b5c3538b8701 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 246, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 246, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 30.795292763143625, + "ssim": 0.9375067540670683, + "lpips": 0.02425140142440796 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 14.92759718890451, + "ssim": 0.4136994091358832, + "lpips": 0.3324988782405853 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 23.87380535539645, + "ssim": 0.84858913143935, + "lpips": 0.03306284919381142 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 36.105213601449094, + "ssim": 0.9805090412282181, + "lpips": 0.03081936575472355 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 50.57606534665488, + "ssim": 0.9945805941139491, + "lpips": 0.004185985773801804 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 254, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.57421875, + "history_latent_mse": 0.038659851998090744, + "psnr": 26.077139747014293, + "ssim": 0.8761739701408273, + "lpips": 0.04953545331954956 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.2734375, + "history_latent_mse": 0.5824840664863586, + "psnr": 12.248635283878626, + "ssim": 0.31396760458700057, + "lpips": 0.41857290267944336 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.1875, + "history_latent_mse": 0.10260015726089478, + "psnr": 20.520245038532057, + "ssim": 0.7285914801487648, + "lpips": 0.0592556968331337 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.296875, + "history_latent_mse": 0.07674028724431992, + "psnr": 29.46021211542179, + "ssim": 0.9111230598450785, + "lpips": 0.25219181180000305 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.16650390625, + "history_latent_mse": 0.019853703677654266, + "psnr": 48.024081122178536, + "ssim": 0.9904534799893632, + "lpips": 0.010807454586029053 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 262, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.63671875, + "history_latent_mse": 0.07276712357997894, + "psnr": 24.637320864989682, + "ssim": 0.8393325874165112, + "lpips": 0.05822025239467621 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.49609375, + "history_latent_mse": 0.769873321056366, + "psnr": 11.705201931620337, + "ssim": 0.3266116503057603, + "lpips": 0.45681723952293396 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.6953125, + "history_latent_mse": 0.14924362301826477, + "psnr": 19.42610056388248, + "ssim": 0.6784066449129437, + "lpips": 0.08655546605587006 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.765625, + "history_latent_mse": 0.19174079596996307, + "psnr": 30.12903701779924, + "ssim": 0.9290899994233911, + "lpips": 0.19087469577789307 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.99853515625, + "history_latent_mse": 0.02691924013197422, + "psnr": 45.814633800868535, + "ssim": 0.9881413469323787, + "lpips": 0.020711563527584076 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 270, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.5546875, + "history_latent_mse": 0.10069779306650162, + "psnr": 23.85195741122631, + "ssim": 0.7995002707250378, + "lpips": 0.07158798724412918 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.26171875, + "history_latent_mse": 0.7238437533378601, + "psnr": 11.844675236200219, + "ssim": 0.4002364522725105, + "lpips": 0.4265420436859131 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.4296875, + "history_latent_mse": 0.1967291384935379, + "psnr": 18.562017105401367, + "ssim": 0.6423294034995817, + "lpips": 0.10178844630718231 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.875, + "history_latent_mse": 0.14837031066417694, + "psnr": 35.806559206981504, + "ssim": 0.9671508288212802, + "lpips": 0.10184541344642639 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.1787109375, + "history_latent_mse": 0.03450794890522957, + "psnr": 43.54366550296782, + "ssim": 0.9855874672044546, + "lpips": 0.036848120391368866 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 278, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.546875, + "history_latent_mse": 0.11091525107622147, + "psnr": 24.004649356844574, + "ssim": 0.7966355665473962, + "lpips": 0.07679988443851471 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.68299800157547, + "psnr": 11.7613169032568, + "ssim": 0.4097034452986453, + "lpips": 0.43709826469421387 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.640625, + "history_latent_mse": 0.22867286205291748, + "psnr": 17.750020132978467, + "ssim": 0.6164277478990445, + "lpips": 0.13044783473014832 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.609375, + "history_latent_mse": 0.1446436196565628, + "psnr": 27.948190906873066, + "ssim": 0.9134572848821165, + "lpips": 0.3038695454597473 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.373046875, + "history_latent_mse": 0.045348186045885086, + "psnr": 42.306728901512436, + "ssim": 0.9818618972453651, + "lpips": 0.05488590523600578 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 286, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.796875, + "history_latent_mse": 0.1163577064871788, + "psnr": 22.47124803152844, + "ssim": 0.7620968860241726, + "lpips": 0.11789532005786896 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.900390625, + "history_latent_mse": 0.6821216940879822, + "psnr": 11.262878325577226, + "ssim": 0.32177303226422693, + "lpips": 0.5545772910118103 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.46875, + "history_latent_mse": 0.3025282025337219, + "psnr": 15.965412993652466, + "ssim": 0.5780602587260784, + "lpips": 0.230415478348732 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.9921875, + "history_latent_mse": 0.31402844190597534, + "psnr": 31.869456102837994, + "ssim": 0.9380825262366355, + "lpips": 0.2047765851020813 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.228515625, + "history_latent_mse": 0.052217308431863785, + "psnr": 40.97729132068973, + "ssim": 0.9792064085226156, + "lpips": 0.0739535391330719 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 294, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.01171875, + "history_latent_mse": 0.1660754531621933, + "psnr": 21.540056109047686, + "ssim": 0.7272121153988813, + "lpips": 0.14757977426052094 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.578125, + "history_latent_mse": 0.9128252863883972, + "psnr": 11.47047461740097, + "ssim": 0.30778240207986063, + "lpips": 0.5249556303024292 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.71875, + "history_latent_mse": 0.4028487205505371, + "psnr": 15.067657198299475, + "ssim": 0.5617287941662996, + "lpips": 0.3014380931854248 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8359375, + "history_latent_mse": 0.19198690354824066, + "psnr": 37.0504156321183, + "ssim": 0.962663102343449, + "lpips": 0.09624835848808289 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.37548828125, + "history_latent_mse": 0.0604068785905838, + "psnr": 39.924448936369366, + "ssim": 0.9768579765169199, + "lpips": 0.09258417785167694 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 302, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.19921875, + "history_latent_mse": 0.17928554117679596, + "psnr": 21.096733242670588, + "ssim": 0.7059633364054707, + "lpips": 0.15332575142383575 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.546875, + "history_latent_mse": 0.8675854802131653, + "psnr": 10.813820183120535, + "ssim": 0.2624991159728946, + "lpips": 0.5749101042747498 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.4765625, + "history_latent_mse": 0.41944554448127747, + "psnr": 14.823632764928785, + "ssim": 0.546739921271526, + "lpips": 0.32546621561050415 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.162109375, + "history_latent_mse": 0.09312323480844498, + "psnr": 30.816213828884223, + "ssim": 0.9265774235065023, + "lpips": 0.2373543381690979 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.408203125, + "history_latent_mse": 0.06636345386505127, + "psnr": 39.294905632287474, + "ssim": 0.973885508256321, + "lpips": 0.11613824963569641 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 310, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6328125, + "history_latent_mse": 0.20221467316150665, + "psnr": 20.922290576785258, + "ssim": 0.7036680945235159, + "lpips": 0.16210044920444489 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.9512791633605957, + "psnr": 10.089184114935655, + "ssim": 0.2703140505870954, + "lpips": 0.5560033917427063 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.375, + "history_latent_mse": 0.4388410747051239, + "psnr": 14.310735757595554, + "ssim": 0.5100947719082833, + "lpips": 0.3630756139755249 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.515625, + "history_latent_mse": 0.20001985132694244, + "psnr": 32.87999346923638, + "ssim": 0.9418147167781654, + "lpips": 0.20846149325370789 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.47265625, + "history_latent_mse": 0.07635728269815445, + "psnr": 38.62656847354744, + "ssim": 0.9712397778586223, + "lpips": 0.13548386096954346 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 318, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 0.21223938465118408, + "psnr": 20.7731838937849, + "ssim": 0.6959392802473751, + "lpips": 0.17094674706459045 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.109375, + "history_latent_mse": 0.9865174889564514, + "psnr": 10.257317358368812, + "ssim": 0.27152351204212, + "lpips": 0.5460131168365479 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.125, + "history_latent_mse": 0.513868510723114, + "psnr": 13.910189074104856, + "ssim": 0.49246216499060247, + "lpips": 0.3760426342487335 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.9765625, + "history_latent_mse": 0.18698781728744507, + "psnr": 26.838834804039298, + "ssim": 0.8905632674979225, + "lpips": 0.3378138542175293 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.703125, + "history_latent_mse": 0.08435074239969254, + "psnr": 37.57432826927823, + "ssim": 0.968203491067738, + "lpips": 0.1566171944141388 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 326, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.546875, + "history_latent_mse": 0.21525204181671143, + "psnr": 20.615004161017264, + "ssim": 0.6899218839296365, + "lpips": 0.17407608032226562 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.828125, + "history_latent_mse": 0.9760610461235046, + "psnr": 10.611070254677255, + "ssim": 0.3370428584644505, + "lpips": 0.5099510550498962 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.61328125, + "history_latent_mse": 0.4933498799800873, + "psnr": 13.714209461493835, + "ssim": 0.49060955162134434, + "lpips": 0.3900112211704254 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.26953125, + "history_latent_mse": 0.24737699329853058, + "psnr": 32.84371445031275, + "ssim": 0.9528876540209911, + "lpips": 0.15910649299621582 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.796875, + "history_latent_mse": 0.09140972048044205, + "psnr": 36.792551445389606, + "ssim": 0.965090161744937, + "lpips": 0.17435982823371887 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 334, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.015625, + "history_latent_mse": 0.22390036284923553, + "psnr": 20.936774485081816, + "ssim": 0.6901082527662892, + "lpips": 0.19902464747428894 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.9296875, + "history_latent_mse": 0.8181533217430115, + "psnr": 11.986492283430497, + "ssim": 0.3276161174014442, + "lpips": 0.4934006631374359 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.296875, + "history_latent_mse": 0.6053687930107117, + "psnr": 14.19300944640209, + "ssim": 0.5471067363541945, + "lpips": 0.37414366006851196 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.71484375, + "history_latent_mse": 0.14907339215278625, + "psnr": 35.58189141241811, + "ssim": 0.9601521443612173, + "lpips": 0.13156986236572266 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.78125, + "history_latent_mse": 0.10063919425010681, + "psnr": 36.181377918737894, + "ssim": 0.9619490665384449, + "lpips": 0.1956077516078949 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 342, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.625, + "history_latent_mse": 0.253597229719162, + "psnr": 21.07719894216505, + "ssim": 0.689939200965541, + "lpips": 0.1993277370929718 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.6807262301445007, + "psnr": 13.591770036297534, + "ssim": 0.2601015329112619, + "lpips": 0.5946899056434631 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.25, + "history_latent_mse": 0.6496118307113647, + "psnr": 13.929625065262437, + "ssim": 0.5338751857299172, + "lpips": 0.3498308062553406 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.6416015625, + "history_latent_mse": 0.0811961367726326, + "psnr": 29.97683382666193, + "ssim": 0.9292874005651764, + "lpips": 0.22914019227027893 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.734375, + "history_latent_mse": 0.11122220009565353, + "psnr": 35.70224335088462, + "ssim": 0.9584138095744668, + "lpips": 0.21002613008022308 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 350, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.23828125, + "history_latent_mse": 0.2163238525390625, + "psnr": 21.22820523343357, + "ssim": 0.6983207396008659, + "lpips": 0.19284312427043915 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.21875, + "history_latent_mse": 1.0081082582473755, + "psnr": 12.676828872390278, + "ssim": 0.24277204818411496, + "lpips": 0.5822117328643799 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.140625, + "history_latent_mse": 0.4972709119319916, + "psnr": 13.713293537261379, + "ssim": 0.516658899509432, + "lpips": 0.3576262295246124 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.5625, + "history_latent_mse": 0.1940355747938156, + "psnr": 26.69610910824879, + "ssim": 0.8983222727977823, + "lpips": 0.3241650462150574 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8671875, + "history_latent_mse": 0.11973786354064941, + "psnr": 35.268247257179254, + "ssim": 0.9547754358754289, + "lpips": 0.22233764827251434 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 358, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.1875, + "history_latent_mse": 0.21211999654769897, + "psnr": 21.034397626285582, + "ssim": 0.6913025623496647, + "lpips": 0.19303707778453827 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5625, + "history_latent_mse": 0.821051299571991, + "psnr": 12.197060524398882, + "ssim": 0.2886713112562056, + "lpips": 0.6050335168838501 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.3125, + "history_latent_mse": 0.513745129108429, + "psnr": 13.881312754421955, + "ssim": 0.5201492304643792, + "lpips": 0.36251479387283325 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.2626953125, + "history_latent_mse": 0.2522940933704376, + "psnr": 30.71490610549273, + "ssim": 0.9447489617158062, + "lpips": 0.19720987975597382 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.828125, + "history_latent_mse": 0.1282929927110672, + "psnr": 34.700039507429985, + "ssim": 0.95206767732851, + "lpips": 0.2371719479560852 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0eba15ca54cfdc52ad5f848b3aea2b6e340fc1c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce77e6a3106a6163138b70b3b26804b7d12837d23a52a5c9f44161939489e0e +size 1308275 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..d45a5908d1e91f6904da5afc4768cb638b33d3cd --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.04854125902056694, + "wrist/video/latent_future_mse": 0.04854125902056694, + "video/psnr": 30.795292763143625, + "wrist/video/psnr": 30.795292763143625, + "video/ssim": 0.9375067540670683, + "wrist/video/ssim": 0.9375067540670683, + "video/lpips": 0.02425140142440796, + "wrist/video/lpips": 0.02425140142440796, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.6281946897506714, + "wrist/wrist_left/latent_future_mse": 0.6281946897506714, + "wrist_left/psnr": 14.92759718890451, + "wrist/wrist_left/psnr": 14.92759718890451, + "wrist_left/ssim": 0.4136994091358832, + "wrist/wrist_left/ssim": 0.4136994091358832, + "wrist_left/lpips": 0.3324988782405853, + "wrist/wrist_left/lpips": 0.3324988782405853, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.10491997003555298, + "wrist/wrist_right/latent_future_mse": 0.10491997003555298, + "wrist_right/psnr": 23.87380535539645, + "wrist/wrist_right/psnr": 23.87380535539645, + "wrist_right/ssim": 0.84858913143935, + "wrist/wrist_right/ssim": 0.84858913143935, + "wrist_right/lpips": 0.03306284919381142, + "wrist/wrist_right/lpips": 0.03306284919381142, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.08559242635965347, + "wrist/tactile_left/latent_future_mse": 0.08559242635965347, + "tactile_left/psnr": 36.105213601449094, + "wrist/tactile_left/psnr": 36.105213601449094, + "tactile_left/ssim": 0.9805090412282181, + "wrist/tactile_left/ssim": 0.9805090412282181, + "tactile_left/lpips": 0.03081936575472355, + "wrist/tactile_left/lpips": 0.03081936575472355, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.014824005775153637, + "wrist/tactile_right/latent_future_mse": 0.014824005775153637, + "tactile_right/psnr": 50.57606534665488, + "wrist/tactile_right/psnr": 50.57606534665488, + "tactile_right/ssim": 0.9945805941139491, + "wrist/tactile_right/ssim": 0.9945805941139491, + "tactile_right/lpips": 0.004185985773801804, + "wrist/tactile_right/lpips": 0.004185985773801804, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 246, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.04854125902056694, + "psnr": 30.795292763143625, + "ssim": 0.9375067540670683, + "lpips": 0.02425140142440796 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.6281946897506714, + "psnr": 14.92759718890451, + "ssim": 0.4136994091358832, + "lpips": 0.3324988782405853 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.10491997003555298, + "psnr": 23.87380535539645, + "ssim": 0.84858913143935, + "lpips": 0.03306284919381142 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08559242635965347, + "psnr": 36.105213601449094, + "ssim": 0.9805090412282181, + "lpips": 0.03081936575472355 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.014824005775153637, + "psnr": 50.57606534665488, + "ssim": 0.9945805941139491, + "lpips": 0.004185985773801804 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/case.json new file mode 100644 index 0000000000000000000000000000000000000000..a3ae31a282633c551bfe268fd595d901af12efb7 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/case.json @@ -0,0 +1,164 @@ +{ + "task": 15, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "full", + "index": 9819, + "t_start": 432, + "start_seconds": 28.8, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.8359375, + "long_future_contact_fraction": 0.825, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "15_motherboard_0911_episode_008_seg00_full", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c37eabddfaaee0648dd1270d45ef0c8969ead15 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35a2a168515c4a2178a7f07600a58c980258a92cd9163e41cf95c2041ffcea07 +size 11035287 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/motherboard_0911_episode_008_seg00_t00432_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/motherboard_0911_episode_008_seg00_t00432_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b62cc81c8f4cf3b15514068675680b2663731e2 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/motherboard_0911_episode_008_seg00_t00432_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3165dc41d909ee6986e67f4626261f964820216e56081b0e1f6ed171af4a48da +size 1978961 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..44d47fe8f5a07cda7c6bfe0f0bf85c99635b2336 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 432, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 432, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 30.21213249615335, + "ssim": 0.9246832556443492, + "lpips": 0.024201437830924988 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 18.684407706007228, + "ssim": 0.5440686808790238, + "lpips": 0.12226273119449615 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 15.944406431715628, + "ssim": 0.5542845741330719, + "lpips": 0.2139267772436142 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.555634707851148, + "ssim": 0.9547180164659093, + "lpips": 0.06950037181377411 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 44.28470445873507, + "ssim": 0.9918869235875629, + "lpips": 0.018876174464821815 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 440, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.9453125, + "history_latent_mse": 0.05137583613395691, + "psnr": 24.364651041284446, + "ssim": 0.8060828668563819, + "lpips": 0.06254396587610245 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.484375, + "history_latent_mse": 0.29738256335258484, + "psnr": 17.25114879934155, + "ssim": 0.4220494305243355, + "lpips": 0.1847894936800003 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.576171875, + "history_latent_mse": 0.525972306728363, + "psnr": 13.585565144075462, + "ssim": 0.47470399929473994, + "lpips": 0.3704395294189453 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.75390625, + "history_latent_mse": 0.0935300961136818, + "psnr": 29.22952682335937, + "ssim": 0.9418486979563321, + "lpips": 0.1145332008600235 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.984375, + "history_latent_mse": 0.027908844873309135, + "psnr": 38.87891584589907, + "ssim": 0.9836002696569228, + "lpips": 0.047796983271837234 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 448, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.6484375, + "history_latent_mse": 0.1027163490653038, + "psnr": 23.69443487905209, + "ssim": 0.7900967680737938, + "lpips": 0.08044761419296265 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.32421875, + "history_latent_mse": 0.3418722152709961, + "psnr": 16.449363028808104, + "ssim": 0.3962415859166461, + "lpips": 0.2285427749156952 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.625, + "history_latent_mse": 0.5316155552864075, + "psnr": 13.452106225172752, + "ssim": 0.38125716277077837, + "lpips": 0.3895080089569092 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.80078125, + "history_latent_mse": 0.10748186707496643, + "psnr": 30.738984251269166, + "ssim": 0.9461426204622629, + "lpips": 0.10732017457485199 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.12109375, + "history_latent_mse": 0.042012955993413925, + "psnr": 35.72101496678856, + "ssim": 0.9744278752323398, + "lpips": 0.10463118553161621 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 456, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.16796875, + "history_latent_mse": 0.12197861820459366, + "psnr": 22.641107060767286, + "ssim": 0.7509822406536326, + "lpips": 0.1145503968000412 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.40625, + "history_latent_mse": 0.42062506079673767, + "psnr": 15.61252651500605, + "ssim": 0.36400256289686217, + "lpips": 0.31129032373428345 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.1171875, + "history_latent_mse": 0.574642539024353, + "psnr": 12.29986218357475, + "ssim": 0.3170054196600638, + "lpips": 0.44422677159309387 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.0, + "history_latent_mse": 0.11462289839982986, + "psnr": 32.450123427823115, + "ssim": 0.9553248959651276, + "lpips": 0.11299257725477219 + }, + "tactile_right": { + "history_latent_max_abs_error": 3.0, + "history_latent_mse": 0.08685940504074097, + "psnr": 35.38470930936101, + "ssim": 0.9691531384058444, + "lpips": 0.1247095912694931 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 464, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.453125, + "history_latent_mse": 0.16741520166397095, + "psnr": 22.279985825421242, + "ssim": 0.7353212294614044, + "lpips": 0.13499289751052856 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.4375, + "history_latent_mse": 0.5514844059944153, + "psnr": 15.064636232234497, + "ssim": 0.34474392822405076, + "lpips": 0.3607896566390991 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.30078125, + "history_latent_mse": 0.6663729548454285, + "psnr": 13.021445123592073, + "ssim": 0.3610671641951063, + "lpips": 0.39925771951675415 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.859375, + "history_latent_mse": 0.10491734743118286, + "psnr": 30.642190172021067, + "ssim": 0.9466048524781072, + "lpips": 0.1511172652244568 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.9453125, + "history_latent_mse": 0.08641978353261948, + "psnr": 41.9422891501243, + "ssim": 0.9814353442184081, + "lpips": 0.05738411843776703 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 472, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.75, + "history_latent_mse": 0.17366069555282593, + "psnr": 21.859112600560373, + "ssim": 0.7190106191247365, + "lpips": 0.15566544234752655 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.296875, + "history_latent_mse": 0.5064476132392883, + "psnr": 14.412417876458974, + "ssim": 0.3089958852046664, + "lpips": 0.42185723781585693 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.8671875, + "history_latent_mse": 0.5888368487358093, + "psnr": 12.981270240533187, + "ssim": 0.30727396192581957, + "lpips": 0.40292978286743164 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.375, + "history_latent_mse": 0.12242767214775085, + "psnr": 29.427338552486717, + "ssim": 0.9275429179290519, + "lpips": 0.23556312918663025 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4432830810546875, + "history_latent_mse": 0.05428663268685341, + "psnr": 33.87229581083544, + "ssim": 0.9609023042169704, + "lpips": 0.18177926540374756 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 480, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.5625, + "history_latent_mse": 0.1978570818901062, + "psnr": 21.136783704852405, + "ssim": 0.6957633563268116, + "lpips": 0.18529056012630463 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8671875, + "history_latent_mse": 0.6348164677619934, + "psnr": 13.469337108065588, + "ssim": 0.288367167084153, + "lpips": 0.46178102493286133 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.921875, + "history_latent_mse": 0.7140262126922607, + "psnr": 13.274121259882454, + "ssim": 0.39912830706066105, + "lpips": 0.4051205515861511 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.73828125, + "history_latent_mse": 0.13431328535079956, + "psnr": 33.91056380860551, + "ssim": 0.9590751084476055, + "lpips": 0.1456911861896515 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.796875, + "history_latent_mse": 0.0991414412856102, + "psnr": 35.83597151476065, + "ssim": 0.968526922044863, + "lpips": 0.1370614618062973 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 488, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.02734375, + "history_latent_mse": 0.2216518670320511, + "psnr": 21.10808154330912, + "ssim": 0.693922477256284, + "lpips": 0.19438588619232178 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.734375, + "history_latent_mse": 0.6845539212226868, + "psnr": 13.462811446037403, + "ssim": 0.31061457292079286, + "lpips": 0.4496174454689026 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.015625, + "history_latent_mse": 0.5746472477912903, + "psnr": 13.679333040445847, + "ssim": 0.4629934977558302, + "lpips": 0.35633403062820435 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.3984375, + "history_latent_mse": 0.14651823043823242, + "psnr": 28.39541011836237, + "ssim": 0.9263061610173249, + "lpips": 0.2588419020175934 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.8203125, + "history_latent_mse": 0.10213237255811691, + "psnr": 39.02271904323471, + "ssim": 0.9746917357372453, + "lpips": 0.10082122683525085 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 496, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.09375, + "history_latent_mse": 0.21827639639377594, + "psnr": 20.978289279467333, + "ssim": 0.6826171975710389, + "lpips": 0.20011377334594727 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.859375, + "history_latent_mse": 0.7079176902770996, + "psnr": 13.411811465288084, + "ssim": 0.29490664953369927, + "lpips": 0.43370771408081055 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.0859375, + "history_latent_mse": 0.517494797706604, + "psnr": 13.83321260521333, + "ssim": 0.4784831297053601, + "lpips": 0.3492093086242676 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.2890625, + "history_latent_mse": 0.25139379501342773, + "psnr": 29.582021222103517, + "ssim": 0.9272187744283609, + "lpips": 0.23934835195541382 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.380859375, + "history_latent_mse": 0.07554133981466293, + "psnr": 38.60208663163579, + "ssim": 0.9724131644140747, + "lpips": 0.11732124537229538 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 504, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.984375, + "history_latent_mse": 0.22561222314834595, + "psnr": 20.851251747901358, + "ssim": 0.6711938394676227, + "lpips": 0.208816260099411 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.328125, + "history_latent_mse": 0.7710349559783936, + "psnr": 13.397783222882616, + "ssim": 0.2777774320431105, + "lpips": 0.45713409781455994 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.15625, + "history_latent_mse": 0.505803644657135, + "psnr": 14.177602396441713, + "ssim": 0.4991443121241438, + "lpips": 0.34690502285957336 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.75, + "history_latent_mse": 0.1896268129348755, + "psnr": 29.70968755645056, + "ssim": 0.9308248242648756, + "lpips": 0.22612211108207703 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.5126953125, + "history_latent_mse": 0.08320692181587219, + "psnr": 37.46886958699639, + "ssim": 0.9694947598749969, + "lpips": 0.14243169128894806 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 512, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.953125, + "history_latent_mse": 0.2354394942522049, + "psnr": 20.858647274871927, + "ssim": 0.6644922836252278, + "lpips": 0.2115623950958252 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.28125, + "history_latent_mse": 0.7327775359153748, + "psnr": 13.651630791851126, + "ssim": 0.305248766749302, + "lpips": 0.44661086797714233 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.39453125, + "history_latent_mse": 0.44463133811950684, + "psnr": 14.300757539800053, + "ssim": 0.5037658248734309, + "lpips": 0.3462887406349182 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.859375, + "history_latent_mse": 0.1623835414648056, + "psnr": 29.71288114085565, + "ssim": 0.9379595896627646, + "lpips": 0.19895273447036743 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.587890625, + "history_latent_mse": 0.09178026765584946, + "psnr": 36.645904446724344, + "ssim": 0.966685255675811, + "lpips": 0.16053955256938934 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 520, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.78125, + "history_latent_mse": 0.23255181312561035, + "psnr": 20.76283136335868, + "ssim": 0.6566986821706188, + "lpips": 0.2161010205745697 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.84375, + "history_latent_mse": 0.700290858745575, + "psnr": 11.389530406380544, + "ssim": 0.3614107560599106, + "lpips": 0.4683923125267029 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.36328125, + "history_latent_mse": 0.46620845794677734, + "psnr": 14.235885736236916, + "ssim": 0.4862983718624966, + "lpips": 0.34109100699424744 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.7265625, + "history_latent_mse": 0.16775840520858765, + "psnr": 31.69778696426901, + "ssim": 0.941159799736913, + "lpips": 0.19120299816131592 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7001953125, + "history_latent_mse": 0.1000535786151886, + "psnr": 35.82251246671312, + "ssim": 0.9634578169266382, + "lpips": 0.17808300256729126 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 528, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.841796875, + "history_latent_mse": 0.2387595772743225, + "psnr": 20.435035034584544, + "ssim": 0.652258685149388, + "lpips": 0.23272806406021118 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.578125, + "history_latent_mse": 0.8430356383323669, + "psnr": 10.368462116337236, + "ssim": 0.3514245202945303, + "lpips": 0.5471752882003784 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.4296875, + "history_latent_mse": 0.5122777819633484, + "psnr": 13.630570755694594, + "ssim": 0.44448161224611016, + "lpips": 0.3608173429965973 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.34375, + "history_latent_mse": 0.15788568556308746, + "psnr": 29.355858440038716, + "ssim": 0.9317106383026895, + "lpips": 0.19475600123405457 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.78125, + "history_latent_mse": 0.116440050303936, + "psnr": 35.28706589750516, + "ssim": 0.9602581160363364, + "lpips": 0.19294187426567078 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 536, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.16015625, + "history_latent_mse": 0.2598450481891632, + "psnr": 20.150557091945572, + "ssim": 0.6487164808313055, + "lpips": 0.25214967131614685 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.15234375, + "history_latent_mse": 0.9366918206214905, + "psnr": 8.684117474787039, + "ssim": 0.31020627841230947, + "lpips": 0.5948623418807983 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6328125, + "history_latent_mse": 0.536710798740387, + "psnr": 13.367459731548399, + "ssim": 0.4365411680403753, + "lpips": 0.38244712352752686 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.890625, + "history_latent_mse": 0.17527128756046295, + "psnr": 31.879434026601977, + "ssim": 0.9379980885518373, + "lpips": 0.20764443278312683 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.611083984375, + "history_latent_mse": 0.11437837034463882, + "psnr": 34.9211870486588, + "ssim": 0.9571673166997581, + "lpips": 0.21038268506526947 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 544, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.30859375, + "history_latent_mse": 0.2484091967344284, + "psnr": 20.163866477256644, + "ssim": 0.645072902113442, + "lpips": 0.24850311875343323 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.109375, + "history_latent_mse": 1.1547969579696655, + "psnr": 8.457836496099837, + "ssim": 0.28147571545420236, + "lpips": 0.6167376637458801 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.203125, + "history_latent_mse": 0.5440744757652283, + "psnr": 13.150410179470475, + "ssim": 0.4119539058579889, + "lpips": 0.38977089524269104 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8671875, + "history_latent_mse": 0.16916656494140625, + "psnr": 28.202087570441915, + "ssim": 0.8917061436838378, + "lpips": 0.3483405113220215 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.701171875, + "history_latent_mse": 0.12363016605377197, + "psnr": 34.43463004955464, + "ssim": 0.9544240574757641, + "lpips": 0.23093026876449585 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..893da3f44cf7b29d8d6f1fe2e8fe3724ba4a9d72 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c535a192557927cf2adf0faa1f2e839c84d0b51450d2d03d272b101484bcbfe +size 1488940 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..da3656ed484c631b2ccb25b8828037148421655b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.05833423510193825, + "wrist/video/latent_future_mse": 0.05833423510193825, + "video/psnr": 30.21213249615335, + "wrist/video/psnr": 30.21213249615335, + "video/ssim": 0.9246832556443492, + "wrist/video/ssim": 0.9246832556443492, + "video/lpips": 0.024201437830924988, + "wrist/video/lpips": 0.024201437830924988, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.33938708901405334, + "wrist/wrist_left/latent_future_mse": 0.33938708901405334, + "wrist_left/psnr": 18.684407706007228, + "wrist/wrist_left/psnr": 18.684407706007228, + "wrist_left/ssim": 0.5440686808790238, + "wrist/wrist_left/ssim": 0.5440686808790238, + "wrist_left/lpips": 0.12226273119449615, + "wrist/wrist_left/lpips": 0.12226273119449615, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.5721608400344849, + "wrist/wrist_right/latent_future_mse": 0.5721608400344849, + "wrist_right/psnr": 15.944406431715628, + "wrist/wrist_right/psnr": 15.944406431715628, + "wrist_right/ssim": 0.5542845741330719, + "wrist/wrist_right/ssim": 0.5542845741330719, + "wrist_right/lpips": 0.2139267772436142, + "wrist/wrist_right/lpips": 0.2139267772436142, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.1147889792919159, + "wrist/tactile_left/latent_future_mse": 0.1147889792919159, + "tactile_left/psnr": 31.555634707851148, + "wrist/tactile_left/psnr": 31.555634707851148, + "tactile_left/ssim": 0.9547180164659093, + "wrist/tactile_left/ssim": 0.9547180164659093, + "tactile_left/lpips": 0.06950037181377411, + "wrist/tactile_left/lpips": 0.06950037181377411, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.025513650849461555, + "wrist/tactile_right/latent_future_mse": 0.025513650849461555, + "tactile_right/psnr": 44.28470445873507, + "wrist/tactile_right/psnr": 44.28470445873507, + "tactile_right/ssim": 0.9918869235875629, + "wrist/tactile_right/ssim": 0.9918869235875629, + "tactile_right/lpips": 0.018876174464821815, + "wrist/tactile_right/lpips": 0.018876174464821815, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0911_episode_008_seg00.pt", + "t_start": 432, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.05833423510193825, + "psnr": 30.21213249615335, + "ssim": 0.9246832556443492, + "lpips": 0.024201437830924988 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.33938708901405334, + "psnr": 18.684407706007228, + "ssim": 0.5440686808790238, + "lpips": 0.12226273119449615 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.5721608400344849, + "psnr": 15.944406431715628, + "ssim": 0.5542845741330719, + "lpips": 0.2139267772436142 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.1147889792919159, + "psnr": 31.555634707851148, + "ssim": 0.9547180164659093, + "lpips": 0.06950037181377411 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.025513650849461555, + "psnr": 44.28470445873507, + "ssim": 0.9918869235875629, + "lpips": 0.018876174464821815 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/case.json new file mode 100644 index 0000000000000000000000000000000000000000..f281adf8de43bdec96e9b8e17878459973ddd095 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/case.json @@ -0,0 +1,164 @@ +{ + "task": 16, + "episode": "motherboard_0912_episode_002.pt", + "category": "none", + "index": 11187, + "t_start": 750, + "start_seconds": 50.0, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.3671875, + "long_future_contact_fraction": 0.39166666666666666, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 65, + "long_label": "none_at_start", + "output_subdir": "16_motherboard_0912_episode_002_none", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df12c20a0ac6b1c775d3c5382496f0054e58d5aa --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9378753c150da626bea3e808d81af14bcc0b301c49476f6c78cb91c5f78196b +size 7879886 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/motherboard_0912_episode_002_t00750_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/motherboard_0912_episode_002_t00750_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0b02985457462d186f272924be014e47b65e7da --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/motherboard_0912_episode_002_t00750_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f651447a31ae84e0a0b7c3fedde1fd7c3ebe7c15dd7cce8ae7a97e98f6c37d7a +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..18af0bb03108e543582afe8ab71f670106a42988 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 750, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 750, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 29.826995303462084, + "ssim": 0.9339680561401201, + "lpips": 0.02761567011475563 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 22.64956943983406, + "ssim": 0.8456670242005284, + "lpips": 0.06932871043682098 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 31.419793533688743, + "ssim": 0.933000893367617, + "lpips": 0.03438973426818848 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 50.66471949450162, + "ssim": 0.9947452134317746, + "lpips": 0.0034115284215658903 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 52.71458013071562, + "ssim": 0.9957780256909837, + "lpips": 0.002500636503100395 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 758, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.1875, + "history_latent_mse": 0.04864555597305298, + "psnr": 27.707981050606403, + "ssim": 0.9181209580844167, + "lpips": 0.033478353172540665 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.8203125, + "history_latent_mse": 0.1749914437532425, + "psnr": 21.899081415616543, + "ssim": 0.8284916441904009, + "lpips": 0.0779583603143692 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.0546875, + "history_latent_mse": 0.11388418823480606, + "psnr": 27.45038880388052, + "ssim": 0.8856643366906327, + "lpips": 0.05666637048125267 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.859375, + "history_latent_mse": 0.010176676325500011, + "psnr": 47.879266053951156, + "ssim": 0.9919642418622279, + "lpips": 0.010868743062019348 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.626953125, + "history_latent_mse": 0.010089843533933163, + "psnr": 48.22762961296075, + "ssim": 0.9923939619494855, + "lpips": 0.009737813845276833 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 766, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.26953125, + "history_latent_mse": 0.04962323606014252, + "psnr": 26.67862387330228, + "ssim": 0.9023480392367714, + "lpips": 0.043137602508068085 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.3046875, + "history_latent_mse": 0.18309813737869263, + "psnr": 21.9905352493044, + "ssim": 0.8265381959221811, + "lpips": 0.07596143335103989 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.8984375, + "history_latent_mse": 0.14479416608810425, + "psnr": 24.741351176519004, + "ssim": 0.8565341714520278, + "lpips": 0.07703709602355957 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.958984375, + "history_latent_mse": 0.019939951598644257, + "psnr": 44.924809984090324, + "ssim": 0.9885502517594265, + "lpips": 0.022517435252666473 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.953125, + "history_latent_mse": 0.024447420611977577, + "psnr": 44.43750031471233, + "ssim": 0.9890563525874748, + "lpips": 0.02383711189031601 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 774, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 1.828125, + "history_latent_mse": 0.05574532970786095, + "psnr": 26.30466969269537, + "ssim": 0.8869323542827217, + "lpips": 0.05045442283153534 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.140625, + "history_latent_mse": 0.17984478175640106, + "psnr": 21.796328097743796, + "ssim": 0.82046335736382, + "lpips": 0.07776239514350891 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.71875, + "history_latent_mse": 0.18316014111042023, + "psnr": 23.354930103919344, + "ssim": 0.8345857138515751, + "lpips": 0.09495078027248383 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.970703125, + "history_latent_mse": 0.027789102867245674, + "psnr": 42.322659794186634, + "ssim": 0.9849255003815416, + "lpips": 0.03843323513865471 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.0703125, + "history_latent_mse": 0.03191332146525383, + "psnr": 42.238458421745094, + "ssim": 0.9865079019063556, + "lpips": 0.040867701172828674 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 782, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.08984375, + "history_latent_mse": 0.07564754784107208, + "psnr": 25.698737021224115, + "ssim": 0.8595503431331534, + "lpips": 0.07741370797157288 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.11328125, + "history_latent_mse": 0.18468545377254486, + "psnr": 21.81107398773428, + "ssim": 0.8139424547246057, + "lpips": 0.09259389340877533 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.0546875, + "history_latent_mse": 0.21145224571228027, + "psnr": 23.070202483001047, + "ssim": 0.8208376938855952, + "lpips": 0.10455762594938278 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.26953125, + "history_latent_mse": 0.03778528794646263, + "psnr": 41.1171552685634, + "ssim": 0.9810035688377086, + "lpips": 0.05762293189764023 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.1171875, + "history_latent_mse": 0.041021596640348434, + "psnr": 40.822234845916626, + "ssim": 0.9833446268264754, + "lpips": 0.058111175894737244 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 790, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.390625, + "history_latent_mse": 0.11446201056241989, + "psnr": 25.198680594630854, + "ssim": 0.8368793189293726, + "lpips": 0.09105303138494492 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.1171875, + "history_latent_mse": 0.2167205810546875, + "psnr": 21.43278902898363, + "ssim": 0.804303497759479, + "lpips": 0.12975770235061646 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.78125, + "history_latent_mse": 0.2379438430070877, + "psnr": 21.813456800605508, + "ssim": 0.7861085697056465, + "lpips": 0.1349715292453766 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.236328125, + "history_latent_mse": 0.04420987889170647, + "psnr": 39.942549402914935, + "ssim": 0.9772879241101499, + "lpips": 0.0822947770357132 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.150390625, + "history_latent_mse": 0.051836904138326645, + "psnr": 39.55355679318795, + "ssim": 0.9801623796439118, + "lpips": 0.07793854922056198 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 798, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.71875, + "history_latent_mse": 0.14397628605365753, + "psnr": 24.459207204201302, + "ssim": 0.8041127808756462, + "lpips": 0.09922066330909729 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.3125, + "history_latent_mse": 0.2633431553840637, + "psnr": 20.633038495358903, + "ssim": 0.8019950840257716, + "lpips": 0.17476040124893188 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.953125, + "history_latent_mse": 0.301511287689209, + "psnr": 20.703122088714466, + "ssim": 0.7333931935749526, + "lpips": 0.19576610624790192 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.8671875, + "history_latent_mse": 0.053842782974243164, + "psnr": 38.50986533399325, + "ssim": 0.9709311130135954, + "lpips": 0.11889180541038513 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.3515625, + "history_latent_mse": 0.05711442604660988, + "psnr": 38.36542740990372, + "ssim": 0.9773771758195795, + "lpips": 0.1019149050116539 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 806, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.53515625, + "history_latent_mse": 0.1434130221605301, + "psnr": 23.881746238363874, + "ssim": 0.7801062528274876, + "lpips": 0.11063408851623535 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.013671875, + "history_latent_mse": 0.27097198367118835, + "psnr": 20.006445740152476, + "ssim": 0.798270313761813, + "lpips": 0.1828778088092804 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.1953125, + "history_latent_mse": 0.36307093501091003, + "psnr": 16.28819743783243, + "ssim": 0.526816505567135, + "lpips": 0.33002012968063354 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.94921875, + "history_latent_mse": 0.07560177147388458, + "psnr": 37.74806671163972, + "ssim": 0.9695157458249548, + "lpips": 0.11777397990226746 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.50244140625, + "history_latent_mse": 0.06437096744775772, + "psnr": 37.841038221194, + "ssim": 0.9743875182473383, + "lpips": 0.1205713152885437 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 814, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.140625, + "history_latent_mse": 0.16399309039115906, + "psnr": 22.439369450642957, + "ssim": 0.7395598634784508, + "lpips": 0.14761647582054138 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.7734375, + "history_latent_mse": 0.2715536653995514, + "psnr": 10.032163474475833, + "ssim": 0.45987729883803785, + "lpips": 0.562286376953125 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.421875, + "history_latent_mse": 0.5362206697463989, + "psnr": 15.173302195159323, + "ssim": 0.6144030993293841, + "lpips": 0.36736857891082764 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.3984375, + "history_latent_mse": 0.06849171966314316, + "psnr": 37.30298764927452, + "ssim": 0.9681741739780829, + "lpips": 0.12978605926036835 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.35791015625, + "history_latent_mse": 0.0724043920636177, + "psnr": 37.21515263261903, + "ssim": 0.9714700762890314, + "lpips": 0.1394333839416504 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 822, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.140625, + "history_latent_mse": 0.1912621408700943, + "psnr": 21.450296705308123, + "ssim": 0.7034897241907673, + "lpips": 0.17263871431350708 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.59375, + "history_latent_mse": 1.077052116394043, + "psnr": 7.734682964491365, + "ssim": 0.2876212398176612, + "lpips": 0.7348428964614868 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.724609375, + "history_latent_mse": 0.48955655097961426, + "psnr": 16.679130907188362, + "ssim": 0.7215754748013226, + "lpips": 0.2813987135887146 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.44140625, + "history_latent_mse": 0.06652388721704483, + "psnr": 34.315023989265654, + "ssim": 0.9395286679853848, + "lpips": 0.2537141740322113 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.51953125, + "history_latent_mse": 0.08054038882255554, + "psnr": 36.45859786790579, + "ssim": 0.9685314027853389, + "lpips": 0.16292229294776917 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 830, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9921875, + "history_latent_mse": 0.26569411158561707, + "psnr": 19.440957506890253, + "ssim": 0.6565535651350105, + "lpips": 0.2805670201778412 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.78125, + "history_latent_mse": 1.4511905908584595, + "psnr": 7.566009458390971, + "ssim": 0.2806691345941018, + "lpips": 0.7298996448516846 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6171875, + "history_latent_mse": 0.4075835943222046, + "psnr": 18.841482061764204, + "ssim": 0.7932536684224123, + "lpips": 0.25248920917510986 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.75, + "history_latent_mse": 0.19186186790466309, + "psnr": 32.275291819824396, + "ssim": 0.9319094609222978, + "lpips": 0.2821759283542633 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.65673828125, + "history_latent_mse": 0.0891382098197937, + "psnr": 35.51440986354993, + "ssim": 0.9662258899218619, + "lpips": 0.17773160338401794 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 838, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.234375, + "history_latent_mse": 0.3391338884830475, + "psnr": 19.061334139709228, + "ssim": 0.6404813325451493, + "lpips": 0.3119100332260132 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.80078125, + "history_latent_mse": 1.6354403495788574, + "psnr": 7.178829047181692, + "ssim": 0.2562037935585332, + "lpips": 0.7320892214775085 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.72265625, + "history_latent_mse": 0.33452293276786804, + "psnr": 19.46034220193534, + "ssim": 0.8006957422511594, + "lpips": 0.26051878929138184 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.1640625, + "history_latent_mse": 0.22700518369674683, + "psnr": 32.01292245138755, + "ssim": 0.9186200209935421, + "lpips": 0.3239569365978241 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.890625, + "history_latent_mse": 0.11265446990728378, + "psnr": 34.805942958544385, + "ssim": 0.9606892206192988, + "lpips": 0.2153005599975586 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 846, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.328125, + "history_latent_mse": 0.3504157066345215, + "psnr": 18.92060853090943, + "ssim": 0.6478991509265035, + "lpips": 0.29250964522361755 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.296875, + "history_latent_mse": 1.7134093046188354, + "psnr": 7.5741785974441, + "ssim": 0.30932702425014647, + "lpips": 0.7232282161712646 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.125, + "history_latent_mse": 0.38261139392852783, + "psnr": 19.270039800936313, + "ssim": 0.805797928490656, + "lpips": 0.23856519162654877 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.09375, + "history_latent_mse": 0.21087199449539185, + "psnr": 32.764887421801895, + "ssim": 0.9288490716725647, + "lpips": 0.30027520656585693 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.26171875, + "history_latent_mse": 0.13109742105007172, + "psnr": 34.33276225824276, + "ssim": 0.9585332789092622, + "lpips": 0.2215556502342224 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 854, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.515625, + "history_latent_mse": 0.3637070953845978, + "psnr": 19.117099963181225, + "ssim": 0.6501088002086116, + "lpips": 0.26896753907203674 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.56640625, + "history_latent_mse": 1.5334199666976929, + "psnr": 7.898978710696939, + "ssim": 0.3163298717688059, + "lpips": 0.7054563760757446 + }, + "wrist_right": { + "history_latent_max_abs_error": 6.40625, + "history_latent_mse": 0.3532874584197998, + "psnr": 17.281598521837942, + "ssim": 0.7648084984478065, + "lpips": 0.25534868240356445 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.69921875, + "history_latent_mse": 0.26853346824645996, + "psnr": 33.86220016987813, + "ssim": 0.9466194054420227, + "lpips": 0.2387869656085968 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.1796875, + "history_latent_mse": 0.12665878236293793, + "psnr": 33.759639351877894, + "ssim": 0.9569759394370801, + "lpips": 0.22461268305778503 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 862, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.6953125, + "history_latent_mse": 0.30984482169151306, + "psnr": 19.010655415565637, + "ssim": 0.6419079599085697, + "lpips": 0.28134486079216003 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.15625, + "history_latent_mse": 1.4902859926223755, + "psnr": 7.143878986915761, + "ssim": 0.23936344239325683, + "lpips": 0.7713989019393921 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.216796875, + "history_latent_mse": 0.3702198266983032, + "psnr": 17.060653017164334, + "ssim": 0.741717273772007, + "lpips": 0.33610597252845764 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.1015625, + "history_latent_mse": 0.22869719564914703, + "psnr": 32.25943047652483, + "ssim": 0.9236261995806168, + "lpips": 0.2866414785385132 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8037109375, + "history_latent_mse": 0.12625442445278168, + "psnr": 33.204804725249105, + "ssim": 0.9522031314406408, + "lpips": 0.254537433385849 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c55ec198253ad7a3077ffde38823183a580677fe --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ff3779dc0e165ed1e3d1d376f1a9f880dd8d5f54f40813467f01e0755f1bbf8 +size 864933 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..4f98f434f54bd09b38b2f5fd9dd38f0ce185f0cc --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.06131753325462341, + "wrist/video/latent_future_mse": 0.06131753325462341, + "video/psnr": 29.826995303462084, + "wrist/video/psnr": 29.826995303462084, + "video/ssim": 0.9339680561401201, + "wrist/video/ssim": 0.9339680561401201, + "video/lpips": 0.02761567011475563, + "wrist/video/lpips": 0.02761567011475563, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.18852488696575165, + "wrist/wrist_left/latent_future_mse": 0.18852488696575165, + "wrist_left/psnr": 22.64956943983406, + "wrist/wrist_left/psnr": 22.64956943983406, + "wrist_left/ssim": 0.8456670242005284, + "wrist/wrist_left/ssim": 0.8456670242005284, + "wrist_left/lpips": 0.06932871043682098, + "wrist/wrist_left/lpips": 0.06932871043682098, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.12008321285247803, + "wrist/wrist_right/latent_future_mse": 0.12008321285247803, + "wrist_right/psnr": 31.419793533688743, + "wrist/wrist_right/psnr": 31.419793533688743, + "wrist_right/ssim": 0.933000893367617, + "wrist/wrist_right/ssim": 0.933000893367617, + "wrist_right/lpips": 0.03438973426818848, + "wrist/wrist_right/lpips": 0.03438973426818848, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.0069364868104457855, + "wrist/tactile_left/latent_future_mse": 0.0069364868104457855, + "tactile_left/psnr": 50.66471949450162, + "wrist/tactile_left/psnr": 50.66471949450162, + "tactile_left/ssim": 0.9947452134317746, + "wrist/tactile_left/ssim": 0.9947452134317746, + "tactile_left/lpips": 0.0034115284215658903, + "wrist/tactile_left/lpips": 0.0034115284215658903, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.00535708898678422, + "wrist/tactile_right/latent_future_mse": 0.00535708898678422, + "tactile_right/psnr": 52.71458013071562, + "wrist/tactile_right/psnr": 52.71458013071562, + "tactile_right/ssim": 0.9957780256909837, + "wrist/tactile_right/ssim": 0.9957780256909837, + "tactile_right/lpips": 0.002500636503100395, + "wrist/tactile_right/lpips": 0.002500636503100395, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 750, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.06131753325462341, + "psnr": 29.826995303462084, + "ssim": 0.9339680561401201, + "lpips": 0.02761567011475563 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.18852488696575165, + "psnr": 22.64956943983406, + "ssim": 0.8456670242005284, + "lpips": 0.06932871043682098 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.12008321285247803, + "psnr": 31.419793533688743, + "ssim": 0.933000893367617, + "lpips": 0.03438973426818848 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.0069364868104457855, + "psnr": 50.66471949450162, + "ssim": 0.9947452134317746, + "lpips": 0.0034115284215658903 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.00535708898678422, + "psnr": 52.71458013071562, + "ssim": 0.9957780256909837, + "lpips": 0.002500636503100395 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/case.json new file mode 100644 index 0000000000000000000000000000000000000000..4f2995acb9bfdb782bc5858a3b9f7ead15495881 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/case.json @@ -0,0 +1,168 @@ +{ + "task": 17, + "episode": "motherboard_0912_episode_002.pt", + "category": "onset", + "index": 12543, + "t_start": 2106, + "start_seconds": 140.4, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.75, + "long_future_contact_fraction": 0.8, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "17_motherboard_0912_episode_002_onset", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c7d2b32f52769a6b3b2f81300f78530db6dde27 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44f5e4773c2e4b6880fea45f2473bfcc9749e5c9396f8140801245041ae6cb07 +size 12102302 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/motherboard_0912_episode_002_t02106_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/motherboard_0912_episode_002_t02106_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..8485f9c72712a3022773ff48066eacaf4f117995 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/motherboard_0912_episode_002_t02106_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41c2fc3cea1ae7f6f3fcfc06a8b1306b7ec528403f5cf4c713dccca50dfc9213 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..981205a8a3ab170124ee48454e794703a9189ad1 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 2106, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 2106, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 27.014078442174597, + "ssim": 0.8537734680924082, + "lpips": 0.04305802658200264 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 17.02602718012348, + "ssim": 0.5545452971583498, + "lpips": 0.21118396520614624 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 16.620284818893403, + "ssim": 0.384131424427874, + "lpips": 0.16797195374965668 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 47.67957685225926, + "ssim": 0.9911773458571171, + "lpips": 0.0060229673981666565 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 42.02465889423186, + "ssim": 0.984716980422512, + "lpips": 0.06635585427284241 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 2114, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.5, + "history_latent_mse": 0.09277269244194031, + "psnr": 23.193705783255552, + "ssim": 0.7368620474692847, + "lpips": 0.1026526689529419 + }, + "wrist_left": { + "history_latent_max_abs_error": 3.974609375, + "history_latent_mse": 0.3335951268672943, + "psnr": 12.6803251807406, + "ssim": 0.39257903742452627, + "lpips": 0.37435224652290344 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.078125, + "history_latent_mse": 0.4148177206516266, + "psnr": 16.303833548663544, + "ssim": 0.40099904743344916, + "lpips": 0.2037639021873474 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.59765625, + "history_latent_mse": 0.030755186453461647, + "psnr": 46.25685983725815, + "ssim": 0.988311726529612, + "lpips": 0.014422392472624779 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.466796875, + "history_latent_mse": 0.061797261238098145, + "psnr": 40.188848272419236, + "ssim": 0.9755943922087594, + "lpips": 0.10928108543157578 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 2122, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.0234375, + "history_latent_mse": 0.17026342451572418, + "psnr": 22.64510832299301, + "ssim": 0.7160262832424318, + "lpips": 0.12871530652046204 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.5444633364677429, + "psnr": 12.461681778847034, + "ssim": 0.3058784762227553, + "lpips": 0.4386119842529297 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7890625, + "history_latent_mse": 0.45874789357185364, + "psnr": 16.446700429644856, + "ssim": 0.42067116779253655, + "lpips": 0.2568044066429138 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.111328125, + "history_latent_mse": 0.02397572249174118, + "psnr": 42.61518599532981, + "ssim": 0.982452086260226, + "lpips": 0.03949815034866333 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.3515625, + "history_latent_mse": 0.07052794843912125, + "psnr": 41.99125758164814, + "ssim": 0.9813101586679613, + "lpips": 0.06796708703041077 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 2130, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0234375, + "history_latent_mse": 0.19132567942142487, + "psnr": 21.263367743266535, + "ssim": 0.7088566912514335, + "lpips": 0.18049629032611847 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.5234375, + "history_latent_mse": 0.6521925330162048, + "psnr": 13.67522056008455, + "ssim": 0.3037594071147026, + "lpips": 0.433808833360672 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.203125, + "history_latent_mse": 0.4453120529651642, + "psnr": 13.70063872146973, + "ssim": 0.413014243800197, + "lpips": 0.3991217017173767 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.125, + "history_latent_mse": 0.04960574209690094, + "psnr": 36.2099161076948, + "ssim": 0.9684519983035211, + "lpips": 0.11095166951417923 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.73046875, + "history_latent_mse": 0.054833993315696716, + "psnr": 43.15839705399051, + "ssim": 0.9836181538512934, + "lpips": 0.03823317587375641 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 2138, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7265625, + "history_latent_mse": 0.21702148020267487, + "psnr": 20.26208398350469, + "ssim": 0.7089134906751157, + "lpips": 0.20850397646427155 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.578125, + "history_latent_mse": 0.6371123790740967, + "psnr": 12.821179267702083, + "ssim": 0.3340335941846782, + "lpips": 0.437974214553833 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.359375, + "history_latent_mse": 0.5822781920433044, + "psnr": 12.820229928944176, + "ssim": 0.2906581256944808, + "lpips": 0.4843248128890991 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.015625, + "history_latent_mse": 0.10048343986272812, + "psnr": 40.46461361923542, + "ssim": 0.9782773904450796, + "lpips": 0.06781888753175735 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.666015625, + "history_latent_mse": 0.05322854593396187, + "psnr": 40.95654627846611, + "ssim": 0.9791282492117304, + "lpips": 0.08110180497169495 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 2146, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0859375, + "history_latent_mse": 0.22881287336349487, + "psnr": 19.870245360616124, + "ssim": 0.6938321846920246, + "lpips": 0.2252332866191864 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.875, + "history_latent_mse": 0.5826783180236816, + "psnr": 12.96879289203883, + "ssim": 0.2695432900232294, + "lpips": 0.47767043113708496 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.234375, + "history_latent_mse": 0.80934077501297, + "psnr": 12.232619464257406, + "ssim": 0.3741025243295242, + "lpips": 0.47848302125930786 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.21875, + "history_latent_mse": 0.06819824874401093, + "psnr": 37.876258190286215, + "ssim": 0.9659349319931626, + "lpips": 0.11093461513519287 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.4375, + "history_latent_mse": 0.06363306194543839, + "psnr": 40.79628705757634, + "ssim": 0.9799701960459357, + "lpips": 0.07002884149551392 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 2154, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.0703125, + "history_latent_mse": 0.2336505651473999, + "psnr": 20.002086816249015, + "ssim": 0.6854771654625638, + "lpips": 0.21389327943325043 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.31640625, + "history_latent_mse": 0.6917423605918884, + "psnr": 12.926521692470418, + "ssim": 0.2671856638196575, + "lpips": 0.4820736050605774 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.6422098875045776, + "psnr": 10.82718397660685, + "ssim": 0.3042208206488878, + "lpips": 0.5217235684394836 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3359375, + "history_latent_mse": 0.1204526424407959, + "psnr": 37.28966969802208, + "ssim": 0.9529642518085815, + "lpips": 0.12644833326339722 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.38671875, + "history_latent_mse": 0.059152934700250626, + "psnr": 38.231643266866485, + "ssim": 0.9735880933395056, + "lpips": 0.12291864305734634 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 2162, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.44140625, + "history_latent_mse": 0.2446414977312088, + "psnr": 20.306859231225374, + "ssim": 0.6920691319331336, + "lpips": 0.21108244359493256 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.046875, + "history_latent_mse": 0.6287658214569092, + "psnr": 12.889811755975558, + "ssim": 0.33629780995494063, + "lpips": 0.44039878249168396 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.9921875, + "history_latent_mse": 0.8564720749855042, + "psnr": 12.3373876175361, + "ssim": 0.38051619173182183, + "lpips": 0.5009351372718811 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.35546875, + "history_latent_mse": 0.10649094730615616, + "psnr": 37.857803947658006, + "ssim": 0.9615416870423107, + "lpips": 0.12590958178043365 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.9375, + "history_latent_mse": 0.09986984729766846, + "psnr": 39.054895310613894, + "ssim": 0.9754108962229073, + "lpips": 0.0997622087597847 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 2170, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6953125, + "history_latent_mse": 0.2500828802585602, + "psnr": 20.49214667342536, + "ssim": 0.6933370865918393, + "lpips": 0.19841524958610535 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.278778076171875, + "history_latent_mse": 0.5618012547492981, + "psnr": 12.752937478920177, + "ssim": 0.3525681402818953, + "lpips": 0.43834590911865234 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.26171875, + "history_latent_mse": 0.7204864621162415, + "psnr": 14.00770603978504, + "ssim": 0.3576533320573731, + "lpips": 0.5005643963813782 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.240234375, + "history_latent_mse": 0.1021137610077858, + "psnr": 37.712331489520004, + "ssim": 0.9639373647847203, + "lpips": 0.1318599432706833 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.40625, + "history_latent_mse": 0.07063231617212296, + "psnr": 37.39392405425794, + "ssim": 0.9727040833919718, + "lpips": 0.10178379714488983 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 2178, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.328125, + "history_latent_mse": 0.22403652966022491, + "psnr": 20.495676257459206, + "ssim": 0.6943262091405564, + "lpips": 0.1964721530675888 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.66796875, + "history_latent_mse": 0.6388174295425415, + "psnr": 12.607608961436789, + "ssim": 0.3637026120689203, + "lpips": 0.43883565068244934 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.453125, + "history_latent_mse": 0.8176984786987305, + "psnr": 12.56277057435354, + "ssim": 0.30781582909397914, + "lpips": 0.518544614315033 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.5712890625, + "history_latent_mse": 0.07678048312664032, + "psnr": 36.82484973635878, + "ssim": 0.9635564892942823, + "lpips": 0.14000913500785828 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.140625, + "history_latent_mse": 0.09807994216680527, + "psnr": 36.78447042000245, + "ssim": 0.972636436338566, + "lpips": 0.1095660999417305 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 2186, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.4921875, + "history_latent_mse": 0.22043760120868683, + "psnr": 18.807346114118786, + "ssim": 0.6746218370000718, + "lpips": 0.2178318053483963 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.0546875, + "history_latent_mse": 0.5896202921867371, + "psnr": 12.18217687791942, + "ssim": 0.3650721251436022, + "lpips": 0.44499021768569946 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 0.7953091263771057, + "psnr": 12.200550084286794, + "ssim": 0.29952490858060926, + "lpips": 0.5131115913391113 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.890625, + "history_latent_mse": 0.09518461674451828, + "psnr": 36.32442584283347, + "ssim": 0.9631361881533402, + "lpips": 0.1562928855419159 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8828125, + "history_latent_mse": 0.08057665824890137, + "psnr": 36.00405221267204, + "ssim": 0.9683223762820012, + "lpips": 0.14610856771469116 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 2194, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8671875, + "history_latent_mse": 0.25679609179496765, + "psnr": 18.205331780429123, + "ssim": 0.6562253259709976, + "lpips": 0.23331773281097412 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.37890625, + "history_latent_mse": 0.6829532980918884, + "psnr": 11.123529053504944, + "ssim": 0.2548037511457519, + "lpips": 0.5309352874755859 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.796875, + "history_latent_mse": 0.7600271105766296, + "psnr": 11.761871796344423, + "ssim": 0.3215185172742515, + "lpips": 0.5119346976280212 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.71875, + "history_latent_mse": 0.07704821228981018, + "psnr": 33.71934676984741, + "ssim": 0.9516876133117059, + "lpips": 0.20587241649627686 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.046875, + "history_latent_mse": 0.090538889169693, + "psnr": 35.51975626209679, + "ssim": 0.9692290530977232, + "lpips": 0.12880505621433258 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 2202, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.81640625, + "history_latent_mse": 0.2643772065639496, + "psnr": 18.08613368807344, + "ssim": 0.6430067425221757, + "lpips": 0.24009525775909424 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 0.8713846206665039, + "psnr": 12.174414943610623, + "ssim": 0.29014301091651606, + "lpips": 0.49786877632141113 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7578125, + "history_latent_mse": 0.7882205843925476, + "psnr": 8.892660986654056, + "ssim": 0.27712867198653596, + "lpips": 0.5395231246948242 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.28125, + "history_latent_mse": 0.11134020239114761, + "psnr": 33.97365878570585, + "ssim": 0.952143654356717, + "lpips": 0.20177578926086426 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7421875, + "history_latent_mse": 0.09902206063270569, + "psnr": 34.92386223846347, + "ssim": 0.9652095960739195, + "lpips": 0.1530810296535492 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 2210, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.4296875, + "history_latent_mse": 0.2922210097312927, + "psnr": 17.999030045189002, + "ssim": 0.6381186889779266, + "lpips": 0.26595598459243774 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.33203125, + "history_latent_mse": 0.7394092679023743, + "psnr": 12.202925593866842, + "ssim": 0.41998789593437447, + "lpips": 0.44280290603637695 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.55078125, + "history_latent_mse": 1.071099877357483, + "psnr": 9.180608767392421, + "ssim": 0.26180610156518574, + "lpips": 0.5459426045417786 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.609375, + "history_latent_mse": 0.10244587808847427, + "psnr": 34.63135210401545, + "ssim": 0.9525688980617382, + "lpips": 0.20637813210487366 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7578125, + "history_latent_mse": 0.0992751345038414, + "psnr": 34.55589601332959, + "ssim": 0.9630725542691796, + "lpips": 0.1685653179883957 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 2218, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.671875, + "history_latent_mse": 0.31175222992897034, + "psnr": 18.230740412348286, + "ssim": 0.6281446956418402, + "lpips": 0.27912336587905884 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.4921875, + "history_latent_mse": 0.7248112559318542, + "psnr": 10.715450764121666, + "ssim": 0.30236256973742015, + "lpips": 0.5260438323020935 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 1.0364041328430176, + "psnr": 9.949980314074358, + "ssim": 0.2812316171055831, + "lpips": 0.5410803556442261 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.1640625, + "history_latent_mse": 0.0953962430357933, + "psnr": 33.75476059877858, + "ssim": 0.9454560800488052, + "lpips": 0.22718392312526703 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8203125, + "history_latent_mse": 0.11438552290201187, + "psnr": 33.96333285445411, + "ssim": 0.9617806375457374, + "lpips": 0.18186882138252258 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88e631d0f78c993cb950d149e9b90b9e7bc91ac7 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b354d89230d93332f094eaff5bd675b49f4e82dec814c9651023fce2c7c624f +size 1616214 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..92d4b097783a504f07648c5d21b3b1c9a01b7627 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.09586220979690552, + "wrist/video/latent_future_mse": 0.09586220979690552, + "video/psnr": 27.014078442174597, + "wrist/video/psnr": 27.014078442174597, + "video/ssim": 0.8537734680924082, + "wrist/video/ssim": 0.8537734680924082, + "video/lpips": 0.04305802658200264, + "wrist/video/lpips": 0.04305802658200264, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.3449423611164093, + "wrist/wrist_left/latent_future_mse": 0.3449423611164093, + "wrist_left/psnr": 17.02602718012348, + "wrist/wrist_left/psnr": 17.02602718012348, + "wrist_left/ssim": 0.5545452971583498, + "wrist/wrist_left/ssim": 0.5545452971583498, + "wrist_left/lpips": 0.21118396520614624, + "wrist/wrist_left/lpips": 0.21118396520614624, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.45878592133522034, + "wrist/wrist_right/latent_future_mse": 0.45878592133522034, + "wrist_right/psnr": 16.620284818893403, + "wrist/wrist_right/psnr": 16.620284818893403, + "wrist_right/ssim": 0.384131424427874, + "wrist/wrist_right/ssim": 0.384131424427874, + "wrist_right/lpips": 0.16797195374965668, + "wrist/wrist_right/lpips": 0.16797195374965668, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.03156638517975807, + "wrist/tactile_left/latent_future_mse": 0.03156638517975807, + "tactile_left/psnr": 47.67957685225926, + "wrist/tactile_left/psnr": 47.67957685225926, + "tactile_left/ssim": 0.9911773458571171, + "wrist/tactile_left/ssim": 0.9911773458571171, + "tactile_left/lpips": 0.0060229673981666565, + "wrist/tactile_left/lpips": 0.0060229673981666565, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.062396418303251266, + "wrist/tactile_right/latent_future_mse": 0.062396418303251266, + "tactile_right/psnr": 42.02465889423186, + "wrist/tactile_right/psnr": 42.02465889423186, + "tactile_right/ssim": 0.984716980422512, + "wrist/tactile_right/ssim": 0.984716980422512, + "tactile_right/lpips": 0.06635585427284241, + "wrist/tactile_right/lpips": 0.06635585427284241, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 2106, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.09586220979690552, + "psnr": 27.014078442174597, + "ssim": 0.8537734680924082, + "lpips": 0.04305802658200264 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.3449423611164093, + "psnr": 17.02602718012348, + "ssim": 0.5545452971583498, + "lpips": 0.21118396520614624 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.45878592133522034, + "psnr": 16.620284818893403, + "ssim": 0.384131424427874, + "lpips": 0.16797195374965668 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.03156638517975807, + "psnr": 47.67957685225926, + "ssim": 0.9911773458571171, + "lpips": 0.0060229673981666565 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.062396418303251266, + "psnr": 42.02465889423186, + "ssim": 0.984716980422512, + "lpips": 0.06635585427284241 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/case.json new file mode 100644 index 0000000000000000000000000000000000000000..931f5da1fb677c6cad42e58705a786d120cf26a7 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/case.json @@ -0,0 +1,168 @@ +{ + "task": 18, + "episode": "motherboard_0912_episode_002.pt", + "category": "release", + "index": 10654, + "t_start": 217, + "start_seconds": 14.466666666666667, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.9296875, + "long_future_contact_fraction": 0.925, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "18_motherboard_0912_episode_002_release", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e90481a353a46beaa9daee15e2678b6785dc11c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c8944855f256411bad5a9292f4aa699ea776eec5204126cda4b6d46766a63d +size 11609660 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/motherboard_0912_episode_002_t00217_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/motherboard_0912_episode_002_t00217_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..de66ecdc4f15623b3f8f7d56888add359f81efb4 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/motherboard_0912_episode_002_t00217_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bd03a8fada27f82354e5503a9f26df930865941f6d81ff09f29bb17c8ca14fa +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..22e6972e90e50c676a0c45c39f362b787d6e91cf --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 217, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 217, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 21.760641673138515, + "ssim": 0.7452214137316457, + "lpips": 0.1625427007675171 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.860470318122003, + "ssim": 0.5015684165372364, + "lpips": 0.4545535743236542 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 13.28071432869454, + "ssim": 0.47143106020683023, + "lpips": 0.4062356948852539 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 41.627355962770984, + "ssim": 0.9835802582462703, + "lpips": 0.04938223212957382 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 48.41633065521247, + "ssim": 0.9939249185483171, + "lpips": 0.00882115587592125 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 225, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.814453125, + "history_latent_mse": 0.21970494091510773, + "psnr": 20.140505471601998, + "ssim": 0.6676681263775246, + "lpips": 0.22777527570724487 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.2890625, + "history_latent_mse": 0.7119962573051453, + "psnr": 13.639374641185459, + "ssim": 0.37054320776110616, + "lpips": 0.4113002121448517 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.0234375, + "history_latent_mse": 0.6032268404960632, + "psnr": 12.123472245005011, + "ssim": 0.3575668159145335, + "lpips": 0.48164474964141846 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.41015625, + "history_latent_mse": 0.07408691197633743, + "psnr": 41.11334825516049, + "ssim": 0.9728495659446579, + "lpips": 0.07536482810974121 + }, + "tactile_right": { + "history_latent_max_abs_error": 2.7265625, + "history_latent_mse": 0.029988134279847145, + "psnr": 45.82415439785264, + "ssim": 0.9893414408177263, + "lpips": 0.01662234403192997 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 233, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.19140625, + "history_latent_mse": 0.25916382670402527, + "psnr": 18.949729059179127, + "ssim": 0.6424816869650679, + "lpips": 0.29416218400001526 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.375, + "history_latent_mse": 0.6441795229911804, + "psnr": 13.486773099630568, + "ssim": 0.3583102425730368, + "lpips": 0.4436154365539551 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.421875, + "history_latent_mse": 0.7828050255775452, + "psnr": 13.368448380849145, + "ssim": 0.4917498544963558, + "lpips": 0.40680938959121704 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.76171875, + "history_latent_mse": 0.10672455281019211, + "psnr": 38.99338803277449, + "ssim": 0.9601759482288443, + "lpips": 0.11251966655254364 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.84765625, + "history_latent_mse": 0.03365201875567436, + "psnr": 45.3722102045884, + "ssim": 0.9878290442502269, + "lpips": 0.021943088620901108 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 241, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.109375, + "history_latent_mse": 0.3430468738079071, + "psnr": 19.263568142261335, + "ssim": 0.6457276235671502, + "lpips": 0.27917852997779846 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.41015625, + "history_latent_mse": 0.6819506287574768, + "psnr": 13.382524567436981, + "ssim": 0.3631557952336697, + "lpips": 0.4660447835922241 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.7734375, + "history_latent_mse": 0.5567581653594971, + "psnr": 13.76214240462167, + "ssim": 0.4399837335005288, + "lpips": 0.38885366916656494 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.396484375, + "history_latent_mse": 0.12278616428375244, + "psnr": 38.999344437757, + "ssim": 0.9687368809858019, + "lpips": 0.07263458520174026 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.3515625, + "history_latent_mse": 0.0348423533141613, + "psnr": 43.79751292912292, + "ssim": 0.9856540617887479, + "lpips": 0.03738494962453842 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 249, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.25, + "history_latent_mse": 0.2839043140411377, + "psnr": 20.337818952017233, + "ssim": 0.6750040521621276, + "lpips": 0.2334269881248474 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.71875, + "history_latent_mse": 0.6252749562263489, + "psnr": 12.992808992087605, + "ssim": 0.42074324544686686, + "lpips": 0.48223093152046204 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6171875, + "history_latent_mse": 0.5040450692176819, + "psnr": 14.252726135205357, + "ssim": 0.4790169885984039, + "lpips": 0.40865957736968994 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.529296875, + "history_latent_mse": 0.09153049439191818, + "psnr": 38.34330987091194, + "ssim": 0.967528671075207, + "lpips": 0.089942067861557 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.255859375, + "history_latent_mse": 0.042521506547927856, + "psnr": 42.573674938553275, + "ssim": 0.983445281601083, + "lpips": 0.053961534053087234 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 257, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.24480648338794708, + "psnr": 19.832610621800782, + "ssim": 0.6624825339150032, + "lpips": 0.26334360241889954 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.515625, + "history_latent_mse": 0.6778244972229004, + "psnr": 10.947112848408844, + "ssim": 0.3252476478451081, + "lpips": 0.5646216869354248 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.046875, + "history_latent_mse": 0.5969168543815613, + "psnr": 11.698157620924956, + "ssim": 0.36147235970621694, + "lpips": 0.48050934076309204 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3828125, + "history_latent_mse": 0.07551120966672897, + "psnr": 37.57178234598283, + "ssim": 0.9665358410103206, + "lpips": 0.09739766269922256 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.1767578125, + "history_latent_mse": 0.048212919384241104, + "psnr": 41.36189212666064, + "ssim": 0.9808301906946357, + "lpips": 0.07126203179359436 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 265, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.7109375, + "history_latent_mse": 0.27142658829689026, + "psnr": 19.088725671879168, + "ssim": 0.6494011610381127, + "lpips": 0.3001136779785156 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.06787109375, + "history_latent_mse": 0.9343789219856262, + "psnr": 10.247352367568709, + "ssim": 0.3028908363725804, + "lpips": 0.5612272024154663 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.1328125, + "history_latent_mse": 0.6543198227882385, + "psnr": 12.196956313364144, + "ssim": 0.4127305634491031, + "lpips": 0.47836798429489136 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.484375, + "history_latent_mse": 0.09266697615385056, + "psnr": 37.471071931703314, + "ssim": 0.9628342108411656, + "lpips": 0.13135123252868652 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.30859375, + "history_latent_mse": 0.058024387806653976, + "psnr": 40.17950571476483, + "ssim": 0.9781191273912287, + "lpips": 0.08987166732549667 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 273, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.234375, + "history_latent_mse": 0.34509170055389404, + "psnr": 18.653181428861707, + "ssim": 0.6136383253536795, + "lpips": 0.33297014236450195 + }, + "wrist_left": { + "history_latent_max_abs_error": 6.0, + "history_latent_mse": 0.9078071713447571, + "psnr": 10.122368089674822, + "ssim": 0.28204953934040444, + "lpips": 0.5454872846603394 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.04296875, + "history_latent_mse": 0.6905022263526917, + "psnr": 14.186199355712862, + "ssim": 0.44068523780800195, + "lpips": 0.43603450059890747 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.8203125, + "history_latent_mse": 0.1283048689365387, + "psnr": 37.355853402192906, + "ssim": 0.9591111690919807, + "lpips": 0.15703943371772766 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.30029296875, + "history_latent_mse": 0.06661611795425415, + "psnr": 39.61639712085204, + "ssim": 0.97438342265769, + "lpips": 0.10702496767044067 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 281, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.1484375, + "history_latent_mse": 0.36372220516204834, + "psnr": 18.567957965743048, + "ssim": 0.6259133622052626, + "lpips": 0.3196270167827606 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1875, + "history_latent_mse": 0.9046269059181213, + "psnr": 11.531667077884482, + "ssim": 0.3473794623194, + "lpips": 0.5163223743438721 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.0, + "history_latent_mse": 0.5556142926216125, + "psnr": 13.505748636478696, + "ssim": 0.3893769266953317, + "lpips": 0.43455249071121216 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.0390625, + "history_latent_mse": 0.12012337893247604, + "psnr": 37.26478017819117, + "ssim": 0.9598009487952812, + "lpips": 0.16572776436805725 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4736328125, + "history_latent_mse": 0.07343220710754395, + "psnr": 38.75045941238825, + "ssim": 0.9713807445975495, + "lpips": 0.12230309844017029 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 289, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.609375, + "history_latent_mse": 0.35883092880249023, + "psnr": 19.404916985248583, + "ssim": 0.6455304536172985, + "lpips": 0.27299052476882935 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.3828125, + "history_latent_mse": 0.7382151484489441, + "psnr": 12.90826316927593, + "ssim": 0.344185273038674, + "lpips": 0.4898313283920288 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.3046875, + "history_latent_mse": 0.5983229279518127, + "psnr": 12.717727691963844, + "ssim": 0.3185484860501885, + "lpips": 0.4623410701751709 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.2109375, + "history_latent_mse": 0.10660126060247421, + "psnr": 36.60046029408781, + "ssim": 0.9571549952671259, + "lpips": 0.20622628927230835 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.564453125, + "history_latent_mse": 0.08048990368843079, + "psnr": 37.771775726162495, + "ssim": 0.968242578364747, + "lpips": 0.1463935375213623 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 297, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.86328125, + "history_latent_mse": 0.31497111916542053, + "psnr": 19.004183008319387, + "ssim": 0.6459928307736367, + "lpips": 0.30305883288383484 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.46484375, + "history_latent_mse": 0.626753032207489, + "psnr": 12.121975215984655, + "ssim": 0.3812296685213081, + "lpips": 0.5045105218887329 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.64453125, + "history_latent_mse": 0.7180840373039246, + "psnr": 13.241867785179588, + "ssim": 0.45065220728779926, + "lpips": 0.4573001563549042 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.126953125, + "history_latent_mse": 0.14269345998764038, + "psnr": 35.440520704106774, + "ssim": 0.9456500613454792, + "lpips": 0.24083703756332397 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.6630859375, + "history_latent_mse": 0.08899743109941483, + "psnr": 37.05162796909528, + "ssim": 0.965861065824422, + "lpips": 0.160465806722641 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 305, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.9609375, + "history_latent_mse": 0.33893826603889465, + "psnr": 18.5327532147643, + "ssim": 0.6316403777555868, + "lpips": 0.338984876871109 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.4375, + "history_latent_mse": 0.7456153035163879, + "psnr": 10.66723246072672, + "ssim": 0.3192933989295139, + "lpips": 0.5223986506462097 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.5, + "history_latent_mse": 0.6550537943840027, + "psnr": 13.67192592428507, + "ssim": 0.4554012988408176, + "lpips": 0.4381822943687439 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.9296875, + "history_latent_mse": 0.16047285497188568, + "psnr": 34.39293730703571, + "ssim": 0.9343516210874803, + "lpips": 0.2546803653240204 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.69482421875, + "history_latent_mse": 0.09527558088302612, + "psnr": 36.369784990490324, + "ssim": 0.9623341270757058, + "lpips": 0.18310663104057312 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 313, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.8984375, + "history_latent_mse": 0.3568964898586273, + "psnr": 18.474859821674713, + "ssim": 0.6340252379328508, + "lpips": 0.3208377957344055 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.3203125, + "history_latent_mse": 0.8646063804626465, + "psnr": 12.536634404522944, + "ssim": 0.33909287039130803, + "lpips": 0.5164653658866882 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6640625, + "history_latent_mse": 0.6300497651100159, + "psnr": 13.081232521034043, + "ssim": 0.343819955536513, + "lpips": 0.5033243894577026 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.921875, + "history_latent_mse": 0.1461557000875473, + "psnr": 34.634408374111, + "ssim": 0.9445911429625082, + "lpips": 0.2300698608160019 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7734375, + "history_latent_mse": 0.1049686074256897, + "psnr": 35.61787097235618, + "ssim": 0.9577488270554306, + "lpips": 0.22510573267936707 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 321, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.3671875, + "history_latent_mse": 0.36685988306999207, + "psnr": 18.909652720520125, + "ssim": 0.6459207078823306, + "lpips": 0.3030533790588379 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.6945356726646423, + "psnr": 14.30162216782593, + "ssim": 0.3521220982008293, + "lpips": 0.48635536432266235 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.65625, + "history_latent_mse": 0.8107951283454895, + "psnr": 12.742886683007434, + "ssim": 0.3349973142377767, + "lpips": 0.5037884712219238 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.15625, + "history_latent_mse": 0.16017551720142365, + "psnr": 34.222301893716846, + "ssim": 0.9395003292117685, + "lpips": 0.2586207687854767 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.79296875, + "history_latent_mse": 0.1287270337343216, + "psnr": 34.778571732988915, + "ssim": 0.9541131783971037, + "lpips": 0.245049387216568 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 329, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.203125, + "history_latent_mse": 0.287699818611145, + "psnr": 18.635521411918436, + "ssim": 0.6276777252684698, + "lpips": 0.3157117962837219 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.662109375, + "history_latent_mse": 0.6810634732246399, + "psnr": 13.972862126771094, + "ssim": 0.3329031578002466, + "lpips": 0.45227545499801636 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.90625, + "history_latent_mse": 0.697955846786499, + "psnr": 11.273008281049627, + "ssim": 0.34968797249307104, + "lpips": 0.5203717947006226 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.08984375, + "history_latent_mse": 0.18305504322052002, + "psnr": 33.394704966846604, + "ssim": 0.935169462779175, + "lpips": 0.27094870805740356 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8046875, + "history_latent_mse": 0.15205946564674377, + "psnr": 34.30235303442076, + "ssim": 0.9512387364142227, + "lpips": 0.2640737295150757 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..851466b70a8b53e81514d25173ddc881a1638a3c --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eed8da122260d855e4d4e5ca5f2dd161a3b1435f85b4e5ba58de9643484e9c7 +size 1464417 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..2a878d6ea2b861854052475b21eef41087f7fc08 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.2364676594734192, + "wrist/video/latent_future_mse": 0.2364676594734192, + "video/psnr": 21.760641673138515, + "wrist/video/psnr": 21.760641673138515, + "video/ssim": 0.7452214137316457, + "wrist/video/ssim": 0.7452214137316457, + "video/lpips": 0.1625427007675171, + "wrist/video/lpips": 0.1625427007675171, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.8056521415710449, + "wrist/wrist_left/latent_future_mse": 0.8056521415710449, + "wrist_left/psnr": 13.860470318122003, + "wrist/wrist_left/psnr": 13.860470318122003, + "wrist_left/ssim": 0.5015684165372364, + "wrist/wrist_left/ssim": 0.5015684165372364, + "wrist_left/lpips": 0.4545535743236542, + "wrist/wrist_left/lpips": 0.4545535743236542, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.6791622042655945, + "wrist/wrist_right/latent_future_mse": 0.6791622042655945, + "wrist_right/psnr": 13.28071432869454, + "wrist/wrist_right/psnr": 13.28071432869454, + "wrist_right/ssim": 0.47143106020683023, + "wrist/wrist_right/ssim": 0.47143106020683023, + "wrist_right/lpips": 0.4062356948852539, + "wrist/wrist_right/lpips": 0.4062356948852539, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.08753391355276108, + "wrist/tactile_left/latent_future_mse": 0.08753391355276108, + "tactile_left/psnr": 41.627355962770984, + "wrist/tactile_left/psnr": 41.627355962770984, + "tactile_left/ssim": 0.9835802582462703, + "wrist/tactile_left/ssim": 0.9835802582462703, + "tactile_left/lpips": 0.04938223212957382, + "wrist/tactile_left/lpips": 0.04938223212957382, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.025439344346523285, + "wrist/tactile_right/latent_future_mse": 0.025439344346523285, + "tactile_right/psnr": 48.41633065521247, + "wrist/tactile_right/psnr": 48.41633065521247, + "tactile_right/ssim": 0.9939249185483171, + "wrist/tactile_right/ssim": 0.9939249185483171, + "tactile_right/lpips": 0.00882115587592125, + "wrist/tactile_right/lpips": 0.00882115587592125, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 217, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.2364676594734192, + "psnr": 21.760641673138515, + "ssim": 0.7452214137316457, + "lpips": 0.1625427007675171 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.8056521415710449, + "psnr": 13.860470318122003, + "ssim": 0.5015684165372364, + "lpips": 0.4545535743236542 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.6791622042655945, + "psnr": 13.28071432869454, + "ssim": 0.47143106020683023, + "lpips": 0.4062356948852539 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.08753391355276108, + "psnr": 41.627355962770984, + "ssim": 0.9835802582462703, + "lpips": 0.04938223212957382 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.025439344346523285, + "psnr": 48.41633065521247, + "ssim": 0.9939249185483171, + "lpips": 0.00882115587592125 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/case.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/case.json new file mode 100644 index 0000000000000000000000000000000000000000..036dfd94b458d993a69fe97e02b10c1ff2d8b533 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/case.json @@ -0,0 +1,164 @@ +{ + "task": 19, + "episode": "motherboard_0912_episode_002.pt", + "category": "full", + "index": 11381, + "t_start": 944, + "start_seconds": 62.93333333333333, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 1.0, + "long_future_contact_fraction": 1.0, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "19_motherboard_0912_episode_002_full", + "rollout_windows": 15 +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/complete.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/complete.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b0dedd1bd6546c59ba764dcf3a54dddf0ddc38 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/complete.json @@ -0,0 +1 @@ +{"short_frames": 16, "long_frames": 128, "rollout_windows": 15} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/long_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/long_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..915f75da38c77ef49dfed92207f352d4b4f29c8b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/long_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338c99d30b63d515c048ab28e93da60997560fbf191a6c5b652ae46f28cd9cc1 +size 10188589 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/motherboard_0912_episode_002_t00944_pred.pt b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/motherboard_0912_episode_002_t00944_pred.pt new file mode 100644 index 0000000000000000000000000000000000000000..50b3549cbd18d63dc42c939634272ca7416e667a --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/motherboard_0912_episode_002_t00944_pred.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efc2f625d0c2687e738351bec3a98707d0d372d9122034a051eaa31f1424fdc5 +size 1978519 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/reference_rollout.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/reference_rollout.json new file mode 100644 index 0000000000000000000000000000000000000000..fff4956eaa3de3464cbf819be06321a1421715a2 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/reference_rollout.json @@ -0,0 +1,661 @@ +[ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 944, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "hide_wrist": false, + "windows": [ + { + "window": 0, + "t_start": 944, + "history_source": "recording", + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 24.681463575563292, + "ssim": 0.819499279912464, + "lpips": 0.07029199600219727 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 17.497804448618954, + "ssim": 0.47679504985244964, + "lpips": 0.31007200479507446 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 29.26793113448044, + "ssim": 0.9340524927075055, + "lpips": 0.048112958669662476 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 38.45417666049245, + "ssim": 0.9587584262145483, + "lpips": 0.07753477990627289 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "psnr": 48.37651614492312, + "ssim": 0.9920276303233837, + "lpips": 0.02037353813648224 + } + }, + "action": {} + }, + { + "window": 1, + "t_start": 952, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 2.984375, + "history_latent_mse": 0.10711588710546494, + "psnr": 21.84431273359289, + "ssim": 0.7380769667613667, + "lpips": 0.16159018874168396 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.359375, + "history_latent_mse": 0.4219270646572113, + "psnr": 13.560122892239804, + "ssim": 0.307946580813359, + "lpips": 0.4417189359664917 + }, + "wrist_right": { + "history_latent_max_abs_error": 2.515625, + "history_latent_mse": 0.13571497797966003, + "psnr": 24.63430394567899, + "ssim": 0.8775901051215428, + "lpips": 0.09218951314687729 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.28125, + "history_latent_mse": 0.09322979301214218, + "psnr": 39.16935489567372, + "ssim": 0.9704536626734409, + "lpips": 0.08338074386119843 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.60546875, + "history_latent_mse": 0.03158937022089958, + "psnr": 45.759823444122915, + "ssim": 0.9866081430208622, + "lpips": 0.026533760130405426 + } + }, + "action": {} + }, + { + "window": 2, + "t_start": 960, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.0, + "history_latent_mse": 0.1581316441297531, + "psnr": 22.15187600307528, + "ssim": 0.7387594841753731, + "lpips": 0.1315494328737259 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.6875, + "history_latent_mse": 0.6511651873588562, + "psnr": 14.895212991642165, + "ssim": 0.3539747030994428, + "lpips": 0.4762258529663086 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.5546875, + "history_latent_mse": 0.17332059144973755, + "psnr": 23.800192529685916, + "ssim": 0.8737160995463389, + "lpips": 0.09141986072063446 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.6796875, + "history_latent_mse": 0.10304980725049973, + "psnr": 39.9534983498748, + "ssim": 0.9709203354242688, + "lpips": 0.07402217388153076 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.58984375, + "history_latent_mse": 0.04370441660284996, + "psnr": 44.40396804165392, + "ssim": 0.9857128832310202, + "lpips": 0.024067465215921402 + } + }, + "action": {} + }, + { + "window": 3, + "t_start": 968, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.15625, + "history_latent_mse": 0.19232608377933502, + "psnr": 20.254315514390562, + "ssim": 0.6956434525306676, + "lpips": 0.24157142639160156 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.01953125, + "history_latent_mse": 0.6145458221435547, + "psnr": 14.094648645629666, + "ssim": 0.37096989193789626, + "lpips": 0.4897575378417969 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.6875, + "history_latent_mse": 0.17755722999572754, + "psnr": 24.202943069753825, + "ssim": 0.8606755920546025, + "lpips": 0.12734374403953552 + }, + "tactile_left": { + "history_latent_max_abs_error": 1.765625, + "history_latent_mse": 0.06772350519895554, + "psnr": 38.56219366928813, + "ssim": 0.9616246833043581, + "lpips": 0.09923327714204788 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.4765625, + "history_latent_mse": 0.0396367572247982, + "psnr": 43.201040185944166, + "ssim": 0.9840834060033671, + "lpips": 0.04085715860128403 + } + }, + "action": {} + }, + { + "window": 4, + "t_start": 976, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.515625, + "history_latent_mse": 0.2530117630958557, + "psnr": 19.5092861840386, + "ssim": 0.6857320888111613, + "lpips": 0.2631455659866333 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.890625, + "history_latent_mse": 0.7384024262428284, + "psnr": 12.978584779734446, + "ssim": 0.36466760290248446, + "lpips": 0.5100364685058594 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.34375, + "history_latent_mse": 0.21682780981063843, + "psnr": 24.345268687178027, + "ssim": 0.8667352453922046, + "lpips": 0.1386127769947052 + }, + "tactile_left": { + "history_latent_max_abs_error": 4.12109375, + "history_latent_mse": 0.1142655611038208, + "psnr": 36.797370807768104, + "ssim": 0.9433949438411136, + "lpips": 0.13873663544654846 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.36474609375, + "history_latent_mse": 0.04671946167945862, + "psnr": 41.848737062801725, + "ssim": 0.982096733472348, + "lpips": 0.05999534949660301 + } + }, + "action": {} + }, + { + "window": 5, + "t_start": 984, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.171875, + "history_latent_mse": 0.29442474246025085, + "psnr": 18.299127236739164, + "ssim": 0.6819478466537496, + "lpips": 0.28184470534324646 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.03125, + "history_latent_mse": 0.6977937817573547, + "psnr": 10.950061682631784, + "ssim": 0.2932447269251388, + "lpips": 0.5619039535522461 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.19140625, + "history_latent_mse": 0.21525879204273224, + "psnr": 19.35900607376387, + "ssim": 0.8081588626124363, + "lpips": 0.21622435748577118 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.921875, + "history_latent_mse": 0.11318793892860413, + "psnr": 36.76043478855387, + "ssim": 0.949754002423749, + "lpips": 0.13441747426986694 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.306640625, + "history_latent_mse": 0.050490740686655045, + "psnr": 40.34838461531863, + "ssim": 0.979009182583015, + "lpips": 0.08503374457359314 + } + }, + "action": {} + }, + { + "window": 6, + "t_start": 992, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.34375, + "history_latent_mse": 0.31924572587013245, + "psnr": 18.05247654546153, + "ssim": 0.6666016890993367, + "lpips": 0.29747867584228516 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.375, + "history_latent_mse": 0.8739915490150452, + "psnr": 9.865609785045136, + "ssim": 0.24517893304607755, + "lpips": 0.5892624855041504 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.421875, + "history_latent_mse": 0.3671722412109375, + "psnr": 20.0530376321808, + "ssim": 0.8010409141612734, + "lpips": 0.23567749559879303 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.51171875, + "history_latent_mse": 0.1040213406085968, + "psnr": 36.97116440553558, + "ssim": 0.9602432641786144, + "lpips": 0.11719538271427155 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.525390625, + "history_latent_mse": 0.05996396020054817, + "psnr": 39.178670621441796, + "ssim": 0.9765449901304037, + "lpips": 0.10608111321926117 + } + }, + "action": {} + }, + { + "window": 7, + "t_start": 1000, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.0390625, + "history_latent_mse": 0.30631619691848755, + "psnr": 18.54421847261493, + "ssim": 0.6557202848947283, + "lpips": 0.30667048692703247 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.8203125, + "history_latent_mse": 0.966468870639801, + "psnr": 13.563801778868845, + "ssim": 0.360486992492696, + "lpips": 0.5174843072891235 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.6630859375, + "history_latent_mse": 0.3852219879627228, + "psnr": 20.90559112866854, + "ssim": 0.829616298582195, + "lpips": 0.18549436330795288 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.15625, + "history_latent_mse": 0.09682580083608627, + "psnr": 37.16651610222278, + "ssim": 0.9613449169503236, + "lpips": 0.12124384939670563 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.443359375, + "history_latent_mse": 0.06491666287183762, + "psnr": 38.645674795942654, + "ssim": 0.9745283304052705, + "lpips": 0.11956681311130524 + } + }, + "action": {} + }, + { + "window": 8, + "t_start": 1008, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 5.4921875, + "history_latent_mse": 0.35144302248954773, + "psnr": 18.701042651586118, + "ssim": 0.6406627482128475, + "lpips": 0.31831109523773193 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.1796875, + "history_latent_mse": 0.72796630859375, + "psnr": 10.807100096986176, + "ssim": 0.33151147040986306, + "lpips": 0.5599737763404846 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.4765625, + "history_latent_mse": 0.2546648681163788, + "psnr": 20.2588988971605, + "ssim": 0.7979271453545638, + "lpips": 0.2626277208328247 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3623046875, + "history_latent_mse": 0.095887191593647, + "psnr": 35.96578377187178, + "ssim": 0.9557285194087822, + "lpips": 0.14946585893630981 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.36865234375, + "history_latent_mse": 0.07341336458921432, + "psnr": 37.6262094171036, + "ssim": 0.9721295683842046, + "lpips": 0.13267965614795685 + } + }, + "action": {} + }, + { + "window": 9, + "t_start": 1016, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.546875, + "history_latent_mse": 0.343233585357666, + "psnr": 18.12331746140542, + "ssim": 0.6328309516839951, + "lpips": 0.327944815158844 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.25390625, + "history_latent_mse": 1.0044093132019043, + "psnr": 11.217858514632855, + "ssim": 0.30334887288258094, + "lpips": 0.5584498643875122 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.9375, + "history_latent_mse": 0.3499092161655426, + "psnr": 20.195968585490434, + "ssim": 0.8074160526557437, + "lpips": 0.20154134929180145 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.3984375, + "history_latent_mse": 0.11152013391256332, + "psnr": 33.52860054643388, + "ssim": 0.9305277495362383, + "lpips": 0.19159723818302155 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.94091796875, + "history_latent_mse": 0.08544278889894485, + "psnr": 35.89267005015252, + "ssim": 0.9690972869143721, + "lpips": 0.15245294570922852 + } + }, + "action": {} + }, + { + "window": 10, + "t_start": 1024, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.6875, + "history_latent_mse": 0.36735105514526367, + "psnr": 16.56527676831867, + "ssim": 0.6139342401306186, + "lpips": 0.40910154581069946 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.322265625, + "history_latent_mse": 0.8481959700584412, + "psnr": 12.760421966494624, + "ssim": 0.29306858808683234, + "lpips": 0.52278733253479 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.5703125, + "history_latent_mse": 0.29110655188560486, + "psnr": 19.447293706779064, + "ssim": 0.8182153869431791, + "lpips": 0.16214250028133392 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.12109375, + "history_latent_mse": 0.15821678936481476, + "psnr": 33.371720684788976, + "ssim": 0.9333098351701967, + "lpips": 0.20022174715995789 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.89453125, + "history_latent_mse": 0.08658342808485031, + "psnr": 34.9602856819983, + "ssim": 0.9663304348676893, + "lpips": 0.1628963053226471 + } + }, + "action": {} + }, + { + "window": 11, + "t_start": 1032, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.78125, + "history_latent_mse": 0.4624047577381134, + "psnr": 15.731855744401464, + "ssim": 0.5821540813032305, + "lpips": 0.45474064350128174 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.640625, + "history_latent_mse": 0.7326492667198181, + "psnr": 11.302612929715579, + "ssim": 0.2649686434755568, + "lpips": 0.5242798328399658 + }, + "wrist_right": { + "history_latent_max_abs_error": 4.9453125, + "history_latent_mse": 0.2658441364765167, + "psnr": 18.710720945255478, + "ssim": 0.7952948841351002, + "lpips": 0.21997332572937012 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.37890625, + "history_latent_mse": 0.1426345556974411, + "psnr": 34.122823240820665, + "ssim": 0.9447340773610553, + "lpips": 0.20182457566261292 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.74609375, + "history_latent_mse": 0.08419465273618698, + "psnr": 34.47113063436918, + "ssim": 0.9648089778307101, + "lpips": 0.17052921652793884 + } + }, + "action": {} + }, + { + "window": 12, + "t_start": 1040, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.640625, + "history_latent_mse": 0.5132301449775696, + "psnr": 15.983674443366356, + "ssim": 0.5711128938061049, + "lpips": 0.46561115980148315 + }, + "wrist_left": { + "history_latent_max_abs_error": 4.375, + "history_latent_mse": 0.7983984351158142, + "psnr": 12.856503377537702, + "ssim": 0.31272353390744084, + "lpips": 0.5393034219741821 + }, + "wrist_right": { + "history_latent_max_abs_error": 5.09375, + "history_latent_mse": 0.33544811606407166, + "psnr": 19.14090374576665, + "ssim": 0.8081156587285867, + "lpips": 0.19741612672805786 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.78515625, + "history_latent_mse": 0.14761357009410858, + "psnr": 33.700728592511325, + "ssim": 0.9410422313603682, + "lpips": 0.21224264800548553 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.794921875, + "history_latent_mse": 0.09147722274065018, + "psnr": 34.26875634978037, + "ssim": 0.9650981540610412, + "lpips": 0.1699458360671997 + } + }, + "action": {} + }, + { + "window": 13, + "t_start": 1048, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 3.6953125, + "history_latent_mse": 0.4933485686779022, + "psnr": 17.082590248257137, + "ssim": 0.5884080536834421, + "lpips": 0.41664260625839233 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.359375, + "history_latent_mse": 0.7197220325469971, + "psnr": 13.882910373636218, + "ssim": 0.3438341225138739, + "lpips": 0.5017337799072266 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.71875, + "history_latent_mse": 0.26054465770721436, + "psnr": 18.905310365713266, + "ssim": 0.7913534802180995, + "lpips": 0.2362925112247467 + }, + "tactile_left": { + "history_latent_max_abs_error": 2.453125, + "history_latent_mse": 0.1428777277469635, + "psnr": 32.57784813393219, + "ssim": 0.9247986940084365, + "lpips": 0.2569653391838074 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.7578125, + "history_latent_mse": 0.09122272580862045, + "psnr": 33.93541215457305, + "ssim": 0.9639032022742243, + "lpips": 0.18089117109775543 + } + }, + "action": {} + }, + { + "window": 14, + "t_start": 1056, + "history_source": "previous_model_prediction", + "image": { + "video": { + "history_latent_max_abs_error": 4.5390625, + "history_latent_mse": 0.42323532700538635, + "psnr": 16.772021704011294, + "ssim": 0.5842499395855403, + "lpips": 0.4377226233482361 + }, + "wrist_left": { + "history_latent_max_abs_error": 5.734375, + "history_latent_mse": 0.7215422987937927, + "psnr": 14.05022370831259, + "ssim": 0.36217781141797434, + "lpips": 0.49938422441482544 + }, + "wrist_right": { + "history_latent_max_abs_error": 3.9296875, + "history_latent_mse": 0.3391540050506592, + "psnr": 19.22053573149978, + "ssim": 0.8064005946207718, + "lpips": 0.20410440862178802 + }, + "tactile_left": { + "history_latent_max_abs_error": 3.546875, + "history_latent_mse": 0.22963808476924896, + "psnr": 32.42793898054949, + "ssim": 0.9224002080856801, + "lpips": 0.26480305194854736 + }, + "tactile_right": { + "history_latent_max_abs_error": 1.8359375, + "history_latent_mse": 0.0955912172794342, + "psnr": 33.316405135925635, + "ssim": 0.9610501846505999, + "lpips": 0.20218218863010406 + } + }, + "action": {} + } + ] + } +] \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/short_gt_pred.mp4 b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/short_gt_pred.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aab5a4078d08b52a03d717fe813231189162bc7b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/short_gt_pred.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b310f068261e36e7e9a8118e3b4e6c606d0f0ce8c54bac92bb926aac156556d +size 1325087 diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/window_eval.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/window_eval.json new file mode 100644 index 0000000000000000000000000000000000000000..8fe88f1638c94a627bdb70b981fd61e2d9f0087b --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/window_eval.json @@ -0,0 +1,209 @@ +{ + "config": { + "run_name": "nano_action_causal_umi_sensoronly_noforce_v1", + "seed": 0, + "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs", + "data": { + "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents", + "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions", + "action_file": "sensor_actions_fps15_noforce.pt", + "view_action_file": "view_actions_fps15.pt", + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "optional_streams": [], + "action_norm": "quantile", + "train_window_stride": 1, + "val_window_stride": 50, + "stats_window_stride": 8, + "action_stats": null, + "num_workers": 4 + }, + "model": { + "language_model": "nano", + "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano", + "strict_load": true, + "dtype": "bfloat16", + "attention_backend": "flex", + "flex_preference": "flex_triton", + "temporal_mode": "causal", + "attention_topology": "all_streams_causal", + "task_mode": "action_full", + "active_streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right", + "action_left", + "action_right" + ], + "init_tactile_from_rgb": true, + "init_action_from_nano": { + "domain": "umi" + }, + "train_time_embedder": false, + "train_rgb_projections": false, + "lora": { + "enable": true, + "rank": 16, + "alpha": 32, + "targets": "attn_ffn" + } + }, + "training": { + "prediction_strategy": "causal_teacher_forcing", + "init_from_checkpoint": null, + "missing_stream_training": false, + "wrist_dropout": { + "probability": 0.0, + "mode": "paired", + "scope": "whole_clip" + }, + "batching": "group_by_active_streams", + "sampling": { + "mode": "uniform", + "steps_per_epoch": null + }, + "val_hide_wrist": false, + "epochs": 20, + "batch_size": 2, + "lr": 5e-05, + "weight_decay": 0.01, + "warmup_steps": 100, + "grad_clip": 1.0, + "amp": "bf16", + "sigma_distribution": "waver", + "sigma_shift": 3.0, + "loss_weights": {}, + "log_every": 20, + "ckpt_every_epochs": 1, + "activation_checkpointing": true + }, + "inference": { + "prediction_strategy": "causal_autoregressive" + } + }, + "summary": { + "video/history_latent_max_abs_error": 0.0, + "wrist/video/history_latent_max_abs_error": 0.0, + "video/history_latent_mse": 0.0, + "wrist/video/history_latent_mse": 0.0, + "video/latent_future_mse": 0.11321314424276352, + "wrist/video/latent_future_mse": 0.11321314424276352, + "video/psnr": 24.681463575563292, + "wrist/video/psnr": 24.681463575563292, + "video/ssim": 0.819499279912464, + "wrist/video/ssim": 0.819499279912464, + "video/lpips": 0.07029199600219727, + "wrist/video/lpips": 0.07029199600219727, + "wrist_left/history_latent_max_abs_error": 0.0, + "wrist/wrist_left/history_latent_max_abs_error": 0.0, + "wrist_left/history_latent_mse": 0.0, + "wrist/wrist_left/history_latent_mse": 0.0, + "wrist_left/latent_future_mse": 0.4533505439758301, + "wrist/wrist_left/latent_future_mse": 0.4533505439758301, + "wrist_left/psnr": 17.497804448618954, + "wrist/wrist_left/psnr": 17.497804448618954, + "wrist_left/ssim": 0.47679504985244964, + "wrist/wrist_left/ssim": 0.47679504985244964, + "wrist_left/lpips": 0.31007200479507446, + "wrist/wrist_left/lpips": 0.31007200479507446, + "wrist_right/history_latent_max_abs_error": 0.0, + "wrist/wrist_right/history_latent_max_abs_error": 0.0, + "wrist_right/history_latent_mse": 0.0, + "wrist/wrist_right/history_latent_mse": 0.0, + "wrist_right/latent_future_mse": 0.14882566034793854, + "wrist/wrist_right/latent_future_mse": 0.14882566034793854, + "wrist_right/psnr": 29.26793113448044, + "wrist/wrist_right/psnr": 29.26793113448044, + "wrist_right/ssim": 0.9340524927075055, + "wrist/wrist_right/ssim": 0.9340524927075055, + "wrist_right/lpips": 0.048112958669662476, + "wrist/wrist_right/lpips": 0.048112958669662476, + "tactile_left/history_latent_max_abs_error": 0.0, + "wrist/tactile_left/history_latent_max_abs_error": 0.0, + "tactile_left/history_latent_mse": 0.0, + "wrist/tactile_left/history_latent_mse": 0.0, + "tactile_left/latent_future_mse": 0.11228052526712418, + "wrist/tactile_left/latent_future_mse": 0.11228052526712418, + "tactile_left/psnr": 38.45417666049245, + "wrist/tactile_left/psnr": 38.45417666049245, + "tactile_left/ssim": 0.9587584262145483, + "wrist/tactile_left/ssim": 0.9587584262145483, + "tactile_left/lpips": 0.07753477990627289, + "wrist/tactile_left/lpips": 0.07753477990627289, + "tactile_right/history_latent_max_abs_error": 0.0, + "wrist/tactile_right/history_latent_max_abs_error": 0.0, + "tactile_right/history_latent_mse": 0.0, + "wrist/tactile_right/history_latent_mse": 0.0, + "tactile_right/latent_future_mse": 0.028353022411465645, + "wrist/tactile_right/latent_future_mse": 0.028353022411465645, + "tactile_right/psnr": 48.37651614492312, + "wrist/tactile_right/psnr": 48.37651614492312, + "tactile_right/ssim": 0.9920276303233837, + "wrist/tactile_right/ssim": 0.9920276303233837, + "tactile_right/lpips": 0.02037353813648224, + "wrist/tactile_right/lpips": 0.02037353813648224, + "num_windows": 1, + "num_windows_with_wrist": 1 + }, + "results": [ + { + "episode": "motherboard_0912_episode_002.pt", + "t_start": 944, + "seed": 0, + "baseline": null, + "hide_wrist": false, + "has_wrist": true, + "streams": [ + "video", + "wrist_left", + "wrist_right", + "tactile_left", + "tactile_right" + ], + "image": { + "video": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.11321314424276352, + "psnr": 24.681463575563292, + "ssim": 0.819499279912464, + "lpips": 0.07029199600219727 + }, + "wrist_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.4533505439758301, + "psnr": 17.497804448618954, + "ssim": 0.47679504985244964, + "lpips": 0.31007200479507446 + }, + "wrist_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.14882566034793854, + "psnr": 29.26793113448044, + "ssim": 0.9340524927075055, + "lpips": 0.048112958669662476 + }, + "tactile_left": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.11228052526712418, + "psnr": 38.45417666049245, + "ssim": 0.9587584262145483, + "lpips": 0.07753477990627289 + }, + "tactile_right": { + "history_latent_max_abs_error": 0.0, + "history_latent_mse": 0.0, + "latent_future_mse": 0.028353022411465645, + "psnr": 48.37651614492312, + "ssim": 0.9920276303233837, + "lpips": 0.02037353813648224 + } + }, + "action": {} + } + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/eval_split.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/eval_split.json new file mode 100644 index 0000000000000000000000000000000000000000..d405ce184e9aa1d83101b1cf553d845a848044a3 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/eval_split.json @@ -0,0 +1,49 @@ +{ + "name": "mixed_mb_wrist_holdout", + "note": "Rebuilt 2026-09-17 on React f186f5f6 and again 2026-09-18 on 990a6787 (OptiTrack two-state flicker repair; pose-only, force unchanged): the 25 motherboard 0911/0912 episodes were regenerated in mixed_mb_actions/ from the repaired OptiTrack poses and the v8 (0-15 N) force channel. The 32 mode1_v2 episodes (0510/0511/0519, no wrist) were DROPPED because upstream removed their force columns from old_data (no v8 force exists for them); their latents remain in mixed_mb_latents but are unused. val = 0911_ep002_seg00 + 0912_ep003 (in-dist, wrist); test = 0911_ep008_seg00 + 0912_ep002 (in-dist) + the three val0909 episodes (OOD hardware/calibration session, never in train; NOTE their force is still the pre-v8 calibration, max 7.87 N, so test metrics on them use a different force scale). Every remaining episode has wrist cameras. Pre-rebuild file: mixed_mb_wrist_holdout.pre_0917.json; pre-rebuild actions: mixed_mb_actions_pre_0917/.", + "counts": { + "train_windows": 71485, + "train_episodes": 21, + "train_windows_with_wrist": 71485, + "val_windows": 5411, + "val_episodes": 2, + "val_windows_with_wrist": 5411, + "test_windows": 16699, + "test_episodes": 5, + "test_windows_with_wrist": 16699 + }, + "train": [ + "motherboard_0911_episode_000.pt", + "motherboard_0911_episode_001_seg00.pt", + "motherboard_0911_episode_001_seg01.pt", + "motherboard_0911_episode_003_seg00.pt", + "motherboard_0911_episode_004_seg00.pt", + "motherboard_0911_episode_004_seg01.pt", + "motherboard_0911_episode_005_seg00.pt", + "motherboard_0911_episode_005_seg01.pt", + "motherboard_0911_episode_005_seg02.pt", + "motherboard_0911_episode_005_seg03.pt", + "motherboard_0911_episode_005_seg04.pt", + "motherboard_0911_episode_005_seg05.pt", + "motherboard_0911_episode_006_seg00.pt", + "motherboard_0911_episode_006_seg01.pt", + "motherboard_0911_episode_007_seg00.pt", + "motherboard_0911_episode_007_seg01.pt", + "motherboard_0911_episode_007_seg02.pt", + "motherboard_0911_episode_007_seg03.pt", + "motherboard_0911_episode_007_seg04.pt", + "motherboard_0912_episode_000_seg00.pt", + "motherboard_0912_episode_000_seg01.pt" + ], + "val": [ + "motherboard_0911_episode_002_seg00.pt", + "motherboard_0912_episode_003.pt" + ], + "test": [ + "motherboard_0909_episode_000.pt", + "motherboard_0909_episode_001.pt", + "motherboard_0909_episode_002.pt", + "motherboard_0911_episode_008_seg00.pt", + "motherboard_0912_episode_002.pt" + ] +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/manifest.json b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb29f2a1503658cbd4dd22e33fd385d9b80fde6 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/manifest.json @@ -0,0 +1,3383 @@ +{ + "checkpoint": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/epoch_009.pt", + "split": "test", + "window_stride": 1, + "episodes": [ + "motherboard_0909_episode_000.pt", + "motherboard_0909_episode_001.pt", + "motherboard_0909_episode_002.pt", + "motherboard_0911_episode_008_seg00.pt", + "motherboard_0912_episode_002.pt" + ], + "dataset_windows": 16598, + "force_threshold_N": 0.5, + "force_rule": "max(left,right) > 0.5 N", + "fps": 15, + "history_frames": 8, + "short_future_frames": 8, + "rollout_windows": 15, + "long_future_frames": 120, + "long_total_frames": 128, + "solver": "unipc", + "denoising_steps_per_latent_block": 35, + "seed": 0, + "categories_defined_on": "initial 16-frame window; see per-case long contact statistics", + "calibration_note": "0909 uses pre-v8 force calibration; 0911/0912 use v8, as documented by source split.", + "eligible_long_counts": { + "motherboard_0909_episode_000.pt": { + "none": 445, + "onset": 252, + "release": 255, + "full": 1535 + }, + "motherboard_0909_episode_001.pt": { + "none": 237, + "onset": 311, + "release": 310, + "full": 1302 + }, + "motherboard_0909_episode_002.pt": { + "none": 92, + "onset": 298, + "release": 299, + "full": 916 + }, + "motherboard_0911_episode_008_seg00.pt": { + "none": 3, + "onset": 104, + "release": 91, + "full": 365 + }, + "motherboard_0912_episode_002.pt": { + "none": 172, + "onset": 350, + "release": 292, + "full": 3897 + } + }, + "cases": [ + { + "task": 0, + "episode": "motherboard_0909_episode_000.pt", + "category": "none", + "index": 1134, + "t_start": 1134, + "start_seconds": 75.6, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.015625, + "long_future_contact_fraction": 0.016666666666666666, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0 + ], + "first_contact_tick": 125, + "long_label": "none_at_start", + "output_subdir": "00_motherboard_0909_episode_000_none", + "rollout_windows": 15 + }, + { + "task": 1, + "episode": "motherboard_0909_episode_000.pt", + "category": "onset", + "index": 1825, + "t_start": 1825, + "start_seconds": 121.66666666666667, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.7265625, + "long_future_contact_fraction": 0.775, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "01_motherboard_0909_episode_000_onset", + "rollout_windows": 15 + }, + { + "task": 2, + "episode": "motherboard_0909_episode_000.pt", + "category": "release", + "index": 3072, + "t_start": 3072, + "start_seconds": 204.8, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.7734375, + "long_future_contact_fraction": 0.7583333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "02_motherboard_0909_episode_000_release", + "rollout_windows": 15 + }, + { + "task": 3, + "episode": "motherboard_0909_episode_000.pt", + "category": "full", + "index": 2531, + "t_start": 2531, + "start_seconds": 168.73333333333332, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.9296875, + "long_future_contact_fraction": 0.925, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "03_motherboard_0909_episode_000_full", + "rollout_windows": 15 + }, + { + "task": 4, + "episode": "motherboard_0909_episode_001.pt", + "category": "none", + "index": 3319, + "t_start": 3, + "start_seconds": 0.2, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.0, + "long_future_contact_fraction": 0.0, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": null, + "long_label": "none_throughout", + "output_subdir": "04_motherboard_0909_episode_001_none", + "rollout_windows": 15 + }, + { + "task": 5, + "episode": "motherboard_0909_episode_001.pt", + "category": "onset", + "index": 4059, + "t_start": 760, + "start_seconds": 50.666666666666664, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6953125, + "long_future_contact_fraction": 0.7416666666666667, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "05_motherboard_0909_episode_001_onset", + "rollout_windows": 15 + }, + { + "task": 6, + "episode": "motherboard_0909_episode_001.pt", + "category": "release", + "index": 6112, + "t_start": 2829, + "start_seconds": 188.6, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6953125, + "long_future_contact_fraction": 0.675, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "06_motherboard_0909_episode_001_release", + "rollout_windows": 15 + }, + { + "task": 7, + "episode": "motherboard_0909_episode_001.pt", + "category": "full", + "index": 3783, + "t_start": 467, + "start_seconds": 31.133333333333333, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 1.0, + "long_future_contact_fraction": 1.0, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "07_motherboard_0909_episode_001_full", + "rollout_windows": 15 + }, + { + "task": 8, + "episode": "motherboard_0909_episode_002.pt", + "category": "none", + "index": 9178, + "t_start": 2600, + "start_seconds": 173.33333333333334, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.296875, + "long_future_contact_fraction": 0.31666666666666665, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "first_contact_tick": 48, + "long_label": "none_at_start", + "output_subdir": "08_motherboard_0909_episode_002_none", + "rollout_windows": 15 + }, + { + "task": 9, + "episode": "motherboard_0909_episode_002.pt", + "category": "onset", + "index": 8502, + "t_start": 1924, + "start_seconds": 128.26666666666668, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6796875, + "long_future_contact_fraction": 0.725, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "09_motherboard_0909_episode_002_onset", + "rollout_windows": 15 + }, + { + "task": 10, + "episode": "motherboard_0909_episode_002.pt", + "category": "release", + "index": 7670, + "t_start": 1051, + "start_seconds": 70.06666666666666, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.703125, + "long_future_contact_fraction": 0.6833333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "10_motherboard_0909_episode_002_release", + "rollout_windows": 15 + }, + { + "task": 11, + "episode": "motherboard_0909_episode_002.pt", + "category": "full", + "index": 8366, + "t_start": 1788, + "start_seconds": 119.2, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.9609375, + "long_future_contact_fraction": 0.9583333333333334, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "11_motherboard_0909_episode_002_full", + "rollout_windows": 15 + }, + { + "task": 12, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "none", + "index": 10300, + "t_start": 913, + "start_seconds": 60.86666666666667, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.2734375, + "long_future_contact_fraction": 0.2916666666666667, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "first_contact_tick": 18, + "long_label": "none_at_start", + "output_subdir": "12_motherboard_0911_episode_008_seg00_none", + "rollout_windows": 15 + }, + { + "task": 13, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "onset", + "index": 10113, + "t_start": 726, + "start_seconds": 48.4, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.515625, + "long_future_contact_fraction": 0.55, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "13_motherboard_0911_episode_008_seg00_onset", + "rollout_windows": 15 + }, + { + "task": 14, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "release", + "index": 9633, + "t_start": 246, + "start_seconds": 16.4, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.6796875, + "long_future_contact_fraction": 0.6583333333333333, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "14_motherboard_0911_episode_008_seg00_release", + "rollout_windows": 15 + }, + { + "task": 15, + "episode": "motherboard_0911_episode_008_seg00.pt", + "category": "full", + "index": 9819, + "t_start": 432, + "start_seconds": 28.8, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.8359375, + "long_future_contact_fraction": 0.825, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "15_motherboard_0911_episode_008_seg00_full", + "rollout_windows": 15 + }, + { + "task": 16, + "episode": "motherboard_0912_episode_002.pt", + "category": "none", + "index": 11187, + "t_start": 750, + "start_seconds": 50.0, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 0.3671875, + "long_future_contact_fraction": 0.39166666666666666, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 65, + "long_label": "none_at_start", + "output_subdir": "16_motherboard_0912_episode_002_none", + "rollout_windows": 15 + }, + { + "task": 17, + "episode": "motherboard_0912_episode_002.pt", + "category": "onset", + "index": 12543, + "t_start": 2106, + "start_seconds": 140.4, + "short_contact": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.75, + "long_future_contact_fraction": 0.8, + "contact_over_128_frames": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1 + ], + "first_contact_tick": 11, + "long_label": "onset", + "output_subdir": "17_motherboard_0912_episode_002_onset", + "rollout_windows": 15 + }, + { + "task": 18, + "episode": "motherboard_0912_episode_002.pt", + "category": "release", + "index": 10654, + "t_start": 217, + "start_seconds": 14.466666666666667, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + "short_transition_ticks": [ + 11 + ], + "future_transition_ticks": [ + 11 + ], + "long_contact_fraction": 0.9296875, + "long_future_contact_fraction": 0.925, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "release", + "output_subdir": "18_motherboard_0912_episode_002_release", + "rollout_windows": 15 + }, + { + "task": 19, + "episode": "motherboard_0912_episode_002.pt", + "category": "full", + "index": 11381, + "t_start": 944, + "start_seconds": 62.93333333333333, + "short_contact": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "short_transition_ticks": [], + "future_transition_ticks": [], + "long_contact_fraction": 1.0, + "long_future_contact_fraction": 1.0, + "contact_over_128_frames": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "first_contact_tick": 0, + "long_label": "full", + "output_subdir": "19_motherboard_0912_episode_002_full", + "rollout_windows": 15 + } + ], + "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json", + "paired_selection_from": "runs/nano_action_causal_umi_all_v1/contact_epoch_019_test/manifest.json", + "requested_split_file": "/group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_mb_wrist_holdout.json" +} \ No newline at end of file diff --git a/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/summary.md b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/summary.md new file mode 100644 index 0000000000000000000000000000000000000000..f6d98b4d269ddc309917c28b6093e75640c7fe75 --- /dev/null +++ b/results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/summary.md @@ -0,0 +1,21 @@ +# nano_action_causal_umi_sensoronly_noforce_v1 test_epoch_009 (20/20 cases complete) -- motherboard 20-case contact test manifest (same cases as umi_all e019) + +### short horizon: 8 -> 8 frames (n=20) + +| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ | +|---|---:|---:|---:| +| video | 27.60 | 0.8773 | 0.0478 | +| wrist_left | 17.59 | 0.5299 | 0.2598 | +| wrist_right | 20.39 | 0.6776 | 0.1715 | +| tactile_left | 41.12 | 0.9777 | 0.0397 | +| tactile_right | 45.88 | 0.9888 | 0.0253 | + +### long horizon: mean over 15 chained windows (n=20) + +| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ | +|---|---:|---:|---:| +| video | 20.84 | 0.7060 | 0.2026 | +| wrist_left | 13.35 | 0.3879 | 0.4723 | +| wrist_right | 14.16 | 0.4465 | 0.4319 | +| tactile_left | 35.29 | 0.9486 | 0.2010 | +| tactile_right | 37.00 | 0.9602 | 0.1602 |