WitneyWW commited on
Commit
ffae4ee
·
verified ·
1 Parent(s): fc95770

add nano_action_causal_umi_sensoronly_noforce_v1 motherboard test results (20 cases, epoch_009); strips (png) omitted

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +40 -0
  2. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/case.json +164 -0
  3. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/complete.json +1 -0
  4. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 +3 -0
  5. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/motherboard_0909_episode_000_t01134_pred.pt +3 -0
  6. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/reference_rollout.json +661 -0
  7. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 +3 -0
  8. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/window_eval.json +209 -0
  9. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/case.json +168 -0
  10. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/complete.json +1 -0
  11. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 +3 -0
  12. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/motherboard_0909_episode_000_t01825_pred.pt +3 -0
  13. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/reference_rollout.json +661 -0
  14. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 +3 -0
  15. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/window_eval.json +209 -0
  16. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/case.json +168 -0
  17. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/complete.json +1 -0
  18. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 +3 -0
  19. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/motherboard_0909_episode_000_t03072_pred.pt +3 -0
  20. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/reference_rollout.json +661 -0
  21. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 +3 -0
  22. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/window_eval.json +209 -0
  23. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/case.json +164 -0
  24. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/complete.json +1 -0
  25. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 +3 -0
  26. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/motherboard_0909_episode_000_t02531_pred.pt +3 -0
  27. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/reference_rollout.json +661 -0
  28. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 +3 -0
  29. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/window_eval.json +209 -0
  30. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/case.json +164 -0
  31. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/complete.json +1 -0
  32. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 +3 -0
  33. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/motherboard_0909_episode_001_t00003_pred.pt +3 -0
  34. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/reference_rollout.json +661 -0
  35. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 +3 -0
  36. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/window_eval.json +209 -0
  37. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/case.json +168 -0
  38. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/complete.json +1 -0
  39. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 +3 -0
  40. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/motherboard_0909_episode_001_t00760_pred.pt +3 -0
  41. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/reference_rollout.json +661 -0
  42. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 +3 -0
  43. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/window_eval.json +209 -0
  44. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/case.json +168 -0
  45. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/complete.json +1 -0
  46. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 +3 -0
  47. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/motherboard_0909_episode_001_t02829_pred.pt +3 -0
  48. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/reference_rollout.json +661 -0
  49. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 +3 -0
  50. results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/window_eval.json +209 -0
.gitattributes CHANGED
@@ -875,3 +875,43 @@ _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT
875
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_left.png filter=lfs diff=lfs merge=lfs -text
876
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_right.png filter=lfs diff=lfs merge=lfs -text
877
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
875
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_left.png filter=lfs diff=lfs merge=lfs -text
876
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/pushT_0912_episode_010_seg03_t00418_wrist_right.png filter=lfs diff=lfs merge=lfs -text
877
  _selftest/results/nano_action_causal_pusht_sensoronly_v1/test_epoch_019/29_pushT_0912_episode_010_seg03_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
878
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
879
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
880
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
881
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
882
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
883
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
884
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
885
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
886
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
887
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
888
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
889
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
890
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
891
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
892
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
893
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/07_motherboard_0909_episode_001_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
894
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
895
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/08_motherboard_0909_episode_002_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
896
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
897
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/09_motherboard_0909_episode_002_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
898
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
899
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/10_motherboard_0909_episode_002_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
900
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
901
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/11_motherboard_0909_episode_002_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
902
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
903
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/12_motherboard_0911_episode_008_seg00_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
904
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
905
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/13_motherboard_0911_episode_008_seg00_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
906
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
907
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/14_motherboard_0911_episode_008_seg00_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
908
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
909
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/15_motherboard_0911_episode_008_seg00_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
910
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
911
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/16_motherboard_0912_episode_002_none/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
912
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
913
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/17_motherboard_0912_episode_002_onset/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
914
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
915
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/18_motherboard_0912_episode_002_release/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
916
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/long_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
917
+ results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/19_motherboard_0912_episode_002_full/short_gt_pred.mp4 filter=lfs diff=lfs merge=lfs -text
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/case.json ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 0,
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "category": "none",
5
+ "index": 1134,
6
+ "t_start": 1134,
7
+ "start_seconds": 75.6,
8
+ "short_contact": [
9
+ 0,
10
+ 0,
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 0,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0
25
+ ],
26
+ "short_transition_ticks": [],
27
+ "future_transition_ticks": [],
28
+ "long_contact_fraction": 0.015625,
29
+ "long_future_contact_fraction": 0.016666666666666666,
30
+ "contact_over_128_frames": [
31
+ 0,
32
+ 0,
33
+ 0,
34
+ 0,
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 0,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 0,
59
+ 0,
60
+ 0,
61
+ 0,
62
+ 0,
63
+ 0,
64
+ 0,
65
+ 0,
66
+ 0,
67
+ 0,
68
+ 0,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 0,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 0,
82
+ 0,
83
+ 0,
84
+ 0,
85
+ 0,
86
+ 0,
87
+ 0,
88
+ 0,
89
+ 0,
90
+ 0,
91
+ 0,
92
+ 0,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 0,
98
+ 0,
99
+ 0,
100
+ 0,
101
+ 0,
102
+ 0,
103
+ 0,
104
+ 0,
105
+ 0,
106
+ 0,
107
+ 0,
108
+ 0,
109
+ 0,
110
+ 0,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 0,
115
+ 0,
116
+ 0,
117
+ 0,
118
+ 0,
119
+ 0,
120
+ 0,
121
+ 0,
122
+ 0,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 0,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 0,
132
+ 0,
133
+ 0,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 0,
141
+ 0,
142
+ 0,
143
+ 0,
144
+ 0,
145
+ 0,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 0,
155
+ 0,
156
+ 1,
157
+ 1,
158
+ 0
159
+ ],
160
+ "first_contact_tick": 125,
161
+ "long_label": "none_at_start",
162
+ "output_subdir": "00_motherboard_0909_episode_000_none",
163
+ "rollout_windows": 15
164
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:714dff21d089ab0704ddfbd44d1973286728628fe074079c2fa46da4a93b2f06
3
+ size 7552869
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/motherboard_0909_episode_000_t01134_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a1d930a777e0fa90a5fcb5dda9fc83df647db7cc8f407c4ff095f71c9258630
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "t_start": 1134,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 1134,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 28.118625935497626,
23
+ "ssim": 0.8981439036327015,
24
+ "lpips": 0.02593047171831131
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 33.22567826043501,
30
+ "ssim": 0.9705809567376241,
31
+ "lpips": 0.028415463864803314
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 16.614699992395067,
37
+ "ssim": 0.6578296037120219,
38
+ "lpips": 0.13281607627868652
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 48.21878437631539,
44
+ "ssim": 0.9935058084502003,
45
+ "lpips": 0.004225203301757574
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 44.9079643326986,
51
+ "ssim": 0.98389798483938,
52
+ "lpips": 0.023684388026595116
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 1142,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 2.875,
64
+ "history_latent_mse": 0.07125979661941528,
65
+ "psnr": 22.299567988857767,
66
+ "ssim": 0.7740196461835518,
67
+ "lpips": 0.08631207048892975
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 3.4609375,
71
+ "history_latent_mse": 0.07626854628324509,
72
+ "psnr": 27.307863321953803,
73
+ "ssim": 0.9027393856884827,
74
+ "lpips": 0.07224798202514648
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 4.8515625,
78
+ "history_latent_mse": 0.4535127580165863,
79
+ "psnr": 11.077167592783326,
80
+ "ssim": 0.35698008339236,
81
+ "lpips": 0.3966830372810364
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 0.693359375,
85
+ "history_latent_mse": 0.00990868080407381,
86
+ "psnr": 45.312933483226345,
87
+ "ssim": 0.9895434031357497,
88
+ "lpips": 0.016883008182048798
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 1.4931640625,
92
+ "history_latent_mse": 0.03798486664891243,
93
+ "psnr": 42.18905422712518,
94
+ "ssim": 0.9758137255932309,
95
+ "lpips": 0.049356162548065186
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 1150,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 3.1796875,
107
+ "history_latent_mse": 0.1578950434923172,
108
+ "psnr": 20.496099549491138,
109
+ "ssim": 0.7374106287444411,
110
+ "lpips": 0.1451627016067505
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 3.3125,
114
+ "history_latent_mse": 0.16328312456607819,
115
+ "psnr": 24.916684340802206,
116
+ "ssim": 0.8430255329021106,
117
+ "lpips": 0.126552015542984
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 4.90625,
121
+ "history_latent_mse": 0.802646815776825,
122
+ "psnr": 10.756574512710799,
123
+ "ssim": 0.3366269340047899,
124
+ "lpips": 0.45835721492767334
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 0.92529296875,
128
+ "history_latent_mse": 0.021422749385237694,
129
+ "psnr": 43.282370651116224,
130
+ "ssim": 0.9847204751128106,
131
+ "lpips": 0.03882313519716263
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 1.390625,
135
+ "history_latent_mse": 0.05414335057139397,
136
+ "psnr": 41.48125187958886,
137
+ "ssim": 0.9737438472038825,
138
+ "lpips": 0.07059751451015472
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 1158,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 3.09375,
150
+ "history_latent_mse": 0.186897411942482,
151
+ "psnr": 20.132890376872112,
152
+ "ssim": 0.7151798327633927,
153
+ "lpips": 0.15986736118793488
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 3.6640625,
157
+ "history_latent_mse": 0.2769032120704651,
158
+ "psnr": 22.686464420994497,
159
+ "ssim": 0.7762327048393174,
160
+ "lpips": 0.15843285620212555
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 5.046875,
164
+ "history_latent_mse": 0.7685828804969788,
165
+ "psnr": 11.316598704771296,
166
+ "ssim": 0.27996114215063156,
167
+ "lpips": 0.46487122774124146
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 0.947509765625,
171
+ "history_latent_mse": 0.031877338886260986,
172
+ "psnr": 41.39235837791431,
173
+ "ssim": 0.9798823696250478,
174
+ "lpips": 0.0765223354101181
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 1.28125,
178
+ "history_latent_mse": 0.05412241816520691,
179
+ "psnr": 40.054445265286745,
180
+ "ssim": 0.9706170962246483,
181
+ "lpips": 0.09510093927383423
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 1166,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 3.4375,
193
+ "history_latent_mse": 0.19426828622817993,
194
+ "psnr": 19.521938358523744,
195
+ "ssim": 0.711563055300819,
196
+ "lpips": 0.17140625417232513
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 3.9921875,
200
+ "history_latent_mse": 0.3184545934200287,
201
+ "psnr": 21.595329401906657,
202
+ "ssim": 0.7292829632491972,
203
+ "lpips": 0.19176793098449707
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 5.03125,
207
+ "history_latent_mse": 0.8025609850883484,
208
+ "psnr": 11.085940117735175,
209
+ "ssim": 0.26395832446885137,
210
+ "lpips": 0.4824531674385071
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 1.044921875,
214
+ "history_latent_mse": 0.04302438721060753,
215
+ "psnr": 39.914173602008624,
216
+ "ssim": 0.9754849562365518,
217
+ "lpips": 0.11436552554368973
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 2.2421875,
221
+ "history_latent_mse": 0.07244016975164413,
222
+ "psnr": 38.92377956111862,
223
+ "ssim": 0.9673908140162797,
224
+ "lpips": 0.11861604452133179
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 1174,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 3.96875,
236
+ "history_latent_mse": 0.21044723689556122,
237
+ "psnr": 19.172171786625192,
238
+ "ssim": 0.704517872746727,
239
+ "lpips": 0.18479205667972565
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 3.82421875,
243
+ "history_latent_mse": 0.4116292893886566,
244
+ "psnr": 22.70440798830937,
245
+ "ssim": 0.7446463412912648,
246
+ "lpips": 0.19933201372623444
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 5.109375,
250
+ "history_latent_mse": 0.7731020450592041,
251
+ "psnr": 10.901734091153113,
252
+ "ssim": 0.250261526565162,
253
+ "lpips": 0.5134169459342957
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 1.259765625,
257
+ "history_latent_mse": 0.051759108901023865,
258
+ "psnr": 39.16722800912323,
259
+ "ssim": 0.9711905790082329,
260
+ "lpips": 0.14375180006027222
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 1.66015625,
264
+ "history_latent_mse": 0.0699012503027916,
265
+ "psnr": 38.23995041546551,
266
+ "ssim": 0.9635273440027555,
267
+ "lpips": 0.13030634820461273
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 1182,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 4.421875,
279
+ "history_latent_mse": 0.22228819131851196,
280
+ "psnr": 19.06247131517149,
281
+ "ssim": 0.6924016519226703,
282
+ "lpips": 0.18284600973129272
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 3.93505859375,
286
+ "history_latent_mse": 0.37236669659614563,
287
+ "psnr": 22.312370155604597,
288
+ "ssim": 0.7331450899010764,
289
+ "lpips": 0.21978585422039032
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 4.921875,
293
+ "history_latent_mse": 0.8304977416992188,
294
+ "psnr": 10.793835822812479,
295
+ "ssim": 0.24135036171360116,
296
+ "lpips": 0.4925817549228668
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 1.2421875,
300
+ "history_latent_mse": 0.0620768703520298,
301
+ "psnr": 38.341802320897685,
302
+ "ssim": 0.9668993105821726,
303
+ "lpips": 0.17329636216163635
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 2.1953125,
307
+ "history_latent_mse": 0.09515998512506485,
308
+ "psnr": 37.828723643381004,
309
+ "ssim": 0.9592576191761607,
310
+ "lpips": 0.14613281190395355
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 1190,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 4.140625,
322
+ "history_latent_mse": 0.2338160276412964,
323
+ "psnr": 19.07268901263889,
324
+ "ssim": 0.6884140759189079,
325
+ "lpips": 0.1924017369747162
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 4.0859375,
329
+ "history_latent_mse": 0.37667128443717957,
330
+ "psnr": 22.223930212587973,
331
+ "ssim": 0.7211460010588575,
332
+ "lpips": 0.23736724257469177
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 5.11328125,
336
+ "history_latent_mse": 0.9431016445159912,
337
+ "psnr": 11.30551157083791,
338
+ "ssim": 0.24893031947198674,
339
+ "lpips": 0.4797683358192444
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 1.31640625,
343
+ "history_latent_mse": 0.07115178555250168,
344
+ "psnr": 37.64922164261528,
345
+ "ssim": 0.9626026930802947,
346
+ "lpips": 0.20532503724098206
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 1.525390625,
350
+ "history_latent_mse": 0.09037545323371887,
351
+ "psnr": 36.95520132192448,
352
+ "ssim": 0.9554855120856747,
353
+ "lpips": 0.17302285134792328
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 1198,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 4.46875,
365
+ "history_latent_mse": 0.24051082134246826,
366
+ "psnr": 19.06611351386221,
367
+ "ssim": 0.6793518227863704,
368
+ "lpips": 0.2010258138179779
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 3.75,
372
+ "history_latent_mse": 0.3760389983654022,
373
+ "psnr": 21.29236028691256,
374
+ "ssim": 0.6912351388757134,
375
+ "lpips": 0.26064425706863403
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 4.671875,
379
+ "history_latent_mse": 0.9166877865791321,
380
+ "psnr": 11.075601469624822,
381
+ "ssim": 0.2391405727778968,
382
+ "lpips": 0.5041201114654541
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 1.58203125,
386
+ "history_latent_mse": 0.08001843839883804,
387
+ "psnr": 36.95966657540948,
388
+ "ssim": 0.9588211571345229,
389
+ "lpips": 0.22562216222286224
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 2.05859375,
393
+ "history_latent_mse": 0.10932174324989319,
394
+ "psnr": 36.3994679137689,
395
+ "ssim": 0.9515778298865583,
396
+ "lpips": 0.1953134536743164
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 1206,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 5.25,
408
+ "history_latent_mse": 0.2604706883430481,
409
+ "psnr": 18.503925060285766,
410
+ "ssim": 0.644975085064724,
411
+ "lpips": 0.24730649590492249
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 4.498046875,
415
+ "history_latent_mse": 0.4385768175125122,
416
+ "psnr": 19.918945132686176,
417
+ "ssim": 0.6337497940723535,
418
+ "lpips": 0.2764352560043335
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 5.125,
422
+ "history_latent_mse": 0.8151993751525879,
423
+ "psnr": 10.891914773891202,
424
+ "ssim": 0.23601986912850992,
425
+ "lpips": 0.5140556693077087
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 1.509765625,
429
+ "history_latent_mse": 0.0883679911494255,
430
+ "psnr": 36.5672491510434,
431
+ "ssim": 0.9549003553892959,
432
+ "lpips": 0.24738405644893646
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 1.95703125,
436
+ "history_latent_mse": 0.10421737283468246,
437
+ "psnr": 36.06402608273454,
438
+ "ssim": 0.9495501773313848,
439
+ "lpips": 0.19591185450553894
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 1214,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 4.8671875,
451
+ "history_latent_mse": 0.2957853674888611,
452
+ "psnr": 18.30719393323559,
453
+ "ssim": 0.6242453652896285,
454
+ "lpips": 0.26864248514175415
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 4.515625,
458
+ "history_latent_mse": 0.4766908586025238,
459
+ "psnr": 19.296219314166898,
460
+ "ssim": 0.6084653542439014,
461
+ "lpips": 0.29249680042266846
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 5.1171875,
465
+ "history_latent_mse": 0.823030948638916,
466
+ "psnr": 9.929037641527676,
467
+ "ssim": 0.23528381378321522,
468
+ "lpips": 0.5044862627983093
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 1.4716796875,
472
+ "history_latent_mse": 0.09646136313676834,
473
+ "psnr": 36.09376945101387,
474
+ "ssim": 0.9505532080139466,
475
+ "lpips": 0.2651330232620239
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 2.25,
479
+ "history_latent_mse": 0.12446943670511246,
480
+ "psnr": 35.77057445272215,
481
+ "ssim": 0.9483529378923702,
482
+ "lpips": 0.1994127333164215
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 1222,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 4.3203125,
494
+ "history_latent_mse": 0.30306458473205566,
495
+ "psnr": 17.93564056800799,
496
+ "ssim": 0.6129561302921299,
497
+ "lpips": 0.2883601784706116
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 4.0625,
501
+ "history_latent_mse": 0.48860955238342285,
502
+ "psnr": 18.96807842006445,
503
+ "ssim": 0.5910876772669825,
504
+ "lpips": 0.31704819202423096
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 5.703125,
508
+ "history_latent_mse": 0.9537672400474548,
509
+ "psnr": 9.410330656003293,
510
+ "ssim": 0.2376455665181072,
511
+ "lpips": 0.5271283388137817
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 1.58984375,
515
+ "history_latent_mse": 0.10451682657003403,
516
+ "psnr": 35.790241161950455,
517
+ "ssim": 0.9467691672545042,
518
+ "lpips": 0.2830126881599426
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 2.0078125,
522
+ "history_latent_mse": 0.1126982644200325,
523
+ "psnr": 35.091368951824094,
524
+ "ssim": 0.9443291247814589,
525
+ "lpips": 0.2325611114501953
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 1230,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.46875,
537
+ "history_latent_mse": 0.31999102234840393,
538
+ "psnr": 17.928656525741033,
539
+ "ssim": 0.614931015609635,
540
+ "lpips": 0.2899765074253082
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 3.99609375,
544
+ "history_latent_mse": 0.5375950932502747,
545
+ "psnr": 18.303168251730394,
546
+ "ssim": 0.5682538687196317,
547
+ "lpips": 0.3424307405948639
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 6.046875,
551
+ "history_latent_mse": 1.0547698736190796,
552
+ "psnr": 9.388592113479417,
553
+ "ssim": 0.23906650579267608,
554
+ "lpips": 0.5055872201919556
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 1.69921875,
558
+ "history_latent_mse": 0.11771353334188461,
559
+ "psnr": 35.4045823403573,
560
+ "ssim": 0.9431462540202011,
561
+ "lpips": 0.29494112730026245
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 2.671875,
565
+ "history_latent_mse": 0.14025308191776276,
566
+ "psnr": 34.770440023358226,
567
+ "ssim": 0.942512951717249,
568
+ "lpips": 0.22735056281089783
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 1238,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 4.5390625,
580
+ "history_latent_mse": 0.3158722519874573,
581
+ "psnr": 17.91332162646378,
582
+ "ssim": 0.6223806892138105,
583
+ "lpips": 0.2958202362060547
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 4.703125,
587
+ "history_latent_mse": 0.5239625573158264,
588
+ "psnr": 17.56523429353433,
589
+ "ssim": 0.5341453904969209,
590
+ "lpips": 0.3766480088233948
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.5078125,
594
+ "history_latent_mse": 0.934349775314331,
595
+ "psnr": 9.723535208620142,
596
+ "ssim": 0.23142451541698372,
597
+ "lpips": 0.513656735420227
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 1.88671875,
601
+ "history_latent_mse": 0.12004297971725464,
602
+ "psnr": 34.77172152401946,
603
+ "ssim": 0.9387000663377864,
604
+ "lpips": 0.3137485980987549
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 2.19921875,
608
+ "history_latent_mse": 0.13056305050849915,
609
+ "psnr": 34.15736555453252,
610
+ "ssim": 0.9386077918616212,
611
+ "lpips": 0.23681756854057312
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 1246,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 3.51171875,
623
+ "history_latent_mse": 0.2952965199947357,
624
+ "psnr": 17.643402573338243,
625
+ "ssim": 0.6265155324322179,
626
+ "lpips": 0.2974652051925659
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 4.796875,
630
+ "history_latent_mse": 0.6053735017776489,
631
+ "psnr": 16.593013323686236,
632
+ "ssim": 0.5137983982273475,
633
+ "lpips": 0.4190220236778259
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 6.0,
637
+ "history_latent_mse": 0.9508810043334961,
638
+ "psnr": 9.878230801841482,
639
+ "ssim": 0.19101576200356277,
640
+ "lpips": 0.5357831716537476
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 1.9375,
644
+ "history_latent_mse": 0.12828531861305237,
645
+ "psnr": 34.28512883129351,
646
+ "ssim": 0.9350560243901624,
647
+ "lpips": 0.3242204487323761
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 2.458984375,
651
+ "history_latent_mse": 0.13128380477428436,
652
+ "psnr": 33.080836199969895,
653
+ "ssim": 0.9308731355698692,
654
+ "lpips": 0.2695121169090271
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fcf61e43e0a6412de10a31f00ef4aa6dfcfa4a9cdf8b2aae2d144b20c6e01d0f
3
+ size 918193
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/00_motherboard_0909_episode_000_none/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.08200626075267792,
91
+ "wrist/video/latent_future_mse": 0.08200626075267792,
92
+ "video/psnr": 28.118625935497626,
93
+ "wrist/video/psnr": 28.118625935497626,
94
+ "video/ssim": 0.8981439036327015,
95
+ "wrist/video/ssim": 0.8981439036327015,
96
+ "video/lpips": 0.02593047171831131,
97
+ "wrist/video/lpips": 0.02593047171831131,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.08269955962896347,
103
+ "wrist/wrist_left/latent_future_mse": 0.08269955962896347,
104
+ "wrist_left/psnr": 33.22567826043501,
105
+ "wrist/wrist_left/psnr": 33.22567826043501,
106
+ "wrist_left/ssim": 0.9705809567376241,
107
+ "wrist/wrist_left/ssim": 0.9705809567376241,
108
+ "wrist_left/lpips": 0.028415463864803314,
109
+ "wrist/wrist_left/lpips": 0.028415463864803314,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.47750023007392883,
115
+ "wrist/wrist_right/latent_future_mse": 0.47750023007392883,
116
+ "wrist_right/psnr": 16.614699992395067,
117
+ "wrist/wrist_right/psnr": 16.614699992395067,
118
+ "wrist_right/ssim": 0.6578296037120219,
119
+ "wrist/wrist_right/ssim": 0.6578296037120219,
120
+ "wrist_right/lpips": 0.13281607627868652,
121
+ "wrist/wrist_right/lpips": 0.13281607627868652,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.007856979966163635,
127
+ "wrist/tactile_left/latent_future_mse": 0.007856979966163635,
128
+ "tactile_left/psnr": 48.21878437631539,
129
+ "wrist/tactile_left/psnr": 48.21878437631539,
130
+ "tactile_left/ssim": 0.9935058084502003,
131
+ "wrist/tactile_left/ssim": 0.9935058084502003,
132
+ "tactile_left/lpips": 0.004225203301757574,
133
+ "wrist/tactile_left/lpips": 0.004225203301757574,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.03919291868805885,
139
+ "wrist/tactile_right/latent_future_mse": 0.03919291868805885,
140
+ "tactile_right/psnr": 44.9079643326986,
141
+ "wrist/tactile_right/psnr": 44.9079643326986,
142
+ "tactile_right/ssim": 0.98389798483938,
143
+ "wrist/tactile_right/ssim": 0.98389798483938,
144
+ "tactile_right/lpips": 0.023684388026595116,
145
+ "wrist/tactile_right/lpips": 0.023684388026595116,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_000.pt",
152
+ "t_start": 1134,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.08200626075267792,
169
+ "psnr": 28.118625935497626,
170
+ "ssim": 0.8981439036327015,
171
+ "lpips": 0.02593047171831131
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.08269955962896347,
177
+ "psnr": 33.22567826043501,
178
+ "ssim": 0.9705809567376241,
179
+ "lpips": 0.028415463864803314
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.47750023007392883,
185
+ "psnr": 16.614699992395067,
186
+ "ssim": 0.6578296037120219,
187
+ "lpips": 0.13281607627868652
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.007856979966163635,
193
+ "psnr": 48.21878437631539,
194
+ "ssim": 0.9935058084502003,
195
+ "lpips": 0.004225203301757574
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.03919291868805885,
201
+ "psnr": 44.9079643326986,
202
+ "ssim": 0.98389798483938,
203
+ "lpips": 0.023684388026595116
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/case.json ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 1,
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "category": "onset",
5
+ "index": 1825,
6
+ "t_start": 1825,
7
+ "start_seconds": 121.66666666666667,
8
+ "short_contact": [
9
+ 0,
10
+ 0,
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 0,
20
+ 1,
21
+ 1,
22
+ 1,
23
+ 1,
24
+ 1
25
+ ],
26
+ "short_transition_ticks": [
27
+ 11
28
+ ],
29
+ "future_transition_ticks": [
30
+ 11
31
+ ],
32
+ "long_contact_fraction": 0.7265625,
33
+ "long_future_contact_fraction": 0.775,
34
+ "contact_over_128_frames": [
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 0,
109
+ 0,
110
+ 0,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 0,
115
+ 0,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 0,
155
+ 0,
156
+ 0,
157
+ 1,
158
+ 1,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 1
163
+ ],
164
+ "first_contact_tick": 11,
165
+ "long_label": "onset",
166
+ "output_subdir": "01_motherboard_0909_episode_000_onset",
167
+ "rollout_windows": 15
168
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70df5a834c3c49340462c55c679b91256fe24fc5365d57fd4813ce8b2de6cc69
3
+ size 9526143
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/motherboard_0909_episode_000_t01825_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6734064a9940d0d8856e6db42f2085b0011cd908057f98627b4dc5da31497560
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "t_start": 1825,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 1825,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 23.628176260948578,
23
+ "ssim": 0.8456452647494677,
24
+ "lpips": 0.0570581778883934
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 16.974798261240593,
30
+ "ssim": 0.430608030325784,
31
+ "lpips": 0.1560155153274536
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 11.3508814597468,
37
+ "ssim": 0.42637402380382783,
38
+ "lpips": 0.37309491634368896
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 37.21835685193686,
44
+ "ssim": 0.9798802064902503,
45
+ "lpips": 0.055090613663196564
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 31.144626811115348,
51
+ "ssim": 0.9719350201675224,
52
+ "lpips": 0.10720653086900711
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 1833,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 3.453125,
64
+ "history_latent_mse": 0.11924435943365097,
65
+ "psnr": 21.38447871325007,
66
+ "ssim": 0.7454026098413906,
67
+ "lpips": 0.09989136457443237
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 4.703125,
71
+ "history_latent_mse": 0.6149580478668213,
72
+ "psnr": 15.709534166775507,
73
+ "ssim": 0.3481757982898162,
74
+ "lpips": 0.23345258831977844
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 6.75,
78
+ "history_latent_mse": 0.8365908265113831,
79
+ "psnr": 10.51854516089019,
80
+ "ssim": 0.39563176970008596,
81
+ "lpips": 0.4264390170574188
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 3.1328125,
85
+ "history_latent_mse": 0.07114292681217194,
86
+ "psnr": 27.08653562731253,
87
+ "ssim": 0.9231090405566171,
88
+ "lpips": 0.26742565631866455
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 3.8671875,
92
+ "history_latent_mse": 0.102386474609375,
93
+ "psnr": 26.352180183774756,
94
+ "ssim": 0.9428494174808446,
95
+ "lpips": 0.20953956246376038
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 1841,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 3.03125,
107
+ "history_latent_mse": 0.15389490127563477,
108
+ "psnr": 20.58760675670795,
109
+ "ssim": 0.7152439390162091,
110
+ "lpips": 0.13344064354896545
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 4.32421875,
114
+ "history_latent_mse": 0.5376542210578918,
115
+ "psnr": 14.86372024979256,
116
+ "ssim": 0.31986041049769304,
117
+ "lpips": 0.3564751148223877
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 5.5625,
121
+ "history_latent_mse": 0.8208544254302979,
122
+ "psnr": 9.399308772525803,
123
+ "ssim": 0.379937250363267,
124
+ "lpips": 0.4914754331111908
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 3.9609375,
128
+ "history_latent_mse": 0.18323363363742828,
129
+ "psnr": 27.37818445738302,
130
+ "ssim": 0.9299320902182369,
131
+ "lpips": 0.25641900300979614
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 2.9951171875,
135
+ "history_latent_mse": 0.13819794356822968,
136
+ "psnr": 26.38334682961372,
137
+ "ssim": 0.9408343045795153,
138
+ "lpips": 0.21969859302043915
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 1849,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 3.68359375,
150
+ "history_latent_mse": 0.19861239194869995,
151
+ "psnr": 20.758812785152756,
152
+ "ssim": 0.6975106537333856,
153
+ "lpips": 0.1406661719083786
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 5.5,
157
+ "history_latent_mse": 0.8406947255134583,
158
+ "psnr": 12.81062295552455,
159
+ "ssim": 0.2740668538115486,
160
+ "lpips": 0.45320695638656616
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 5.66796875,
164
+ "history_latent_mse": 0.9032319188117981,
165
+ "psnr": 12.964569428178333,
166
+ "ssim": 0.44145546590248474,
167
+ "lpips": 0.46219319105148315
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 3.640625,
171
+ "history_latent_mse": 0.18205487728118896,
172
+ "psnr": 27.98484835314955,
173
+ "ssim": 0.9292138306951223,
174
+ "lpips": 0.2694128751754761
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 3.44140625,
178
+ "history_latent_mse": 0.14041712880134583,
179
+ "psnr": 41.67464345589414,
180
+ "ssim": 0.9743010559371356,
181
+ "lpips": 0.08515998721122742
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 1857,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 3.8359375,
193
+ "history_latent_mse": 0.21089641749858856,
194
+ "psnr": 20.12968986430085,
195
+ "ssim": 0.6729452297464844,
196
+ "lpips": 0.19104285538196564
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 5.78125,
200
+ "history_latent_mse": 0.9549667835235596,
201
+ "psnr": 12.550545095641612,
202
+ "ssim": 0.42146506309265463,
203
+ "lpips": 0.4507025480270386
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 4.8984375,
207
+ "history_latent_mse": 0.7145854830741882,
208
+ "psnr": 11.401634918412771,
209
+ "ssim": 0.28026669964421147,
210
+ "lpips": 0.6360324621200562
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 2.8828125,
214
+ "history_latent_mse": 0.16469396650791168,
215
+ "psnr": 38.39359132029856,
216
+ "ssim": 0.9650768356833449,
217
+ "lpips": 0.1758243292570114
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 1.859375,
221
+ "history_latent_mse": 0.06372665613889694,
222
+ "psnr": 39.74709760258228,
223
+ "ssim": 0.9697240396062548,
224
+ "lpips": 0.11890912055969238
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 1865,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 5.703125,
236
+ "history_latent_mse": 0.26115942001342773,
237
+ "psnr": 19.3840588315676,
238
+ "ssim": 0.6533332812739424,
239
+ "lpips": 0.22177474200725555
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 4.52734375,
243
+ "history_latent_mse": 0.7400798201560974,
244
+ "psnr": 11.641789232035197,
245
+ "ssim": 0.35105336418918753,
246
+ "lpips": 0.49183470010757446
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 5.5703125,
250
+ "history_latent_mse": 1.0202034711837769,
251
+ "psnr": 10.671679805235513,
252
+ "ssim": 0.2757385503072361,
253
+ "lpips": 0.7221993207931519
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 2.2265625,
257
+ "history_latent_mse": 0.092548668384552,
258
+ "psnr": 38.86592262744519,
259
+ "ssim": 0.9639488963676137,
260
+ "lpips": 0.1817716360092163
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 2.01171875,
264
+ "history_latent_mse": 0.08237791061401367,
265
+ "psnr": 28.937859597528977,
266
+ "ssim": 0.9130735440172923,
267
+ "lpips": 0.3372594118118286
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 1873,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 4.234375,
279
+ "history_latent_mse": 0.2853606045246124,
280
+ "psnr": 19.327087853647637,
281
+ "ssim": 0.6527707724798322,
282
+ "lpips": 0.23417320847511292
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 4.5546875,
286
+ "history_latent_mse": 0.9626197814941406,
287
+ "psnr": 10.949636084625007,
288
+ "ssim": 0.29039771588789137,
289
+ "lpips": 0.5295237302780151
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 5.40625,
293
+ "history_latent_mse": 1.1418527364730835,
294
+ "psnr": 9.80005694351729,
295
+ "ssim": 0.27183607856813097,
296
+ "lpips": 0.7450927495956421
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 1.6171875,
300
+ "history_latent_mse": 0.08214738219976425,
301
+ "psnr": 38.044049077303185,
302
+ "ssim": 0.9611224301634556,
303
+ "lpips": 0.19384586811065674
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 3.6328125,
307
+ "history_latent_mse": 0.2798396050930023,
308
+ "psnr": 29.630710620583017,
309
+ "ssim": 0.9067128979478171,
310
+ "lpips": 0.3555738925933838
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 1881,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 3.9765625,
322
+ "history_latent_mse": 0.3022192120552063,
323
+ "psnr": 18.474017428693067,
324
+ "ssim": 0.6382623648501708,
325
+ "lpips": 0.2747034728527069
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 4.9296875,
329
+ "history_latent_mse": 0.9861699938774109,
330
+ "psnr": 9.804466590576641,
331
+ "ssim": 0.239695646269107,
332
+ "lpips": 0.5884726047515869
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 5.171875,
336
+ "history_latent_mse": 1.1945067644119263,
337
+ "psnr": 9.817503092548431,
338
+ "ssim": 0.2559563394471839,
339
+ "lpips": 0.7234587669372559
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 1.546875,
343
+ "history_latent_mse": 0.08684799820184708,
344
+ "psnr": 37.02358126070998,
345
+ "ssim": 0.9571071971857603,
346
+ "lpips": 0.21416965126991272
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 3.890625,
350
+ "history_latent_mse": 0.3355695307254791,
351
+ "psnr": 30.56725607826592,
352
+ "ssim": 0.9054872014132207,
353
+ "lpips": 0.3531824052333832
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 1889,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 4.53125,
365
+ "history_latent_mse": 0.3008727729320526,
366
+ "psnr": 18.334771306226575,
367
+ "ssim": 0.6482847651536765,
368
+ "lpips": 0.28134870529174805
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 5.23046875,
372
+ "history_latent_mse": 1.0673683881759644,
373
+ "psnr": 10.502831838929554,
374
+ "ssim": 0.3037068785100905,
375
+ "lpips": 0.5442860722541809
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 5.40625,
379
+ "history_latent_mse": 1.1673964262008667,
380
+ "psnr": 9.045239052512038,
381
+ "ssim": 0.18471718775414525,
382
+ "lpips": 0.6944596767425537
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 1.7578125,
386
+ "history_latent_mse": 0.09910959005355835,
387
+ "psnr": 36.619810000881145,
388
+ "ssim": 0.9556470391864594,
389
+ "lpips": 0.2256094217300415
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 2.5546875,
393
+ "history_latent_mse": 0.2062147855758667,
394
+ "psnr": 36.28021961420545,
395
+ "ssim": 0.959326061857189,
396
+ "lpips": 0.17003518342971802
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 1897,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 3.765625,
408
+ "history_latent_mse": 0.30186331272125244,
409
+ "psnr": 18.03807525162402,
410
+ "ssim": 0.6454518931632223,
411
+ "lpips": 0.28907760977745056
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 5.875,
415
+ "history_latent_mse": 0.8890538811683655,
416
+ "psnr": 10.39684811802661,
417
+ "ssim": 0.29498948139481107,
418
+ "lpips": 0.5650955438613892
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 6.7734375,
422
+ "history_latent_mse": 1.1140291690826416,
423
+ "psnr": 9.289455988436062,
424
+ "ssim": 0.1557308833748376,
425
+ "lpips": 0.7086127996444702
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 1.6171875,
429
+ "history_latent_mse": 0.09790438413619995,
430
+ "psnr": 36.211948050450346,
431
+ "ssim": 0.9528436610809317,
432
+ "lpips": 0.23762783408164978
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 2.359375,
436
+ "history_latent_mse": 0.14760760962963104,
437
+ "psnr": 34.08932658177453,
438
+ "ssim": 0.9368645450533417,
439
+ "lpips": 0.21939662098884583
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 1905,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 4.25,
451
+ "history_latent_mse": 0.28883758187294006,
452
+ "psnr": 18.00619140224066,
453
+ "ssim": 0.645133041821521,
454
+ "lpips": 0.29140937328338623
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 5.53125,
458
+ "history_latent_mse": 1.0083221197128296,
459
+ "psnr": 10.23226909703255,
460
+ "ssim": 0.28376860883500077,
461
+ "lpips": 0.5770002603530884
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 5.765625,
465
+ "history_latent_mse": 1.3987174034118652,
466
+ "psnr": 9.058174410777262,
467
+ "ssim": 0.14708021609148147,
468
+ "lpips": 0.7367289066314697
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 1.6865234375,
472
+ "history_latent_mse": 0.10319709032773972,
473
+ "psnr": 35.50569664133912,
474
+ "ssim": 0.9492254278084,
475
+ "lpips": 0.24971535801887512
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 2.9677734375,
479
+ "history_latent_mse": 0.23123686015605927,
480
+ "psnr": 30.87939622028728,
481
+ "ssim": 0.8752212486062663,
482
+ "lpips": 0.30587390065193176
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 1913,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 4.140625,
494
+ "history_latent_mse": 0.2750135064125061,
495
+ "psnr": 18.022429446139647,
496
+ "ssim": 0.6455458783856715,
497
+ "lpips": 0.29137641191482544
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 4.76171875,
501
+ "history_latent_mse": 1.0009616613388062,
502
+ "psnr": 10.371446021281326,
503
+ "ssim": 0.3005361599410692,
504
+ "lpips": 0.5711427330970764
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 5.84375,
508
+ "history_latent_mse": 1.2773358821868896,
509
+ "psnr": 9.281254755236965,
510
+ "ssim": 0.1496813331936439,
511
+ "lpips": 0.7296895980834961
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 1.650390625,
515
+ "history_latent_mse": 0.10978487133979797,
516
+ "psnr": 35.15160195860626,
517
+ "ssim": 0.9460204977305288,
518
+ "lpips": 0.2653445899486542
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 3.27734375,
522
+ "history_latent_mse": 0.26345571875572205,
523
+ "psnr": 31.984257250549504,
524
+ "ssim": 0.8987391876954239,
525
+ "lpips": 0.2942860722541809
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 1921,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.42138671875,
537
+ "history_latent_mse": 0.2782556712627411,
538
+ "psnr": 18.08716396623567,
539
+ "ssim": 0.6525679444946482,
540
+ "lpips": 0.2933219075202942
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 4.75,
544
+ "history_latent_mse": 0.9220336079597473,
545
+ "psnr": 10.313384160235174,
546
+ "ssim": 0.30653793556181264,
547
+ "lpips": 0.5786222219467163
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 5.890625,
551
+ "history_latent_mse": 1.3143733739852905,
552
+ "psnr": 9.654781771154955,
553
+ "ssim": 0.16233785932343134,
554
+ "lpips": 0.7144385576248169
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 1.875,
558
+ "history_latent_mse": 0.11926188319921494,
559
+ "psnr": 34.73191641558897,
560
+ "ssim": 0.9422505534304545,
561
+ "lpips": 0.2749377191066742
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 2.84375,
565
+ "history_latent_mse": 0.22833473980426788,
566
+ "psnr": 33.316754080222914,
567
+ "ssim": 0.9276727551495014,
568
+ "lpips": 0.24914956092834473
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 1929,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 3.892578125,
580
+ "history_latent_mse": 0.27144190669059753,
581
+ "psnr": 18.12713456455959,
582
+ "ssim": 0.6514087079592135,
583
+ "lpips": 0.2898755371570587
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 5.0703125,
587
+ "history_latent_mse": 0.9486039280891418,
588
+ "psnr": 10.831998972595065,
589
+ "ssim": 0.34260404403467953,
590
+ "lpips": 0.5509961843490601
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.59375,
594
+ "history_latent_mse": 1.3222182989120483,
595
+ "psnr": 9.676479397509294,
596
+ "ssim": 0.17591857036204672,
597
+ "lpips": 0.6677525043487549
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 1.84716796875,
601
+ "history_latent_mse": 0.12162520736455917,
602
+ "psnr": 34.26179215064562,
603
+ "ssim": 0.9379269060301725,
604
+ "lpips": 0.2951134443283081
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 2.45703125,
608
+ "history_latent_mse": 0.17624182999134064,
609
+ "psnr": 35.574963847934846,
610
+ "ssim": 0.9519700138394485,
611
+ "lpips": 0.17888721823692322
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 1937,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 7.125,
623
+ "history_latent_mse": 0.28930458426475525,
624
+ "psnr": 17.933937036292257,
625
+ "ssim": 0.6501697360484575,
626
+ "lpips": 0.2973931133747101
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 5.9375,
630
+ "history_latent_mse": 0.951725959777832,
631
+ "psnr": 10.521384358361725,
632
+ "ssim": 0.3264058407296058,
633
+ "lpips": 0.5590139031410217
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 5.671875,
637
+ "history_latent_mse": 1.1301602125167847,
638
+ "psnr": 9.693420192053367,
639
+ "ssim": 0.17648837550633995,
640
+ "lpips": 0.7731425762176514
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 2.125,
644
+ "history_latent_mse": 0.12921133637428284,
645
+ "psnr": 34.00654352401008,
646
+ "ssim": 0.9354487104185354,
647
+ "lpips": 0.3045352101325989
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 1.75390625,
651
+ "history_latent_mse": 0.10979020595550537,
652
+ "psnr": 31.363891055101227,
653
+ "ssim": 0.9245440910377623,
654
+ "lpips": 0.280612051486969
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a04544d9b48f7e181852631ba6543ac50e3f6e94c3b26ac4fbcaaeadcd3df1c
3
+ size 1265979
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/01_motherboard_0909_episode_000_onset/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.15089699625968933,
91
+ "wrist/video/latent_future_mse": 0.15089699625968933,
92
+ "video/psnr": 23.628176260948578,
93
+ "wrist/video/psnr": 23.628176260948578,
94
+ "video/ssim": 0.8456452647494677,
95
+ "wrist/video/ssim": 0.8456452647494677,
96
+ "video/lpips": 0.0570581778883934,
97
+ "wrist/video/lpips": 0.0570581778883934,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.6843722462654114,
103
+ "wrist/wrist_left/latent_future_mse": 0.6843722462654114,
104
+ "wrist_left/psnr": 16.974798261240593,
105
+ "wrist/wrist_left/psnr": 16.974798261240593,
106
+ "wrist_left/ssim": 0.430608030325784,
107
+ "wrist/wrist_left/ssim": 0.430608030325784,
108
+ "wrist_left/lpips": 0.1560155153274536,
109
+ "wrist/wrist_left/lpips": 0.1560155153274536,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.9355753064155579,
115
+ "wrist/wrist_right/latent_future_mse": 0.9355753064155579,
116
+ "wrist_right/psnr": 11.3508814597468,
117
+ "wrist/wrist_right/psnr": 11.3508814597468,
118
+ "wrist_right/ssim": 0.42637402380382783,
119
+ "wrist/wrist_right/ssim": 0.42637402380382783,
120
+ "wrist_right/lpips": 0.37309491634368896,
121
+ "wrist/wrist_right/lpips": 0.37309491634368896,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.07507113367319107,
127
+ "wrist/tactile_left/latent_future_mse": 0.07507113367319107,
128
+ "tactile_left/psnr": 37.21835685193686,
129
+ "wrist/tactile_left/psnr": 37.21835685193686,
130
+ "tactile_left/ssim": 0.9798802064902503,
131
+ "wrist/tactile_left/ssim": 0.9798802064902503,
132
+ "tactile_left/lpips": 0.055090613663196564,
133
+ "wrist/tactile_left/lpips": 0.055090613663196564,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.1042751669883728,
139
+ "wrist/tactile_right/latent_future_mse": 0.1042751669883728,
140
+ "tactile_right/psnr": 31.144626811115348,
141
+ "wrist/tactile_right/psnr": 31.144626811115348,
142
+ "tactile_right/ssim": 0.9719350201675224,
143
+ "wrist/tactile_right/ssim": 0.9719350201675224,
144
+ "tactile_right/lpips": 0.10720653086900711,
145
+ "wrist/tactile_right/lpips": 0.10720653086900711,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_000.pt",
152
+ "t_start": 1825,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.15089699625968933,
169
+ "psnr": 23.628176260948578,
170
+ "ssim": 0.8456452647494677,
171
+ "lpips": 0.0570581778883934
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.6843722462654114,
177
+ "psnr": 16.974798261240593,
178
+ "ssim": 0.430608030325784,
179
+ "lpips": 0.1560155153274536
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.9355753064155579,
185
+ "psnr": 11.3508814597468,
186
+ "ssim": 0.42637402380382783,
187
+ "lpips": 0.37309491634368896
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.07507113367319107,
193
+ "psnr": 37.21835685193686,
194
+ "ssim": 0.9798802064902503,
195
+ "lpips": 0.055090613663196564
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.1042751669883728,
201
+ "psnr": 31.144626811115348,
202
+ "ssim": 0.9719350201675224,
203
+ "lpips": 0.10720653086900711
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/case.json ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 2,
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "category": "release",
5
+ "index": 3072,
6
+ "t_start": 3072,
7
+ "start_seconds": 204.8,
8
+ "short_contact": [
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 1,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0
25
+ ],
26
+ "short_transition_ticks": [
27
+ 11
28
+ ],
29
+ "future_transition_ticks": [
30
+ 11
31
+ ],
32
+ "long_contact_fraction": 0.7734375,
33
+ "long_future_contact_fraction": 0.7583333333333333,
34
+ "contact_over_128_frames": [
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 0,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 0,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 0,
132
+ 0,
133
+ 0,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 0,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 0,
159
+ 0,
160
+ 0,
161
+ 0,
162
+ 0
163
+ ],
164
+ "first_contact_tick": 0,
165
+ "long_label": "release",
166
+ "output_subdir": "02_motherboard_0909_episode_000_release",
167
+ "rollout_windows": 15
168
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e98b09b60768a3011884e2aebcc363698237271e882e07b3c939b581f319eb01
3
+ size 9025700
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/motherboard_0909_episode_000_t03072_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd08ce704e39dbcdb41edd46f6924ee2482860068255be037fd1c48f49a699bd
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "t_start": 3072,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 3072,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 29.266572537636293,
23
+ "ssim": 0.9162150885896057,
24
+ "lpips": 0.03114306926727295
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 15.72964153701189,
30
+ "ssim": 0.5237543840369523,
31
+ "lpips": 0.18139484524726868
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 13.913819208591171,
37
+ "ssim": 0.5439860906598222,
38
+ "lpips": 0.30286651849746704
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 42.44182842336647,
44
+ "ssim": 0.9840405275886075,
45
+ "lpips": 0.014103841967880726
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 43.15277773614905,
51
+ "ssim": 0.9819352677548217,
52
+ "lpips": 0.04016111418604851
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 3080,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 2.658203125,
64
+ "history_latent_mse": 0.0524832159280777,
65
+ "psnr": 26.480900577635992,
66
+ "ssim": 0.8568048789627838,
67
+ "lpips": 0.0566643625497818
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 5.59375,
71
+ "history_latent_mse": 0.5854699611663818,
72
+ "psnr": 14.425112517205427,
73
+ "ssim": 0.44195591168688314,
74
+ "lpips": 0.2938387393951416
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 4.796875,
78
+ "history_latent_mse": 0.6425979137420654,
79
+ "psnr": 11.877670430829415,
80
+ "ssim": 0.4240810047598062,
81
+ "lpips": 0.4234185218811035
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 1.53125,
85
+ "history_latent_mse": 0.03738224878907204,
86
+ "psnr": 37.894477381030505,
87
+ "ssim": 0.9701083073248673,
88
+ "lpips": 0.06112324446439743
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 1.41015625,
92
+ "history_latent_mse": 0.05036762356758118,
93
+ "psnr": 42.507885252321444,
94
+ "ssim": 0.9766029615344074,
95
+ "lpips": 0.055503640323877335
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 3088,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 3.1484375,
107
+ "history_latent_mse": 0.07951003313064575,
108
+ "psnr": 24.564863578668753,
109
+ "ssim": 0.7881806175917664,
110
+ "lpips": 0.08079169690608978
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 5.296875,
114
+ "history_latent_mse": 0.6849780082702637,
115
+ "psnr": 12.90153598569784,
116
+ "ssim": 0.38549987983287787,
117
+ "lpips": 0.36882656812667847
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 5.5859375,
121
+ "history_latent_mse": 0.7878640294075012,
122
+ "psnr": 11.660039349859384,
123
+ "ssim": 0.38050448241676854,
124
+ "lpips": 0.5146379470825195
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 2.48828125,
128
+ "history_latent_mse": 0.06754805892705917,
129
+ "psnr": 34.91500774183322,
130
+ "ssim": 0.9589869880474067,
131
+ "lpips": 0.14406634867191315
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 1.48828125,
135
+ "history_latent_mse": 0.055852070450782776,
136
+ "psnr": 35.54515781236948,
137
+ "ssim": 0.9663785536995418,
138
+ "lpips": 0.1137726902961731
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 3096,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 4.890625,
150
+ "history_latent_mse": 0.12277263402938843,
151
+ "psnr": 22.94232235508399,
152
+ "ssim": 0.729770033078336,
153
+ "lpips": 0.10369352996349335
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 5.3984375,
157
+ "history_latent_mse": 0.690626323223114,
158
+ "psnr": 13.911253496880104,
159
+ "ssim": 0.4345450204978216,
160
+ "lpips": 0.34035584330558777
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 5.03125,
164
+ "history_latent_mse": 0.8453990817070007,
165
+ "psnr": 10.980996235036901,
166
+ "ssim": 0.3287601789194513,
167
+ "lpips": 0.5500631332397461
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 2.72265625,
171
+ "history_latent_mse": 0.09262625128030777,
172
+ "psnr": 33.46081177972756,
173
+ "ssim": 0.9531914052591324,
174
+ "lpips": 0.20742003619670868
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 3.34375,
178
+ "history_latent_mse": 0.11502578109502792,
179
+ "psnr": 31.539306887337162,
180
+ "ssim": 0.9472127857497132,
181
+ "lpips": 0.203257218003273
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 3104,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 2.9296875,
193
+ "history_latent_mse": 0.16665054857730865,
194
+ "psnr": 22.04007984596984,
195
+ "ssim": 0.6941891443404591,
196
+ "lpips": 0.124331995844841
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 4.7421875,
200
+ "history_latent_mse": 0.6611791253089905,
201
+ "psnr": 14.176640399708514,
202
+ "ssim": 0.4220857849868887,
203
+ "lpips": 0.3271068036556244
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 6.640625,
207
+ "history_latent_mse": 0.8994737267494202,
208
+ "psnr": 10.491407354591216,
209
+ "ssim": 0.3351243739596609,
210
+ "lpips": 0.5160884857177734
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 2.7421875,
214
+ "history_latent_mse": 0.11128786951303482,
215
+ "psnr": 34.6155133245452,
216
+ "ssim": 0.9577253908695167,
217
+ "lpips": 0.21651971340179443
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 3.1015625,
221
+ "history_latent_mse": 0.11482458561658859,
222
+ "psnr": 31.327422280966797,
223
+ "ssim": 0.942726219103224,
224
+ "lpips": 0.21856579184532166
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 3112,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 3.352783203125,
236
+ "history_latent_mse": 0.18367813527584076,
237
+ "psnr": 21.127098852448007,
238
+ "ssim": 0.6933516038223533,
239
+ "lpips": 0.17727592587471008
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 4.5,
243
+ "history_latent_mse": 0.5865779519081116,
244
+ "psnr": 12.512415844269501,
245
+ "ssim": 0.3622295575154916,
246
+ "lpips": 0.40887606143951416
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 5.40625,
250
+ "history_latent_mse": 0.9623259902000427,
251
+ "psnr": 9.850495173981773,
252
+ "ssim": 0.3399974859369542,
253
+ "lpips": 0.48278534412384033
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 2.390625,
257
+ "history_latent_mse": 0.11763932555913925,
258
+ "psnr": 34.978677140953806,
259
+ "ssim": 0.9573239829106417,
260
+ "lpips": 0.22706487774848938
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 2.70703125,
264
+ "history_latent_mse": 0.13144007325172424,
265
+ "psnr": 33.42194081729072,
266
+ "ssim": 0.9504479238586145,
267
+ "lpips": 0.209794282913208
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 3120,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 3.6640625,
279
+ "history_latent_mse": 0.23482775688171387,
280
+ "psnr": 19.742974716246493,
281
+ "ssim": 0.6568032426650807,
282
+ "lpips": 0.24461683630943298
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 5.9609375,
286
+ "history_latent_mse": 0.7560710310935974,
287
+ "psnr": 11.087393192067854,
288
+ "ssim": 0.30024546702065225,
289
+ "lpips": 0.4932715892791748
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 5.5,
293
+ "history_latent_mse": 0.9448109269142151,
294
+ "psnr": 10.017077900899105,
295
+ "ssim": 0.32922143118481717,
296
+ "lpips": 0.49438393115997314
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 2.60546875,
300
+ "history_latent_mse": 0.10446634143590927,
301
+ "psnr": 35.094482493360594,
302
+ "ssim": 0.9545911342300593,
303
+ "lpips": 0.2322356104850769
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 2.37109375,
307
+ "history_latent_mse": 0.12086028605699539,
308
+ "psnr": 34.39604525779137,
309
+ "ssim": 0.9508120283033794,
310
+ "lpips": 0.20816770195960999
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 3128,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 4.0234375,
322
+ "history_latent_mse": 0.26398757100105286,
323
+ "psnr": 19.490515236262617,
324
+ "ssim": 0.6357417017243336,
325
+ "lpips": 0.2562229037284851
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 5.6015625,
329
+ "history_latent_mse": 0.9454730153083801,
330
+ "psnr": 11.381691494621517,
331
+ "ssim": 0.317029280474494,
332
+ "lpips": 0.46522700786590576
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 4.7890625,
336
+ "history_latent_mse": 0.9471634030342102,
337
+ "psnr": 9.323565723535017,
338
+ "ssim": 0.31195205039034823,
339
+ "lpips": 0.5405911803245544
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 2.15771484375,
343
+ "history_latent_mse": 0.10136154294013977,
344
+ "psnr": 34.641060180658116,
345
+ "ssim": 0.9482866741629074,
346
+ "lpips": 0.260652095079422
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 2.40234375,
350
+ "history_latent_mse": 0.1180509701371193,
351
+ "psnr": 36.045222383582995,
352
+ "ssim": 0.9531250000497351,
353
+ "lpips": 0.2066667228937149
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 3136,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 5.953125,
365
+ "history_latent_mse": 0.2903461456298828,
366
+ "psnr": 19.49002682443959,
367
+ "ssim": 0.6248671674031994,
368
+ "lpips": 0.23285439610481262
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 5.015625,
372
+ "history_latent_mse": 0.8181669116020203,
373
+ "psnr": 11.839906286502913,
374
+ "ssim": 0.34480539758716977,
375
+ "lpips": 0.4296375513076782
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 4.8125,
379
+ "history_latent_mse": 0.9962184429168701,
380
+ "psnr": 8.001149714870516,
381
+ "ssim": 0.26536818740541224,
382
+ "lpips": 0.597160816192627
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 2.546875,
386
+ "history_latent_mse": 0.12413499504327774,
387
+ "psnr": 32.79141311437882,
388
+ "ssim": 0.9389920308518498,
389
+ "lpips": 0.3098233938217163
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 3.171875,
393
+ "history_latent_mse": 0.1330520361661911,
394
+ "psnr": 36.726209603006545,
395
+ "ssim": 0.9541922844243047,
396
+ "lpips": 0.20737692713737488
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 3144,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 3.6788330078125,
408
+ "history_latent_mse": 0.27717462182044983,
409
+ "psnr": 19.398272521696065,
410
+ "ssim": 0.6096253384874204,
411
+ "lpips": 0.22750045359134674
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 5.75,
415
+ "history_latent_mse": 0.8745848536491394,
416
+ "psnr": 10.611273957111177,
417
+ "ssim": 0.32374390172224476,
418
+ "lpips": 0.4980604350566864
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 5.8125,
422
+ "history_latent_mse": 1.1955797672271729,
423
+ "psnr": 6.962309834326984,
424
+ "ssim": 0.2628506639940223,
425
+ "lpips": 0.6147164106369019
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 2.9453125,
429
+ "history_latent_mse": 0.14682048559188843,
430
+ "psnr": 30.486920618259962,
431
+ "ssim": 0.9299237964500228,
432
+ "lpips": 0.33560019731521606
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 2.609375,
436
+ "history_latent_mse": 0.1349182277917862,
437
+ "psnr": 36.70657183525083,
438
+ "ssim": 0.953313381429965,
439
+ "lpips": 0.1944001019001007
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 3152,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 4.15234375,
451
+ "history_latent_mse": 0.299398273229599,
452
+ "psnr": 19.78143673961738,
453
+ "ssim": 0.6238783178037695,
454
+ "lpips": 0.21857289969921112
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 4.9296875,
458
+ "history_latent_mse": 0.8995222449302673,
459
+ "psnr": 12.112046005977927,
460
+ "ssim": 0.31583914984852396,
461
+ "lpips": 0.4641914665699005
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 6.421875,
465
+ "history_latent_mse": 1.4405235052108765,
466
+ "psnr": 8.447949295659507,
467
+ "ssim": 0.2894142125139586,
468
+ "lpips": 0.616000771522522
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 3.146484375,
472
+ "history_latent_mse": 0.18555539846420288,
473
+ "psnr": 35.50311870711727,
474
+ "ssim": 0.9480858985537217,
475
+ "lpips": 0.25570857524871826
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 2.3828125,
479
+ "history_latent_mse": 0.11988797038793564,
480
+ "psnr": 37.18997158102523,
481
+ "ssim": 0.9534102759059571,
482
+ "lpips": 0.17949232459068298
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 3160,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 3.5,
494
+ "history_latent_mse": 0.26305127143859863,
495
+ "psnr": 19.199608709573234,
496
+ "ssim": 0.611331433193068,
497
+ "lpips": 0.24953411519527435
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 4.888671875,
501
+ "history_latent_mse": 0.7852938771247864,
502
+ "psnr": 11.572953367318476,
503
+ "ssim": 0.3238332913964056,
504
+ "lpips": 0.4967406392097473
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 5.1484375,
508
+ "history_latent_mse": 1.145728588104248,
509
+ "psnr": 7.744809140730141,
510
+ "ssim": 0.2059366484145767,
511
+ "lpips": 0.6874375343322754
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 1.99609375,
515
+ "history_latent_mse": 0.1139211654663086,
516
+ "psnr": 35.27091367991169,
517
+ "ssim": 0.9441998356097521,
518
+ "lpips": 0.27397745847702026
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 2.0625,
522
+ "history_latent_mse": 0.10503560304641724,
523
+ "psnr": 36.1373293969658,
524
+ "ssim": 0.9487875146287383,
525
+ "lpips": 0.20599575340747833
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 3168,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.3203125,
537
+ "history_latent_mse": 0.3029901683330536,
538
+ "psnr": 18.914070770327754,
539
+ "ssim": 0.6005699051833406,
540
+ "lpips": 0.26203030347824097
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 5.890625,
544
+ "history_latent_mse": 0.8589582443237305,
545
+ "psnr": 11.431910071869627,
546
+ "ssim": 0.32711357091987503,
547
+ "lpips": 0.5092601776123047
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 5.34375,
551
+ "history_latent_mse": 1.2575098276138306,
552
+ "psnr": 7.438335581507751,
553
+ "ssim": 0.20773836807506874,
554
+ "lpips": 0.7033286094665527
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 2.0,
558
+ "history_latent_mse": 0.11988425999879837,
559
+ "psnr": 35.07300876696385,
560
+ "ssim": 0.9421615157874426,
561
+ "lpips": 0.2858327329158783
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 2.54296875,
565
+ "history_latent_mse": 0.1247773990035057,
566
+ "psnr": 31.765331780163567,
567
+ "ssim": 0.9338935658553225,
568
+ "lpips": 0.32855433225631714
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 3176,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 3.8671875,
580
+ "history_latent_mse": 0.29801949858665466,
581
+ "psnr": 17.743571328599067,
582
+ "ssim": 0.5835674410820922,
583
+ "lpips": 0.32023608684539795
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 5.2421875,
587
+ "history_latent_mse": 0.880891740322113,
588
+ "psnr": 11.860265498249019,
589
+ "ssim": 0.3625592751989118,
590
+ "lpips": 0.4984472990036011
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.3125,
594
+ "history_latent_mse": 1.3354878425598145,
595
+ "psnr": 6.901163704143754,
596
+ "ssim": 0.25132595270990854,
597
+ "lpips": 0.7489310503005981
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 1.81640625,
601
+ "history_latent_mse": 0.12284906953573227,
602
+ "psnr": 34.826902851967375,
603
+ "ssim": 0.9396243239896089,
604
+ "lpips": 0.29613763093948364
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 3.65625,
608
+ "history_latent_mse": 0.18616968393325806,
609
+ "psnr": 32.35612792799206,
610
+ "ssim": 0.9313905496953596,
611
+ "lpips": 0.32109880447387695
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 3184,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 5.2265625,
623
+ "history_latent_mse": 0.3595130741596222,
624
+ "psnr": 17.001402687593565,
625
+ "ssim": 0.5801625562166594,
626
+ "lpips": 0.35640954971313477
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 4.46875,
630
+ "history_latent_mse": 0.8259716033935547,
631
+ "psnr": 11.853545381287065,
632
+ "ssim": 0.37933143170024974,
633
+ "lpips": 0.5165786147117615
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 6.015625,
637
+ "history_latent_mse": 1.6166692972183228,
638
+ "psnr": 6.541689035094549,
639
+ "ssim": 0.24061577418572283,
640
+ "lpips": 0.7551593780517578
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 1.796875,
644
+ "history_latent_mse": 0.1284082978963852,
645
+ "psnr": 34.565221408096896,
646
+ "ssim": 0.9368704220705981,
647
+ "lpips": 0.30383628606796265
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 3.0625,
651
+ "history_latent_mse": 0.17708100378513336,
652
+ "psnr": 34.23701526944256,
653
+ "ssim": 0.9404904003820407,
654
+ "lpips": 0.26351332664489746
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7ac3d49cdc986fd14efc80d5e8963944437f8c445b59f78c452075b47fee9ab
3
+ size 1015121
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/02_motherboard_0909_episode_000_release/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.07159221917390823,
91
+ "wrist/video/latent_future_mse": 0.07159221917390823,
92
+ "video/psnr": 29.266572537636293,
93
+ "wrist/video/psnr": 29.266572537636293,
94
+ "video/ssim": 0.9162150885896057,
95
+ "wrist/video/ssim": 0.9162150885896057,
96
+ "video/lpips": 0.03114306926727295,
97
+ "wrist/video/lpips": 0.03114306926727295,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.658707857131958,
103
+ "wrist/wrist_left/latent_future_mse": 0.658707857131958,
104
+ "wrist_left/psnr": 15.72964153701189,
105
+ "wrist/wrist_left/psnr": 15.72964153701189,
106
+ "wrist_left/ssim": 0.5237543840369523,
107
+ "wrist/wrist_left/ssim": 0.5237543840369523,
108
+ "wrist_left/lpips": 0.18139484524726868,
109
+ "wrist/wrist_left/lpips": 0.18139484524726868,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.7194229960441589,
115
+ "wrist/wrist_right/latent_future_mse": 0.7194229960441589,
116
+ "wrist_right/psnr": 13.913819208591171,
117
+ "wrist/wrist_right/psnr": 13.913819208591171,
118
+ "wrist_right/ssim": 0.5439860906598222,
119
+ "wrist/wrist_right/ssim": 0.5439860906598222,
120
+ "wrist_right/lpips": 0.30286651849746704,
121
+ "wrist/wrist_right/lpips": 0.30286651849746704,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.044789332896471024,
127
+ "wrist/tactile_left/latent_future_mse": 0.044789332896471024,
128
+ "tactile_left/psnr": 42.44182842336647,
129
+ "wrist/tactile_left/psnr": 42.44182842336647,
130
+ "tactile_left/ssim": 0.9840405275886075,
131
+ "wrist/tactile_left/ssim": 0.9840405275886075,
132
+ "tactile_left/lpips": 0.014103841967880726,
133
+ "wrist/tactile_left/lpips": 0.014103841967880726,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.05245906487107277,
139
+ "wrist/tactile_right/latent_future_mse": 0.05245906487107277,
140
+ "tactile_right/psnr": 43.15277773614905,
141
+ "wrist/tactile_right/psnr": 43.15277773614905,
142
+ "tactile_right/ssim": 0.9819352677548217,
143
+ "wrist/tactile_right/ssim": 0.9819352677548217,
144
+ "tactile_right/lpips": 0.04016111418604851,
145
+ "wrist/tactile_right/lpips": 0.04016111418604851,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_000.pt",
152
+ "t_start": 3072,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.07159221917390823,
169
+ "psnr": 29.266572537636293,
170
+ "ssim": 0.9162150885896057,
171
+ "lpips": 0.03114306926727295
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.658707857131958,
177
+ "psnr": 15.72964153701189,
178
+ "ssim": 0.5237543840369523,
179
+ "lpips": 0.18139484524726868
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.7194229960441589,
185
+ "psnr": 13.913819208591171,
186
+ "ssim": 0.5439860906598222,
187
+ "lpips": 0.30286651849746704
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.044789332896471024,
193
+ "psnr": 42.44182842336647,
194
+ "ssim": 0.9840405275886075,
195
+ "lpips": 0.014103841967880726
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.05245906487107277,
201
+ "psnr": 43.15277773614905,
202
+ "ssim": 0.9819352677548217,
203
+ "lpips": 0.04016111418604851
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/case.json ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 3,
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "category": "full",
5
+ "index": 2531,
6
+ "t_start": 2531,
7
+ "start_seconds": 168.73333333333332,
8
+ "short_contact": [
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 1,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 1,
21
+ 1,
22
+ 1,
23
+ 1,
24
+ 1
25
+ ],
26
+ "short_transition_ticks": [],
27
+ "future_transition_ticks": [],
28
+ "long_contact_fraction": 0.9296875,
29
+ "long_future_contact_fraction": 0.925,
30
+ "contact_over_128_frames": [
31
+ 1,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 1,
54
+ 1,
55
+ 1,
56
+ 1,
57
+ 1,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 0,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 0,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 1
159
+ ],
160
+ "first_contact_tick": 0,
161
+ "long_label": "full",
162
+ "output_subdir": "03_motherboard_0909_episode_000_full",
163
+ "rollout_windows": 15
164
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:905ecc7502ec5a7ac205070796140eb1548ce6678a8464ca206854983c7b55cb
3
+ size 9207150
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/motherboard_0909_episode_000_t02531_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f6246e71358c7c6d85690bab70bab37942573d0a7e02e9e73f01c34a8ea20a8
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_000.pt",
4
+ "t_start": 2531,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 2531,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 26.63426720435477,
23
+ "ssim": 0.8708790760218011,
24
+ "lpips": 0.046272218227386475
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 17.216493839691783,
30
+ "ssim": 0.5951449385403667,
31
+ "lpips": 0.17227669060230255
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 17.487450501330382,
37
+ "ssim": 0.5787619882113139,
38
+ "lpips": 0.23368918895721436
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 35.386292833088994,
44
+ "ssim": 0.9682620496470609,
45
+ "lpips": 0.0447380356490612
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 36.69995933064942,
51
+ "ssim": 0.9682634386253878,
52
+ "lpips": 0.08494263142347336
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 2539,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 3.8251953125,
64
+ "history_latent_mse": 0.08070540428161621,
65
+ "psnr": 23.159437122001776,
66
+ "ssim": 0.7823098292669202,
67
+ "lpips": 0.10296832025051117
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 4.515625,
71
+ "history_latent_mse": 0.39089512825012207,
72
+ "psnr": 14.673834614553565,
73
+ "ssim": 0.4183632750673575,
74
+ "lpips": 0.3741599917411804
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 4.8359375,
78
+ "history_latent_mse": 0.527142345905304,
79
+ "psnr": 14.254749471245711,
80
+ "ssim": 0.4171200134046572,
81
+ "lpips": 0.4276435971260071
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 2.78125,
85
+ "history_latent_mse": 0.06735280156135559,
86
+ "psnr": 28.736440877463714,
87
+ "ssim": 0.9332238196872995,
88
+ "lpips": 0.19336062669754028
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 2.609375,
92
+ "history_latent_mse": 0.0874987244606018,
93
+ "psnr": 37.26545928086202,
94
+ "ssim": 0.9663097214724079,
95
+ "lpips": 0.1253584921360016
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 2547,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 3.39453125,
107
+ "history_latent_mse": 0.15959617495536804,
108
+ "psnr": 20.62850763959014,
109
+ "ssim": 0.7172446505481008,
110
+ "lpips": 0.17381739616394043
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 5.953125,
114
+ "history_latent_mse": 0.6985142827033997,
115
+ "psnr": 12.82548513288514,
116
+ "ssim": 0.30840404812611005,
117
+ "lpips": 0.5225712060928345
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 5.46875,
121
+ "history_latent_mse": 0.8651549220085144,
122
+ "psnr": 12.95824588599153,
123
+ "ssim": 0.36013512019582494,
124
+ "lpips": 0.5288517475128174
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 3.3671875,
128
+ "history_latent_mse": 0.15326480567455292,
129
+ "psnr": 29.3778627177778,
130
+ "ssim": 0.9308608957824773,
131
+ "lpips": 0.2518291175365448
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 2.7109375,
135
+ "history_latent_mse": 0.09487878531217575,
136
+ "psnr": 34.76537606447303,
137
+ "ssim": 0.9603698907407732,
138
+ "lpips": 0.1629692018032074
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 2555,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 4.03125,
150
+ "history_latent_mse": 0.24894385039806366,
151
+ "psnr": 17.443917565051784,
152
+ "ssim": 0.6693827035691184,
153
+ "lpips": 0.3044857978820801
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 5.265625,
157
+ "history_latent_mse": 0.9179932475090027,
158
+ "psnr": 10.550608387552627,
159
+ "ssim": 0.2635874614567964,
160
+ "lpips": 0.6275637149810791
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 5.1015625,
164
+ "history_latent_mse": 0.9075899124145508,
165
+ "psnr": 12.775938497835632,
166
+ "ssim": 0.32981508164611417,
167
+ "lpips": 0.5660387277603149
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 2.984375,
171
+ "history_latent_mse": 0.14413142204284668,
172
+ "psnr": 31.98423471974735,
173
+ "ssim": 0.9414440889533799,
174
+ "lpips": 0.23925256729125977
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 2.25,
178
+ "history_latent_mse": 0.11077144742012024,
179
+ "psnr": 33.64269211697651,
180
+ "ssim": 0.9556408978265329,
181
+ "lpips": 0.19989131391048431
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 2563,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 5.0,
193
+ "history_latent_mse": 0.39638951420783997,
194
+ "psnr": 14.73174274772068,
195
+ "ssim": 0.585983706585437,
196
+ "lpips": 0.415120005607605
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 4.8521728515625,
200
+ "history_latent_mse": 1.1219066381454468,
201
+ "psnr": 8.928059073629477,
202
+ "ssim": 0.224150783567039,
203
+ "lpips": 0.7316471338272095
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 5.12890625,
207
+ "history_latent_mse": 1.0076806545257568,
208
+ "psnr": 11.184012805939638,
209
+ "ssim": 0.2882455510490546,
210
+ "lpips": 0.6224209070205688
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 3.0078125,
214
+ "history_latent_mse": 0.11010429263114929,
215
+ "psnr": 33.260507171447415,
216
+ "ssim": 0.9461954082057156,
217
+ "lpips": 0.2397955358028412
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 2.189453125,
221
+ "history_latent_mse": 0.1103661060333252,
222
+ "psnr": 33.35748976734123,
223
+ "ssim": 0.9512292889887657,
224
+ "lpips": 0.21728402376174927
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 2571,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 5.46875,
236
+ "history_latent_mse": 0.4892379939556122,
237
+ "psnr": 14.334460869912744,
238
+ "ssim": 0.5790020853021631,
239
+ "lpips": 0.41715461015701294
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 4.9609375,
243
+ "history_latent_mse": 1.3448820114135742,
244
+ "psnr": 8.799930353471497,
245
+ "ssim": 0.21221004452292824,
246
+ "lpips": 0.716631293296814
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 6.43359375,
250
+ "history_latent_mse": 1.133210301399231,
251
+ "psnr": 11.191955502330352,
252
+ "ssim": 0.28797609721676753,
253
+ "lpips": 0.6440451145172119
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 2.4609375,
257
+ "history_latent_mse": 0.12806642055511475,
258
+ "psnr": 33.859292154893545,
259
+ "ssim": 0.9467556135771815,
260
+ "lpips": 0.24504689872264862
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 2.59375,
264
+ "history_latent_mse": 0.1392369419336319,
265
+ "psnr": 32.49246806577232,
266
+ "ssim": 0.9465398813390566,
267
+ "lpips": 0.23140281438827515
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 2579,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 3.46875,
279
+ "history_latent_mse": 0.4454944431781769,
280
+ "psnr": 14.302113527236903,
281
+ "ssim": 0.5739056927373726,
282
+ "lpips": 0.4254215955734253
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 4.96875,
286
+ "history_latent_mse": 1.3893589973449707,
287
+ "psnr": 8.653165509360154,
288
+ "ssim": 0.22678990052173326,
289
+ "lpips": 0.7852842807769775
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 5.1640625,
293
+ "history_latent_mse": 1.2213624715805054,
294
+ "psnr": 10.658522710345189,
295
+ "ssim": 0.2944790973066083,
296
+ "lpips": 0.6900660991668701
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 3.39453125,
300
+ "history_latent_mse": 0.14844247698783875,
301
+ "psnr": 34.90857239958195,
302
+ "ssim": 0.9470455298839704,
303
+ "lpips": 0.25731635093688965
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 2.7578125,
307
+ "history_latent_mse": 0.170336052775383,
308
+ "psnr": 31.603033853314226,
309
+ "ssim": 0.9423910748516358,
310
+ "lpips": 0.2559289336204529
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 2587,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 4.25,
322
+ "history_latent_mse": 0.5087829232215881,
323
+ "psnr": 14.300809302476104,
324
+ "ssim": 0.5767790701594095,
325
+ "lpips": 0.41574013233184814
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 4.96875,
329
+ "history_latent_mse": 1.5244522094726562,
330
+ "psnr": 8.586503810292381,
331
+ "ssim": 0.20867409667261005,
332
+ "lpips": 0.7698798179626465
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 4.9375,
336
+ "history_latent_mse": 1.2617968320846558,
337
+ "psnr": 10.188565542458578,
338
+ "ssim": 0.25988805688630157,
339
+ "lpips": 0.6863692998886108
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 2.55859375,
343
+ "history_latent_mse": 0.1374787837266922,
344
+ "psnr": 34.29122747192379,
345
+ "ssim": 0.938551057802476,
346
+ "lpips": 0.29119420051574707
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 2.44921875,
350
+ "history_latent_mse": 0.1452258825302124,
351
+ "psnr": 31.989948746552052,
352
+ "ssim": 0.9404435053929187,
353
+ "lpips": 0.25835925340652466
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 2595,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 3.84375,
365
+ "history_latent_mse": 0.5014035701751709,
366
+ "psnr": 14.053325979340862,
367
+ "ssim": 0.5619428936991968,
368
+ "lpips": 0.4426591396331787
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 5.5703125,
372
+ "history_latent_mse": 1.4670085906982422,
373
+ "psnr": 8.45139375971582,
374
+ "ssim": 0.21053082769187503,
375
+ "lpips": 0.7753310203552246
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 6.71875,
379
+ "history_latent_mse": 1.214146375656128,
380
+ "psnr": 8.064663961919155,
381
+ "ssim": 0.216773069440242,
382
+ "lpips": 0.8136317729949951
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 3.0625,
386
+ "history_latent_mse": 0.1508377343416214,
387
+ "psnr": 34.87256996632177,
388
+ "ssim": 0.939595877735945,
389
+ "lpips": 0.2883746027946472
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 3.16015625,
393
+ "history_latent_mse": 0.16494834423065186,
394
+ "psnr": 33.6269204922963,
395
+ "ssim": 0.9425436603443749,
396
+ "lpips": 0.2461204081773758
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 2603,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 4.87109375,
408
+ "history_latent_mse": 0.5396252870559692,
409
+ "psnr": 13.939093041392185,
410
+ "ssim": 0.5519655360276372,
411
+ "lpips": 0.4710853695869446
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 4.9765625,
415
+ "history_latent_mse": 1.5143388509750366,
416
+ "psnr": 8.205375822657874,
417
+ "ssim": 0.2038023523271112,
418
+ "lpips": 0.7657009363174438
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 5.25,
422
+ "history_latent_mse": 1.5066324472427368,
423
+ "psnr": 8.40044824459646,
424
+ "ssim": 0.21608138567351615,
425
+ "lpips": 0.8176727890968323
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 2.65625,
429
+ "history_latent_mse": 0.13566608726978302,
430
+ "psnr": 34.98019656406381,
431
+ "ssim": 0.9396104027733518,
432
+ "lpips": 0.29628050327301025
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 3.0703125,
436
+ "history_latent_mse": 0.144988015294075,
437
+ "psnr": 33.08963139480401,
438
+ "ssim": 0.9410875123563129,
439
+ "lpips": 0.258383572101593
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 2611,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 4.15625,
451
+ "history_latent_mse": 0.5618721842765808,
452
+ "psnr": 14.022926307752998,
453
+ "ssim": 0.5597091502249111,
454
+ "lpips": 0.4689983129501343
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 6.125,
458
+ "history_latent_mse": 1.482651710510254,
459
+ "psnr": 9.247492241476783,
460
+ "ssim": 0.3291559204751646,
461
+ "lpips": 0.6797118782997131
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 5.5625,
465
+ "history_latent_mse": 1.5147126913070679,
466
+ "psnr": 9.54024800688658,
467
+ "ssim": 0.28481042074729657,
468
+ "lpips": 0.7541435956954956
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 2.7109375,
472
+ "history_latent_mse": 0.1551610678434372,
473
+ "psnr": 35.599234237202694,
474
+ "ssim": 0.9427807967268611,
475
+ "lpips": 0.2821560204029083
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 2.765625,
479
+ "history_latent_mse": 0.1496855467557907,
480
+ "psnr": 35.99517843138076,
481
+ "ssim": 0.9486212645952063,
482
+ "lpips": 0.21314403414726257
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 2619,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 4.44140625,
494
+ "history_latent_mse": 0.5302004814147949,
495
+ "psnr": 14.880947695835854,
496
+ "ssim": 0.57533410681392,
497
+ "lpips": 0.45671528577804565
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 5.296875,
501
+ "history_latent_mse": 1.2381802797317505,
502
+ "psnr": 10.703357894145372,
503
+ "ssim": 0.41701729843814095,
504
+ "lpips": 0.5924857258796692
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 5.03125,
508
+ "history_latent_mse": 1.3446159362792969,
509
+ "psnr": 8.821429504322131,
510
+ "ssim": 0.19334200080630412,
511
+ "lpips": 0.814721405506134
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 2.3515625,
515
+ "history_latent_mse": 0.12978319823741913,
516
+ "psnr": 35.598542324647646,
517
+ "ssim": 0.9397215758602377,
518
+ "lpips": 0.2898758053779602
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 2.7109375,
522
+ "history_latent_mse": 0.15500086545944214,
523
+ "psnr": 34.32492532341694,
524
+ "ssim": 0.9427065419641114,
525
+ "lpips": 0.25140976905822754
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 2627,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.3125,
537
+ "history_latent_mse": 0.47130608558654785,
538
+ "psnr": 14.606693310270844,
539
+ "ssim": 0.5531711847126617,
540
+ "lpips": 0.4825262427330017
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 5.046875,
544
+ "history_latent_mse": 1.0705623626708984,
545
+ "psnr": 12.064152210124384,
546
+ "ssim": 0.516445524010195,
547
+ "lpips": 0.5754836797714233
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 5.78125,
551
+ "history_latent_mse": 1.6478544473648071,
552
+ "psnr": 9.411113216018585,
553
+ "ssim": 0.1698900599853368,
554
+ "lpips": 0.8225197792053223
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 1.88671875,
558
+ "history_latent_mse": 0.1231420636177063,
559
+ "psnr": 35.32644705073464,
560
+ "ssim": 0.9375313809980473,
561
+ "lpips": 0.3004121780395508
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 2.640625,
565
+ "history_latent_mse": 0.19039489328861237,
566
+ "psnr": 30.3456437967443,
567
+ "ssim": 0.9275757916035403,
568
+ "lpips": 0.33385026454925537
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 2635,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 4.3203125,
580
+ "history_latent_mse": 0.5852118730545044,
581
+ "psnr": 14.079720521704612,
582
+ "ssim": 0.5082219844715823,
583
+ "lpips": 0.5390398502349854
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 4.8359375,
587
+ "history_latent_mse": 0.8839182257652283,
588
+ "psnr": 12.705496023461402,
589
+ "ssim": 0.5040707284931212,
590
+ "lpips": 0.5906945466995239
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.4765625,
594
+ "history_latent_mse": 1.4683102369308472,
595
+ "psnr": 9.855829945014992,
596
+ "ssim": 0.17281228171489707,
597
+ "lpips": 0.746903121471405
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 2.056884765625,
601
+ "history_latent_mse": 0.12720677256584167,
602
+ "psnr": 35.04194541254678,
603
+ "ssim": 0.9346703476639409,
604
+ "lpips": 0.3099118173122406
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 2.8828125,
608
+ "history_latent_mse": 0.20562660694122314,
609
+ "psnr": 29.746095590532956,
610
+ "ssim": 0.9229609742765656,
611
+ "lpips": 0.3514562249183655
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 2643,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 5.71875,
623
+ "history_latent_mse": 0.6232137084007263,
624
+ "psnr": 14.082933236823468,
625
+ "ssim": 0.5156893884621685,
626
+ "lpips": 0.5388348698616028
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 5.140625,
630
+ "history_latent_mse": 0.8454455733299255,
631
+ "psnr": 12.986729481331821,
632
+ "ssim": 0.5114012672230022,
633
+ "lpips": 0.5878117084503174
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 4.94921875,
637
+ "history_latent_mse": 1.3163434267044067,
638
+ "psnr": 9.973529345020598,
639
+ "ssim": 0.17466833704868473,
640
+ "lpips": 0.684074878692627
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 1.978729248046875,
644
+ "history_latent_mse": 0.13266299664974213,
645
+ "psnr": 34.50713700039933,
646
+ "ssim": 0.9316547931630519,
647
+ "lpips": 0.3206460475921631
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 2.7578125,
651
+ "history_latent_mse": 0.22606706619262695,
652
+ "psnr": 31.080840879215664,
653
+ "ssim": 0.9219519881025942,
654
+ "lpips": 0.3415656089782715
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecd7e7a72f29030b8926dc1564b3a4ce93c6e574ba8b187b6b275d95707acc66
3
+ size 1201037
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/03_motherboard_0909_episode_000_full/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.0986940786242485,
91
+ "wrist/video/latent_future_mse": 0.0986940786242485,
92
+ "video/psnr": 26.63426720435477,
93
+ "wrist/video/psnr": 26.63426720435477,
94
+ "video/ssim": 0.8708790760218011,
95
+ "wrist/video/ssim": 0.8708790760218011,
96
+ "video/lpips": 0.046272218227386475,
97
+ "wrist/video/lpips": 0.046272218227386475,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.4402367174625397,
103
+ "wrist/wrist_left/latent_future_mse": 0.4402367174625397,
104
+ "wrist_left/psnr": 17.216493839691783,
105
+ "wrist/wrist_left/psnr": 17.216493839691783,
106
+ "wrist_left/ssim": 0.5951449385403667,
107
+ "wrist/wrist_left/ssim": 0.5951449385403667,
108
+ "wrist_left/lpips": 0.17227669060230255,
109
+ "wrist/wrist_left/lpips": 0.17227669060230255,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.599057137966156,
115
+ "wrist/wrist_right/latent_future_mse": 0.599057137966156,
116
+ "wrist_right/psnr": 17.487450501330382,
117
+ "wrist/wrist_right/psnr": 17.487450501330382,
118
+ "wrist_right/ssim": 0.5787619882113139,
119
+ "wrist/wrist_right/ssim": 0.5787619882113139,
120
+ "wrist_right/lpips": 0.23368918895721436,
121
+ "wrist/wrist_right/lpips": 0.23368918895721436,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.0710873007774353,
127
+ "wrist/tactile_left/latent_future_mse": 0.0710873007774353,
128
+ "tactile_left/psnr": 35.386292833088994,
129
+ "wrist/tactile_left/psnr": 35.386292833088994,
130
+ "tactile_left/ssim": 0.9682620496470609,
131
+ "wrist/tactile_left/ssim": 0.9682620496470609,
132
+ "tactile_left/lpips": 0.0447380356490612,
133
+ "wrist/tactile_left/lpips": 0.0447380356490612,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.09564441442489624,
139
+ "wrist/tactile_right/latent_future_mse": 0.09564441442489624,
140
+ "tactile_right/psnr": 36.69995933064942,
141
+ "wrist/tactile_right/psnr": 36.69995933064942,
142
+ "tactile_right/ssim": 0.9682634386253878,
143
+ "wrist/tactile_right/ssim": 0.9682634386253878,
144
+ "tactile_right/lpips": 0.08494263142347336,
145
+ "wrist/tactile_right/lpips": 0.08494263142347336,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_000.pt",
152
+ "t_start": 2531,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.0986940786242485,
169
+ "psnr": 26.63426720435477,
170
+ "ssim": 0.8708790760218011,
171
+ "lpips": 0.046272218227386475
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.4402367174625397,
177
+ "psnr": 17.216493839691783,
178
+ "ssim": 0.5951449385403667,
179
+ "lpips": 0.17227669060230255
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.599057137966156,
185
+ "psnr": 17.487450501330382,
186
+ "ssim": 0.5787619882113139,
187
+ "lpips": 0.23368918895721436
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.0710873007774353,
193
+ "psnr": 35.386292833088994,
194
+ "ssim": 0.9682620496470609,
195
+ "lpips": 0.0447380356490612
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.09564441442489624,
201
+ "psnr": 36.69995933064942,
202
+ "ssim": 0.9682634386253878,
203
+ "lpips": 0.08494263142347336
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/case.json ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 4,
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "category": "none",
5
+ "index": 3319,
6
+ "t_start": 3,
7
+ "start_seconds": 0.2,
8
+ "short_contact": [
9
+ 0,
10
+ 0,
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 0,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0
25
+ ],
26
+ "short_transition_ticks": [],
27
+ "future_transition_ticks": [],
28
+ "long_contact_fraction": 0.0,
29
+ "long_future_contact_fraction": 0.0,
30
+ "contact_over_128_frames": [
31
+ 0,
32
+ 0,
33
+ 0,
34
+ 0,
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 0,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 0,
59
+ 0,
60
+ 0,
61
+ 0,
62
+ 0,
63
+ 0,
64
+ 0,
65
+ 0,
66
+ 0,
67
+ 0,
68
+ 0,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 0,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 0,
82
+ 0,
83
+ 0,
84
+ 0,
85
+ 0,
86
+ 0,
87
+ 0,
88
+ 0,
89
+ 0,
90
+ 0,
91
+ 0,
92
+ 0,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 0,
98
+ 0,
99
+ 0,
100
+ 0,
101
+ 0,
102
+ 0,
103
+ 0,
104
+ 0,
105
+ 0,
106
+ 0,
107
+ 0,
108
+ 0,
109
+ 0,
110
+ 0,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 0,
115
+ 0,
116
+ 0,
117
+ 0,
118
+ 0,
119
+ 0,
120
+ 0,
121
+ 0,
122
+ 0,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 0,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 0,
132
+ 0,
133
+ 0,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 0,
141
+ 0,
142
+ 0,
143
+ 0,
144
+ 0,
145
+ 0,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 0,
155
+ 0,
156
+ 0,
157
+ 0,
158
+ 0
159
+ ],
160
+ "first_contact_tick": null,
161
+ "long_label": "none_throughout",
162
+ "output_subdir": "04_motherboard_0909_episode_001_none",
163
+ "rollout_windows": 15
164
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8ab5fc9e5a51129d7d014edfa4eb5efc9d2b055993ddff6c3ff768511e85aba
3
+ size 4678993
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/motherboard_0909_episode_001_t00003_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b783f20cd087d6d9acf43772992bc5d2b7b17dc38c31e8ee9bafb1b4f2b79aec
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "t_start": 3,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 3,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 34.98245723682948,
23
+ "ssim": 0.9629138931467476,
24
+ "lpips": 0.010414157062768936
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 39.17661504835171,
30
+ "ssim": 0.9774012756239003,
31
+ "lpips": 0.014707624912261963
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 41.112583412445105,
37
+ "ssim": 0.980698017129837,
38
+ "lpips": 0.011336026713252068
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 47.88916107867081,
44
+ "ssim": 0.992888085249837,
45
+ "lpips": 0.005414151586592197
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 50.09711170465344,
51
+ "ssim": 0.9943909986189488,
52
+ "lpips": 0.004877166822552681
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 11,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 0.80078125,
64
+ "history_latent_mse": 0.016177501529455185,
65
+ "psnr": 32.65360172062122,
66
+ "ssim": 0.9451018438629752,
67
+ "lpips": 0.018051061779260635
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 2.51953125,
71
+ "history_latent_mse": 0.0688566341996193,
72
+ "psnr": 34.70874410652181,
73
+ "ssim": 0.9550191848141394,
74
+ "lpips": 0.030690759420394897
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 1.609375,
78
+ "history_latent_mse": 0.051370058208703995,
79
+ "psnr": 36.70862851801243,
80
+ "ssim": 0.9665328940070996,
81
+ "lpips": 0.021000880748033524
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 0.7998046875,
85
+ "history_latent_mse": 0.01607528142631054,
86
+ "psnr": 43.30408058688558,
87
+ "ssim": 0.9866618062365831,
88
+ "lpips": 0.02096220850944519
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 0.78125,
92
+ "history_latent_mse": 0.01136635523289442,
93
+ "psnr": 45.19819940703411,
94
+ "ssim": 0.9887515562169498,
95
+ "lpips": 0.029524262994527817
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 19,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 0.859375,
107
+ "history_latent_mse": 0.024617746472358704,
108
+ "psnr": 30.657814377587233,
109
+ "ssim": 0.9261530313214832,
110
+ "lpips": 0.026675928384065628
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 2.75,
114
+ "history_latent_mse": 0.1057824194431305,
115
+ "psnr": 31.89597433160625,
116
+ "ssim": 0.9414306908418408,
117
+ "lpips": 0.052067264914512634
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 2.13671875,
121
+ "history_latent_mse": 0.07896627485752106,
122
+ "psnr": 33.360781933264356,
123
+ "ssim": 0.9495707114799177,
124
+ "lpips": 0.03309304267168045
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 1.2509765625,
128
+ "history_latent_mse": 0.03207814693450928,
129
+ "psnr": 41.132266877457944,
130
+ "ssim": 0.9818701938980837,
131
+ "lpips": 0.0422329306602478
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 1.115234375,
135
+ "history_latent_mse": 0.03048039972782135,
136
+ "psnr": 42.74665725794399,
137
+ "ssim": 0.9834887554005778,
138
+ "lpips": 0.05530240386724472
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 27,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 1.09375,
150
+ "history_latent_mse": 0.03539456054568291,
151
+ "psnr": 29.217249573879712,
152
+ "ssim": 0.9059627175180314,
153
+ "lpips": 0.035778772085905075
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 3.51171875,
157
+ "history_latent_mse": 0.12455450743436813,
158
+ "psnr": 29.53919078110118,
159
+ "ssim": 0.929478994425008,
160
+ "lpips": 0.06983967125415802
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 2.3984375,
164
+ "history_latent_mse": 0.09827543050050735,
165
+ "psnr": 31.221406923166764,
166
+ "ssim": 0.9340979552251343,
167
+ "lpips": 0.04688713699579239
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 1.41015625,
171
+ "history_latent_mse": 0.041825417429208755,
172
+ "psnr": 39.6108188738143,
173
+ "ssim": 0.9777722940812191,
174
+ "lpips": 0.07898698002099991
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 1.083984375,
178
+ "history_latent_mse": 0.03997945412993431,
179
+ "psnr": 41.088363148294775,
180
+ "ssim": 0.9800138270826642,
181
+ "lpips": 0.07595539093017578
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 35,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 1.453125,
193
+ "history_latent_mse": 0.04712721332907677,
194
+ "psnr": 28.073694923796836,
195
+ "ssim": 0.8865141116325409,
196
+ "lpips": 0.047943443059921265
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 3.078125,
200
+ "history_latent_mse": 0.14618082344532013,
201
+ "psnr": 27.72459402619125,
202
+ "ssim": 0.9197344237354016,
203
+ "lpips": 0.0823066234588623
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 2.5234375,
207
+ "history_latent_mse": 0.11575943976640701,
208
+ "psnr": 29.226118852301507,
209
+ "ssim": 0.9202291093930883,
210
+ "lpips": 0.06048545986413956
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 1.326171875,
214
+ "history_latent_mse": 0.050554513931274414,
215
+ "psnr": 38.41573621709636,
216
+ "ssim": 0.9740903761437776,
217
+ "lpips": 0.11617135256528854
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 1.28125,
221
+ "history_latent_mse": 0.048963647335767746,
222
+ "psnr": 39.43631928449023,
223
+ "ssim": 0.9763008183608433,
224
+ "lpips": 0.09465165436267853
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 43,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 1.609375,
236
+ "history_latent_mse": 0.06006389856338501,
237
+ "psnr": 27.02987894339884,
238
+ "ssim": 0.866084420760076,
239
+ "lpips": 0.05735699087381363
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 2.98828125,
243
+ "history_latent_mse": 0.1538659930229187,
244
+ "psnr": 25.91892593024735,
245
+ "ssim": 0.8915482906987001,
246
+ "lpips": 0.13015079498291016
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 2.5859375,
250
+ "history_latent_mse": 0.1415880173444748,
251
+ "psnr": 27.698779252768805,
252
+ "ssim": 0.9043232175014985,
253
+ "lpips": 0.07837487757205963
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 1.37060546875,
257
+ "history_latent_mse": 0.0584084577858448,
258
+ "psnr": 37.7379841467458,
259
+ "ssim": 0.970306988952025,
260
+ "lpips": 0.14769673347473145
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 1.19140625,
264
+ "history_latent_mse": 0.057037342339754105,
265
+ "psnr": 38.661940769686765,
266
+ "ssim": 0.9726057034187406,
267
+ "lpips": 0.11783362179994583
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 51,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 2.08984375,
279
+ "history_latent_mse": 0.07282394915819168,
280
+ "psnr": 26.190869255919587,
281
+ "ssim": 0.8458709432522004,
282
+ "lpips": 0.06871116906404495
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 2.6796875,
286
+ "history_latent_mse": 0.22041599452495575,
287
+ "psnr": 24.700992186275258,
288
+ "ssim": 0.8639734152728547,
289
+ "lpips": 0.16663044691085815
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 3.19921875,
293
+ "history_latent_mse": 0.15733292698860168,
294
+ "psnr": 26.35364796082934,
295
+ "ssim": 0.8889362924344137,
296
+ "lpips": 0.09919115900993347
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 1.458984375,
300
+ "history_latent_mse": 0.06807977706193924,
301
+ "psnr": 36.85889801614486,
302
+ "ssim": 0.9663484443781485,
303
+ "lpips": 0.1788940131664276
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 1.484619140625,
307
+ "history_latent_mse": 0.06784757971763611,
308
+ "psnr": 38.00342742716552,
309
+ "ssim": 0.9689376271909127,
310
+ "lpips": 0.1367427408695221
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 59,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 1.84765625,
322
+ "history_latent_mse": 0.08585706353187561,
323
+ "psnr": 25.4728454101052,
324
+ "ssim": 0.8278488497254033,
325
+ "lpips": 0.08031848073005676
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 2.9453125,
329
+ "history_latent_mse": 0.2418694943189621,
330
+ "psnr": 22.741613755245133,
331
+ "ssim": 0.7890121080437303,
332
+ "lpips": 0.24681398272514343
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 3.06640625,
336
+ "history_latent_mse": 0.18104489147663116,
337
+ "psnr": 25.48509429605201,
338
+ "ssim": 0.863099005353641,
339
+ "lpips": 0.11643271893262863
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 1.423828125,
343
+ "history_latent_mse": 0.07726151496171951,
344
+ "psnr": 36.215608122520976,
345
+ "ssim": 0.9625128813389783,
346
+ "lpips": 0.20342519879341125
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 1.40625,
350
+ "history_latent_mse": 0.07355325669050217,
351
+ "psnr": 37.22816225778982,
352
+ "ssim": 0.9651628246134882,
353
+ "lpips": 0.15830203890800476
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 67,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 2.10546875,
365
+ "history_latent_mse": 0.09972519427537918,
366
+ "psnr": 24.810640359296006,
367
+ "ssim": 0.8092628495916824,
368
+ "lpips": 0.0904068723320961
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 3.546875,
372
+ "history_latent_mse": 0.28614574670791626,
373
+ "psnr": 21.360640624845622,
374
+ "ssim": 0.7470997269319821,
375
+ "lpips": 0.28648388385772705
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 3.71875,
379
+ "history_latent_mse": 0.2093622088432312,
380
+ "psnr": 24.62544191727146,
381
+ "ssim": 0.8383657738887982,
382
+ "lpips": 0.13321352005004883
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 1.44921875,
386
+ "history_latent_mse": 0.08496720343828201,
387
+ "psnr": 35.345682259461014,
388
+ "ssim": 0.9586259166662039,
389
+ "lpips": 0.23060844838619232
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 1.493408203125,
393
+ "history_latent_mse": 0.0822221040725708,
394
+ "psnr": 36.436543148097485,
395
+ "ssim": 0.9616665582813301,
396
+ "lpips": 0.17387701570987701
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 75,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 2.3828125,
408
+ "history_latent_mse": 0.11592298746109009,
409
+ "psnr": 24.177785101261165,
410
+ "ssim": 0.7940892053250914,
411
+ "lpips": 0.09998476505279541
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 3.7578125,
415
+ "history_latent_mse": 0.3163505792617798,
416
+ "psnr": 20.864276155296217,
417
+ "ssim": 0.7295377803498351,
418
+ "lpips": 0.3027108311653137
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 3.46435546875,
422
+ "history_latent_mse": 0.24694335460662842,
423
+ "psnr": 23.87788299042554,
424
+ "ssim": 0.8193955600194242,
425
+ "lpips": 0.15296679735183716
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 1.541015625,
429
+ "history_latent_mse": 0.09362727403640747,
430
+ "psnr": 34.78741039492542,
431
+ "ssim": 0.9547197357495681,
432
+ "lpips": 0.25133681297302246
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 1.525390625,
436
+ "history_latent_mse": 0.08905778080224991,
437
+ "psnr": 35.892239294070905,
438
+ "ssim": 0.9580291924617752,
439
+ "lpips": 0.19143548607826233
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 83,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 2.515625,
451
+ "history_latent_mse": 0.12416509538888931,
452
+ "psnr": 23.771543933786234,
453
+ "ssim": 0.7821418086178215,
454
+ "lpips": 0.10931544005870819
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 2.8984375,
458
+ "history_latent_mse": 0.32416948676109314,
459
+ "psnr": 20.65019768223056,
460
+ "ssim": 0.7196238203856296,
461
+ "lpips": 0.3087509870529175
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 3.71875,
465
+ "history_latent_mse": 0.2711307406425476,
466
+ "psnr": 23.208677977308643,
467
+ "ssim": 0.8031497119100225,
468
+ "lpips": 0.1665535718202591
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 1.548828125,
472
+ "history_latent_mse": 0.10238610953092575,
473
+ "psnr": 34.228455996369156,
474
+ "ssim": 0.950545647103486,
475
+ "lpips": 0.2692863941192627
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 1.56640625,
479
+ "history_latent_mse": 0.09811829775571823,
480
+ "psnr": 35.527784126835904,
481
+ "ssim": 0.9543523143871042,
482
+ "lpips": 0.20773538947105408
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 91,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 2.625,
494
+ "history_latent_mse": 0.1330309510231018,
495
+ "psnr": 23.2575131675048,
496
+ "ssim": 0.7703655601357615,
497
+ "lpips": 0.11754289269447327
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 3.478515625,
501
+ "history_latent_mse": 0.3368115723133087,
502
+ "psnr": 20.464269335672892,
503
+ "ssim": 0.7121922812898331,
504
+ "lpips": 0.3153577446937561
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 3.625,
508
+ "history_latent_mse": 0.2756876051425934,
509
+ "psnr": 22.60791053573388,
510
+ "ssim": 0.7901893278748782,
511
+ "lpips": 0.17656481266021729
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 1.630859375,
515
+ "history_latent_mse": 0.11201604455709457,
516
+ "psnr": 33.760866912896724,
517
+ "ssim": 0.9461824642387036,
518
+ "lpips": 0.28509899973869324
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 1.71875,
522
+ "history_latent_mse": 0.10582425445318222,
523
+ "psnr": 35.03517923183785,
524
+ "ssim": 0.9508115855691162,
525
+ "lpips": 0.22077180445194244
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 99,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 2.8515625,
537
+ "history_latent_mse": 0.14295947551727295,
538
+ "psnr": 22.767795813713203,
539
+ "ssim": 0.7567105433703114,
540
+ "lpips": 0.12600049376487732
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 3.6171875,
544
+ "history_latent_mse": 0.33440685272216797,
545
+ "psnr": 20.289214883884632,
546
+ "ssim": 0.7069400382975393,
547
+ "lpips": 0.32493889331817627
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 3.7578125,
551
+ "history_latent_mse": 0.2922215163707733,
552
+ "psnr": 22.042752144313628,
553
+ "ssim": 0.7740129932497972,
554
+ "lpips": 0.19255070388317108
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 1.6728515625,
558
+ "history_latent_mse": 0.1231573224067688,
559
+ "psnr": 33.411619844836665,
560
+ "ssim": 0.9415519517741281,
561
+ "lpips": 0.2940673232078552
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 1.83203125,
565
+ "history_latent_mse": 0.11262484639883041,
566
+ "psnr": 34.4821552377769,
567
+ "ssim": 0.9460351792023809,
568
+ "lpips": 0.2379671335220337
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 107,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 3.1015625,
580
+ "history_latent_mse": 0.15253527462482452,
581
+ "psnr": 22.41080653955966,
582
+ "ssim": 0.7440511207261107,
583
+ "lpips": 0.13436497747898102
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 4.140625,
587
+ "history_latent_mse": 0.3626815378665924,
588
+ "psnr": 20.165464811224048,
589
+ "ssim": 0.6996546904626508,
590
+ "lpips": 0.32388269901275635
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 3.658935546875,
594
+ "history_latent_mse": 0.304993599653244,
595
+ "psnr": 21.357920729699682,
596
+ "ssim": 0.7567205235876333,
597
+ "lpips": 0.20732323825359344
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 1.8359375,
601
+ "history_latent_mse": 0.12264937162399292,
602
+ "psnr": 32.91540619581552,
603
+ "ssim": 0.9368707716007975,
604
+ "lpips": 0.3127726912498474
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 1.978515625,
608
+ "history_latent_mse": 0.11911370605230331,
609
+ "psnr": 33.81251976800809,
610
+ "ssim": 0.9419957173487334,
611
+ "lpips": 0.25386711955070496
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 115,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 3.2265625,
623
+ "history_latent_mse": 0.16395385563373566,
624
+ "psnr": 21.352153566226427,
625
+ "ssim": 0.7121591690641549,
626
+ "lpips": 0.1606350988149643
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 3.08984375,
630
+ "history_latent_mse": 0.3456915318965912,
631
+ "psnr": 20.105780881761074,
632
+ "ssim": 0.6982235168499099,
633
+ "lpips": 0.3303559422492981
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 3.734375,
637
+ "history_latent_mse": 0.3107913136482239,
638
+ "psnr": 20.490728589145085,
639
+ "ssim": 0.7300526933251967,
640
+ "lpips": 0.24053047597408295
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 2.033203125,
644
+ "history_latent_mse": 0.131675586104393,
645
+ "psnr": 32.407308936669715,
646
+ "ssim": 0.9330809563942799,
647
+ "lpips": 0.32732903957366943
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 2.2421875,
651
+ "history_latent_mse": 0.12655149400234222,
652
+ "psnr": 33.08481006743399,
653
+ "ssim": 0.938295244543474,
654
+ "lpips": 0.2669263780117035
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06b1e62289dfad8a71829f168aa8dcc8d3b4e52d2244868cc24c33f0dddcc078
3
+ size 578075
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/04_motherboard_0909_episode_001_none/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.03628881648182869,
91
+ "wrist/video/latent_future_mse": 0.03628881648182869,
92
+ "video/psnr": 34.98245723682948,
93
+ "wrist/video/psnr": 34.98245723682948,
94
+ "video/ssim": 0.9629138931467476,
95
+ "wrist/video/ssim": 0.9629138931467476,
96
+ "video/lpips": 0.010414157062768936,
97
+ "wrist/video/lpips": 0.010414157062768936,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.0688975527882576,
103
+ "wrist/wrist_left/latent_future_mse": 0.0688975527882576,
104
+ "wrist_left/psnr": 39.17661504835171,
105
+ "wrist/wrist_left/psnr": 39.17661504835171,
106
+ "wrist_left/ssim": 0.9774012756239003,
107
+ "wrist/wrist_left/ssim": 0.9774012756239003,
108
+ "wrist_left/lpips": 0.014707624912261963,
109
+ "wrist/wrist_left/lpips": 0.014707624912261963,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.05638621747493744,
115
+ "wrist/wrist_right/latent_future_mse": 0.05638621747493744,
116
+ "wrist_right/psnr": 41.112583412445105,
117
+ "wrist/wrist_right/psnr": 41.112583412445105,
118
+ "wrist_right/ssim": 0.980698017129837,
119
+ "wrist/wrist_right/ssim": 0.980698017129837,
120
+ "wrist_right/lpips": 0.011336026713252068,
121
+ "wrist/wrist_right/lpips": 0.011336026713252068,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.013817056082189083,
127
+ "wrist/tactile_left/latent_future_mse": 0.013817056082189083,
128
+ "tactile_left/psnr": 47.88916107867081,
129
+ "wrist/tactile_left/psnr": 47.88916107867081,
130
+ "tactile_left/ssim": 0.992888085249837,
131
+ "wrist/tactile_left/ssim": 0.992888085249837,
132
+ "tactile_left/lpips": 0.005414151586592197,
133
+ "wrist/tactile_left/lpips": 0.005414151586592197,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.00848926417529583,
139
+ "wrist/tactile_right/latent_future_mse": 0.00848926417529583,
140
+ "tactile_right/psnr": 50.09711170465344,
141
+ "wrist/tactile_right/psnr": 50.09711170465344,
142
+ "tactile_right/ssim": 0.9943909986189488,
143
+ "wrist/tactile_right/ssim": 0.9943909986189488,
144
+ "tactile_right/lpips": 0.004877166822552681,
145
+ "wrist/tactile_right/lpips": 0.004877166822552681,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_001.pt",
152
+ "t_start": 3,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.03628881648182869,
169
+ "psnr": 34.98245723682948,
170
+ "ssim": 0.9629138931467476,
171
+ "lpips": 0.010414157062768936
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.0688975527882576,
177
+ "psnr": 39.17661504835171,
178
+ "ssim": 0.9774012756239003,
179
+ "lpips": 0.014707624912261963
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.05638621747493744,
185
+ "psnr": 41.112583412445105,
186
+ "ssim": 0.980698017129837,
187
+ "lpips": 0.011336026713252068
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.013817056082189083,
193
+ "psnr": 47.88916107867081,
194
+ "ssim": 0.992888085249837,
195
+ "lpips": 0.005414151586592197
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.00848926417529583,
201
+ "psnr": 50.09711170465344,
202
+ "ssim": 0.9943909986189488,
203
+ "lpips": 0.004877166822552681
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/case.json ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 5,
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "category": "onset",
5
+ "index": 4059,
6
+ "t_start": 760,
7
+ "start_seconds": 50.666666666666664,
8
+ "short_contact": [
9
+ 0,
10
+ 0,
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 0,
20
+ 1,
21
+ 1,
22
+ 1,
23
+ 1,
24
+ 1
25
+ ],
26
+ "short_transition_ticks": [
27
+ 11
28
+ ],
29
+ "future_transition_ticks": [
30
+ 11
31
+ ],
32
+ "long_contact_fraction": 0.6953125,
33
+ "long_future_contact_fraction": 0.7416666666666667,
34
+ "contact_over_128_frames": [
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 1,
47
+ 1,
48
+ 1,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 1,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 1,
63
+ 1,
64
+ 1,
65
+ 1,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 0,
73
+ 0,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 0,
82
+ 0,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 1,
107
+ 1,
108
+ 1,
109
+ 1,
110
+ 1,
111
+ 1,
112
+ 1,
113
+ 1,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 0,
133
+ 0,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 0,
158
+ 0,
159
+ 0,
160
+ 0,
161
+ 0,
162
+ 0
163
+ ],
164
+ "first_contact_tick": 11,
165
+ "long_label": "onset",
166
+ "output_subdir": "05_motherboard_0909_episode_001_onset",
167
+ "rollout_windows": 15
168
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:efd25e4c8d5dc4a3ca2022e9125977e0dda87ddfda30496c0256577693bc7072
3
+ size 9794147
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/motherboard_0909_episode_001_t00760_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebb1f433278c24e0a40f64c711ce45ff1da495a2a7f2d9f685fb24702cc2f2b7
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "t_start": 760,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 760,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 23.97182098554074,
23
+ "ssim": 0.8125829526693039,
24
+ "lpips": 0.07511236518621445
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 13.948032842082776,
30
+ "ssim": 0.3378560447266604,
31
+ "lpips": 0.47823843359947205
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 15.361403592715266,
37
+ "ssim": 0.5478371830024589,
38
+ "lpips": 0.3370887041091919
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 38.742343650877785,
44
+ "ssim": 0.9835824167841523,
45
+ "lpips": 0.041552357375621796
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 46.984432773655556,
51
+ "ssim": 0.9918726462661673,
52
+ "lpips": 0.013785661198198795
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 768,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 4.828125,
64
+ "history_latent_mse": 0.16472817957401276,
65
+ "psnr": 20.773689788720986,
66
+ "ssim": 0.702239032431079,
67
+ "lpips": 0.161298930644989
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 4.749755859375,
71
+ "history_latent_mse": 0.8072547912597656,
72
+ "psnr": 12.957019796188575,
73
+ "ssim": 0.33489133909746094,
74
+ "lpips": 0.49681586027145386
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 5.0,
78
+ "history_latent_mse": 0.5608716607093811,
79
+ "psnr": 13.357670605569608,
80
+ "ssim": 0.4937799383731884,
81
+ "lpips": 0.3790629208087921
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 2.64453125,
85
+ "history_latent_mse": 0.06477810442447662,
86
+ "psnr": 41.98333673446485,
87
+ "ssim": 0.981422978192573,
88
+ "lpips": 0.036314770579338074
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 1.2216796875,
92
+ "history_latent_mse": 0.025710830464959145,
93
+ "psnr": 44.02224942484943,
94
+ "ssim": 0.9845782055239337,
95
+ "lpips": 0.0436430424451828
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 776,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 4.0390625,
107
+ "history_latent_mse": 0.22434532642364502,
108
+ "psnr": 21.411604121735902,
109
+ "ssim": 0.7241759497755744,
110
+ "lpips": 0.1670190840959549
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 5.0703125,
114
+ "history_latent_mse": 0.7591099739074707,
115
+ "psnr": 13.382551202141775,
116
+ "ssim": 0.2598375119677453,
117
+ "lpips": 0.5377963185310364
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 4.75,
121
+ "history_latent_mse": 0.6611424088478088,
122
+ "psnr": 12.686662470364169,
123
+ "ssim": 0.4378278147681438,
124
+ "lpips": 0.4320305585861206
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 2.667236328125,
128
+ "history_latent_mse": 0.053133945912122726,
129
+ "psnr": 29.017501133794287,
130
+ "ssim": 0.9454906762667659,
131
+ "lpips": 0.19702446460723877
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 1.048828125,
135
+ "history_latent_mse": 0.035533007234334946,
136
+ "psnr": 43.24735895355716,
137
+ "ssim": 0.9815280666806907,
138
+ "lpips": 0.06259869039058685
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 784,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 4.625,
150
+ "history_latent_mse": 0.18156300485134125,
151
+ "psnr": 19.813532405047198,
152
+ "ssim": 0.6956227564452014,
153
+ "lpips": 0.24994304776191711
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 4.640625,
157
+ "history_latent_mse": 0.7792689204216003,
158
+ "psnr": 10.840011900492755,
159
+ "ssim": 0.2175243299314709,
160
+ "lpips": 0.5774753093719482
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 4.625,
164
+ "history_latent_mse": 0.710976779460907,
165
+ "psnr": 12.708692467527909,
166
+ "ssim": 0.46053409200751866,
167
+ "lpips": 0.47454896569252014
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 4.546875,
171
+ "history_latent_mse": 0.15830831229686737,
172
+ "psnr": 29.84664517899878,
173
+ "ssim": 0.953936999401324,
174
+ "lpips": 0.19601863622665405
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 1.177734375,
178
+ "history_latent_mse": 0.04353208467364311,
179
+ "psnr": 42.08288312723045,
180
+ "ssim": 0.9780972762830105,
181
+ "lpips": 0.08162233233451843
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 792,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 5.640625,
193
+ "history_latent_mse": 0.25846076011657715,
194
+ "psnr": 19.261874864915804,
195
+ "ssim": 0.6692308307134727,
196
+ "lpips": 0.2730814516544342
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 6.453125,
200
+ "history_latent_mse": 0.8949176669120789,
201
+ "psnr": 9.92673343964626,
202
+ "ssim": 0.4358654626353847,
203
+ "lpips": 0.5115290880203247
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 4.8984375,
207
+ "history_latent_mse": 0.7173665165901184,
208
+ "psnr": 9.293844716863788,
209
+ "ssim": 0.23060218251391312,
210
+ "lpips": 0.5908918380737305
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 4.4140625,
214
+ "history_latent_mse": 0.15500132739543915,
215
+ "psnr": 39.90042011569596,
216
+ "ssim": 0.9705844551989551,
217
+ "lpips": 0.14060094952583313
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 1.287109375,
221
+ "history_latent_mse": 0.05181732773780823,
222
+ "psnr": 40.93387182630329,
223
+ "ssim": 0.9750033902078178,
224
+ "lpips": 0.09806308150291443
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 800,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 5.015625,
236
+ "history_latent_mse": 0.2749159634113312,
237
+ "psnr": 19.2806336321409,
238
+ "ssim": 0.6734096140701664,
239
+ "lpips": 0.25466710329055786
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 5.1328125,
243
+ "history_latent_mse": 0.9686599373817444,
244
+ "psnr": 10.464444482360339,
245
+ "ssim": 0.4894071927984617,
246
+ "lpips": 0.5146087408065796
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 5.3828125,
250
+ "history_latent_mse": 1.1812862157821655,
251
+ "psnr": 8.351546856531277,
252
+ "ssim": 0.17845092278843294,
253
+ "lpips": 0.6624883413314819
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 1.47265625,
257
+ "history_latent_mse": 0.06376153975725174,
258
+ "psnr": 39.330011846542305,
259
+ "ssim": 0.9668452695236465,
260
+ "lpips": 0.162667915225029
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 1.2783203125,
264
+ "history_latent_mse": 0.05738365277647972,
265
+ "psnr": 33.65080702739904,
266
+ "ssim": 0.9520888476662168,
267
+ "lpips": 0.2124367356300354
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 808,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 3.63330078125,
279
+ "history_latent_mse": 0.2533222436904907,
280
+ "psnr": 19.185930506613264,
281
+ "ssim": 0.6614148823732806,
282
+ "lpips": 0.28340551257133484
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 5.21875,
286
+ "history_latent_mse": 1.0155400037765503,
287
+ "psnr": 11.5018379751468,
288
+ "ssim": 0.4703773290802423,
289
+ "lpips": 0.5402319431304932
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 5.578125,
293
+ "history_latent_mse": 1.164634108543396,
294
+ "psnr": 8.047975165782235,
295
+ "ssim": 0.17040814911619756,
296
+ "lpips": 0.6948565244674683
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 1.580078125,
300
+ "history_latent_mse": 0.0728769525885582,
301
+ "psnr": 38.43696015877503,
302
+ "ssim": 0.9633539665800088,
303
+ "lpips": 0.18397042155265808
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 3.5390625,
307
+ "history_latent_mse": 0.15528421103954315,
308
+ "psnr": 30.482760363891877,
309
+ "ssim": 0.9316210718878324,
310
+ "lpips": 0.29477837681770325
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 816,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 4.16796875,
322
+ "history_latent_mse": 0.2984682321548462,
323
+ "psnr": 18.461471730632585,
324
+ "ssim": 0.6432727285054268,
325
+ "lpips": 0.3496913015842438
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 5.109375,
329
+ "history_latent_mse": 0.872347891330719,
330
+ "psnr": 10.769764364051005,
331
+ "ssim": 0.3029838169425373,
332
+ "lpips": 0.6494001150131226
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 5.40625,
336
+ "history_latent_mse": 1.3400568962097168,
337
+ "psnr": 7.740216071696815,
338
+ "ssim": 0.16791392220948567,
339
+ "lpips": 0.7081996202468872
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 1.5224609375,
343
+ "history_latent_mse": 0.08166665583848953,
344
+ "psnr": 37.54734938192997,
345
+ "ssim": 0.9595712611877427,
346
+ "lpips": 0.20131734013557434
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 2.8125,
350
+ "history_latent_mse": 0.16986556351184845,
351
+ "psnr": 29.36502662780875,
352
+ "ssim": 0.9217778719544037,
353
+ "lpips": 0.3236025273799896
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 824,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 3.828125,
365
+ "history_latent_mse": 0.2938782274723053,
366
+ "psnr": 18.5235528992905,
367
+ "ssim": 0.6452774613458347,
368
+ "lpips": 0.34617170691490173
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 6.6640625,
372
+ "history_latent_mse": 1.1267484426498413,
373
+ "psnr": 10.979310461236569,
374
+ "ssim": 0.24758757208566562,
375
+ "lpips": 0.721572995185852
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 6.578125,
379
+ "history_latent_mse": 1.5104466676712036,
380
+ "psnr": 7.804949675021238,
381
+ "ssim": 0.15574708281079835,
382
+ "lpips": 0.7038141489028931
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 1.8984375,
386
+ "history_latent_mse": 0.10589944571256638,
387
+ "psnr": 33.30858080413044,
388
+ "ssim": 0.9326610898617956,
389
+ "lpips": 0.2946099638938904
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 3.0390625,
393
+ "history_latent_mse": 0.17906594276428223,
394
+ "psnr": 29.386979883751515,
395
+ "ssim": 0.9234396478463931,
396
+ "lpips": 0.31560975313186646
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 832,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 4.6484375,
408
+ "history_latent_mse": 0.29150208830833435,
409
+ "psnr": 18.801432301325395,
410
+ "ssim": 0.6654753952910855,
411
+ "lpips": 0.3038299083709717
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 6.6875,
415
+ "history_latent_mse": 1.4942198991775513,
416
+ "psnr": 7.578415182211341,
417
+ "ssim": 0.15813444732251838,
418
+ "lpips": 0.7655599117279053
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 7.2734375,
422
+ "history_latent_mse": 1.5599931478500366,
423
+ "psnr": 8.69173064918552,
424
+ "ssim": 0.14549984755213147,
425
+ "lpips": 0.6497141718864441
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 2.7578125,
429
+ "history_latent_mse": 0.22963522374629974,
430
+ "psnr": 34.0053449152069,
431
+ "ssim": 0.9345576496650239,
432
+ "lpips": 0.3088558316230774
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 3.10546875,
436
+ "history_latent_mse": 0.1841275691986084,
437
+ "psnr": 30.267839570359605,
438
+ "ssim": 0.9280124234953808,
439
+ "lpips": 0.30279654264450073
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 840,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 4.1484375,
451
+ "history_latent_mse": 0.3664903938770294,
452
+ "psnr": 18.4862279240429,
453
+ "ssim": 0.6609155744072859,
454
+ "lpips": 0.2633398771286011
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 5.109375,
458
+ "history_latent_mse": 1.6551098823547363,
459
+ "psnr": 7.660092129532145,
460
+ "ssim": 0.2063683648179046,
461
+ "lpips": 0.7334326505661011
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 5.6875,
465
+ "history_latent_mse": 1.2903972864151,
466
+ "psnr": 9.634885592697719,
467
+ "ssim": 0.21263329836904776,
468
+ "lpips": 0.6240497827529907
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 2.625,
472
+ "history_latent_mse": 0.14765022695064545,
473
+ "psnr": 33.23548299191736,
474
+ "ssim": 0.9318444574210466,
475
+ "lpips": 0.30578720569610596
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 3.171875,
479
+ "history_latent_mse": 0.17726582288742065,
480
+ "psnr": 37.76077328802814,
481
+ "ssim": 0.9590818585007767,
482
+ "lpips": 0.18256664276123047
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 848,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 4.2265625,
494
+ "history_latent_mse": 0.3046681582927704,
495
+ "psnr": 18.709655100884643,
496
+ "ssim": 0.650614385218887,
497
+ "lpips": 0.234390988945961
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 5.203125,
501
+ "history_latent_mse": 1.516667366027832,
502
+ "psnr": 8.81480209739876,
503
+ "ssim": 0.2781303169195639,
504
+ "lpips": 0.6655868291854858
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 6.75,
508
+ "history_latent_mse": 1.1167012453079224,
509
+ "psnr": 10.98071122293361,
510
+ "ssim": 0.2703117738891817,
511
+ "lpips": 0.5898345112800598
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 2.890625,
515
+ "history_latent_mse": 0.1776418834924698,
516
+ "psnr": 35.887101881085634,
517
+ "ssim": 0.9509251859429928,
518
+ "lpips": 0.2477835714817047
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 1.6748046875,
522
+ "history_latent_mse": 0.09520227462053299,
523
+ "psnr": 35.238608750384515,
524
+ "ssim": 0.9507887189950697,
525
+ "lpips": 0.22817659378051758
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 856,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.3046875,
537
+ "history_latent_mse": 0.27612948417663574,
538
+ "psnr": 18.590045602181135,
539
+ "ssim": 0.652070265922557,
540
+ "lpips": 0.23579567670822144
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 5.6875,
544
+ "history_latent_mse": 1.2576309442520142,
545
+ "psnr": 9.062799913712063,
546
+ "ssim": 0.27118083845238455,
547
+ "lpips": 0.6443774104118347
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 5.640625,
551
+ "history_latent_mse": 1.0636368989944458,
552
+ "psnr": 10.750034044458946,
553
+ "ssim": 0.3228667446565132,
554
+ "lpips": 0.5970298051834106
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 2.02734375,
558
+ "history_latent_mse": 0.13855737447738647,
559
+ "psnr": 35.77824613720884,
560
+ "ssim": 0.9494368221539021,
561
+ "lpips": 0.25469911098480225
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 3.5859375,
565
+ "history_latent_mse": 0.14308881759643555,
566
+ "psnr": 32.26957627504644,
567
+ "ssim": 0.9360240426541555,
568
+ "lpips": 0.314947247505188
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 864,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 4.1484375,
580
+ "history_latent_mse": 0.2701523005962372,
581
+ "psnr": 18.504458378406493,
582
+ "ssim": 0.6583827715134457,
583
+ "lpips": 0.25816187262535095
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 5.4921875,
587
+ "history_latent_mse": 1.231392741203308,
588
+ "psnr": 8.901461708555818,
589
+ "ssim": 0.25798425067663955,
590
+ "lpips": 0.6264577507972717
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.2578125,
594
+ "history_latent_mse": 0.9444217681884766,
595
+ "psnr": 9.285897667881601,
596
+ "ssim": 0.28747021138644513,
597
+ "lpips": 0.6289656758308411
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 1.6640625,
601
+ "history_latent_mse": 0.11717787384986877,
602
+ "psnr": 35.4269862753644,
603
+ "ssim": 0.9457311159669539,
604
+ "lpips": 0.27127423882484436
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 2.71875,
608
+ "history_latent_mse": 0.2028433233499527,
609
+ "psnr": 31.57250570206226,
610
+ "ssim": 0.9245300013350644,
611
+ "lpips": 0.3654765486717224
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 872,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 4.8125,
623
+ "history_latent_mse": 0.2978976368904114,
624
+ "psnr": 17.773418873175515,
625
+ "ssim": 0.6404226633787303,
626
+ "lpips": 0.3210606575012207
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 6.28125,
630
+ "history_latent_mse": 1.4542804956436157,
631
+ "psnr": 8.89524846520872,
632
+ "ssim": 0.27624061718745085,
633
+ "lpips": 0.6418129801750183
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 5.453125,
637
+ "history_latent_mse": 1.1367557048797607,
638
+ "psnr": 8.078108348028705,
639
+ "ssim": 0.2556396720105939,
640
+ "lpips": 0.670830488204956
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 1.7421875,
644
+ "history_latent_mse": 0.12477728724479675,
645
+ "psnr": 35.10459493823618,
646
+ "ssim": 0.9424944582689578,
647
+ "lpips": 0.28546005487442017
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 2.8359375,
651
+ "history_latent_mse": 0.2300664782524109,
652
+ "psnr": 34.46117796678137,
653
+ "ssim": 0.9412823137088673,
654
+ "lpips": 0.2880580425262451
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8508f7a4e233120bcd928e8e2f95d80204cb6f88a0ea3665cb15fc30cb0ba4b3
3
+ size 1316592
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/05_motherboard_0909_episode_001_onset/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.19504790008068085,
91
+ "wrist/video/latent_future_mse": 0.19504790008068085,
92
+ "video/psnr": 23.97182098554074,
93
+ "wrist/video/psnr": 23.97182098554074,
94
+ "video/ssim": 0.8125829526693039,
95
+ "wrist/video/ssim": 0.8125829526693039,
96
+ "video/lpips": 0.07511236518621445,
97
+ "wrist/video/lpips": 0.07511236518621445,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 0.9129447340965271,
103
+ "wrist/wrist_left/latent_future_mse": 0.9129447340965271,
104
+ "wrist_left/psnr": 13.948032842082776,
105
+ "wrist/wrist_left/psnr": 13.948032842082776,
106
+ "wrist_left/ssim": 0.3378560447266604,
107
+ "wrist/wrist_left/ssim": 0.3378560447266604,
108
+ "wrist_left/lpips": 0.47823843359947205,
109
+ "wrist/wrist_left/lpips": 0.47823843359947205,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.6459246277809143,
115
+ "wrist/wrist_right/latent_future_mse": 0.6459246277809143,
116
+ "wrist_right/psnr": 15.361403592715266,
117
+ "wrist/wrist_right/psnr": 15.361403592715266,
118
+ "wrist_right/ssim": 0.5478371830024589,
119
+ "wrist/wrist_right/ssim": 0.5478371830024589,
120
+ "wrist_right/lpips": 0.3370887041091919,
121
+ "wrist/wrist_right/lpips": 0.3370887041091919,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.08018997311592102,
127
+ "wrist/tactile_left/latent_future_mse": 0.08018997311592102,
128
+ "tactile_left/psnr": 38.742343650877785,
129
+ "wrist/tactile_left/psnr": 38.742343650877785,
130
+ "tactile_left/ssim": 0.9835824167841523,
131
+ "wrist/tactile_left/ssim": 0.9835824167841523,
132
+ "tactile_left/lpips": 0.041552357375621796,
133
+ "wrist/tactile_left/lpips": 0.041552357375621796,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.022527383640408516,
139
+ "wrist/tactile_right/latent_future_mse": 0.022527383640408516,
140
+ "tactile_right/psnr": 46.984432773655556,
141
+ "wrist/tactile_right/psnr": 46.984432773655556,
142
+ "tactile_right/ssim": 0.9918726462661673,
143
+ "wrist/tactile_right/ssim": 0.9918726462661673,
144
+ "tactile_right/lpips": 0.013785661198198795,
145
+ "wrist/tactile_right/lpips": 0.013785661198198795,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_001.pt",
152
+ "t_start": 760,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.19504790008068085,
169
+ "psnr": 23.97182098554074,
170
+ "ssim": 0.8125829526693039,
171
+ "lpips": 0.07511236518621445
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 0.9129447340965271,
177
+ "psnr": 13.948032842082776,
178
+ "ssim": 0.3378560447266604,
179
+ "lpips": 0.47823843359947205
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.6459246277809143,
185
+ "psnr": 15.361403592715266,
186
+ "ssim": 0.5478371830024589,
187
+ "lpips": 0.3370887041091919
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.08018997311592102,
193
+ "psnr": 38.742343650877785,
194
+ "ssim": 0.9835824167841523,
195
+ "lpips": 0.041552357375621796
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.022527383640408516,
201
+ "psnr": 46.984432773655556,
202
+ "ssim": 0.9918726462661673,
203
+ "lpips": 0.013785661198198795
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/case.json ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": 6,
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "category": "release",
5
+ "index": 6112,
6
+ "t_start": 2829,
7
+ "start_seconds": 188.6,
8
+ "short_contact": [
9
+ 1,
10
+ 1,
11
+ 1,
12
+ 1,
13
+ 1,
14
+ 1,
15
+ 1,
16
+ 1,
17
+ 1,
18
+ 1,
19
+ 1,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0
25
+ ],
26
+ "short_transition_ticks": [
27
+ 11
28
+ ],
29
+ "future_transition_ticks": [
30
+ 11
31
+ ],
32
+ "long_contact_fraction": 0.6953125,
33
+ "long_future_contact_fraction": 0.675,
34
+ "contact_over_128_frames": [
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 1,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 0,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 0,
59
+ 0,
60
+ 0,
61
+ 0,
62
+ 0,
63
+ 0,
64
+ 0,
65
+ 0,
66
+ 1,
67
+ 1,
68
+ 1,
69
+ 1,
70
+ 1,
71
+ 1,
72
+ 1,
73
+ 1,
74
+ 1,
75
+ 1,
76
+ 1,
77
+ 1,
78
+ 1,
79
+ 1,
80
+ 1,
81
+ 1,
82
+ 1,
83
+ 1,
84
+ 1,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 1,
90
+ 1,
91
+ 1,
92
+ 1,
93
+ 1,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ 1,
98
+ 1,
99
+ 1,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 1,
104
+ 1,
105
+ 0,
106
+ 0,
107
+ 0,
108
+ 0,
109
+ 0,
110
+ 0,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 1,
118
+ 1,
119
+ 1,
120
+ 1,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 0,
145
+ 0,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 1,
155
+ 1,
156
+ 1,
157
+ 1,
158
+ 1,
159
+ 1,
160
+ 1,
161
+ 1,
162
+ 1
163
+ ],
164
+ "first_contact_tick": 0,
165
+ "long_label": "release",
166
+ "output_subdir": "06_motherboard_0909_episode_001_release",
167
+ "rollout_windows": 15
168
+ }
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/complete.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"short_frames": 16, "long_frames": 128, "rollout_windows": 15}
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/long_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20c9fd5f94287c73f2541a8d89687770ae361b845b90fee22b4e046caf86e266
3
+ size 8171715
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/motherboard_0909_episode_001_t02829_pred.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:405160612d929df4ed208ad723babcf18f25e656f3202d9757a0c8a6e3bd8a03
3
+ size 1978519
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/reference_rollout.json ADDED
@@ -0,0 +1,661 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "episode": "motherboard_0909_episode_001.pt",
4
+ "t_start": 2829,
5
+ "streams": [
6
+ "video",
7
+ "wrist_left",
8
+ "wrist_right",
9
+ "tactile_left",
10
+ "tactile_right"
11
+ ],
12
+ "hide_wrist": false,
13
+ "windows": [
14
+ {
15
+ "window": 0,
16
+ "t_start": 2829,
17
+ "history_source": "recording",
18
+ "image": {
19
+ "video": {
20
+ "history_latent_max_abs_error": 0.0,
21
+ "history_latent_mse": 0.0,
22
+ "psnr": 25.756199167378753,
23
+ "ssim": 0.835266090630362,
24
+ "lpips": 0.05219259858131409
25
+ },
26
+ "wrist_left": {
27
+ "history_latent_max_abs_error": 0.0,
28
+ "history_latent_mse": 0.0,
29
+ "psnr": 9.795115082654975,
30
+ "ssim": 0.3088627337972464,
31
+ "lpips": 0.5185638070106506
32
+ },
33
+ "wrist_right": {
34
+ "history_latent_max_abs_error": 0.0,
35
+ "history_latent_mse": 0.0,
36
+ "psnr": 21.236576841817993,
37
+ "ssim": 0.833024592249541,
38
+ "lpips": 0.11528465896844864
39
+ },
40
+ "tactile_left": {
41
+ "history_latent_max_abs_error": 0.0,
42
+ "history_latent_mse": 0.0,
43
+ "psnr": 37.94824790738401,
44
+ "ssim": 0.9632617663474357,
45
+ "lpips": 0.08573136478662491
46
+ },
47
+ "tactile_right": {
48
+ "history_latent_max_abs_error": 0.0,
49
+ "history_latent_mse": 0.0,
50
+ "psnr": 47.52862412781565,
51
+ "ssim": 0.9924801051994103,
52
+ "lpips": 0.013260400854051113
53
+ }
54
+ },
55
+ "action": {}
56
+ },
57
+ {
58
+ "window": 1,
59
+ "t_start": 2837,
60
+ "history_source": "previous_model_prediction",
61
+ "image": {
62
+ "video": {
63
+ "history_latent_max_abs_error": 3.515625,
64
+ "history_latent_mse": 0.12307489663362503,
65
+ "psnr": 23.72925858408616,
66
+ "ssim": 0.7761899281313098,
67
+ "lpips": 0.07246850430965424
68
+ },
69
+ "wrist_left": {
70
+ "history_latent_max_abs_error": 6.65625,
71
+ "history_latent_mse": 1.2207019329071045,
72
+ "psnr": 8.630960419020989,
73
+ "ssim": 0.20055540644528894,
74
+ "lpips": 0.6255948543548584
75
+ },
76
+ "wrist_right": {
77
+ "history_latent_max_abs_error": 5.84375,
78
+ "history_latent_mse": 0.31101498007774353,
79
+ "psnr": 19.73899343110421,
80
+ "ssim": 0.7430886145307394,
81
+ "lpips": 0.15666377544403076
82
+ },
83
+ "tactile_left": {
84
+ "history_latent_max_abs_error": 1.93359375,
85
+ "history_latent_mse": 0.09051945060491562,
86
+ "psnr": 37.439019622510806,
87
+ "ssim": 0.9635076161667111,
88
+ "lpips": 0.12076577544212341
89
+ },
90
+ "tactile_right": {
91
+ "history_latent_max_abs_error": 1.159423828125,
92
+ "history_latent_mse": 0.024197274819016457,
93
+ "psnr": 44.48048027060615,
94
+ "ssim": 0.9852560510154088,
95
+ "lpips": 0.04280985891819
96
+ }
97
+ },
98
+ "action": {}
99
+ },
100
+ {
101
+ "window": 2,
102
+ "t_start": 2845,
103
+ "history_source": "previous_model_prediction",
104
+ "image": {
105
+ "video": {
106
+ "history_latent_max_abs_error": 3.35546875,
107
+ "history_latent_mse": 0.14202453196048737,
108
+ "psnr": 23.43751343681503,
109
+ "ssim": 0.7771850841248189,
110
+ "lpips": 0.07657304406166077
111
+ },
112
+ "wrist_left": {
113
+ "history_latent_max_abs_error": 7.609375,
114
+ "history_latent_mse": 1.2571264505386353,
115
+ "psnr": 9.566423281198027,
116
+ "ssim": 0.15526217428360162,
117
+ "lpips": 0.6325905323028564
118
+ },
119
+ "wrist_right": {
120
+ "history_latent_max_abs_error": 4.90625,
121
+ "history_latent_mse": 0.4207431375980377,
122
+ "psnr": 17.3419850995055,
123
+ "ssim": 0.6332227410823139,
124
+ "lpips": 0.2565159797668457
125
+ },
126
+ "tactile_left": {
127
+ "history_latent_max_abs_error": 1.828125,
128
+ "history_latent_mse": 0.09976337105035782,
129
+ "psnr": 35.387877081236866,
130
+ "ssim": 0.9514584495842969,
131
+ "lpips": 0.18000397086143494
132
+ },
133
+ "tactile_right": {
134
+ "history_latent_max_abs_error": 0.9580078125,
135
+ "history_latent_mse": 0.032493047416210175,
136
+ "psnr": 43.35396284367206,
137
+ "ssim": 0.9817948579709243,
138
+ "lpips": 0.06435484439134598
139
+ }
140
+ },
141
+ "action": {}
142
+ },
143
+ {
144
+ "window": 3,
145
+ "t_start": 2853,
146
+ "history_source": "previous_model_prediction",
147
+ "image": {
148
+ "video": {
149
+ "history_latent_max_abs_error": 3.48046875,
150
+ "history_latent_mse": 0.13837173581123352,
151
+ "psnr": 23.269153282176926,
152
+ "ssim": 0.7713606991305408,
153
+ "lpips": 0.08427079766988754
154
+ },
155
+ "wrist_left": {
156
+ "history_latent_max_abs_error": 6.3515625,
157
+ "history_latent_mse": 1.243075966835022,
158
+ "psnr": 9.803793547610336,
159
+ "ssim": 0.15565791287500588,
160
+ "lpips": 0.6375032663345337
161
+ },
162
+ "wrist_right": {
163
+ "history_latent_max_abs_error": 5.328125,
164
+ "history_latent_mse": 0.5453107357025146,
165
+ "psnr": 15.749417675969656,
166
+ "ssim": 0.5873199043996946,
167
+ "lpips": 0.3279944062232971
168
+ },
169
+ "tactile_left": {
170
+ "history_latent_max_abs_error": 2.701171875,
171
+ "history_latent_mse": 0.12913911044597626,
172
+ "psnr": 30.941367570492837,
173
+ "ssim": 0.8596609505029358,
174
+ "lpips": 0.29118359088897705
175
+ },
176
+ "tactile_right": {
177
+ "history_latent_max_abs_error": 1.150146484375,
178
+ "history_latent_mse": 0.04036927968263626,
179
+ "psnr": 42.60904908641229,
180
+ "ssim": 0.978562387292111,
181
+ "lpips": 0.08306442201137543
182
+ }
183
+ },
184
+ "action": {}
185
+ },
186
+ {
187
+ "window": 4,
188
+ "t_start": 2861,
189
+ "history_source": "previous_model_prediction",
190
+ "image": {
191
+ "video": {
192
+ "history_latent_max_abs_error": 3.484375,
193
+ "history_latent_mse": 0.15298156440258026,
194
+ "psnr": 22.931096522383424,
195
+ "ssim": 0.7530826154687198,
196
+ "lpips": 0.0916934460401535
197
+ },
198
+ "wrist_left": {
199
+ "history_latent_max_abs_error": 6.34375,
200
+ "history_latent_mse": 1.1576999425888062,
201
+ "psnr": 9.602714612953354,
202
+ "ssim": 0.12942120882511385,
203
+ "lpips": 0.6547681093215942
204
+ },
205
+ "wrist_right": {
206
+ "history_latent_max_abs_error": 4.8203125,
207
+ "history_latent_mse": 0.5596463680267334,
208
+ "psnr": 15.209122319573844,
209
+ "ssim": 0.5545493435156822,
210
+ "lpips": 0.35842907428741455
211
+ },
212
+ "tactile_left": {
213
+ "history_latent_max_abs_error": 3.390625,
214
+ "history_latent_mse": 0.2723928987979889,
215
+ "psnr": 29.160411673850604,
216
+ "ssim": 0.7936044125988544,
217
+ "lpips": 0.3303782641887665
218
+ },
219
+ "tactile_right": {
220
+ "history_latent_max_abs_error": 1.1348876953125,
221
+ "history_latent_mse": 0.048756178468465805,
222
+ "psnr": 41.11739088054814,
223
+ "ssim": 0.9749281700328197,
224
+ "lpips": 0.10401212424039841
225
+ }
226
+ },
227
+ "action": {}
228
+ },
229
+ {
230
+ "window": 5,
231
+ "t_start": 2869,
232
+ "history_source": "previous_model_prediction",
233
+ "image": {
234
+ "video": {
235
+ "history_latent_max_abs_error": 3.4609375,
236
+ "history_latent_mse": 0.16085605323314667,
237
+ "psnr": 22.353856441894724,
238
+ "ssim": 0.7333473397406705,
239
+ "lpips": 0.11020157486200333
240
+ },
241
+ "wrist_left": {
242
+ "history_latent_max_abs_error": 5.859375,
243
+ "history_latent_mse": 1.2130208015441895,
244
+ "psnr": 9.827940443273292,
245
+ "ssim": 0.120363783339094,
246
+ "lpips": 0.6839362978935242
247
+ },
248
+ "wrist_right": {
249
+ "history_latent_max_abs_error": 4.7021484375,
250
+ "history_latent_mse": 0.5513512492179871,
251
+ "psnr": 14.426776491518376,
252
+ "ssim": 0.5260012032368425,
253
+ "lpips": 0.40512168407440186
254
+ },
255
+ "tactile_left": {
256
+ "history_latent_max_abs_error": 2.60546875,
257
+ "history_latent_mse": 0.3037930428981781,
258
+ "psnr": 29.655984399543414,
259
+ "ssim": 0.8017136302310763,
260
+ "lpips": 0.3270813524723053
261
+ },
262
+ "tactile_right": {
263
+ "history_latent_max_abs_error": 1.400390625,
264
+ "history_latent_mse": 0.05523292347788811,
265
+ "psnr": 40.29634074004274,
266
+ "ssim": 0.9717637859116596,
267
+ "lpips": 0.12469758838415146
268
+ }
269
+ },
270
+ "action": {}
271
+ },
272
+ {
273
+ "window": 6,
274
+ "t_start": 2877,
275
+ "history_source": "previous_model_prediction",
276
+ "image": {
277
+ "video": {
278
+ "history_latent_max_abs_error": 3.78125,
279
+ "history_latent_mse": 0.1902128905057907,
280
+ "psnr": 19.4039788749634,
281
+ "ssim": 0.6855616918287396,
282
+ "lpips": 0.2106633484363556
283
+ },
284
+ "wrist_left": {
285
+ "history_latent_max_abs_error": 5.7734375,
286
+ "history_latent_mse": 1.1584187746047974,
287
+ "psnr": 10.287976859647348,
288
+ "ssim": 0.16378251630219276,
289
+ "lpips": 0.7025153636932373
290
+ },
291
+ "wrist_right": {
292
+ "history_latent_max_abs_error": 4.7265625,
293
+ "history_latent_mse": 0.6200301051139832,
294
+ "psnr": 12.910240263466672,
295
+ "ssim": 0.4701324883426373,
296
+ "lpips": 0.48982176184654236
297
+ },
298
+ "tactile_left": {
299
+ "history_latent_max_abs_error": 3.4921875,
300
+ "history_latent_mse": 0.3263384997844696,
301
+ "psnr": 29.934611188531743,
302
+ "ssim": 0.8139848020527061,
303
+ "lpips": 0.33004093170166016
304
+ },
305
+ "tactile_right": {
306
+ "history_latent_max_abs_error": 1.466552734375,
307
+ "history_latent_mse": 0.06368327885866165,
308
+ "psnr": 39.24159367842562,
309
+ "ssim": 0.9684283080090664,
310
+ "lpips": 0.13924658298492432
311
+ }
312
+ },
313
+ "action": {}
314
+ },
315
+ {
316
+ "window": 7,
317
+ "t_start": 2885,
318
+ "history_source": "previous_model_prediction",
319
+ "image": {
320
+ "video": {
321
+ "history_latent_max_abs_error": 3.703125,
322
+ "history_latent_mse": 0.30108970403671265,
323
+ "psnr": 17.764057073614584,
324
+ "ssim": 0.6620262777791566,
325
+ "lpips": 0.2986719608306885
326
+ },
327
+ "wrist_left": {
328
+ "history_latent_max_abs_error": 6.78125,
329
+ "history_latent_mse": 1.2482184171676636,
330
+ "psnr": 10.03961572574659,
331
+ "ssim": 0.18423219634043292,
332
+ "lpips": 0.704594612121582
333
+ },
334
+ "wrist_right": {
335
+ "history_latent_max_abs_error": 6.859375,
336
+ "history_latent_mse": 0.8195632100105286,
337
+ "psnr": 11.468674621173065,
338
+ "ssim": 0.42334443091871987,
339
+ "lpips": 0.5219416618347168
340
+ },
341
+ "tactile_left": {
342
+ "history_latent_max_abs_error": 3.125,
343
+ "history_latent_mse": 0.3108655512332916,
344
+ "psnr": 31.394821750293097,
345
+ "ssim": 0.875660407564387,
346
+ "lpips": 0.29375094175338745
347
+ },
348
+ "tactile_right": {
349
+ "history_latent_max_abs_error": 1.36328125,
350
+ "history_latent_mse": 0.06977532804012299,
351
+ "psnr": 38.478987549231675,
352
+ "ssim": 0.9648117161060317,
353
+ "lpips": 0.1598256230354309
354
+ }
355
+ },
356
+ "action": {}
357
+ },
358
+ {
359
+ "window": 8,
360
+ "t_start": 2893,
361
+ "history_source": "previous_model_prediction",
362
+ "image": {
363
+ "video": {
364
+ "history_latent_max_abs_error": 3.78515625,
365
+ "history_latent_mse": 0.3189527690410614,
366
+ "psnr": 17.48441877434705,
367
+ "ssim": 0.6254595757164626,
368
+ "lpips": 0.3275079131126404
369
+ },
370
+ "wrist_left": {
371
+ "history_latent_max_abs_error": 5.9140625,
372
+ "history_latent_mse": 1.1481460332870483,
373
+ "psnr": 9.389247640325724,
374
+ "ssim": 0.1767917168409729,
375
+ "lpips": 0.6874648332595825
376
+ },
377
+ "wrist_right": {
378
+ "history_latent_max_abs_error": 4.6875,
379
+ "history_latent_mse": 0.8640062808990479,
380
+ "psnr": 11.339433764633501,
381
+ "ssim": 0.42499702225197766,
382
+ "lpips": 0.5273013710975647
383
+ },
384
+ "tactile_left": {
385
+ "history_latent_max_abs_error": 3.20703125,
386
+ "history_latent_mse": 0.3068641722202301,
387
+ "psnr": 33.69861295645229,
388
+ "ssim": 0.9492698384083883,
389
+ "lpips": 0.24424457550048828
390
+ },
391
+ "tactile_right": {
392
+ "history_latent_max_abs_error": 1.4892578125,
393
+ "history_latent_mse": 0.07717099785804749,
394
+ "psnr": 37.740329840455,
395
+ "ssim": 0.9618718538699986,
396
+ "lpips": 0.17238739132881165
397
+ }
398
+ },
399
+ "action": {}
400
+ },
401
+ {
402
+ "window": 9,
403
+ "t_start": 2901,
404
+ "history_source": "previous_model_prediction",
405
+ "image": {
406
+ "video": {
407
+ "history_latent_max_abs_error": 4.34765625,
408
+ "history_latent_mse": 0.3440483808517456,
409
+ "psnr": 16.933706100900928,
410
+ "ssim": 0.6001300345545135,
411
+ "lpips": 0.3501172959804535
412
+ },
413
+ "wrist_left": {
414
+ "history_latent_max_abs_error": 6.5859375,
415
+ "history_latent_mse": 1.1391674280166626,
416
+ "psnr": 10.52694284469202,
417
+ "ssim": 0.20861702685694775,
418
+ "lpips": 0.6568442583084106
419
+ },
420
+ "wrist_right": {
421
+ "history_latent_max_abs_error": 6.0234375,
422
+ "history_latent_mse": 0.8517677783966064,
423
+ "psnr": 11.265056864826166,
424
+ "ssim": 0.4087446973819553,
425
+ "lpips": 0.5313889980316162
426
+ },
427
+ "tactile_left": {
428
+ "history_latent_max_abs_error": 2.1015625,
429
+ "history_latent_mse": 0.13151998817920685,
430
+ "psnr": 30.829325047674693,
431
+ "ssim": 0.9280656745489044,
432
+ "lpips": 0.31298744678497314
433
+ },
434
+ "tactile_right": {
435
+ "history_latent_max_abs_error": 1.48779296875,
436
+ "history_latent_mse": 0.08445736765861511,
437
+ "psnr": 37.40243412292781,
438
+ "ssim": 0.9589167978689157,
439
+ "lpips": 0.1800118386745453
440
+ }
441
+ },
442
+ "action": {}
443
+ },
444
+ {
445
+ "window": 10,
446
+ "t_start": 2909,
447
+ "history_source": "previous_model_prediction",
448
+ "image": {
449
+ "video": {
450
+ "history_latent_max_abs_error": 3.96875,
451
+ "history_latent_mse": 0.37753233313560486,
452
+ "psnr": 18.03464233729001,
453
+ "ssim": 0.6383288619058014,
454
+ "lpips": 0.28571292757987976
455
+ },
456
+ "wrist_left": {
457
+ "history_latent_max_abs_error": 6.65625,
458
+ "history_latent_mse": 1.238336443901062,
459
+ "psnr": 10.779121594349029,
460
+ "ssim": 0.2390221962666336,
461
+ "lpips": 0.654449462890625
462
+ },
463
+ "wrist_right": {
464
+ "history_latent_max_abs_error": 5.375,
465
+ "history_latent_mse": 0.8353622555732727,
466
+ "psnr": 11.476342685836116,
467
+ "ssim": 0.42685966672093406,
468
+ "lpips": 0.5434567928314209
469
+ },
470
+ "tactile_left": {
471
+ "history_latent_max_abs_error": 2.71875,
472
+ "history_latent_mse": 0.22170914709568024,
473
+ "psnr": 30.056813774227344,
474
+ "ssim": 0.9215089027334331,
475
+ "lpips": 0.3433040380477905
476
+ },
477
+ "tactile_right": {
478
+ "history_latent_max_abs_error": 1.7998046875,
479
+ "history_latent_mse": 0.09339123964309692,
480
+ "psnr": 37.12693345464745,
481
+ "ssim": 0.9559216184310354,
482
+ "lpips": 0.18842431902885437
483
+ }
484
+ },
485
+ "action": {}
486
+ },
487
+ {
488
+ "window": 11,
489
+ "t_start": 2917,
490
+ "history_source": "previous_model_prediction",
491
+ "image": {
492
+ "video": {
493
+ "history_latent_max_abs_error": 4.484375,
494
+ "history_latent_mse": 0.32711321115493774,
495
+ "psnr": 19.61492843296683,
496
+ "ssim": 0.6587162664755082,
497
+ "lpips": 0.23245057463645935
498
+ },
499
+ "wrist_left": {
500
+ "history_latent_max_abs_error": 6.703125,
501
+ "history_latent_mse": 1.164857029914856,
502
+ "psnr": 9.9483687954496,
503
+ "ssim": 0.2461362383961195,
504
+ "lpips": 0.6712664365768433
505
+ },
506
+ "wrist_right": {
507
+ "history_latent_max_abs_error": 4.9375,
508
+ "history_latent_mse": 0.9033206105232239,
509
+ "psnr": 12.222441214228827,
510
+ "ssim": 0.4488215279057117,
511
+ "lpips": 0.5212984085083008
512
+ },
513
+ "tactile_left": {
514
+ "history_latent_max_abs_error": 3.234375,
515
+ "history_latent_mse": 0.2137620896100998,
516
+ "psnr": 30.74562702996214,
517
+ "ssim": 0.9239795544912861,
518
+ "lpips": 0.34028100967407227
519
+ },
520
+ "tactile_right": {
521
+ "history_latent_max_abs_error": 1.720458984375,
522
+ "history_latent_mse": 0.09752737730741501,
523
+ "psnr": 36.76620970072229,
524
+ "ssim": 0.9528357943842705,
525
+ "lpips": 0.2019694447517395
526
+ }
527
+ },
528
+ "action": {}
529
+ },
530
+ {
531
+ "window": 12,
532
+ "t_start": 2925,
533
+ "history_source": "previous_model_prediction",
534
+ "image": {
535
+ "video": {
536
+ "history_latent_max_abs_error": 4.21875,
537
+ "history_latent_mse": 0.3174107074737549,
538
+ "psnr": 17.630640348833893,
539
+ "ssim": 0.6352566776951215,
540
+ "lpips": 0.3112691640853882
541
+ },
542
+ "wrist_left": {
543
+ "history_latent_max_abs_error": 5.875,
544
+ "history_latent_mse": 1.1844148635864258,
545
+ "psnr": 9.926584509785014,
546
+ "ssim": 0.2263229233906025,
547
+ "lpips": 0.6708772778511047
548
+ },
549
+ "wrist_right": {
550
+ "history_latent_max_abs_error": 5.2890625,
551
+ "history_latent_mse": 0.8612427711486816,
552
+ "psnr": 11.288683097340076,
553
+ "ssim": 0.39453047331131896,
554
+ "lpips": 0.5532624125480652
555
+ },
556
+ "tactile_left": {
557
+ "history_latent_max_abs_error": 3.26953125,
558
+ "history_latent_mse": 0.23575620353221893,
559
+ "psnr": 31.900122410023602,
560
+ "ssim": 0.9339039367245892,
561
+ "lpips": 0.3118135333061218
562
+ },
563
+ "tactile_right": {
564
+ "history_latent_max_abs_error": 1.818359375,
565
+ "history_latent_mse": 0.10420220345258713,
566
+ "psnr": 36.18968641226627,
567
+ "ssim": 0.9493757037877266,
568
+ "lpips": 0.2170557975769043
569
+ }
570
+ },
571
+ "action": {}
572
+ },
573
+ {
574
+ "window": 13,
575
+ "t_start": 2933,
576
+ "history_source": "previous_model_prediction",
577
+ "image": {
578
+ "video": {
579
+ "history_latent_max_abs_error": 3.6982421875,
580
+ "history_latent_mse": 0.3512437045574188,
581
+ "psnr": 17.38577659661196,
582
+ "ssim": 0.6256218854209,
583
+ "lpips": 0.33145877718925476
584
+ },
585
+ "wrist_left": {
586
+ "history_latent_max_abs_error": 5.8837890625,
587
+ "history_latent_mse": 1.0546153783798218,
588
+ "psnr": 9.186732640274805,
589
+ "ssim": 0.22645646590277574,
590
+ "lpips": 0.7241584062576294
591
+ },
592
+ "wrist_right": {
593
+ "history_latent_max_abs_error": 5.828125,
594
+ "history_latent_mse": 0.9743571877479553,
595
+ "psnr": 11.340736695156577,
596
+ "ssim": 0.3807836173228526,
597
+ "lpips": 0.5694361925125122
598
+ },
599
+ "tactile_left": {
600
+ "history_latent_max_abs_error": 2.517578125,
601
+ "history_latent_mse": 0.20394068956375122,
602
+ "psnr": 32.77276578471606,
603
+ "ssim": 0.9385675745388503,
604
+ "lpips": 0.29230186343193054
605
+ },
606
+ "tactile_right": {
607
+ "history_latent_max_abs_error": 1.87109375,
608
+ "history_latent_mse": 0.11122607439756393,
609
+ "psnr": 35.49436248095677,
610
+ "ssim": 0.9463017571242149,
611
+ "lpips": 0.23368960618972778
612
+ }
613
+ },
614
+ "action": {}
615
+ },
616
+ {
617
+ "window": 14,
618
+ "t_start": 2941,
619
+ "history_source": "previous_model_prediction",
620
+ "image": {
621
+ "video": {
622
+ "history_latent_max_abs_error": 3.8125,
623
+ "history_latent_mse": 0.31654423475265503,
624
+ "psnr": 17.264726033259684,
625
+ "ssim": 0.6227012440019195,
626
+ "lpips": 0.34232401847839355
627
+ },
628
+ "wrist_left": {
629
+ "history_latent_max_abs_error": 5.65625,
630
+ "history_latent_mse": 1.1066244840621948,
631
+ "psnr": 9.659802896710838,
632
+ "ssim": 0.2412243192254273,
633
+ "lpips": 0.7216579914093018
634
+ },
635
+ "wrist_right": {
636
+ "history_latent_max_abs_error": 4.46875,
637
+ "history_latent_mse": 0.8346479535102844,
638
+ "psnr": 11.25499090369395,
639
+ "ssim": 0.3682623444970198,
640
+ "lpips": 0.5757245421409607
641
+ },
642
+ "tactile_left": {
643
+ "history_latent_max_abs_error": 2.1630859375,
644
+ "history_latent_mse": 0.14695937931537628,
645
+ "psnr": 28.03061389141117,
646
+ "ssim": 0.890513159166643,
647
+ "lpips": 0.40389591455459595
648
+ },
649
+ "tactile_right": {
650
+ "history_latent_max_abs_error": 1.921875,
651
+ "history_latent_mse": 0.11754439026117325,
652
+ "psnr": 34.854910955501516,
653
+ "ssim": 0.9429673473778937,
654
+ "lpips": 0.24597874283790588
655
+ }
656
+ },
657
+ "action": {}
658
+ }
659
+ ]
660
+ }
661
+ ]
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/short_gt_pred.mp4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:621c80cb39cecb67d779dc27dd75fa929c05edab9e00529a6c8ebe921ffab4b0
3
+ size 1013997
results/nano_action_causal_umi_sensoronly_noforce_v1/test_epoch_009/06_motherboard_0909_episode_001_release/window_eval.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "run_name": "nano_action_causal_umi_sensoronly_noforce_v1",
4
+ "seed": 0,
5
+ "out_dir": "/group2/ct/weihanx/world_model/vtm_cosmos/runs",
6
+ "data": {
7
+ "latent_root": "/group2/ct/weihanx/world_model/mixed_mb_latents",
8
+ "action_root": "/group2/ct/weihanx/world_model/mixed_mb_actions",
9
+ "action_file": "sensor_actions_fps15_noforce.pt",
10
+ "view_action_file": "view_actions_fps15.pt",
11
+ "split_file": "/music-shared-disk/group/ct/weihanx/world_model/vtm_cosmos/runs/nano_action_causal_umi_sensoronly_noforce_v1/contact_epoch_009_test/eval_split.json",
12
+ "optional_streams": [],
13
+ "action_norm": "quantile",
14
+ "train_window_stride": 1,
15
+ "val_window_stride": 50,
16
+ "stats_window_stride": 8,
17
+ "action_stats": null,
18
+ "num_workers": 4
19
+ },
20
+ "model": {
21
+ "language_model": "nano",
22
+ "nano_checkpoint": "/group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano",
23
+ "strict_load": true,
24
+ "dtype": "bfloat16",
25
+ "attention_backend": "flex",
26
+ "flex_preference": "flex_triton",
27
+ "temporal_mode": "causal",
28
+ "attention_topology": "all_streams_causal",
29
+ "task_mode": "action_full",
30
+ "active_streams": [
31
+ "video",
32
+ "wrist_left",
33
+ "wrist_right",
34
+ "tactile_left",
35
+ "tactile_right",
36
+ "action_left",
37
+ "action_right"
38
+ ],
39
+ "init_tactile_from_rgb": true,
40
+ "init_action_from_nano": {
41
+ "domain": "umi"
42
+ },
43
+ "train_time_embedder": false,
44
+ "train_rgb_projections": false,
45
+ "lora": {
46
+ "enable": true,
47
+ "rank": 16,
48
+ "alpha": 32,
49
+ "targets": "attn_ffn"
50
+ }
51
+ },
52
+ "training": {
53
+ "prediction_strategy": "causal_teacher_forcing",
54
+ "init_from_checkpoint": null,
55
+ "missing_stream_training": false,
56
+ "wrist_dropout": {
57
+ "probability": 0.0,
58
+ "mode": "paired",
59
+ "scope": "whole_clip"
60
+ },
61
+ "batching": "group_by_active_streams",
62
+ "sampling": {
63
+ "mode": "uniform",
64
+ "steps_per_epoch": null
65
+ },
66
+ "val_hide_wrist": false,
67
+ "epochs": 20,
68
+ "batch_size": 2,
69
+ "lr": 5e-05,
70
+ "weight_decay": 0.01,
71
+ "warmup_steps": 100,
72
+ "grad_clip": 1.0,
73
+ "amp": "bf16",
74
+ "sigma_distribution": "waver",
75
+ "sigma_shift": 3.0,
76
+ "loss_weights": {},
77
+ "log_every": 20,
78
+ "ckpt_every_epochs": 1,
79
+ "activation_checkpointing": true
80
+ },
81
+ "inference": {
82
+ "prediction_strategy": "causal_autoregressive"
83
+ }
84
+ },
85
+ "summary": {
86
+ "video/history_latent_max_abs_error": 0.0,
87
+ "wrist/video/history_latent_max_abs_error": 0.0,
88
+ "video/history_latent_mse": 0.0,
89
+ "wrist/video/history_latent_mse": 0.0,
90
+ "video/latent_future_mse": 0.14344173669815063,
91
+ "wrist/video/latent_future_mse": 0.14344173669815063,
92
+ "video/psnr": 25.756199167378753,
93
+ "wrist/video/psnr": 25.756199167378753,
94
+ "video/ssim": 0.835266090630362,
95
+ "wrist/video/ssim": 0.835266090630362,
96
+ "video/lpips": 0.05219259858131409,
97
+ "wrist/video/lpips": 0.05219259858131409,
98
+ "wrist_left/history_latent_max_abs_error": 0.0,
99
+ "wrist/wrist_left/history_latent_max_abs_error": 0.0,
100
+ "wrist_left/history_latent_mse": 0.0,
101
+ "wrist/wrist_left/history_latent_mse": 0.0,
102
+ "wrist_left/latent_future_mse": 1.364241600036621,
103
+ "wrist/wrist_left/latent_future_mse": 1.364241600036621,
104
+ "wrist_left/psnr": 9.795115082654975,
105
+ "wrist/wrist_left/psnr": 9.795115082654975,
106
+ "wrist_left/ssim": 0.3088627337972464,
107
+ "wrist/wrist_left/ssim": 0.3088627337972464,
108
+ "wrist_left/lpips": 0.5185638070106506,
109
+ "wrist/wrist_left/lpips": 0.5185638070106506,
110
+ "wrist_right/history_latent_max_abs_error": 0.0,
111
+ "wrist/wrist_right/history_latent_max_abs_error": 0.0,
112
+ "wrist_right/history_latent_mse": 0.0,
113
+ "wrist/wrist_right/history_latent_mse": 0.0,
114
+ "wrist_right/latent_future_mse": 0.3349973261356354,
115
+ "wrist/wrist_right/latent_future_mse": 0.3349973261356354,
116
+ "wrist_right/psnr": 21.236576841817993,
117
+ "wrist/wrist_right/psnr": 21.236576841817993,
118
+ "wrist_right/ssim": 0.833024592249541,
119
+ "wrist/wrist_right/ssim": 0.833024592249541,
120
+ "wrist_right/lpips": 0.11528465896844864,
121
+ "wrist/wrist_right/lpips": 0.11528465896844864,
122
+ "tactile_left/history_latent_max_abs_error": 0.0,
123
+ "wrist/tactile_left/history_latent_max_abs_error": 0.0,
124
+ "tactile_left/history_latent_mse": 0.0,
125
+ "wrist/tactile_left/history_latent_mse": 0.0,
126
+ "tactile_left/latent_future_mse": 0.13295195996761322,
127
+ "wrist/tactile_left/latent_future_mse": 0.13295195996761322,
128
+ "tactile_left/psnr": 37.94824790738401,
129
+ "wrist/tactile_left/psnr": 37.94824790738401,
130
+ "tactile_left/ssim": 0.9632617663474357,
131
+ "wrist/tactile_left/ssim": 0.9632617663474357,
132
+ "tactile_left/lpips": 0.08573136478662491,
133
+ "wrist/tactile_left/lpips": 0.08573136478662491,
134
+ "tactile_right/history_latent_max_abs_error": 0.0,
135
+ "wrist/tactile_right/history_latent_max_abs_error": 0.0,
136
+ "tactile_right/history_latent_mse": 0.0,
137
+ "wrist/tactile_right/history_latent_mse": 0.0,
138
+ "tactile_right/latent_future_mse": 0.020939955487847328,
139
+ "wrist/tactile_right/latent_future_mse": 0.020939955487847328,
140
+ "tactile_right/psnr": 47.52862412781565,
141
+ "wrist/tactile_right/psnr": 47.52862412781565,
142
+ "tactile_right/ssim": 0.9924801051994103,
143
+ "wrist/tactile_right/ssim": 0.9924801051994103,
144
+ "tactile_right/lpips": 0.013260400854051113,
145
+ "wrist/tactile_right/lpips": 0.013260400854051113,
146
+ "num_windows": 1,
147
+ "num_windows_with_wrist": 1
148
+ },
149
+ "results": [
150
+ {
151
+ "episode": "motherboard_0909_episode_001.pt",
152
+ "t_start": 2829,
153
+ "seed": 0,
154
+ "baseline": null,
155
+ "hide_wrist": false,
156
+ "has_wrist": true,
157
+ "streams": [
158
+ "video",
159
+ "wrist_left",
160
+ "wrist_right",
161
+ "tactile_left",
162
+ "tactile_right"
163
+ ],
164
+ "image": {
165
+ "video": {
166
+ "history_latent_max_abs_error": 0.0,
167
+ "history_latent_mse": 0.0,
168
+ "latent_future_mse": 0.14344173669815063,
169
+ "psnr": 25.756199167378753,
170
+ "ssim": 0.835266090630362,
171
+ "lpips": 0.05219259858131409
172
+ },
173
+ "wrist_left": {
174
+ "history_latent_max_abs_error": 0.0,
175
+ "history_latent_mse": 0.0,
176
+ "latent_future_mse": 1.364241600036621,
177
+ "psnr": 9.795115082654975,
178
+ "ssim": 0.3088627337972464,
179
+ "lpips": 0.5185638070106506
180
+ },
181
+ "wrist_right": {
182
+ "history_latent_max_abs_error": 0.0,
183
+ "history_latent_mse": 0.0,
184
+ "latent_future_mse": 0.3349973261356354,
185
+ "psnr": 21.236576841817993,
186
+ "ssim": 0.833024592249541,
187
+ "lpips": 0.11528465896844864
188
+ },
189
+ "tactile_left": {
190
+ "history_latent_max_abs_error": 0.0,
191
+ "history_latent_mse": 0.0,
192
+ "latent_future_mse": 0.13295195996761322,
193
+ "psnr": 37.94824790738401,
194
+ "ssim": 0.9632617663474357,
195
+ "lpips": 0.08573136478662491
196
+ },
197
+ "tactile_right": {
198
+ "history_latent_max_abs_error": 0.0,
199
+ "history_latent_mse": 0.0,
200
+ "latent_future_mse": 0.020939955487847328,
201
+ "psnr": 47.52862412781565,
202
+ "ssim": 0.9924801051994103,
203
+ "lpips": 0.013260400854051113
204
+ }
205
+ },
206
+ "action": {}
207
+ }
208
+ ]
209
+ }