WitneyWW commited on
Commit
e07bf5e
·
verified ·
1 Parent(s): 1d167b0

add checkpoints (vae LoRA, pusht_v1 e019, rope_v1 e019, umi_sensoronly_noforce_v1 e013)

Browse files
checkpoints/nano_action_causal_pusht_v1/action_normalizer.json ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": 2,
3
+ "sensor": {
4
+ "mode": "quantile",
5
+ "mean": [
6
+ -0.0003890679217875004,
7
+ 0.0008608214557170868,
8
+ 0.0005704499781131744,
9
+ 0.9973807334899902,
10
+ 0.0025033093988895416,
11
+ -0.0008984990417957306,
12
+ -0.0028330646455287933,
13
+ 0.9974104166030884,
14
+ 0.002793760970234871
15
+ ],
16
+ "std": [
17
+ 0.01522023230791092,
18
+ 0.013312933966517448,
19
+ 0.017298635095357895,
20
+ 0.0026190578937530518,
21
+ 0.06116945669054985,
22
+ 0.06694568693637848,
23
+ 0.06128935143351555,
24
+ 0.002589315176010132,
25
+ 0.06364484876394272
26
+ ],
27
+ "meta": {
28
+ "rows": 207000,
29
+ "episodes": [
30
+ "pushT_0910_episode_000_seg00.pt",
31
+ "pushT_0910_episode_001_seg00.pt",
32
+ "pushT_0910_episode_001_seg01.pt",
33
+ "pushT_0910_episode_001_seg02.pt",
34
+ "pushT_0910_episode_003_seg00.pt",
35
+ "pushT_0910_episode_003_seg01.pt",
36
+ "pushT_0910_episode_003_seg02.pt",
37
+ "pushT_0910_episode_003_seg03.pt",
38
+ "pushT_0910_episode_003_seg04.pt",
39
+ "pushT_0910_episode_003_seg05.pt",
40
+ "pushT_0910_episode_003_seg06.pt",
41
+ "pushT_0910_episode_003_seg07.pt",
42
+ "pushT_0910_episode_004_seg00.pt",
43
+ "pushT_0910_episode_004_seg01.pt",
44
+ "pushT_0910_episode_004_seg02.pt",
45
+ "pushT_0910_episode_004_seg03.pt",
46
+ "pushT_0911_episode_000_seg00.pt",
47
+ "pushT_0911_episode_001_seg00.pt",
48
+ "pushT_0911_episode_002_seg00.pt",
49
+ "pushT_0911_episode_002_seg01.pt",
50
+ "pushT_0911_episode_002_seg02.pt",
51
+ "pushT_0911_episode_002_seg03.pt",
52
+ "pushT_0911_episode_004_seg00.pt",
53
+ "pushT_0911_episode_004_seg01.pt",
54
+ "pushT_0911_episode_004_seg02.pt",
55
+ "pushT_0911_episode_004_seg03.pt",
56
+ "pushT_0911_episode_004_seg04.pt",
57
+ "pushT_0911_episode_004_seg05.pt",
58
+ "pushT_0912_episode_001_seg00.pt",
59
+ "pushT_0912_episode_002_seg00.pt",
60
+ "pushT_0912_episode_002_seg01.pt",
61
+ "pushT_0912_episode_002_seg02.pt",
62
+ "pushT_0912_episode_002_seg03.pt",
63
+ "pushT_0912_episode_002_seg04.pt",
64
+ "pushT_0912_episode_004_seg00.pt",
65
+ "pushT_0912_episode_005_seg00.pt",
66
+ "pushT_0912_episode_005_seg01.pt",
67
+ "pushT_0912_episode_006_seg00.pt",
68
+ "pushT_0912_episode_007_seg00.pt",
69
+ "pushT_0912_episode_007_seg01.pt",
70
+ "pushT_0912_episode_007_seg02.pt",
71
+ "pushT_0912_episode_007_seg03.pt",
72
+ "pushT_0912_episode_008_seg00.pt",
73
+ "pushT_0912_episode_008_seg01.pt",
74
+ "pushT_0912_episode_008_seg02.pt",
75
+ "pushT_0912_episode_008_seg03.pt",
76
+ "pushT_0912_episode_009_seg00.pt",
77
+ "pushT_0915_episode_000_seg00.pt",
78
+ "pushT_0915_episode_000_seg01.pt",
79
+ "pushT_0915_episode_000_seg02.pt",
80
+ "pushT_0915_episode_000_seg03.pt",
81
+ "pushT_0915_episode_000_seg04.pt",
82
+ "pushT_0915_episode_000_seg05.pt",
83
+ "pushT_0915_episode_000_seg06.pt",
84
+ "pushT_0915_episode_000_seg07.pt",
85
+ "pushT_0915_episode_000_seg08.pt"
86
+ ],
87
+ "streams": [
88
+ "action_left",
89
+ "action_right"
90
+ ],
91
+ "kind": "sensor",
92
+ "hands": "left+right stacked"
93
+ },
94
+ "q01": [
95
+ -0.015609300695359707,
96
+ -0.012452112510800362,
97
+ -0.01672818511724472,
98
+ 0.9947617053985596,
99
+ -0.05866614729166031,
100
+ -0.06784418225288391,
101
+ -0.06412241607904434,
102
+ 0.9948211312294006,
103
+ -0.0608510859310627
104
+ ],
105
+ "q99": [
106
+ 0.014831164851784706,
107
+ 0.014173755422234535,
108
+ 0.01786908507347107,
109
+ 0.9999998211860657,
110
+ 0.06367276608943939,
111
+ 0.06604718416929245,
112
+ 0.058456286787986755,
113
+ 0.9999997615814209,
114
+ 0.06643860787153244
115
+ ]
116
+ },
117
+ "view": {
118
+ "mode": "quantile",
119
+ "mean": [
120
+ 0.49366551637649536,
121
+ 0.5795469880104065,
122
+ 0.7682432532310486,
123
+ 0.512519121170044,
124
+ 0.5563752055168152,
125
+ 0.8493101596832275,
126
+ 0.5399284362792969,
127
+ 0.482826828956604,
128
+ 0.7413364052772522
129
+ ],
130
+ "std": [
131
+ 0.28291356563568115,
132
+ 0.2295605093240738,
133
+ 0.04556626081466675,
134
+ 0.24206016957759857,
135
+ 0.200794979929924,
136
+ 0.06341215968132019,
137
+ 0.2992331385612488,
138
+ 0.26479488611221313,
139
+ 0.07175266742706299
140
+ ],
141
+ "meta": {
142
+ "rows": 220800,
143
+ "episodes": [
144
+ "pushT_0910_episode_000_seg00.pt",
145
+ "pushT_0910_episode_001_seg00.pt",
146
+ "pushT_0910_episode_001_seg01.pt",
147
+ "pushT_0910_episode_001_seg02.pt",
148
+ "pushT_0910_episode_003_seg00.pt",
149
+ "pushT_0910_episode_003_seg01.pt",
150
+ "pushT_0910_episode_003_seg02.pt",
151
+ "pushT_0910_episode_003_seg03.pt",
152
+ "pushT_0910_episode_003_seg04.pt",
153
+ "pushT_0910_episode_003_seg05.pt",
154
+ "pushT_0910_episode_003_seg06.pt",
155
+ "pushT_0910_episode_003_seg07.pt",
156
+ "pushT_0910_episode_004_seg00.pt",
157
+ "pushT_0910_episode_004_seg01.pt",
158
+ "pushT_0910_episode_004_seg02.pt",
159
+ "pushT_0910_episode_004_seg03.pt",
160
+ "pushT_0911_episode_000_seg00.pt",
161
+ "pushT_0911_episode_001_seg00.pt",
162
+ "pushT_0911_episode_002_seg00.pt",
163
+ "pushT_0911_episode_002_seg01.pt",
164
+ "pushT_0911_episode_002_seg02.pt",
165
+ "pushT_0911_episode_002_seg03.pt",
166
+ "pushT_0911_episode_004_seg00.pt",
167
+ "pushT_0911_episode_004_seg01.pt",
168
+ "pushT_0911_episode_004_seg02.pt",
169
+ "pushT_0911_episode_004_seg03.pt",
170
+ "pushT_0911_episode_004_seg04.pt",
171
+ "pushT_0911_episode_004_seg05.pt",
172
+ "pushT_0912_episode_001_seg00.pt",
173
+ "pushT_0912_episode_002_seg00.pt",
174
+ "pushT_0912_episode_002_seg01.pt",
175
+ "pushT_0912_episode_002_seg02.pt",
176
+ "pushT_0912_episode_002_seg03.pt",
177
+ "pushT_0912_episode_002_seg04.pt",
178
+ "pushT_0912_episode_004_seg00.pt",
179
+ "pushT_0912_episode_005_seg00.pt",
180
+ "pushT_0912_episode_005_seg01.pt",
181
+ "pushT_0912_episode_006_seg00.pt",
182
+ "pushT_0912_episode_007_seg00.pt",
183
+ "pushT_0912_episode_007_seg01.pt",
184
+ "pushT_0912_episode_007_seg02.pt",
185
+ "pushT_0912_episode_007_seg03.pt",
186
+ "pushT_0912_episode_008_seg00.pt",
187
+ "pushT_0912_episode_008_seg01.pt",
188
+ "pushT_0912_episode_008_seg02.pt",
189
+ "pushT_0912_episode_008_seg03.pt",
190
+ "pushT_0912_episode_009_seg00.pt",
191
+ "pushT_0915_episode_000_seg00.pt",
192
+ "pushT_0915_episode_000_seg01.pt",
193
+ "pushT_0915_episode_000_seg02.pt",
194
+ "pushT_0915_episode_000_seg03.pt",
195
+ "pushT_0915_episode_000_seg04.pt",
196
+ "pushT_0915_episode_000_seg05.pt",
197
+ "pushT_0915_episode_000_seg06.pt",
198
+ "pushT_0915_episode_000_seg07.pt",
199
+ "pushT_0915_episode_000_seg08.pt"
200
+ ],
201
+ "streams": [
202
+ "view_action_left",
203
+ "view_action_right"
204
+ ],
205
+ "kind": "view",
206
+ "hands": "left+right stacked"
207
+ },
208
+ "q01": [
209
+ 0.21075193583965302,
210
+ 0.3499864637851715,
211
+ 0.7226769924163818,
212
+ 0.27045896649360657,
213
+ 0.35558024048805237,
214
+ 0.7858980298042297,
215
+ 0.24069532752037048,
216
+ 0.21803195774555206,
217
+ 0.6695837378501892
218
+ ],
219
+ "q99": [
220
+ 0.7765790820121765,
221
+ 0.8091074824333191,
222
+ 0.8138095140457153,
223
+ 0.7545793056488037,
224
+ 0.7571702003479004,
225
+ 0.9127223491668701,
226
+ 0.8391615748405457,
227
+ 0.7476217150688171,
228
+ 0.8130890727043152
229
+ ]
230
+ }
231
+ }
checkpoints/nano_action_causal_pusht_v1/config.yaml ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_name: nano_action_causal_pusht_v1
2
+ seed: 0
3
+ out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
4
+ data:
5
+ latent_root: /group2/ct/weihanx/world_model/mixed_pushT_latents
6
+ action_root: /group2/ct/weihanx/world_model/mixed_pushT_actions
7
+ action_file: sensor_actions_with_dforce_fps15.pt
8
+ view_action_file: view_actions_fps15.pt
9
+ split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_pusht_wrist_holdout.json
10
+ optional_streams: []
11
+ action_norm: quantile
12
+ train_window_stride: 1
13
+ val_window_stride: 50
14
+ stats_window_stride: 8
15
+ action_stats: null
16
+ num_workers: 4
17
+ model:
18
+ language_model: nano
19
+ nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
20
+ strict_load: true
21
+ dtype: bfloat16
22
+ attention_backend: flex
23
+ flex_preference: flex_triton
24
+ temporal_mode: causal
25
+ attention_topology: all_streams_causal
26
+ task_mode: action_full
27
+ init_tactile_from_rgb: true
28
+ init_action_from_nano:
29
+ domain: umi
30
+ train_time_embedder: false
31
+ train_rgb_projections: false
32
+ lora:
33
+ enable: true
34
+ rank: 16
35
+ alpha: 32
36
+ targets: attn_ffn
37
+ training:
38
+ prediction_strategy: causal_teacher_forcing
39
+ init_from_checkpoint: null
40
+ missing_stream_training: false
41
+ wrist_dropout:
42
+ probability: 0.0
43
+ mode: paired
44
+ scope: whole_clip
45
+ batching: group_by_active_streams
46
+ sampling:
47
+ mode: uniform
48
+ steps_per_epoch: null
49
+ val_hide_wrist: false
50
+ epochs: 20
51
+ batch_size: 2
52
+ lr: 5.0e-05
53
+ weight_decay: 0.01
54
+ warmup_steps: 100
55
+ grad_clip: 1.0
56
+ amp: bf16
57
+ sigma_distribution: waver
58
+ sigma_shift: 3.0
59
+ loss_weights: {}
60
+ log_every: 20
61
+ ckpt_every_epochs: 1
62
+ activation_checkpointing: true
63
+ inference:
64
+ prediction_strategy: causal_autoregressive
checkpoints/nano_action_causal_pusht_v1/data_manifest.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fingerprint": "ea6ddd54206fd5c3cc51c7927a83e6d1ff0f7c2615d878830d1eaad5200a725d",
3
+ "windows": [
4
+ 55199,
5
+ 0
6
+ ],
7
+ "batch_size": 2,
8
+ "world_size": 4,
9
+ "steps_per_epoch": 6900,
10
+ "seed": 0,
11
+ "sampling_mode": "uniform",
12
+ "validation_fingerprint": "c751f783dc5ea9804c83f0c55c81853adbae5144d33250b51a4bb5dffc789a6d"
13
+ }
checkpoints/nano_action_causal_pusht_v1/epoch_019.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c8f9c4f2eaf46b56c890d3f787afb7cb9cdb141158ff0ee9bf158b9e916edfc
3
+ size 544776495
checkpoints/nano_action_causal_pusht_v1/metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/nano_action_causal_pusht_v1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/nano_action_causal_rope_v1/action_normalizer.json ADDED
@@ -0,0 +1,225 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": 2,
3
+ "sensor": {
4
+ "mode": "quantile",
5
+ "mean": [
6
+ -0.0004158033989369869,
7
+ 0.0009414111264050007,
8
+ 0.00019630556926131248,
9
+ 0.9966022968292236,
10
+ 0.003917371854186058,
11
+ -0.0016839727759361267,
12
+ -0.0035519488155841827,
13
+ 0.9968067407608032,
14
+ 0.004057098180055618
15
+ ],
16
+ "std": [
17
+ 0.013511914759874344,
18
+ 0.014263447374105453,
19
+ 0.015576692298054695,
20
+ 0.003397703170776367,
21
+ 0.06636063754558563,
22
+ 0.06976140290498734,
23
+ 0.06645658612251282,
24
+ 0.0031932294368743896,
25
+ 0.06916747987270355
26
+ ],
27
+ "meta": {
28
+ "rows": 289470,
29
+ "episodes": [
30
+ "rope_0911_episode_000_seg00.pt",
31
+ "rope_0911_episode_000_seg01.pt",
32
+ "rope_0911_episode_000_seg02.pt",
33
+ "rope_0911_episode_000_seg03.pt",
34
+ "rope_0911_episode_003_seg00.pt",
35
+ "rope_0911_episode_003_seg01.pt",
36
+ "rope_0911_episode_003_seg02.pt",
37
+ "rope_0911_episode_004.pt",
38
+ "rope_0911_episode_006_seg00.pt",
39
+ "rope_0911_episode_006_seg01.pt",
40
+ "rope_0911_episode_006_seg02.pt",
41
+ "rope_0911_episode_006_seg03.pt",
42
+ "rope_0911_episode_007_seg00.pt",
43
+ "rope_0911_episode_007_seg01.pt",
44
+ "rope_0911_episode_007_seg02.pt",
45
+ "rope_0911_episode_007_seg03.pt",
46
+ "rope_0914_episode_001_seg00.pt",
47
+ "rope_0914_episode_002_seg00.pt",
48
+ "rope_0914_episode_002_seg01.pt",
49
+ "rope_0914_episode_002_seg02.pt",
50
+ "rope_0914_episode_002_seg03.pt",
51
+ "rope_0914_episode_002_seg04.pt",
52
+ "rope_0914_episode_005_seg00.pt",
53
+ "rope_0914_episode_005_seg01.pt",
54
+ "rope_0914_episode_005_seg02.pt",
55
+ "rope_0914_episode_005_seg03.pt",
56
+ "rope_0914_episode_005_seg04.pt",
57
+ "rope_0914_episode_005_seg05.pt",
58
+ "rope_0914_episode_005_seg06.pt",
59
+ "rope_0914_episode_005_seg07.pt",
60
+ "rope_0914_episode_005_seg08.pt",
61
+ "rope_0914_episode_005_seg09.pt",
62
+ "rope_0914_episode_005_seg10.pt",
63
+ "rope_0915_episode_000_seg00.pt",
64
+ "rope_0915_episode_000_seg01.pt",
65
+ "rope_0915_episode_000_seg02.pt",
66
+ "rope_0916_episode_000_seg00.pt",
67
+ "rope_0916_episode_000_seg01.pt",
68
+ "rope_0916_episode_000_seg02.pt",
69
+ "rope_0916_episode_000_seg03.pt",
70
+ "rope_0916_episode_000_seg04.pt",
71
+ "rope_0916_episode_000_seg05.pt",
72
+ "rope_0916_episode_000_seg06.pt",
73
+ "rope_0916_episode_001_seg00.pt",
74
+ "rope_0916_episode_001_seg01.pt",
75
+ "rope_0916_episode_001_seg02.pt",
76
+ "rope_0916_episode_001_seg03.pt",
77
+ "rope_0916_episode_001_seg04.pt",
78
+ "rope_0916_episode_001_seg05.pt",
79
+ "rope_0916_episode_001_seg06.pt",
80
+ "rope_0916_episode_001_seg07.pt",
81
+ "rope_0916_episode_001_seg08.pt",
82
+ "rope_0916_episode_001_seg09.pt"
83
+ ],
84
+ "streams": [
85
+ "action_left",
86
+ "action_right"
87
+ ],
88
+ "kind": "sensor",
89
+ "hands": "left+right stacked"
90
+ },
91
+ "q01": [
92
+ -0.013927718624472618,
93
+ -0.013322035782039165,
94
+ -0.01538038719445467,
95
+ 0.9932045936584473,
96
+ -0.062443267554044724,
97
+ -0.07144537568092346,
98
+ -0.0700085312128067,
99
+ 0.9936135411262512,
100
+ -0.06511038541793823
101
+ ],
102
+ "q99": [
103
+ 0.013096111826598644,
104
+ 0.015204858034849167,
105
+ 0.015772998332977295,
106
+ 1.0,
107
+ 0.07027801126241684,
108
+ 0.06807743012905121,
109
+ 0.06290463358163834,
110
+ 1.0,
111
+ 0.07322458177804947
112
+ ]
113
+ },
114
+ "view": {
115
+ "mode": "quantile",
116
+ "mean": [
117
+ 0.4700682759284973,
118
+ 0.5895602703094482,
119
+ 0.7756980657577515,
120
+ 0.4884106516838074,
121
+ 0.5661192536354065,
122
+ 0.867732584476471,
123
+ 0.536643922328949,
124
+ 0.5083187818527222,
125
+ 0.7565478682518005
126
+ ],
127
+ "std": [
128
+ 0.32166892290115356,
129
+ 0.2662782669067383,
130
+ 0.04422447085380554,
131
+ 0.30136144161224365,
132
+ 0.24191366136074066,
133
+ 0.04571503400802612,
134
+ 0.3374778628349304,
135
+ 0.3240416944026947,
136
+ 0.05939453840255737
137
+ ],
138
+ "meta": {
139
+ "rows": 308768,
140
+ "episodes": [
141
+ "rope_0911_episode_000_seg00.pt",
142
+ "rope_0911_episode_000_seg01.pt",
143
+ "rope_0911_episode_000_seg02.pt",
144
+ "rope_0911_episode_000_seg03.pt",
145
+ "rope_0911_episode_003_seg00.pt",
146
+ "rope_0911_episode_003_seg01.pt",
147
+ "rope_0911_episode_003_seg02.pt",
148
+ "rope_0911_episode_004.pt",
149
+ "rope_0911_episode_006_seg00.pt",
150
+ "rope_0911_episode_006_seg01.pt",
151
+ "rope_0911_episode_006_seg02.pt",
152
+ "rope_0911_episode_006_seg03.pt",
153
+ "rope_0911_episode_007_seg00.pt",
154
+ "rope_0911_episode_007_seg01.pt",
155
+ "rope_0911_episode_007_seg02.pt",
156
+ "rope_0911_episode_007_seg03.pt",
157
+ "rope_0914_episode_001_seg00.pt",
158
+ "rope_0914_episode_002_seg00.pt",
159
+ "rope_0914_episode_002_seg01.pt",
160
+ "rope_0914_episode_002_seg02.pt",
161
+ "rope_0914_episode_002_seg03.pt",
162
+ "rope_0914_episode_002_seg04.pt",
163
+ "rope_0914_episode_005_seg00.pt",
164
+ "rope_0914_episode_005_seg01.pt",
165
+ "rope_0914_episode_005_seg02.pt",
166
+ "rope_0914_episode_005_seg03.pt",
167
+ "rope_0914_episode_005_seg04.pt",
168
+ "rope_0914_episode_005_seg05.pt",
169
+ "rope_0914_episode_005_seg06.pt",
170
+ "rope_0914_episode_005_seg07.pt",
171
+ "rope_0914_episode_005_seg08.pt",
172
+ "rope_0914_episode_005_seg09.pt",
173
+ "rope_0914_episode_005_seg10.pt",
174
+ "rope_0915_episode_000_seg00.pt",
175
+ "rope_0915_episode_000_seg01.pt",
176
+ "rope_0915_episode_000_seg02.pt",
177
+ "rope_0916_episode_000_seg00.pt",
178
+ "rope_0916_episode_000_seg01.pt",
179
+ "rope_0916_episode_000_seg02.pt",
180
+ "rope_0916_episode_000_seg03.pt",
181
+ "rope_0916_episode_000_seg04.pt",
182
+ "rope_0916_episode_000_seg05.pt",
183
+ "rope_0916_episode_000_seg06.pt",
184
+ "rope_0916_episode_001_seg00.pt",
185
+ "rope_0916_episode_001_seg01.pt",
186
+ "rope_0916_episode_001_seg02.pt",
187
+ "rope_0916_episode_001_seg03.pt",
188
+ "rope_0916_episode_001_seg04.pt",
189
+ "rope_0916_episode_001_seg05.pt",
190
+ "rope_0916_episode_001_seg06.pt",
191
+ "rope_0916_episode_001_seg07.pt",
192
+ "rope_0916_episode_001_seg08.pt",
193
+ "rope_0916_episode_001_seg09.pt"
194
+ ],
195
+ "streams": [
196
+ "view_action_left",
197
+ "view_action_right"
198
+ ],
199
+ "kind": "view",
200
+ "hands": "left+right stacked"
201
+ },
202
+ "q01": [
203
+ 0.14839932322502136,
204
+ 0.32328200340270996,
205
+ 0.7314735651016235,
206
+ 0.18704919517040253,
207
+ 0.324205607175827,
208
+ 0.8220175504684448,
209
+ 0.19916608929634094,
210
+ 0.18427711725234985,
211
+ 0.6971533298492432
212
+ ],
213
+ "q99": [
214
+ 0.7917371988296509,
215
+ 0.8558385372161865,
216
+ 0.8199225068092346,
217
+ 0.789772093296051,
218
+ 0.8080329298973083,
219
+ 0.9134476184844971,
220
+ 0.8741217851638794,
221
+ 0.8323605060577393,
222
+ 0.8159424066543579
223
+ ]
224
+ }
225
+ }
checkpoints/nano_action_causal_rope_v1/config.yaml ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_name: nano_action_causal_rope_v1
2
+ seed: 0
3
+ out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
4
+ data:
5
+ latent_root: /group2/ct/weihanx/world_model/mixed_rope_latents
6
+ action_root: /group2/ct/weihanx/world_model/mixed_rope_actions
7
+ action_file: sensor_actions_with_dforce_fps15.pt
8
+ view_action_file: view_actions_fps15.pt
9
+ split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_rope_wrist_holdout.json
10
+ optional_streams: []
11
+ action_norm: quantile
12
+ train_window_stride: 1
13
+ val_window_stride: 50
14
+ stats_window_stride: 8
15
+ action_stats: null
16
+ num_workers: 4
17
+ model:
18
+ language_model: nano
19
+ nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
20
+ strict_load: true
21
+ dtype: bfloat16
22
+ attention_backend: flex
23
+ flex_preference: flex_triton
24
+ temporal_mode: causal
25
+ attention_topology: all_streams_causal
26
+ task_mode: action_full
27
+ init_tactile_from_rgb: true
28
+ init_action_from_nano:
29
+ domain: umi
30
+ train_time_embedder: false
31
+ train_rgb_projections: false
32
+ lora:
33
+ enable: true
34
+ rank: 16
35
+ alpha: 32
36
+ targets: attn_ffn
37
+ training:
38
+ prediction_strategy: causal_teacher_forcing
39
+ init_from_checkpoint: null
40
+ missing_stream_training: false
41
+ wrist_dropout:
42
+ probability: 0.0
43
+ mode: paired
44
+ scope: whole_clip
45
+ batching: group_by_active_streams
46
+ sampling:
47
+ mode: uniform
48
+ steps_per_epoch: null
49
+ val_hide_wrist: false
50
+ epochs: 20
51
+ batch_size: 2
52
+ lr: 5.0e-05
53
+ weight_decay: 0.01
54
+ warmup_steps: 100
55
+ grad_clip: 1.0
56
+ amp: bf16
57
+ sigma_distribution: waver
58
+ sigma_shift: 3.0
59
+ loss_weights: {}
60
+ log_every: 20
61
+ ckpt_every_epochs: 1
62
+ activation_checkpointing: true
63
+ inference:
64
+ prediction_strategy: causal_autoregressive
checkpoints/nano_action_causal_rope_v1/data_manifest.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fingerprint": "05585f27e1418918d9be4fff4fc8bc95d2049de8f5ce2a978c82a2e5ab8c520e",
3
+ "windows": [
4
+ 77188,
5
+ 0
6
+ ],
7
+ "batch_size": 2,
8
+ "world_size": 4,
9
+ "steps_per_epoch": 9648,
10
+ "seed": 0,
11
+ "sampling_mode": "uniform",
12
+ "validation_fingerprint": "b221879dfbe78b51c7b89cd9e3a9954ff49bf62cbae6d722bd1cf19869f19b28"
13
+ }
checkpoints/nano_action_causal_rope_v1/epoch_019.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:849387fb39d1323bc9d25d96c9c3c30123f232aba1e88274f355581a9da95b74
3
+ size 544776495
checkpoints/nano_action_causal_rope_v1/metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/nano_action_causal_rope_v1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/action_normalizer.json ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": 2,
3
+ "sensor": {
4
+ "mode": "quantile",
5
+ "mean": [
6
+ -1.3721641153097153e-05,
7
+ 0.002626492641866207,
8
+ 0.0005233986303210258,
9
+ 0.9972736239433289,
10
+ 0.004918001592159271,
11
+ 0.001323983073234558,
12
+ -0.005411813035607338,
13
+ 0.9968388080596924,
14
+ 0.0037902556359767914
15
+ ],
16
+ "std": [
17
+ 0.010827474296092987,
18
+ 0.01424520742148161,
19
+ 0.012470426969230175,
20
+ 0.0027263760566711426,
21
+ 0.0642358735203743,
22
+ 0.05420434847474098,
23
+ 0.06445430219173431,
24
+ 0.0031611621379852295,
25
+ 0.06770282983779907
26
+ ],
27
+ "meta": {
28
+ "rows": 267720,
29
+ "episodes": [
30
+ "motherboard_0911_episode_000.pt",
31
+ "motherboard_0911_episode_001_seg00.pt",
32
+ "motherboard_0911_episode_001_seg01.pt",
33
+ "motherboard_0911_episode_003_seg00.pt",
34
+ "motherboard_0911_episode_004_seg00.pt",
35
+ "motherboard_0911_episode_004_seg01.pt",
36
+ "motherboard_0911_episode_005_seg00.pt",
37
+ "motherboard_0911_episode_005_seg01.pt",
38
+ "motherboard_0911_episode_005_seg02.pt",
39
+ "motherboard_0911_episode_005_seg03.pt",
40
+ "motherboard_0911_episode_005_seg04.pt",
41
+ "motherboard_0911_episode_005_seg05.pt",
42
+ "motherboard_0911_episode_006_seg00.pt",
43
+ "motherboard_0911_episode_006_seg01.pt",
44
+ "motherboard_0911_episode_007_seg00.pt",
45
+ "motherboard_0911_episode_007_seg01.pt",
46
+ "motherboard_0911_episode_007_seg02.pt",
47
+ "motherboard_0911_episode_007_seg03.pt",
48
+ "motherboard_0911_episode_007_seg04.pt",
49
+ "motherboard_0912_episode_000_seg00.pt",
50
+ "motherboard_0912_episode_000_seg01.pt"
51
+ ],
52
+ "streams": [
53
+ "action_left",
54
+ "action_right"
55
+ ],
56
+ "kind": "sensor",
57
+ "hands": "left+right stacked"
58
+ },
59
+ "q01": [
60
+ -0.010841195471584797,
61
+ -0.011618714779615402,
62
+ -0.01194702833890915,
63
+ 0.9945472478866577,
64
+ -0.05931787192821503,
65
+ -0.052880365401506424,
66
+ -0.0698661133646965,
67
+ 0.9936776757240295,
68
+ -0.06391257792711258
69
+ ],
70
+ "q99": [
71
+ 0.010813752189278603,
72
+ 0.016871700063347816,
73
+ 0.0129938255995512,
74
+ 1.0,
75
+ 0.06915387511253357,
76
+ 0.05552833154797554,
77
+ 0.05904248729348183,
78
+ 1.0,
79
+ 0.07149308919906616
80
+ ]
81
+ },
82
+ "view": {
83
+ "mode": "quantile",
84
+ "mean": [
85
+ 0.44803303480148315,
86
+ 0.5292316675186157,
87
+ 0.7715288400650024,
88
+ 0.47830602526664734,
89
+ 0.5115014314651489,
90
+ 0.8584882616996765,
91
+ 0.5024852752685547,
92
+ 0.44486939907073975,
93
+ 0.7453709840774536
94
+ ],
95
+ "std": [
96
+ 0.322518527507782,
97
+ 0.2845557928085327,
98
+ 0.04919716715812683,
99
+ 0.28917667269706726,
100
+ 0.26094764471054077,
101
+ 0.0553051233291626,
102
+ 0.32752352952957153,
103
+ 0.32817262411117554,
104
+ 0.07070961594581604
105
+ ],
106
+ "meta": {
107
+ "rows": 285568,
108
+ "episodes": [
109
+ "motherboard_0911_episode_000.pt",
110
+ "motherboard_0911_episode_001_seg00.pt",
111
+ "motherboard_0911_episode_001_seg01.pt",
112
+ "motherboard_0911_episode_003_seg00.pt",
113
+ "motherboard_0911_episode_004_seg00.pt",
114
+ "motherboard_0911_episode_004_seg01.pt",
115
+ "motherboard_0911_episode_005_seg00.pt",
116
+ "motherboard_0911_episode_005_seg01.pt",
117
+ "motherboard_0911_episode_005_seg02.pt",
118
+ "motherboard_0911_episode_005_seg03.pt",
119
+ "motherboard_0911_episode_005_seg04.pt",
120
+ "motherboard_0911_episode_005_seg05.pt",
121
+ "motherboard_0911_episode_006_seg00.pt",
122
+ "motherboard_0911_episode_006_seg01.pt",
123
+ "motherboard_0911_episode_007_seg00.pt",
124
+ "motherboard_0911_episode_007_seg01.pt",
125
+ "motherboard_0911_episode_007_seg02.pt",
126
+ "motherboard_0911_episode_007_seg03.pt",
127
+ "motherboard_0911_episode_007_seg04.pt",
128
+ "motherboard_0912_episode_000_seg00.pt",
129
+ "motherboard_0912_episode_000_seg01.pt"
130
+ ],
131
+ "streams": [
132
+ "view_action_left",
133
+ "view_action_right"
134
+ ],
135
+ "kind": "view",
136
+ "hands": "left+right stacked"
137
+ },
138
+ "q01": [
139
+ 0.12551453709602356,
140
+ 0.24467585980892181,
141
+ 0.722331702709198,
142
+ 0.18912935256958008,
143
+ 0.25055375695228577,
144
+ 0.8031831383705139,
145
+ 0.17496176064014435,
146
+ 0.1166967824101448,
147
+ 0.6746613383293152
148
+ ],
149
+ "q99": [
150
+ 0.7705515623092651,
151
+ 0.8137874603271484,
152
+ 0.8207260370254517,
153
+ 0.7674826979637146,
154
+ 0.7724490761756897,
155
+ 0.9137933850288391,
156
+ 0.8300088047981262,
157
+ 0.7730420231819153,
158
+ 0.8160805702209473
159
+ ]
160
+ }
161
+ }
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/config.yaml ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run_name: nano_action_causal_umi_sensoronly_noforce_v1
2
+ seed: 0
3
+ out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
4
+ data:
5
+ latent_root: /group2/ct/weihanx/world_model/mixed_mb_latents
6
+ action_root: /group2/ct/weihanx/world_model/mixed_mb_actions
7
+ action_file: sensor_actions_fps15_noforce.pt
8
+ view_action_file: view_actions_fps15.pt
9
+ split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_mb_wrist_holdout.json
10
+ optional_streams: []
11
+ action_norm: quantile
12
+ train_window_stride: 1
13
+ val_window_stride: 50
14
+ stats_window_stride: 8
15
+ action_stats: null
16
+ num_workers: 4
17
+ model:
18
+ language_model: nano
19
+ nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
20
+ strict_load: true
21
+ dtype: bfloat16
22
+ attention_backend: flex
23
+ flex_preference: flex_triton
24
+ temporal_mode: causal
25
+ attention_topology: all_streams_causal
26
+ task_mode: action_full
27
+ active_streams:
28
+ - video
29
+ - wrist_left
30
+ - wrist_right
31
+ - tactile_left
32
+ - tactile_right
33
+ - action_left
34
+ - action_right
35
+ init_tactile_from_rgb: true
36
+ init_action_from_nano:
37
+ domain: umi
38
+ train_time_embedder: false
39
+ train_rgb_projections: false
40
+ lora:
41
+ enable: true
42
+ rank: 16
43
+ alpha: 32
44
+ targets: attn_ffn
45
+ training:
46
+ prediction_strategy: causal_teacher_forcing
47
+ init_from_checkpoint: null
48
+ missing_stream_training: false
49
+ wrist_dropout:
50
+ probability: 0.0
51
+ mode: paired
52
+ scope: whole_clip
53
+ batching: group_by_active_streams
54
+ sampling:
55
+ mode: uniform
56
+ steps_per_epoch: null
57
+ val_hide_wrist: false
58
+ epochs: 20
59
+ batch_size: 2
60
+ lr: 5.0e-05
61
+ weight_decay: 0.01
62
+ warmup_steps: 100
63
+ grad_clip: 1.0
64
+ amp: bf16
65
+ sigma_distribution: waver
66
+ sigma_shift: 3.0
67
+ loss_weights: {}
68
+ log_every: 20
69
+ ckpt_every_epochs: 1
70
+ activation_checkpointing: true
71
+ inference:
72
+ prediction_strategy: causal_autoregressive
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/data_manifest.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fingerprint": "8f3e74e1fc09cc4d6a4c53ce13858c2834a3544c8de704321ad4ff8bde174ad3",
3
+ "windows": [
4
+ 71386,
5
+ 0
6
+ ],
7
+ "batch_size": 2,
8
+ "world_size": 4,
9
+ "steps_per_epoch": 8923,
10
+ "seed": 0,
11
+ "sampling_mode": "uniform",
12
+ "validation_fingerprint": "d9b76ca212840e64c814357151266c0d2be613aa01adf092e6a74ad8f7007881"
13
+ }
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/epoch_013.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c0519e896cc147631cb8ff6d9ae1f46ca30f72b8205c9ae2539a3d3df87d2aa
3
+ size 544282855
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/train.log ADDED
The diff for this file is too large to render. See raw diff
 
checkpoints/vae/lora_last.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8fae543fb989596ea9f19dab1ece229d2f03201b0872ecfe3eb2ed978df6b8da
3
+ size 122466955
checkpoints/vae/train_config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vae_pth": "/group2/ct/weihanx/Wan-AI/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
3
+ "csv": "/group2/ct/weihanx/world_model/data_preprocessing/freetacman_tactile_shuffled_filtered.csv",
4
+ "out": "./lora_out_refflow",
5
+ "num_frames": 17,
6
+ "height": 256,
7
+ "width": 256,
8
+ "val_size": 64,
9
+ "num_workers": 8,
10
+ "fps": 6.0,
11
+ "src_fps": 30.0,
12
+ "lora_r": 32,
13
+ "lora_alpha": 32.0,
14
+ "lora_dropout": 0.0,
15
+ "lora_targets": "all",
16
+ "batch_size": 1,
17
+ "grad_accum": 4,
18
+ "lr": 0.0001,
19
+ "weight_decay": 0.0,
20
+ "steps": 4000,
21
+ "warmup": 100,
22
+ "grad_clip": 1.0,
23
+ "lpips_weight": 0.1,
24
+ "edge_weight": 1.0,
25
+ "flow_weight": 1.0,
26
+ "flow_floor": 1.5,
27
+ "flow_suffix": "_refflow.npz",
28
+ "log_every": 20,
29
+ "val_every": 500,
30
+ "save_every": 1000,
31
+ "dtype": "bf16",
32
+ "seed": 0
33
+ }