Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
add checkpoints (vae LoRA, pusht_v1 e019, rope_v1 e019, umi_sensoronly_noforce_v1 e013)
Browse files- checkpoints/nano_action_causal_pusht_v1/action_normalizer.json +231 -0
- checkpoints/nano_action_causal_pusht_v1/config.yaml +64 -0
- checkpoints/nano_action_causal_pusht_v1/data_manifest.json +13 -0
- checkpoints/nano_action_causal_pusht_v1/epoch_019.pt +3 -0
- checkpoints/nano_action_causal_pusht_v1/metrics.jsonl +0 -0
- checkpoints/nano_action_causal_pusht_v1/train.log +0 -0
- checkpoints/nano_action_causal_rope_v1/action_normalizer.json +225 -0
- checkpoints/nano_action_causal_rope_v1/config.yaml +64 -0
- checkpoints/nano_action_causal_rope_v1/data_manifest.json +13 -0
- checkpoints/nano_action_causal_rope_v1/epoch_019.pt +3 -0
- checkpoints/nano_action_causal_rope_v1/metrics.jsonl +0 -0
- checkpoints/nano_action_causal_rope_v1/train.log +0 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/action_normalizer.json +161 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/config.yaml +72 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/data_manifest.json +13 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/epoch_013.pt +3 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/metrics.jsonl +0 -0
- checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/train.log +0 -0
- checkpoints/vae/lora_last.pt +3 -0
- checkpoints/vae/train_config.json +33 -0
checkpoints/nano_action_causal_pusht_v1/action_normalizer.json
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": 2,
|
| 3 |
+
"sensor": {
|
| 4 |
+
"mode": "quantile",
|
| 5 |
+
"mean": [
|
| 6 |
+
-0.0003890679217875004,
|
| 7 |
+
0.0008608214557170868,
|
| 8 |
+
0.0005704499781131744,
|
| 9 |
+
0.9973807334899902,
|
| 10 |
+
0.0025033093988895416,
|
| 11 |
+
-0.0008984990417957306,
|
| 12 |
+
-0.0028330646455287933,
|
| 13 |
+
0.9974104166030884,
|
| 14 |
+
0.002793760970234871
|
| 15 |
+
],
|
| 16 |
+
"std": [
|
| 17 |
+
0.01522023230791092,
|
| 18 |
+
0.013312933966517448,
|
| 19 |
+
0.017298635095357895,
|
| 20 |
+
0.0026190578937530518,
|
| 21 |
+
0.06116945669054985,
|
| 22 |
+
0.06694568693637848,
|
| 23 |
+
0.06128935143351555,
|
| 24 |
+
0.002589315176010132,
|
| 25 |
+
0.06364484876394272
|
| 26 |
+
],
|
| 27 |
+
"meta": {
|
| 28 |
+
"rows": 207000,
|
| 29 |
+
"episodes": [
|
| 30 |
+
"pushT_0910_episode_000_seg00.pt",
|
| 31 |
+
"pushT_0910_episode_001_seg00.pt",
|
| 32 |
+
"pushT_0910_episode_001_seg01.pt",
|
| 33 |
+
"pushT_0910_episode_001_seg02.pt",
|
| 34 |
+
"pushT_0910_episode_003_seg00.pt",
|
| 35 |
+
"pushT_0910_episode_003_seg01.pt",
|
| 36 |
+
"pushT_0910_episode_003_seg02.pt",
|
| 37 |
+
"pushT_0910_episode_003_seg03.pt",
|
| 38 |
+
"pushT_0910_episode_003_seg04.pt",
|
| 39 |
+
"pushT_0910_episode_003_seg05.pt",
|
| 40 |
+
"pushT_0910_episode_003_seg06.pt",
|
| 41 |
+
"pushT_0910_episode_003_seg07.pt",
|
| 42 |
+
"pushT_0910_episode_004_seg00.pt",
|
| 43 |
+
"pushT_0910_episode_004_seg01.pt",
|
| 44 |
+
"pushT_0910_episode_004_seg02.pt",
|
| 45 |
+
"pushT_0910_episode_004_seg03.pt",
|
| 46 |
+
"pushT_0911_episode_000_seg00.pt",
|
| 47 |
+
"pushT_0911_episode_001_seg00.pt",
|
| 48 |
+
"pushT_0911_episode_002_seg00.pt",
|
| 49 |
+
"pushT_0911_episode_002_seg01.pt",
|
| 50 |
+
"pushT_0911_episode_002_seg02.pt",
|
| 51 |
+
"pushT_0911_episode_002_seg03.pt",
|
| 52 |
+
"pushT_0911_episode_004_seg00.pt",
|
| 53 |
+
"pushT_0911_episode_004_seg01.pt",
|
| 54 |
+
"pushT_0911_episode_004_seg02.pt",
|
| 55 |
+
"pushT_0911_episode_004_seg03.pt",
|
| 56 |
+
"pushT_0911_episode_004_seg04.pt",
|
| 57 |
+
"pushT_0911_episode_004_seg05.pt",
|
| 58 |
+
"pushT_0912_episode_001_seg00.pt",
|
| 59 |
+
"pushT_0912_episode_002_seg00.pt",
|
| 60 |
+
"pushT_0912_episode_002_seg01.pt",
|
| 61 |
+
"pushT_0912_episode_002_seg02.pt",
|
| 62 |
+
"pushT_0912_episode_002_seg03.pt",
|
| 63 |
+
"pushT_0912_episode_002_seg04.pt",
|
| 64 |
+
"pushT_0912_episode_004_seg00.pt",
|
| 65 |
+
"pushT_0912_episode_005_seg00.pt",
|
| 66 |
+
"pushT_0912_episode_005_seg01.pt",
|
| 67 |
+
"pushT_0912_episode_006_seg00.pt",
|
| 68 |
+
"pushT_0912_episode_007_seg00.pt",
|
| 69 |
+
"pushT_0912_episode_007_seg01.pt",
|
| 70 |
+
"pushT_0912_episode_007_seg02.pt",
|
| 71 |
+
"pushT_0912_episode_007_seg03.pt",
|
| 72 |
+
"pushT_0912_episode_008_seg00.pt",
|
| 73 |
+
"pushT_0912_episode_008_seg01.pt",
|
| 74 |
+
"pushT_0912_episode_008_seg02.pt",
|
| 75 |
+
"pushT_0912_episode_008_seg03.pt",
|
| 76 |
+
"pushT_0912_episode_009_seg00.pt",
|
| 77 |
+
"pushT_0915_episode_000_seg00.pt",
|
| 78 |
+
"pushT_0915_episode_000_seg01.pt",
|
| 79 |
+
"pushT_0915_episode_000_seg02.pt",
|
| 80 |
+
"pushT_0915_episode_000_seg03.pt",
|
| 81 |
+
"pushT_0915_episode_000_seg04.pt",
|
| 82 |
+
"pushT_0915_episode_000_seg05.pt",
|
| 83 |
+
"pushT_0915_episode_000_seg06.pt",
|
| 84 |
+
"pushT_0915_episode_000_seg07.pt",
|
| 85 |
+
"pushT_0915_episode_000_seg08.pt"
|
| 86 |
+
],
|
| 87 |
+
"streams": [
|
| 88 |
+
"action_left",
|
| 89 |
+
"action_right"
|
| 90 |
+
],
|
| 91 |
+
"kind": "sensor",
|
| 92 |
+
"hands": "left+right stacked"
|
| 93 |
+
},
|
| 94 |
+
"q01": [
|
| 95 |
+
-0.015609300695359707,
|
| 96 |
+
-0.012452112510800362,
|
| 97 |
+
-0.01672818511724472,
|
| 98 |
+
0.9947617053985596,
|
| 99 |
+
-0.05866614729166031,
|
| 100 |
+
-0.06784418225288391,
|
| 101 |
+
-0.06412241607904434,
|
| 102 |
+
0.9948211312294006,
|
| 103 |
+
-0.0608510859310627
|
| 104 |
+
],
|
| 105 |
+
"q99": [
|
| 106 |
+
0.014831164851784706,
|
| 107 |
+
0.014173755422234535,
|
| 108 |
+
0.01786908507347107,
|
| 109 |
+
0.9999998211860657,
|
| 110 |
+
0.06367276608943939,
|
| 111 |
+
0.06604718416929245,
|
| 112 |
+
0.058456286787986755,
|
| 113 |
+
0.9999997615814209,
|
| 114 |
+
0.06643860787153244
|
| 115 |
+
]
|
| 116 |
+
},
|
| 117 |
+
"view": {
|
| 118 |
+
"mode": "quantile",
|
| 119 |
+
"mean": [
|
| 120 |
+
0.49366551637649536,
|
| 121 |
+
0.5795469880104065,
|
| 122 |
+
0.7682432532310486,
|
| 123 |
+
0.512519121170044,
|
| 124 |
+
0.5563752055168152,
|
| 125 |
+
0.8493101596832275,
|
| 126 |
+
0.5399284362792969,
|
| 127 |
+
0.482826828956604,
|
| 128 |
+
0.7413364052772522
|
| 129 |
+
],
|
| 130 |
+
"std": [
|
| 131 |
+
0.28291356563568115,
|
| 132 |
+
0.2295605093240738,
|
| 133 |
+
0.04556626081466675,
|
| 134 |
+
0.24206016957759857,
|
| 135 |
+
0.200794979929924,
|
| 136 |
+
0.06341215968132019,
|
| 137 |
+
0.2992331385612488,
|
| 138 |
+
0.26479488611221313,
|
| 139 |
+
0.07175266742706299
|
| 140 |
+
],
|
| 141 |
+
"meta": {
|
| 142 |
+
"rows": 220800,
|
| 143 |
+
"episodes": [
|
| 144 |
+
"pushT_0910_episode_000_seg00.pt",
|
| 145 |
+
"pushT_0910_episode_001_seg00.pt",
|
| 146 |
+
"pushT_0910_episode_001_seg01.pt",
|
| 147 |
+
"pushT_0910_episode_001_seg02.pt",
|
| 148 |
+
"pushT_0910_episode_003_seg00.pt",
|
| 149 |
+
"pushT_0910_episode_003_seg01.pt",
|
| 150 |
+
"pushT_0910_episode_003_seg02.pt",
|
| 151 |
+
"pushT_0910_episode_003_seg03.pt",
|
| 152 |
+
"pushT_0910_episode_003_seg04.pt",
|
| 153 |
+
"pushT_0910_episode_003_seg05.pt",
|
| 154 |
+
"pushT_0910_episode_003_seg06.pt",
|
| 155 |
+
"pushT_0910_episode_003_seg07.pt",
|
| 156 |
+
"pushT_0910_episode_004_seg00.pt",
|
| 157 |
+
"pushT_0910_episode_004_seg01.pt",
|
| 158 |
+
"pushT_0910_episode_004_seg02.pt",
|
| 159 |
+
"pushT_0910_episode_004_seg03.pt",
|
| 160 |
+
"pushT_0911_episode_000_seg00.pt",
|
| 161 |
+
"pushT_0911_episode_001_seg00.pt",
|
| 162 |
+
"pushT_0911_episode_002_seg00.pt",
|
| 163 |
+
"pushT_0911_episode_002_seg01.pt",
|
| 164 |
+
"pushT_0911_episode_002_seg02.pt",
|
| 165 |
+
"pushT_0911_episode_002_seg03.pt",
|
| 166 |
+
"pushT_0911_episode_004_seg00.pt",
|
| 167 |
+
"pushT_0911_episode_004_seg01.pt",
|
| 168 |
+
"pushT_0911_episode_004_seg02.pt",
|
| 169 |
+
"pushT_0911_episode_004_seg03.pt",
|
| 170 |
+
"pushT_0911_episode_004_seg04.pt",
|
| 171 |
+
"pushT_0911_episode_004_seg05.pt",
|
| 172 |
+
"pushT_0912_episode_001_seg00.pt",
|
| 173 |
+
"pushT_0912_episode_002_seg00.pt",
|
| 174 |
+
"pushT_0912_episode_002_seg01.pt",
|
| 175 |
+
"pushT_0912_episode_002_seg02.pt",
|
| 176 |
+
"pushT_0912_episode_002_seg03.pt",
|
| 177 |
+
"pushT_0912_episode_002_seg04.pt",
|
| 178 |
+
"pushT_0912_episode_004_seg00.pt",
|
| 179 |
+
"pushT_0912_episode_005_seg00.pt",
|
| 180 |
+
"pushT_0912_episode_005_seg01.pt",
|
| 181 |
+
"pushT_0912_episode_006_seg00.pt",
|
| 182 |
+
"pushT_0912_episode_007_seg00.pt",
|
| 183 |
+
"pushT_0912_episode_007_seg01.pt",
|
| 184 |
+
"pushT_0912_episode_007_seg02.pt",
|
| 185 |
+
"pushT_0912_episode_007_seg03.pt",
|
| 186 |
+
"pushT_0912_episode_008_seg00.pt",
|
| 187 |
+
"pushT_0912_episode_008_seg01.pt",
|
| 188 |
+
"pushT_0912_episode_008_seg02.pt",
|
| 189 |
+
"pushT_0912_episode_008_seg03.pt",
|
| 190 |
+
"pushT_0912_episode_009_seg00.pt",
|
| 191 |
+
"pushT_0915_episode_000_seg00.pt",
|
| 192 |
+
"pushT_0915_episode_000_seg01.pt",
|
| 193 |
+
"pushT_0915_episode_000_seg02.pt",
|
| 194 |
+
"pushT_0915_episode_000_seg03.pt",
|
| 195 |
+
"pushT_0915_episode_000_seg04.pt",
|
| 196 |
+
"pushT_0915_episode_000_seg05.pt",
|
| 197 |
+
"pushT_0915_episode_000_seg06.pt",
|
| 198 |
+
"pushT_0915_episode_000_seg07.pt",
|
| 199 |
+
"pushT_0915_episode_000_seg08.pt"
|
| 200 |
+
],
|
| 201 |
+
"streams": [
|
| 202 |
+
"view_action_left",
|
| 203 |
+
"view_action_right"
|
| 204 |
+
],
|
| 205 |
+
"kind": "view",
|
| 206 |
+
"hands": "left+right stacked"
|
| 207 |
+
},
|
| 208 |
+
"q01": [
|
| 209 |
+
0.21075193583965302,
|
| 210 |
+
0.3499864637851715,
|
| 211 |
+
0.7226769924163818,
|
| 212 |
+
0.27045896649360657,
|
| 213 |
+
0.35558024048805237,
|
| 214 |
+
0.7858980298042297,
|
| 215 |
+
0.24069532752037048,
|
| 216 |
+
0.21803195774555206,
|
| 217 |
+
0.6695837378501892
|
| 218 |
+
],
|
| 219 |
+
"q99": [
|
| 220 |
+
0.7765790820121765,
|
| 221 |
+
0.8091074824333191,
|
| 222 |
+
0.8138095140457153,
|
| 223 |
+
0.7545793056488037,
|
| 224 |
+
0.7571702003479004,
|
| 225 |
+
0.9127223491668701,
|
| 226 |
+
0.8391615748405457,
|
| 227 |
+
0.7476217150688171,
|
| 228 |
+
0.8130890727043152
|
| 229 |
+
]
|
| 230 |
+
}
|
| 231 |
+
}
|
checkpoints/nano_action_causal_pusht_v1/config.yaml
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: nano_action_causal_pusht_v1
|
| 2 |
+
seed: 0
|
| 3 |
+
out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
|
| 4 |
+
data:
|
| 5 |
+
latent_root: /group2/ct/weihanx/world_model/mixed_pushT_latents
|
| 6 |
+
action_root: /group2/ct/weihanx/world_model/mixed_pushT_actions
|
| 7 |
+
action_file: sensor_actions_with_dforce_fps15.pt
|
| 8 |
+
view_action_file: view_actions_fps15.pt
|
| 9 |
+
split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_pusht_wrist_holdout.json
|
| 10 |
+
optional_streams: []
|
| 11 |
+
action_norm: quantile
|
| 12 |
+
train_window_stride: 1
|
| 13 |
+
val_window_stride: 50
|
| 14 |
+
stats_window_stride: 8
|
| 15 |
+
action_stats: null
|
| 16 |
+
num_workers: 4
|
| 17 |
+
model:
|
| 18 |
+
language_model: nano
|
| 19 |
+
nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
|
| 20 |
+
strict_load: true
|
| 21 |
+
dtype: bfloat16
|
| 22 |
+
attention_backend: flex
|
| 23 |
+
flex_preference: flex_triton
|
| 24 |
+
temporal_mode: causal
|
| 25 |
+
attention_topology: all_streams_causal
|
| 26 |
+
task_mode: action_full
|
| 27 |
+
init_tactile_from_rgb: true
|
| 28 |
+
init_action_from_nano:
|
| 29 |
+
domain: umi
|
| 30 |
+
train_time_embedder: false
|
| 31 |
+
train_rgb_projections: false
|
| 32 |
+
lora:
|
| 33 |
+
enable: true
|
| 34 |
+
rank: 16
|
| 35 |
+
alpha: 32
|
| 36 |
+
targets: attn_ffn
|
| 37 |
+
training:
|
| 38 |
+
prediction_strategy: causal_teacher_forcing
|
| 39 |
+
init_from_checkpoint: null
|
| 40 |
+
missing_stream_training: false
|
| 41 |
+
wrist_dropout:
|
| 42 |
+
probability: 0.0
|
| 43 |
+
mode: paired
|
| 44 |
+
scope: whole_clip
|
| 45 |
+
batching: group_by_active_streams
|
| 46 |
+
sampling:
|
| 47 |
+
mode: uniform
|
| 48 |
+
steps_per_epoch: null
|
| 49 |
+
val_hide_wrist: false
|
| 50 |
+
epochs: 20
|
| 51 |
+
batch_size: 2
|
| 52 |
+
lr: 5.0e-05
|
| 53 |
+
weight_decay: 0.01
|
| 54 |
+
warmup_steps: 100
|
| 55 |
+
grad_clip: 1.0
|
| 56 |
+
amp: bf16
|
| 57 |
+
sigma_distribution: waver
|
| 58 |
+
sigma_shift: 3.0
|
| 59 |
+
loss_weights: {}
|
| 60 |
+
log_every: 20
|
| 61 |
+
ckpt_every_epochs: 1
|
| 62 |
+
activation_checkpointing: true
|
| 63 |
+
inference:
|
| 64 |
+
prediction_strategy: causal_autoregressive
|
checkpoints/nano_action_causal_pusht_v1/data_manifest.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fingerprint": "ea6ddd54206fd5c3cc51c7927a83e6d1ff0f7c2615d878830d1eaad5200a725d",
|
| 3 |
+
"windows": [
|
| 4 |
+
55199,
|
| 5 |
+
0
|
| 6 |
+
],
|
| 7 |
+
"batch_size": 2,
|
| 8 |
+
"world_size": 4,
|
| 9 |
+
"steps_per_epoch": 6900,
|
| 10 |
+
"seed": 0,
|
| 11 |
+
"sampling_mode": "uniform",
|
| 12 |
+
"validation_fingerprint": "c751f783dc5ea9804c83f0c55c81853adbae5144d33250b51a4bb5dffc789a6d"
|
| 13 |
+
}
|
checkpoints/nano_action_causal_pusht_v1/epoch_019.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2c8f9c4f2eaf46b56c890d3f787afb7cb9cdb141158ff0ee9bf158b9e916edfc
|
| 3 |
+
size 544776495
|
checkpoints/nano_action_causal_pusht_v1/metrics.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/nano_action_causal_pusht_v1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/nano_action_causal_rope_v1/action_normalizer.json
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": 2,
|
| 3 |
+
"sensor": {
|
| 4 |
+
"mode": "quantile",
|
| 5 |
+
"mean": [
|
| 6 |
+
-0.0004158033989369869,
|
| 7 |
+
0.0009414111264050007,
|
| 8 |
+
0.00019630556926131248,
|
| 9 |
+
0.9966022968292236,
|
| 10 |
+
0.003917371854186058,
|
| 11 |
+
-0.0016839727759361267,
|
| 12 |
+
-0.0035519488155841827,
|
| 13 |
+
0.9968067407608032,
|
| 14 |
+
0.004057098180055618
|
| 15 |
+
],
|
| 16 |
+
"std": [
|
| 17 |
+
0.013511914759874344,
|
| 18 |
+
0.014263447374105453,
|
| 19 |
+
0.015576692298054695,
|
| 20 |
+
0.003397703170776367,
|
| 21 |
+
0.06636063754558563,
|
| 22 |
+
0.06976140290498734,
|
| 23 |
+
0.06645658612251282,
|
| 24 |
+
0.0031932294368743896,
|
| 25 |
+
0.06916747987270355
|
| 26 |
+
],
|
| 27 |
+
"meta": {
|
| 28 |
+
"rows": 289470,
|
| 29 |
+
"episodes": [
|
| 30 |
+
"rope_0911_episode_000_seg00.pt",
|
| 31 |
+
"rope_0911_episode_000_seg01.pt",
|
| 32 |
+
"rope_0911_episode_000_seg02.pt",
|
| 33 |
+
"rope_0911_episode_000_seg03.pt",
|
| 34 |
+
"rope_0911_episode_003_seg00.pt",
|
| 35 |
+
"rope_0911_episode_003_seg01.pt",
|
| 36 |
+
"rope_0911_episode_003_seg02.pt",
|
| 37 |
+
"rope_0911_episode_004.pt",
|
| 38 |
+
"rope_0911_episode_006_seg00.pt",
|
| 39 |
+
"rope_0911_episode_006_seg01.pt",
|
| 40 |
+
"rope_0911_episode_006_seg02.pt",
|
| 41 |
+
"rope_0911_episode_006_seg03.pt",
|
| 42 |
+
"rope_0911_episode_007_seg00.pt",
|
| 43 |
+
"rope_0911_episode_007_seg01.pt",
|
| 44 |
+
"rope_0911_episode_007_seg02.pt",
|
| 45 |
+
"rope_0911_episode_007_seg03.pt",
|
| 46 |
+
"rope_0914_episode_001_seg00.pt",
|
| 47 |
+
"rope_0914_episode_002_seg00.pt",
|
| 48 |
+
"rope_0914_episode_002_seg01.pt",
|
| 49 |
+
"rope_0914_episode_002_seg02.pt",
|
| 50 |
+
"rope_0914_episode_002_seg03.pt",
|
| 51 |
+
"rope_0914_episode_002_seg04.pt",
|
| 52 |
+
"rope_0914_episode_005_seg00.pt",
|
| 53 |
+
"rope_0914_episode_005_seg01.pt",
|
| 54 |
+
"rope_0914_episode_005_seg02.pt",
|
| 55 |
+
"rope_0914_episode_005_seg03.pt",
|
| 56 |
+
"rope_0914_episode_005_seg04.pt",
|
| 57 |
+
"rope_0914_episode_005_seg05.pt",
|
| 58 |
+
"rope_0914_episode_005_seg06.pt",
|
| 59 |
+
"rope_0914_episode_005_seg07.pt",
|
| 60 |
+
"rope_0914_episode_005_seg08.pt",
|
| 61 |
+
"rope_0914_episode_005_seg09.pt",
|
| 62 |
+
"rope_0914_episode_005_seg10.pt",
|
| 63 |
+
"rope_0915_episode_000_seg00.pt",
|
| 64 |
+
"rope_0915_episode_000_seg01.pt",
|
| 65 |
+
"rope_0915_episode_000_seg02.pt",
|
| 66 |
+
"rope_0916_episode_000_seg00.pt",
|
| 67 |
+
"rope_0916_episode_000_seg01.pt",
|
| 68 |
+
"rope_0916_episode_000_seg02.pt",
|
| 69 |
+
"rope_0916_episode_000_seg03.pt",
|
| 70 |
+
"rope_0916_episode_000_seg04.pt",
|
| 71 |
+
"rope_0916_episode_000_seg05.pt",
|
| 72 |
+
"rope_0916_episode_000_seg06.pt",
|
| 73 |
+
"rope_0916_episode_001_seg00.pt",
|
| 74 |
+
"rope_0916_episode_001_seg01.pt",
|
| 75 |
+
"rope_0916_episode_001_seg02.pt",
|
| 76 |
+
"rope_0916_episode_001_seg03.pt",
|
| 77 |
+
"rope_0916_episode_001_seg04.pt",
|
| 78 |
+
"rope_0916_episode_001_seg05.pt",
|
| 79 |
+
"rope_0916_episode_001_seg06.pt",
|
| 80 |
+
"rope_0916_episode_001_seg07.pt",
|
| 81 |
+
"rope_0916_episode_001_seg08.pt",
|
| 82 |
+
"rope_0916_episode_001_seg09.pt"
|
| 83 |
+
],
|
| 84 |
+
"streams": [
|
| 85 |
+
"action_left",
|
| 86 |
+
"action_right"
|
| 87 |
+
],
|
| 88 |
+
"kind": "sensor",
|
| 89 |
+
"hands": "left+right stacked"
|
| 90 |
+
},
|
| 91 |
+
"q01": [
|
| 92 |
+
-0.013927718624472618,
|
| 93 |
+
-0.013322035782039165,
|
| 94 |
+
-0.01538038719445467,
|
| 95 |
+
0.9932045936584473,
|
| 96 |
+
-0.062443267554044724,
|
| 97 |
+
-0.07144537568092346,
|
| 98 |
+
-0.0700085312128067,
|
| 99 |
+
0.9936135411262512,
|
| 100 |
+
-0.06511038541793823
|
| 101 |
+
],
|
| 102 |
+
"q99": [
|
| 103 |
+
0.013096111826598644,
|
| 104 |
+
0.015204858034849167,
|
| 105 |
+
0.015772998332977295,
|
| 106 |
+
1.0,
|
| 107 |
+
0.07027801126241684,
|
| 108 |
+
0.06807743012905121,
|
| 109 |
+
0.06290463358163834,
|
| 110 |
+
1.0,
|
| 111 |
+
0.07322458177804947
|
| 112 |
+
]
|
| 113 |
+
},
|
| 114 |
+
"view": {
|
| 115 |
+
"mode": "quantile",
|
| 116 |
+
"mean": [
|
| 117 |
+
0.4700682759284973,
|
| 118 |
+
0.5895602703094482,
|
| 119 |
+
0.7756980657577515,
|
| 120 |
+
0.4884106516838074,
|
| 121 |
+
0.5661192536354065,
|
| 122 |
+
0.867732584476471,
|
| 123 |
+
0.536643922328949,
|
| 124 |
+
0.5083187818527222,
|
| 125 |
+
0.7565478682518005
|
| 126 |
+
],
|
| 127 |
+
"std": [
|
| 128 |
+
0.32166892290115356,
|
| 129 |
+
0.2662782669067383,
|
| 130 |
+
0.04422447085380554,
|
| 131 |
+
0.30136144161224365,
|
| 132 |
+
0.24191366136074066,
|
| 133 |
+
0.04571503400802612,
|
| 134 |
+
0.3374778628349304,
|
| 135 |
+
0.3240416944026947,
|
| 136 |
+
0.05939453840255737
|
| 137 |
+
],
|
| 138 |
+
"meta": {
|
| 139 |
+
"rows": 308768,
|
| 140 |
+
"episodes": [
|
| 141 |
+
"rope_0911_episode_000_seg00.pt",
|
| 142 |
+
"rope_0911_episode_000_seg01.pt",
|
| 143 |
+
"rope_0911_episode_000_seg02.pt",
|
| 144 |
+
"rope_0911_episode_000_seg03.pt",
|
| 145 |
+
"rope_0911_episode_003_seg00.pt",
|
| 146 |
+
"rope_0911_episode_003_seg01.pt",
|
| 147 |
+
"rope_0911_episode_003_seg02.pt",
|
| 148 |
+
"rope_0911_episode_004.pt",
|
| 149 |
+
"rope_0911_episode_006_seg00.pt",
|
| 150 |
+
"rope_0911_episode_006_seg01.pt",
|
| 151 |
+
"rope_0911_episode_006_seg02.pt",
|
| 152 |
+
"rope_0911_episode_006_seg03.pt",
|
| 153 |
+
"rope_0911_episode_007_seg00.pt",
|
| 154 |
+
"rope_0911_episode_007_seg01.pt",
|
| 155 |
+
"rope_0911_episode_007_seg02.pt",
|
| 156 |
+
"rope_0911_episode_007_seg03.pt",
|
| 157 |
+
"rope_0914_episode_001_seg00.pt",
|
| 158 |
+
"rope_0914_episode_002_seg00.pt",
|
| 159 |
+
"rope_0914_episode_002_seg01.pt",
|
| 160 |
+
"rope_0914_episode_002_seg02.pt",
|
| 161 |
+
"rope_0914_episode_002_seg03.pt",
|
| 162 |
+
"rope_0914_episode_002_seg04.pt",
|
| 163 |
+
"rope_0914_episode_005_seg00.pt",
|
| 164 |
+
"rope_0914_episode_005_seg01.pt",
|
| 165 |
+
"rope_0914_episode_005_seg02.pt",
|
| 166 |
+
"rope_0914_episode_005_seg03.pt",
|
| 167 |
+
"rope_0914_episode_005_seg04.pt",
|
| 168 |
+
"rope_0914_episode_005_seg05.pt",
|
| 169 |
+
"rope_0914_episode_005_seg06.pt",
|
| 170 |
+
"rope_0914_episode_005_seg07.pt",
|
| 171 |
+
"rope_0914_episode_005_seg08.pt",
|
| 172 |
+
"rope_0914_episode_005_seg09.pt",
|
| 173 |
+
"rope_0914_episode_005_seg10.pt",
|
| 174 |
+
"rope_0915_episode_000_seg00.pt",
|
| 175 |
+
"rope_0915_episode_000_seg01.pt",
|
| 176 |
+
"rope_0915_episode_000_seg02.pt",
|
| 177 |
+
"rope_0916_episode_000_seg00.pt",
|
| 178 |
+
"rope_0916_episode_000_seg01.pt",
|
| 179 |
+
"rope_0916_episode_000_seg02.pt",
|
| 180 |
+
"rope_0916_episode_000_seg03.pt",
|
| 181 |
+
"rope_0916_episode_000_seg04.pt",
|
| 182 |
+
"rope_0916_episode_000_seg05.pt",
|
| 183 |
+
"rope_0916_episode_000_seg06.pt",
|
| 184 |
+
"rope_0916_episode_001_seg00.pt",
|
| 185 |
+
"rope_0916_episode_001_seg01.pt",
|
| 186 |
+
"rope_0916_episode_001_seg02.pt",
|
| 187 |
+
"rope_0916_episode_001_seg03.pt",
|
| 188 |
+
"rope_0916_episode_001_seg04.pt",
|
| 189 |
+
"rope_0916_episode_001_seg05.pt",
|
| 190 |
+
"rope_0916_episode_001_seg06.pt",
|
| 191 |
+
"rope_0916_episode_001_seg07.pt",
|
| 192 |
+
"rope_0916_episode_001_seg08.pt",
|
| 193 |
+
"rope_0916_episode_001_seg09.pt"
|
| 194 |
+
],
|
| 195 |
+
"streams": [
|
| 196 |
+
"view_action_left",
|
| 197 |
+
"view_action_right"
|
| 198 |
+
],
|
| 199 |
+
"kind": "view",
|
| 200 |
+
"hands": "left+right stacked"
|
| 201 |
+
},
|
| 202 |
+
"q01": [
|
| 203 |
+
0.14839932322502136,
|
| 204 |
+
0.32328200340270996,
|
| 205 |
+
0.7314735651016235,
|
| 206 |
+
0.18704919517040253,
|
| 207 |
+
0.324205607175827,
|
| 208 |
+
0.8220175504684448,
|
| 209 |
+
0.19916608929634094,
|
| 210 |
+
0.18427711725234985,
|
| 211 |
+
0.6971533298492432
|
| 212 |
+
],
|
| 213 |
+
"q99": [
|
| 214 |
+
0.7917371988296509,
|
| 215 |
+
0.8558385372161865,
|
| 216 |
+
0.8199225068092346,
|
| 217 |
+
0.789772093296051,
|
| 218 |
+
0.8080329298973083,
|
| 219 |
+
0.9134476184844971,
|
| 220 |
+
0.8741217851638794,
|
| 221 |
+
0.8323605060577393,
|
| 222 |
+
0.8159424066543579
|
| 223 |
+
]
|
| 224 |
+
}
|
| 225 |
+
}
|
checkpoints/nano_action_causal_rope_v1/config.yaml
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: nano_action_causal_rope_v1
|
| 2 |
+
seed: 0
|
| 3 |
+
out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
|
| 4 |
+
data:
|
| 5 |
+
latent_root: /group2/ct/weihanx/world_model/mixed_rope_latents
|
| 6 |
+
action_root: /group2/ct/weihanx/world_model/mixed_rope_actions
|
| 7 |
+
action_file: sensor_actions_with_dforce_fps15.pt
|
| 8 |
+
view_action_file: view_actions_fps15.pt
|
| 9 |
+
split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_rope_wrist_holdout.json
|
| 10 |
+
optional_streams: []
|
| 11 |
+
action_norm: quantile
|
| 12 |
+
train_window_stride: 1
|
| 13 |
+
val_window_stride: 50
|
| 14 |
+
stats_window_stride: 8
|
| 15 |
+
action_stats: null
|
| 16 |
+
num_workers: 4
|
| 17 |
+
model:
|
| 18 |
+
language_model: nano
|
| 19 |
+
nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
|
| 20 |
+
strict_load: true
|
| 21 |
+
dtype: bfloat16
|
| 22 |
+
attention_backend: flex
|
| 23 |
+
flex_preference: flex_triton
|
| 24 |
+
temporal_mode: causal
|
| 25 |
+
attention_topology: all_streams_causal
|
| 26 |
+
task_mode: action_full
|
| 27 |
+
init_tactile_from_rgb: true
|
| 28 |
+
init_action_from_nano:
|
| 29 |
+
domain: umi
|
| 30 |
+
train_time_embedder: false
|
| 31 |
+
train_rgb_projections: false
|
| 32 |
+
lora:
|
| 33 |
+
enable: true
|
| 34 |
+
rank: 16
|
| 35 |
+
alpha: 32
|
| 36 |
+
targets: attn_ffn
|
| 37 |
+
training:
|
| 38 |
+
prediction_strategy: causal_teacher_forcing
|
| 39 |
+
init_from_checkpoint: null
|
| 40 |
+
missing_stream_training: false
|
| 41 |
+
wrist_dropout:
|
| 42 |
+
probability: 0.0
|
| 43 |
+
mode: paired
|
| 44 |
+
scope: whole_clip
|
| 45 |
+
batching: group_by_active_streams
|
| 46 |
+
sampling:
|
| 47 |
+
mode: uniform
|
| 48 |
+
steps_per_epoch: null
|
| 49 |
+
val_hide_wrist: false
|
| 50 |
+
epochs: 20
|
| 51 |
+
batch_size: 2
|
| 52 |
+
lr: 5.0e-05
|
| 53 |
+
weight_decay: 0.01
|
| 54 |
+
warmup_steps: 100
|
| 55 |
+
grad_clip: 1.0
|
| 56 |
+
amp: bf16
|
| 57 |
+
sigma_distribution: waver
|
| 58 |
+
sigma_shift: 3.0
|
| 59 |
+
loss_weights: {}
|
| 60 |
+
log_every: 20
|
| 61 |
+
ckpt_every_epochs: 1
|
| 62 |
+
activation_checkpointing: true
|
| 63 |
+
inference:
|
| 64 |
+
prediction_strategy: causal_autoregressive
|
checkpoints/nano_action_causal_rope_v1/data_manifest.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fingerprint": "05585f27e1418918d9be4fff4fc8bc95d2049de8f5ce2a978c82a2e5ab8c520e",
|
| 3 |
+
"windows": [
|
| 4 |
+
77188,
|
| 5 |
+
0
|
| 6 |
+
],
|
| 7 |
+
"batch_size": 2,
|
| 8 |
+
"world_size": 4,
|
| 9 |
+
"steps_per_epoch": 9648,
|
| 10 |
+
"seed": 0,
|
| 11 |
+
"sampling_mode": "uniform",
|
| 12 |
+
"validation_fingerprint": "b221879dfbe78b51c7b89cd9e3a9954ff49bf62cbae6d722bd1cf19869f19b28"
|
| 13 |
+
}
|
checkpoints/nano_action_causal_rope_v1/epoch_019.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:849387fb39d1323bc9d25d96c9c3c30123f232aba1e88274f355581a9da95b74
|
| 3 |
+
size 544776495
|
checkpoints/nano_action_causal_rope_v1/metrics.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/nano_action_causal_rope_v1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/action_normalizer.json
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": 2,
|
| 3 |
+
"sensor": {
|
| 4 |
+
"mode": "quantile",
|
| 5 |
+
"mean": [
|
| 6 |
+
-1.3721641153097153e-05,
|
| 7 |
+
0.002626492641866207,
|
| 8 |
+
0.0005233986303210258,
|
| 9 |
+
0.9972736239433289,
|
| 10 |
+
0.004918001592159271,
|
| 11 |
+
0.001323983073234558,
|
| 12 |
+
-0.005411813035607338,
|
| 13 |
+
0.9968388080596924,
|
| 14 |
+
0.0037902556359767914
|
| 15 |
+
],
|
| 16 |
+
"std": [
|
| 17 |
+
0.010827474296092987,
|
| 18 |
+
0.01424520742148161,
|
| 19 |
+
0.012470426969230175,
|
| 20 |
+
0.0027263760566711426,
|
| 21 |
+
0.0642358735203743,
|
| 22 |
+
0.05420434847474098,
|
| 23 |
+
0.06445430219173431,
|
| 24 |
+
0.0031611621379852295,
|
| 25 |
+
0.06770282983779907
|
| 26 |
+
],
|
| 27 |
+
"meta": {
|
| 28 |
+
"rows": 267720,
|
| 29 |
+
"episodes": [
|
| 30 |
+
"motherboard_0911_episode_000.pt",
|
| 31 |
+
"motherboard_0911_episode_001_seg00.pt",
|
| 32 |
+
"motherboard_0911_episode_001_seg01.pt",
|
| 33 |
+
"motherboard_0911_episode_003_seg00.pt",
|
| 34 |
+
"motherboard_0911_episode_004_seg00.pt",
|
| 35 |
+
"motherboard_0911_episode_004_seg01.pt",
|
| 36 |
+
"motherboard_0911_episode_005_seg00.pt",
|
| 37 |
+
"motherboard_0911_episode_005_seg01.pt",
|
| 38 |
+
"motherboard_0911_episode_005_seg02.pt",
|
| 39 |
+
"motherboard_0911_episode_005_seg03.pt",
|
| 40 |
+
"motherboard_0911_episode_005_seg04.pt",
|
| 41 |
+
"motherboard_0911_episode_005_seg05.pt",
|
| 42 |
+
"motherboard_0911_episode_006_seg00.pt",
|
| 43 |
+
"motherboard_0911_episode_006_seg01.pt",
|
| 44 |
+
"motherboard_0911_episode_007_seg00.pt",
|
| 45 |
+
"motherboard_0911_episode_007_seg01.pt",
|
| 46 |
+
"motherboard_0911_episode_007_seg02.pt",
|
| 47 |
+
"motherboard_0911_episode_007_seg03.pt",
|
| 48 |
+
"motherboard_0911_episode_007_seg04.pt",
|
| 49 |
+
"motherboard_0912_episode_000_seg00.pt",
|
| 50 |
+
"motherboard_0912_episode_000_seg01.pt"
|
| 51 |
+
],
|
| 52 |
+
"streams": [
|
| 53 |
+
"action_left",
|
| 54 |
+
"action_right"
|
| 55 |
+
],
|
| 56 |
+
"kind": "sensor",
|
| 57 |
+
"hands": "left+right stacked"
|
| 58 |
+
},
|
| 59 |
+
"q01": [
|
| 60 |
+
-0.010841195471584797,
|
| 61 |
+
-0.011618714779615402,
|
| 62 |
+
-0.01194702833890915,
|
| 63 |
+
0.9945472478866577,
|
| 64 |
+
-0.05931787192821503,
|
| 65 |
+
-0.052880365401506424,
|
| 66 |
+
-0.0698661133646965,
|
| 67 |
+
0.9936776757240295,
|
| 68 |
+
-0.06391257792711258
|
| 69 |
+
],
|
| 70 |
+
"q99": [
|
| 71 |
+
0.010813752189278603,
|
| 72 |
+
0.016871700063347816,
|
| 73 |
+
0.0129938255995512,
|
| 74 |
+
1.0,
|
| 75 |
+
0.06915387511253357,
|
| 76 |
+
0.05552833154797554,
|
| 77 |
+
0.05904248729348183,
|
| 78 |
+
1.0,
|
| 79 |
+
0.07149308919906616
|
| 80 |
+
]
|
| 81 |
+
},
|
| 82 |
+
"view": {
|
| 83 |
+
"mode": "quantile",
|
| 84 |
+
"mean": [
|
| 85 |
+
0.44803303480148315,
|
| 86 |
+
0.5292316675186157,
|
| 87 |
+
0.7715288400650024,
|
| 88 |
+
0.47830602526664734,
|
| 89 |
+
0.5115014314651489,
|
| 90 |
+
0.8584882616996765,
|
| 91 |
+
0.5024852752685547,
|
| 92 |
+
0.44486939907073975,
|
| 93 |
+
0.7453709840774536
|
| 94 |
+
],
|
| 95 |
+
"std": [
|
| 96 |
+
0.322518527507782,
|
| 97 |
+
0.2845557928085327,
|
| 98 |
+
0.04919716715812683,
|
| 99 |
+
0.28917667269706726,
|
| 100 |
+
0.26094764471054077,
|
| 101 |
+
0.0553051233291626,
|
| 102 |
+
0.32752352952957153,
|
| 103 |
+
0.32817262411117554,
|
| 104 |
+
0.07070961594581604
|
| 105 |
+
],
|
| 106 |
+
"meta": {
|
| 107 |
+
"rows": 285568,
|
| 108 |
+
"episodes": [
|
| 109 |
+
"motherboard_0911_episode_000.pt",
|
| 110 |
+
"motherboard_0911_episode_001_seg00.pt",
|
| 111 |
+
"motherboard_0911_episode_001_seg01.pt",
|
| 112 |
+
"motherboard_0911_episode_003_seg00.pt",
|
| 113 |
+
"motherboard_0911_episode_004_seg00.pt",
|
| 114 |
+
"motherboard_0911_episode_004_seg01.pt",
|
| 115 |
+
"motherboard_0911_episode_005_seg00.pt",
|
| 116 |
+
"motherboard_0911_episode_005_seg01.pt",
|
| 117 |
+
"motherboard_0911_episode_005_seg02.pt",
|
| 118 |
+
"motherboard_0911_episode_005_seg03.pt",
|
| 119 |
+
"motherboard_0911_episode_005_seg04.pt",
|
| 120 |
+
"motherboard_0911_episode_005_seg05.pt",
|
| 121 |
+
"motherboard_0911_episode_006_seg00.pt",
|
| 122 |
+
"motherboard_0911_episode_006_seg01.pt",
|
| 123 |
+
"motherboard_0911_episode_007_seg00.pt",
|
| 124 |
+
"motherboard_0911_episode_007_seg01.pt",
|
| 125 |
+
"motherboard_0911_episode_007_seg02.pt",
|
| 126 |
+
"motherboard_0911_episode_007_seg03.pt",
|
| 127 |
+
"motherboard_0911_episode_007_seg04.pt",
|
| 128 |
+
"motherboard_0912_episode_000_seg00.pt",
|
| 129 |
+
"motherboard_0912_episode_000_seg01.pt"
|
| 130 |
+
],
|
| 131 |
+
"streams": [
|
| 132 |
+
"view_action_left",
|
| 133 |
+
"view_action_right"
|
| 134 |
+
],
|
| 135 |
+
"kind": "view",
|
| 136 |
+
"hands": "left+right stacked"
|
| 137 |
+
},
|
| 138 |
+
"q01": [
|
| 139 |
+
0.12551453709602356,
|
| 140 |
+
0.24467585980892181,
|
| 141 |
+
0.722331702709198,
|
| 142 |
+
0.18912935256958008,
|
| 143 |
+
0.25055375695228577,
|
| 144 |
+
0.8031831383705139,
|
| 145 |
+
0.17496176064014435,
|
| 146 |
+
0.1166967824101448,
|
| 147 |
+
0.6746613383293152
|
| 148 |
+
],
|
| 149 |
+
"q99": [
|
| 150 |
+
0.7705515623092651,
|
| 151 |
+
0.8137874603271484,
|
| 152 |
+
0.8207260370254517,
|
| 153 |
+
0.7674826979637146,
|
| 154 |
+
0.7724490761756897,
|
| 155 |
+
0.9137933850288391,
|
| 156 |
+
0.8300088047981262,
|
| 157 |
+
0.7730420231819153,
|
| 158 |
+
0.8160805702209473
|
| 159 |
+
]
|
| 160 |
+
}
|
| 161 |
+
}
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/config.yaml
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: nano_action_causal_umi_sensoronly_noforce_v1
|
| 2 |
+
seed: 0
|
| 3 |
+
out_dir: /group2/ct/weihanx/world_model/vtm_cosmos/runs
|
| 4 |
+
data:
|
| 5 |
+
latent_root: /group2/ct/weihanx/world_model/mixed_mb_latents
|
| 6 |
+
action_root: /group2/ct/weihanx/world_model/mixed_mb_actions
|
| 7 |
+
action_file: sensor_actions_fps15_noforce.pt
|
| 8 |
+
view_action_file: view_actions_fps15.pt
|
| 9 |
+
split_file: /group2/ct/weihanx/world_model/vm_diffusion/config/splits/mixed_mb_wrist_holdout.json
|
| 10 |
+
optional_streams: []
|
| 11 |
+
action_norm: quantile
|
| 12 |
+
train_window_stride: 1
|
| 13 |
+
val_window_stride: 50
|
| 14 |
+
stats_window_stride: 8
|
| 15 |
+
action_stats: null
|
| 16 |
+
num_workers: 4
|
| 17 |
+
model:
|
| 18 |
+
language_model: nano
|
| 19 |
+
nano_checkpoint: /group2/ct/weihanx/world_model/checkpoints/hf/Cosmos3-Nano
|
| 20 |
+
strict_load: true
|
| 21 |
+
dtype: bfloat16
|
| 22 |
+
attention_backend: flex
|
| 23 |
+
flex_preference: flex_triton
|
| 24 |
+
temporal_mode: causal
|
| 25 |
+
attention_topology: all_streams_causal
|
| 26 |
+
task_mode: action_full
|
| 27 |
+
active_streams:
|
| 28 |
+
- video
|
| 29 |
+
- wrist_left
|
| 30 |
+
- wrist_right
|
| 31 |
+
- tactile_left
|
| 32 |
+
- tactile_right
|
| 33 |
+
- action_left
|
| 34 |
+
- action_right
|
| 35 |
+
init_tactile_from_rgb: true
|
| 36 |
+
init_action_from_nano:
|
| 37 |
+
domain: umi
|
| 38 |
+
train_time_embedder: false
|
| 39 |
+
train_rgb_projections: false
|
| 40 |
+
lora:
|
| 41 |
+
enable: true
|
| 42 |
+
rank: 16
|
| 43 |
+
alpha: 32
|
| 44 |
+
targets: attn_ffn
|
| 45 |
+
training:
|
| 46 |
+
prediction_strategy: causal_teacher_forcing
|
| 47 |
+
init_from_checkpoint: null
|
| 48 |
+
missing_stream_training: false
|
| 49 |
+
wrist_dropout:
|
| 50 |
+
probability: 0.0
|
| 51 |
+
mode: paired
|
| 52 |
+
scope: whole_clip
|
| 53 |
+
batching: group_by_active_streams
|
| 54 |
+
sampling:
|
| 55 |
+
mode: uniform
|
| 56 |
+
steps_per_epoch: null
|
| 57 |
+
val_hide_wrist: false
|
| 58 |
+
epochs: 20
|
| 59 |
+
batch_size: 2
|
| 60 |
+
lr: 5.0e-05
|
| 61 |
+
weight_decay: 0.01
|
| 62 |
+
warmup_steps: 100
|
| 63 |
+
grad_clip: 1.0
|
| 64 |
+
amp: bf16
|
| 65 |
+
sigma_distribution: waver
|
| 66 |
+
sigma_shift: 3.0
|
| 67 |
+
loss_weights: {}
|
| 68 |
+
log_every: 20
|
| 69 |
+
ckpt_every_epochs: 1
|
| 70 |
+
activation_checkpointing: true
|
| 71 |
+
inference:
|
| 72 |
+
prediction_strategy: causal_autoregressive
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/data_manifest.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fingerprint": "8f3e74e1fc09cc4d6a4c53ce13858c2834a3544c8de704321ad4ff8bde174ad3",
|
| 3 |
+
"windows": [
|
| 4 |
+
71386,
|
| 5 |
+
0
|
| 6 |
+
],
|
| 7 |
+
"batch_size": 2,
|
| 8 |
+
"world_size": 4,
|
| 9 |
+
"steps_per_epoch": 8923,
|
| 10 |
+
"seed": 0,
|
| 11 |
+
"sampling_mode": "uniform",
|
| 12 |
+
"validation_fingerprint": "d9b76ca212840e64c814357151266c0d2be613aa01adf092e6a74ad8f7007881"
|
| 13 |
+
}
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/epoch_013.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1c0519e896cc147631cb8ff6d9ae1f46ca30f72b8205c9ae2539a3d3df87d2aa
|
| 3 |
+
size 544282855
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/metrics.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/nano_action_causal_umi_sensoronly_noforce_v1/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/vae/lora_last.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8fae543fb989596ea9f19dab1ece229d2f03201b0872ecfe3eb2ed978df6b8da
|
| 3 |
+
size 122466955
|
checkpoints/vae/train_config.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"vae_pth": "/group2/ct/weihanx/Wan-AI/Wan2.2-TI2V-5B/Wan2.2_VAE.pth",
|
| 3 |
+
"csv": "/group2/ct/weihanx/world_model/data_preprocessing/freetacman_tactile_shuffled_filtered.csv",
|
| 4 |
+
"out": "./lora_out_refflow",
|
| 5 |
+
"num_frames": 17,
|
| 6 |
+
"height": 256,
|
| 7 |
+
"width": 256,
|
| 8 |
+
"val_size": 64,
|
| 9 |
+
"num_workers": 8,
|
| 10 |
+
"fps": 6.0,
|
| 11 |
+
"src_fps": 30.0,
|
| 12 |
+
"lora_r": 32,
|
| 13 |
+
"lora_alpha": 32.0,
|
| 14 |
+
"lora_dropout": 0.0,
|
| 15 |
+
"lora_targets": "all",
|
| 16 |
+
"batch_size": 1,
|
| 17 |
+
"grad_accum": 4,
|
| 18 |
+
"lr": 0.0001,
|
| 19 |
+
"weight_decay": 0.0,
|
| 20 |
+
"steps": 4000,
|
| 21 |
+
"warmup": 100,
|
| 22 |
+
"grad_clip": 1.0,
|
| 23 |
+
"lpips_weight": 0.1,
|
| 24 |
+
"edge_weight": 1.0,
|
| 25 |
+
"flow_weight": 1.0,
|
| 26 |
+
"flow_floor": 1.5,
|
| 27 |
+
"flow_suffix": "_refflow.npz",
|
| 28 |
+
"log_every": 20,
|
| 29 |
+
"val_every": 500,
|
| 30 |
+
"save_every": 1000,
|
| 31 |
+
"dtype": "bf16",
|
| 32 |
+
"seed": 0
|
| 33 |
+
}
|