mishok43 commited on
Commit
c84e762
·
verified ·
1 Parent(s): 927f202

Add files using upload-large-folder tool

Browse files
tokenizers/k600-semantok/config.json ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "semantok-tokenizer/1",
3
+ "semantok": {
4
+ "name": "k600-semantok",
5
+ "arm": "semantok",
6
+ "dataset": "k600",
7
+ "tokenizer_step": 200000,
8
+ "dino_input": true,
9
+ "global_token": true,
10
+ "dino_frame_agg": "avg",
11
+ "global_token_mode": "per_frame"
12
+ },
13
+ "videoflextok": {
14
+ "video_flex_tok": {
15
+ "_target_": "videoflextok.resamplers.video_wrappers.VideoFlowMatchingResampler",
16
+ "vae": {
17
+ "_target_": "videoflextok.compressors.vidtok.VidTokVAE",
18
+ "tokenizer_id": "vidtok_kl_causal_488_16chn",
19
+ "images_read_key": "rgb",
20
+ "vae_latents_read_key": "vae_latents_reconst",
21
+ "vae_latents_write_key": "vae_latents",
22
+ "images_reconst_write_key": "rgb_reconst",
23
+ "force_vae_encode": false
24
+ },
25
+ "encoder": {
26
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
27
+ "module_dict": {
28
+ "enc_channels_to_last": {
29
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
30
+ "read_key": "vae_latents",
31
+ "write_key": "vae_latents_bthwc",
32
+ "per_sample_op": {
33
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
34
+ "_partial_": true
35
+ }
36
+ },
37
+ "enc_patch_emb": {
38
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
39
+ "input_tensor_list_read_key": "enc_dino_vae_bthwc",
40
+ "patches_list_write_key": "enc_vae_latents_patched",
41
+ "n_patches_write_key": "enc_n_patches",
42
+ "channels_in": 1040,
43
+ "dim": 1152,
44
+ "patch_sizes": [
45
+ 1,
46
+ 1,
47
+ 1
48
+ ],
49
+ "flatten_patches": false
50
+ },
51
+ "enc_posemb_module": {
52
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
53
+ "read_key": "enc_vae_latents_patched",
54
+ "write_key": "enc_vae_latents_patched",
55
+ "dim": 1152,
56
+ "max_sizes": [
57
+ 16,
58
+ 16,
59
+ 16
60
+ ],
61
+ "posemb_type": "learnable_sum",
62
+ "posemb_scaling": "absolute"
63
+ },
64
+ "enc_register_module": {
65
+ "_target_": "videoflextok.model.preprocessors.registers.RegistersTemporal2D",
66
+ "input_tensor_list_read_key": "enc_vae_latents_patched",
67
+ "register_sizes_read_write_key": "enc_register_sizes",
68
+ "registers_write_key": "enc_registers",
69
+ "dim": 1152,
70
+ "t_max": 5,
71
+ "n_min": 256,
72
+ "n_max": 256,
73
+ "size_sampling_mode": "uniform",
74
+ "ordering_mode": "nested"
75
+ },
76
+ "enc_seq_packer": {
77
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
78
+ "input_list_read_keys": [
79
+ "enc_vae_latents_patched",
80
+ "enc_registers"
81
+ ],
82
+ "packed_seq_write_key": "enc_packed_seq",
83
+ "block_mask_write_key": "enc_block_mask",
84
+ "inner_packed_shapes_write_key": "enc_ps_inner",
85
+ "outer_packed_shapes_write_key": "enc_ps_outer",
86
+ "mask_mode": "time_block_causal_last",
87
+ "pad_to_multiple": 128
88
+ },
89
+ "enc_transformer": {
90
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
91
+ "input_seq_read_key": "enc_packed_seq",
92
+ "output_seq_write_key": "enc_packed_seq",
93
+ "dim": 1152,
94
+ "depth": 18,
95
+ "block_mask_read_key": "enc_block_mask",
96
+ "use_act_checkpoint": false
97
+ },
98
+ "enc_unpacker": {
99
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
100
+ "packed_seq_read_key": "enc_packed_seq",
101
+ "inner_seq_write_keys": [
102
+ "enc_vae_latents_patched",
103
+ "enc_registers"
104
+ ],
105
+ "inner_packed_shapes_read_key": "enc_ps_inner",
106
+ "outer_packed_shapes_read_key": "enc_ps_outer"
107
+ },
108
+ "enc_to_latents": {
109
+ "_target_": "videoflextok.model.postprocessors.heads.LinearHead",
110
+ "read_key": "enc_registers",
111
+ "write_key": "enc_registers",
112
+ "dim": 1152,
113
+ "dim_out": 6,
114
+ "use_mup_readout": false,
115
+ "weight_init_style": "zero",
116
+ "dtype_override": null
117
+ }
118
+ }
119
+ },
120
+ "regularizer": {
121
+ "_target_": "videoflextok.regularizers.quantize_fsq.FSQ",
122
+ "latents_read_key": "enc_registers",
123
+ "quants_write_key": "enc_registers_quant",
124
+ "tokens_write_key": "tokens",
125
+ "levels": [
126
+ 8,
127
+ 8,
128
+ 8,
129
+ 5,
130
+ 5,
131
+ 5
132
+ ],
133
+ "drop_quant_p": 0.0,
134
+ "packed_call": false
135
+ },
136
+ "decoder": {
137
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
138
+ "module_dict": {
139
+ "dec_from_latents": {
140
+ "_target_": "videoflextok.model.preprocessors.linear.LinearLayer",
141
+ "read_key": "enc_registers_quant",
142
+ "write_key": "dec_registers_proj",
143
+ "dim_in": 6,
144
+ "dim": 1152
145
+ },
146
+ "dec_registers_posemb_module": {
147
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
148
+ "read_key": "dec_registers_proj",
149
+ "write_key": "dec_registers_proj",
150
+ "dim": 1152,
151
+ "max_sizes": [
152
+ 256,
153
+ 256
154
+ ],
155
+ "posemb_type": "learnable_sum",
156
+ "posemb_scaling": "absolute"
157
+ },
158
+ "dec_nested_dropout": {
159
+ "_target_": "videoflextok.model.preprocessors.token_dropout.MaskedNestedDropout",
160
+ "read_write_key": "dec_registers_proj",
161
+ "dim": 1152,
162
+ "size_sampling_mode": "pow2"
163
+ },
164
+ "dec_latent_dropout": {
165
+ "_target_": "videoflextok.model.preprocessors.nullcond.LearnedNullCond",
166
+ "read_write_key": "dec_registers_proj",
167
+ "dim": 1152,
168
+ "dropout_prob": 0.2
169
+ },
170
+ "dec_noise_channels_to_last": {
171
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
172
+ "read_key": "vae_latents_noised",
173
+ "write_key": "vae_latents_noised_bthwc",
174
+ "per_sample_op": {
175
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
176
+ "_partial_": true
177
+ }
178
+ },
179
+ "dec_noise_patch_emb": {
180
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
181
+ "input_tensor_list_read_key": "vae_latents_noised_bthwc",
182
+ "patches_list_write_key": "vae_latents_noised_patched",
183
+ "n_patches_write_key": "dec_n_patches",
184
+ "channels_in": 16,
185
+ "dim": 1152,
186
+ "patch_sizes": [
187
+ 1,
188
+ 1,
189
+ 1
190
+ ],
191
+ "flatten_patches": false
192
+ },
193
+ "dec_patches_posemb_module": {
194
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
195
+ "read_key": "vae_latents_noised_patched",
196
+ "write_key": "dec_patches",
197
+ "dim": 1152,
198
+ "max_sizes": [
199
+ 16,
200
+ 16,
201
+ 16
202
+ ],
203
+ "posemb_type": "learnable_sum",
204
+ "posemb_scaling": "absolute"
205
+ },
206
+ "dec_seq_packer": {
207
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
208
+ "input_list_read_keys": [
209
+ "dec_registers_proj",
210
+ "dec_patches"
211
+ ],
212
+ "packed_seq_write_key": "dec_packed_seq",
213
+ "block_mask_write_key": "dec_block_mask",
214
+ "inner_packed_shapes_write_key": "dec_ps_inner",
215
+ "outer_packed_shapes_write_key": "dec_ps_outer",
216
+ "emb_packing_fn_write_key": "emb_packing_fn",
217
+ "mask_mode": "time_block_causal",
218
+ "pad_to_multiple": 128,
219
+ "per_subseq_embs": true,
220
+ "repa_block_mask_write_key": "dec_repa_block_mask",
221
+ "repa_mask_mode": "time_block_causal",
222
+ "repa_patch_index": 1
223
+ },
224
+ "dec_time_embedder": {
225
+ "_target_": "videoflextok.model.preprocessors.time_embedding.TimestepEmbedder",
226
+ "timesteps_read_key": "timesteps",
227
+ "time_embedding_write_key": "dec_temb",
228
+ "dim": 1152,
229
+ "frequency_embedding_size": 256,
230
+ "max_timestep": 1000.0
231
+ },
232
+ "dec_transformer": {
233
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
234
+ "input_seq_read_key": "dec_packed_seq",
235
+ "output_seq_write_key": "dec_packed_seq",
236
+ "dim": 1152,
237
+ "depth": 18,
238
+ "block_mask_read_key": "dec_block_mask",
239
+ "adaLN_emb_read_key": "dec_temb",
240
+ "adaLN_packing_fn_read_key": "emb_packing_fn",
241
+ "adaLN_expansion": 2,
242
+ "use_act_checkpoint": false,
243
+ "intermediate_layer_write_key": "dec_packed_seq_repa_layer",
244
+ "intermediate_layers": [
245
+ 1
246
+ ]
247
+ },
248
+ "dec_unpacker": {
249
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
250
+ "packed_seq_read_key": "dec_packed_seq",
251
+ "inner_seq_write_keys": [
252
+ "dec_registers_proj",
253
+ "dec_patches"
254
+ ],
255
+ "inner_packed_shapes_read_key": "dec_ps_inner",
256
+ "outer_packed_shapes_read_key": "dec_ps_outer"
257
+ },
258
+ "dec_repa_unpacker": {
259
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
260
+ "packed_seq_read_key": "dec_packed_seq_repa_layer",
261
+ "inner_seq_write_keys": [
262
+ "dec_registers_repa_layer",
263
+ "dec_patches_repa_layer"
264
+ ],
265
+ "inner_packed_shapes_read_key": "dec_ps_inner",
266
+ "outer_packed_shapes_read_key": "dec_ps_outer"
267
+ },
268
+ "dec_repa_proj": {
269
+ "_target_": "videoflextok.model.postprocessors.heads.TransformerHead",
270
+ "read_key": "dec_patches_repa_layer",
271
+ "write_key": "dec_repa_pred",
272
+ "dim": 1152,
273
+ "dim_out": 1024,
274
+ "depth": 2,
275
+ "use_mup_readout": false,
276
+ "weight_init_style": "xavier",
277
+ "block_mask_read_key": "dec_repa_block_mask"
278
+ },
279
+ "dec_to_patches": {
280
+ "_target_": "videoflextok.model.postprocessors.heads.ToPatchesLinearHead",
281
+ "read_key": "dec_patches",
282
+ "write_key": "dec_patches",
283
+ "dim": 1152,
284
+ "channels_out": 16,
285
+ "patch_sizes": [
286
+ 1,
287
+ 1,
288
+ 1
289
+ ],
290
+ "use_mup_readout": false,
291
+ "weight_init_style": "zero",
292
+ "adaLN_emb_read_key": "dec_temb"
293
+ },
294
+ "dec_channels_to_first": {
295
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
296
+ "read_key": "dec_patches",
297
+ "write_key": "vae_latents_reconst",
298
+ "per_sample_op": {
299
+ "_target_": "videoflextok.model.utils.dict_ops.channels_last_to_first",
300
+ "_partial_": true
301
+ }
302
+ }
303
+ }
304
+ },
305
+ "flow_matching_noise_module": {
306
+ "_target_": "videoflextok.flow_matching.noise_modules.MinRFNoiseModule",
307
+ "clean_images_read_key": "vae_latents",
308
+ "noised_images_write_key": "vae_latents_noised",
309
+ "noise_write_key": "flow_noise",
310
+ "timesteps_write_key": "timesteps",
311
+ "sigmas_write_key": "sigmas",
312
+ "ln": false,
313
+ "stratisfied": false,
314
+ "mode_scale": 0.25
315
+ },
316
+ "pipeline": {
317
+ "_target_": "videoflextok.flow_matching.pipelines.VideoMinRFPipeline",
318
+ "_partial_": true,
319
+ "target_sizes_read_key": null,
320
+ "latents_read_key": "enc_registers_quant",
321
+ "timesteps_read_key": "timesteps",
322
+ "noised_videos_read_key": "vae_latents_noised",
323
+ "reconst_write_key": "vae_latents_reconst",
324
+ "out_channels": 16
325
+ }
326
+ },
327
+ "chunk_size": 17,
328
+ "overlap_size_frames": 0,
329
+ "vae_video_sizes": [
330
+ 5,
331
+ 16,
332
+ 16
333
+ ],
334
+ "video_preprocess_args": {
335
+ "size": 128,
336
+ "chunk_size": 17,
337
+ "overlap_size_frames": 0
338
+ }
339
+ },
340
+ "hub_vae": {
341
+ "repo": "EPFL-VILAB/videoflextok_d18_d18_k600",
342
+ "revision": "b1e468168ba7a1869206d3fc74e369c095895722"
343
+ }
344
+ }
tokenizers/k600-semantok/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c58bcee2dee218418355dad8102380a9370b3af77a0cc6212d3f0a72231523c8
3
+ size 1804312076
tokenizers/k600-videoflextok/config.json ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "semantok-tokenizer/1",
3
+ "semantok": {
4
+ "name": "k600-videoflextok",
5
+ "arm": "videoflextok",
6
+ "dataset": "k600",
7
+ "tokenizer_step": 200000,
8
+ "dino_input": false,
9
+ "global_token": false,
10
+ "dino_frame_agg": null,
11
+ "global_token_mode": null
12
+ },
13
+ "videoflextok": {
14
+ "video_flex_tok": {
15
+ "_target_": "videoflextok.resamplers.video_wrappers.VideoFlowMatchingResampler",
16
+ "vae": {
17
+ "_target_": "videoflextok.compressors.vidtok.VidTokVAE",
18
+ "tokenizer_id": "vidtok_kl_causal_488_16chn",
19
+ "images_read_key": "rgb",
20
+ "vae_latents_read_key": "vae_latents_reconst",
21
+ "vae_latents_write_key": "vae_latents",
22
+ "images_reconst_write_key": "rgb_reconst",
23
+ "force_vae_encode": false
24
+ },
25
+ "encoder": {
26
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
27
+ "module_dict": {
28
+ "enc_channels_to_last": {
29
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
30
+ "read_key": "vae_latents",
31
+ "write_key": "vae_latents_bthwc",
32
+ "per_sample_op": {
33
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
34
+ "_partial_": true
35
+ }
36
+ },
37
+ "enc_patch_emb": {
38
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
39
+ "input_tensor_list_read_key": "vae_latents_bthwc",
40
+ "patches_list_write_key": "enc_vae_latents_patched",
41
+ "n_patches_write_key": "enc_n_patches",
42
+ "channels_in": 16,
43
+ "dim": 1152,
44
+ "patch_sizes": [
45
+ 1,
46
+ 1,
47
+ 1
48
+ ],
49
+ "flatten_patches": false
50
+ },
51
+ "enc_posemb_module": {
52
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
53
+ "read_key": "enc_vae_latents_patched",
54
+ "write_key": "enc_vae_latents_patched",
55
+ "dim": 1152,
56
+ "max_sizes": [
57
+ 16,
58
+ 16,
59
+ 16
60
+ ],
61
+ "posemb_type": "learnable_sum",
62
+ "posemb_scaling": "absolute"
63
+ },
64
+ "enc_register_module": {
65
+ "_target_": "videoflextok.model.preprocessors.registers.RegistersTemporal2D",
66
+ "input_tensor_list_read_key": "enc_vae_latents_patched",
67
+ "register_sizes_read_write_key": "enc_register_sizes",
68
+ "registers_write_key": "enc_registers",
69
+ "dim": 1152,
70
+ "t_max": 5,
71
+ "n_min": 256,
72
+ "n_max": 256,
73
+ "size_sampling_mode": "uniform",
74
+ "ordering_mode": "nested"
75
+ },
76
+ "enc_seq_packer": {
77
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
78
+ "input_list_read_keys": [
79
+ "enc_vae_latents_patched",
80
+ "enc_registers"
81
+ ],
82
+ "packed_seq_write_key": "enc_packed_seq",
83
+ "block_mask_write_key": "enc_block_mask",
84
+ "inner_packed_shapes_write_key": "enc_ps_inner",
85
+ "outer_packed_shapes_write_key": "enc_ps_outer",
86
+ "mask_mode": "time_block_causal_last",
87
+ "pad_to_multiple": 128
88
+ },
89
+ "enc_transformer": {
90
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
91
+ "input_seq_read_key": "enc_packed_seq",
92
+ "output_seq_write_key": "enc_packed_seq",
93
+ "dim": 1152,
94
+ "depth": 18,
95
+ "block_mask_read_key": "enc_block_mask",
96
+ "use_act_checkpoint": false
97
+ },
98
+ "enc_unpacker": {
99
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
100
+ "packed_seq_read_key": "enc_packed_seq",
101
+ "inner_seq_write_keys": [
102
+ "enc_vae_latents_patched",
103
+ "enc_registers"
104
+ ],
105
+ "inner_packed_shapes_read_key": "enc_ps_inner",
106
+ "outer_packed_shapes_read_key": "enc_ps_outer"
107
+ },
108
+ "enc_to_latents": {
109
+ "_target_": "videoflextok.model.postprocessors.heads.LinearHead",
110
+ "read_key": "enc_registers",
111
+ "write_key": "enc_registers",
112
+ "dim": 1152,
113
+ "dim_out": 6,
114
+ "use_mup_readout": false,
115
+ "weight_init_style": "zero",
116
+ "dtype_override": null
117
+ }
118
+ }
119
+ },
120
+ "regularizer": {
121
+ "_target_": "videoflextok.regularizers.quantize_fsq.FSQ",
122
+ "latents_read_key": "enc_registers",
123
+ "quants_write_key": "enc_registers_quant",
124
+ "tokens_write_key": "tokens",
125
+ "levels": [
126
+ 8,
127
+ 8,
128
+ 8,
129
+ 5,
130
+ 5,
131
+ 5
132
+ ],
133
+ "drop_quant_p": 0.0,
134
+ "packed_call": false
135
+ },
136
+ "decoder": {
137
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
138
+ "module_dict": {
139
+ "dec_from_latents": {
140
+ "_target_": "videoflextok.model.preprocessors.linear.LinearLayer",
141
+ "read_key": "enc_registers_quant",
142
+ "write_key": "dec_registers_proj",
143
+ "dim_in": 6,
144
+ "dim": 1152
145
+ },
146
+ "dec_registers_posemb_module": {
147
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
148
+ "read_key": "dec_registers_proj",
149
+ "write_key": "dec_registers_proj",
150
+ "dim": 1152,
151
+ "max_sizes": [
152
+ 256,
153
+ 256
154
+ ],
155
+ "posemb_type": "learnable_sum",
156
+ "posemb_scaling": "absolute"
157
+ },
158
+ "dec_nested_dropout": {
159
+ "_target_": "videoflextok.model.preprocessors.token_dropout.MaskedNestedDropout",
160
+ "read_write_key": "dec_registers_proj",
161
+ "dim": 1152,
162
+ "size_sampling_mode": "pow2"
163
+ },
164
+ "dec_latent_dropout": {
165
+ "_target_": "videoflextok.model.preprocessors.nullcond.LearnedNullCond",
166
+ "read_write_key": "dec_registers_proj",
167
+ "dim": 1152,
168
+ "dropout_prob": 0.2
169
+ },
170
+ "dec_noise_channels_to_last": {
171
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
172
+ "read_key": "vae_latents_noised",
173
+ "write_key": "vae_latents_noised_bthwc",
174
+ "per_sample_op": {
175
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
176
+ "_partial_": true
177
+ }
178
+ },
179
+ "dec_noise_patch_emb": {
180
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
181
+ "input_tensor_list_read_key": "vae_latents_noised_bthwc",
182
+ "patches_list_write_key": "vae_latents_noised_patched",
183
+ "n_patches_write_key": "dec_n_patches",
184
+ "channels_in": 16,
185
+ "dim": 1152,
186
+ "patch_sizes": [
187
+ 1,
188
+ 1,
189
+ 1
190
+ ],
191
+ "flatten_patches": false
192
+ },
193
+ "dec_patches_posemb_module": {
194
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
195
+ "read_key": "vae_latents_noised_patched",
196
+ "write_key": "dec_patches",
197
+ "dim": 1152,
198
+ "max_sizes": [
199
+ 16,
200
+ 16,
201
+ 16
202
+ ],
203
+ "posemb_type": "learnable_sum",
204
+ "posemb_scaling": "absolute"
205
+ },
206
+ "dec_seq_packer": {
207
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
208
+ "input_list_read_keys": [
209
+ "dec_registers_proj",
210
+ "dec_patches"
211
+ ],
212
+ "packed_seq_write_key": "dec_packed_seq",
213
+ "block_mask_write_key": "dec_block_mask",
214
+ "inner_packed_shapes_write_key": "dec_ps_inner",
215
+ "outer_packed_shapes_write_key": "dec_ps_outer",
216
+ "emb_packing_fn_write_key": "emb_packing_fn",
217
+ "mask_mode": "time_block_causal",
218
+ "pad_to_multiple": 128,
219
+ "per_subseq_embs": true,
220
+ "repa_block_mask_write_key": "dec_repa_block_mask",
221
+ "repa_mask_mode": "time_block_causal",
222
+ "repa_patch_index": 1
223
+ },
224
+ "dec_time_embedder": {
225
+ "_target_": "videoflextok.model.preprocessors.time_embedding.TimestepEmbedder",
226
+ "timesteps_read_key": "timesteps",
227
+ "time_embedding_write_key": "dec_temb",
228
+ "dim": 1152,
229
+ "frequency_embedding_size": 256,
230
+ "max_timestep": 1000.0
231
+ },
232
+ "dec_transformer": {
233
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
234
+ "input_seq_read_key": "dec_packed_seq",
235
+ "output_seq_write_key": "dec_packed_seq",
236
+ "dim": 1152,
237
+ "depth": 18,
238
+ "block_mask_read_key": "dec_block_mask",
239
+ "adaLN_emb_read_key": "dec_temb",
240
+ "adaLN_packing_fn_read_key": "emb_packing_fn",
241
+ "adaLN_expansion": 2,
242
+ "use_act_checkpoint": false,
243
+ "intermediate_layer_write_key": "dec_packed_seq_repa_layer",
244
+ "intermediate_layers": [
245
+ 1
246
+ ]
247
+ },
248
+ "dec_unpacker": {
249
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
250
+ "packed_seq_read_key": "dec_packed_seq",
251
+ "inner_seq_write_keys": [
252
+ "dec_registers_proj",
253
+ "dec_patches"
254
+ ],
255
+ "inner_packed_shapes_read_key": "dec_ps_inner",
256
+ "outer_packed_shapes_read_key": "dec_ps_outer"
257
+ },
258
+ "dec_repa_unpacker": {
259
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
260
+ "packed_seq_read_key": "dec_packed_seq_repa_layer",
261
+ "inner_seq_write_keys": [
262
+ "dec_registers_repa_layer",
263
+ "dec_patches_repa_layer"
264
+ ],
265
+ "inner_packed_shapes_read_key": "dec_ps_inner",
266
+ "outer_packed_shapes_read_key": "dec_ps_outer"
267
+ },
268
+ "dec_repa_proj": {
269
+ "_target_": "videoflextok.model.postprocessors.heads.TransformerHead",
270
+ "read_key": "dec_patches_repa_layer",
271
+ "write_key": "dec_repa_pred",
272
+ "dim": 1152,
273
+ "dim_out": 1024,
274
+ "depth": 2,
275
+ "use_mup_readout": false,
276
+ "weight_init_style": "xavier",
277
+ "block_mask_read_key": "dec_repa_block_mask"
278
+ },
279
+ "dec_to_patches": {
280
+ "_target_": "videoflextok.model.postprocessors.heads.ToPatchesLinearHead",
281
+ "read_key": "dec_patches",
282
+ "write_key": "dec_patches",
283
+ "dim": 1152,
284
+ "channels_out": 16,
285
+ "patch_sizes": [
286
+ 1,
287
+ 1,
288
+ 1
289
+ ],
290
+ "use_mup_readout": false,
291
+ "weight_init_style": "zero",
292
+ "adaLN_emb_read_key": "dec_temb"
293
+ },
294
+ "dec_channels_to_first": {
295
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
296
+ "read_key": "dec_patches",
297
+ "write_key": "vae_latents_reconst",
298
+ "per_sample_op": {
299
+ "_target_": "videoflextok.model.utils.dict_ops.channels_last_to_first",
300
+ "_partial_": true
301
+ }
302
+ }
303
+ }
304
+ },
305
+ "flow_matching_noise_module": {
306
+ "_target_": "videoflextok.flow_matching.noise_modules.MinRFNoiseModule",
307
+ "clean_images_read_key": "vae_latents",
308
+ "noised_images_write_key": "vae_latents_noised",
309
+ "noise_write_key": "flow_noise",
310
+ "timesteps_write_key": "timesteps",
311
+ "sigmas_write_key": "sigmas",
312
+ "ln": false,
313
+ "stratisfied": false,
314
+ "mode_scale": 0.25
315
+ },
316
+ "pipeline": {
317
+ "_target_": "videoflextok.flow_matching.pipelines.VideoMinRFPipeline",
318
+ "_partial_": true,
319
+ "target_sizes_read_key": null,
320
+ "latents_read_key": "enc_registers_quant",
321
+ "timesteps_read_key": "timesteps",
322
+ "noised_videos_read_key": "vae_latents_noised",
323
+ "reconst_write_key": "vae_latents_reconst",
324
+ "out_channels": 16
325
+ }
326
+ },
327
+ "chunk_size": 17,
328
+ "overlap_size_frames": 0,
329
+ "vae_video_sizes": [
330
+ 5,
331
+ 16,
332
+ 16
333
+ ],
334
+ "video_preprocess_args": {
335
+ "size": 128,
336
+ "chunk_size": 17,
337
+ "overlap_size_frames": 0
338
+ }
339
+ },
340
+ "hub_vae": {
341
+ "repo": "EPFL-VILAB/videoflextok_d18_d18_k600",
342
+ "revision": "b1e468168ba7a1869206d3fc74e369c095895722"
343
+ }
344
+ }
tokenizers/k600-videoflextok/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc39ca1f3a4289c7b19e57ade7665b705a918651f673d5c7df2b87ab6600ca96
3
+ size 1799590964
tokenizers/uco3d-semantok/config.json ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "semantok-tokenizer/1",
3
+ "semantok": {
4
+ "name": "uco3d-semantok",
5
+ "arm": "semantok",
6
+ "dataset": "uco3d",
7
+ "tokenizer_step": 100000,
8
+ "dino_input": true,
9
+ "global_token": true,
10
+ "dino_frame_agg": "avg",
11
+ "global_token_mode": "frame0"
12
+ },
13
+ "videoflextok": {
14
+ "video_flex_tok": {
15
+ "_target_": "videoflextok.resamplers.video_wrappers.VideoFlowMatchingResampler",
16
+ "vae": {
17
+ "_target_": "videoflextok.compressors.vidtok.VidTokVAE",
18
+ "tokenizer_id": "vidtok_kl_causal_488_16chn",
19
+ "images_read_key": "rgb",
20
+ "vae_latents_read_key": "vae_latents_reconst",
21
+ "vae_latents_write_key": "vae_latents",
22
+ "images_reconst_write_key": "rgb_reconst",
23
+ "force_vae_encode": false
24
+ },
25
+ "encoder": {
26
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
27
+ "module_dict": {
28
+ "enc_channels_to_last": {
29
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
30
+ "read_key": "vae_latents",
31
+ "write_key": "vae_latents_bthwc",
32
+ "per_sample_op": {
33
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
34
+ "_partial_": true
35
+ }
36
+ },
37
+ "enc_patch_emb": {
38
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
39
+ "input_tensor_list_read_key": "enc_dino_vae_bthwc",
40
+ "patches_list_write_key": "enc_vae_latents_patched",
41
+ "n_patches_write_key": "enc_n_patches",
42
+ "channels_in": 1040,
43
+ "dim": 1152,
44
+ "patch_sizes": [
45
+ 1,
46
+ 1,
47
+ 1
48
+ ],
49
+ "flatten_patches": false
50
+ },
51
+ "enc_posemb_module": {
52
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
53
+ "read_key": "enc_vae_latents_patched",
54
+ "write_key": "enc_vae_latents_patched",
55
+ "dim": 1152,
56
+ "max_sizes": [
57
+ 16,
58
+ 16,
59
+ 16
60
+ ],
61
+ "posemb_type": "learnable_sum",
62
+ "posemb_scaling": "absolute"
63
+ },
64
+ "enc_register_module": {
65
+ "_target_": "videoflextok.model.preprocessors.registers.RegistersTemporal2D",
66
+ "input_tensor_list_read_key": "enc_vae_latents_patched",
67
+ "register_sizes_read_write_key": "enc_register_sizes",
68
+ "registers_write_key": "enc_registers",
69
+ "dim": 1152,
70
+ "t_max": 5,
71
+ "n_min": 256,
72
+ "n_max": 256,
73
+ "size_sampling_mode": "uniform",
74
+ "ordering_mode": "nested"
75
+ },
76
+ "enc_seq_packer": {
77
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
78
+ "input_list_read_keys": [
79
+ "enc_vae_latents_patched",
80
+ "enc_registers"
81
+ ],
82
+ "packed_seq_write_key": "enc_packed_seq",
83
+ "block_mask_write_key": "enc_block_mask",
84
+ "inner_packed_shapes_write_key": "enc_ps_inner",
85
+ "outer_packed_shapes_write_key": "enc_ps_outer",
86
+ "mask_mode": "time_block_causal_last",
87
+ "pad_to_multiple": 128
88
+ },
89
+ "enc_transformer": {
90
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
91
+ "input_seq_read_key": "enc_packed_seq",
92
+ "output_seq_write_key": "enc_packed_seq",
93
+ "dim": 1152,
94
+ "depth": 18,
95
+ "block_mask_read_key": "enc_block_mask",
96
+ "use_act_checkpoint": false
97
+ },
98
+ "enc_unpacker": {
99
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
100
+ "packed_seq_read_key": "enc_packed_seq",
101
+ "inner_seq_write_keys": [
102
+ "enc_vae_latents_patched",
103
+ "enc_registers"
104
+ ],
105
+ "inner_packed_shapes_read_key": "enc_ps_inner",
106
+ "outer_packed_shapes_read_key": "enc_ps_outer"
107
+ },
108
+ "enc_to_latents": {
109
+ "_target_": "videoflextok.model.postprocessors.heads.LinearHead",
110
+ "read_key": "enc_registers",
111
+ "write_key": "enc_registers",
112
+ "dim": 1152,
113
+ "dim_out": 6,
114
+ "use_mup_readout": false,
115
+ "weight_init_style": "zero",
116
+ "dtype_override": null
117
+ }
118
+ }
119
+ },
120
+ "regularizer": {
121
+ "_target_": "videoflextok.regularizers.quantize_fsq.FSQ",
122
+ "latents_read_key": "enc_registers",
123
+ "quants_write_key": "enc_registers_quant",
124
+ "tokens_write_key": "tokens",
125
+ "levels": [
126
+ 8,
127
+ 8,
128
+ 8,
129
+ 5,
130
+ 5,
131
+ 5
132
+ ],
133
+ "drop_quant_p": 0.0,
134
+ "packed_call": false
135
+ },
136
+ "decoder": {
137
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
138
+ "module_dict": {
139
+ "dec_from_latents": {
140
+ "_target_": "videoflextok.model.preprocessors.linear.LinearLayer",
141
+ "read_key": "enc_registers_quant",
142
+ "write_key": "dec_registers_proj",
143
+ "dim_in": 6,
144
+ "dim": 1152
145
+ },
146
+ "dec_registers_posemb_module": {
147
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
148
+ "read_key": "dec_registers_proj",
149
+ "write_key": "dec_registers_proj",
150
+ "dim": 1152,
151
+ "max_sizes": [
152
+ 256,
153
+ 256
154
+ ],
155
+ "posemb_type": "learnable_sum",
156
+ "posemb_scaling": "absolute"
157
+ },
158
+ "dec_nested_dropout": {
159
+ "_target_": "videoflextok.model.preprocessors.token_dropout.MaskedNestedDropout",
160
+ "read_write_key": "dec_registers_proj",
161
+ "dim": 1152,
162
+ "size_sampling_mode": "pow2"
163
+ },
164
+ "dec_latent_dropout": {
165
+ "_target_": "videoflextok.model.preprocessors.nullcond.LearnedNullCond",
166
+ "read_write_key": "dec_registers_proj",
167
+ "dim": 1152,
168
+ "dropout_prob": 0.2
169
+ },
170
+ "dec_noise_channels_to_last": {
171
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
172
+ "read_key": "vae_latents_noised",
173
+ "write_key": "vae_latents_noised_bthwc",
174
+ "per_sample_op": {
175
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
176
+ "_partial_": true
177
+ }
178
+ },
179
+ "dec_noise_patch_emb": {
180
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
181
+ "input_tensor_list_read_key": "vae_latents_noised_bthwc",
182
+ "patches_list_write_key": "vae_latents_noised_patched",
183
+ "n_patches_write_key": "dec_n_patches",
184
+ "channels_in": 16,
185
+ "dim": 1152,
186
+ "patch_sizes": [
187
+ 1,
188
+ 1,
189
+ 1
190
+ ],
191
+ "flatten_patches": false
192
+ },
193
+ "dec_patches_posemb_module": {
194
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
195
+ "read_key": "vae_latents_noised_patched",
196
+ "write_key": "dec_patches",
197
+ "dim": 1152,
198
+ "max_sizes": [
199
+ 16,
200
+ 16,
201
+ 16
202
+ ],
203
+ "posemb_type": "learnable_sum",
204
+ "posemb_scaling": "absolute"
205
+ },
206
+ "dec_seq_packer": {
207
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
208
+ "input_list_read_keys": [
209
+ "dec_registers_proj",
210
+ "dec_patches"
211
+ ],
212
+ "packed_seq_write_key": "dec_packed_seq",
213
+ "block_mask_write_key": "dec_block_mask",
214
+ "inner_packed_shapes_write_key": "dec_ps_inner",
215
+ "outer_packed_shapes_write_key": "dec_ps_outer",
216
+ "emb_packing_fn_write_key": "emb_packing_fn",
217
+ "mask_mode": "time_block_causal",
218
+ "pad_to_multiple": 128,
219
+ "per_subseq_embs": true,
220
+ "repa_block_mask_write_key": "dec_repa_block_mask",
221
+ "repa_mask_mode": "time_block_causal",
222
+ "repa_patch_index": 1
223
+ },
224
+ "dec_time_embedder": {
225
+ "_target_": "videoflextok.model.preprocessors.time_embedding.TimestepEmbedder",
226
+ "timesteps_read_key": "timesteps",
227
+ "time_embedding_write_key": "dec_temb",
228
+ "dim": 1152,
229
+ "frequency_embedding_size": 256,
230
+ "max_timestep": 1000.0
231
+ },
232
+ "dec_transformer": {
233
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
234
+ "input_seq_read_key": "dec_packed_seq",
235
+ "output_seq_write_key": "dec_packed_seq",
236
+ "dim": 1152,
237
+ "depth": 18,
238
+ "block_mask_read_key": "dec_block_mask",
239
+ "adaLN_emb_read_key": "dec_temb",
240
+ "adaLN_packing_fn_read_key": "emb_packing_fn",
241
+ "adaLN_expansion": 2,
242
+ "use_act_checkpoint": false,
243
+ "intermediate_layer_write_key": "dec_packed_seq_repa_layer",
244
+ "intermediate_layers": [
245
+ 1
246
+ ]
247
+ },
248
+ "dec_unpacker": {
249
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
250
+ "packed_seq_read_key": "dec_packed_seq",
251
+ "inner_seq_write_keys": [
252
+ "dec_registers_proj",
253
+ "dec_patches"
254
+ ],
255
+ "inner_packed_shapes_read_key": "dec_ps_inner",
256
+ "outer_packed_shapes_read_key": "dec_ps_outer"
257
+ },
258
+ "dec_repa_unpacker": {
259
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
260
+ "packed_seq_read_key": "dec_packed_seq_repa_layer",
261
+ "inner_seq_write_keys": [
262
+ "dec_registers_repa_layer",
263
+ "dec_patches_repa_layer"
264
+ ],
265
+ "inner_packed_shapes_read_key": "dec_ps_inner",
266
+ "outer_packed_shapes_read_key": "dec_ps_outer"
267
+ },
268
+ "dec_repa_proj": {
269
+ "_target_": "videoflextok.model.postprocessors.heads.TransformerHead",
270
+ "read_key": "dec_patches_repa_layer",
271
+ "write_key": "dec_repa_pred",
272
+ "dim": 1152,
273
+ "dim_out": 1024,
274
+ "depth": 2,
275
+ "use_mup_readout": false,
276
+ "weight_init_style": "xavier",
277
+ "block_mask_read_key": "dec_repa_block_mask"
278
+ },
279
+ "dec_to_patches": {
280
+ "_target_": "videoflextok.model.postprocessors.heads.ToPatchesLinearHead",
281
+ "read_key": "dec_patches",
282
+ "write_key": "dec_patches",
283
+ "dim": 1152,
284
+ "channels_out": 16,
285
+ "patch_sizes": [
286
+ 1,
287
+ 1,
288
+ 1
289
+ ],
290
+ "use_mup_readout": false,
291
+ "weight_init_style": "zero",
292
+ "adaLN_emb_read_key": "dec_temb"
293
+ },
294
+ "dec_channels_to_first": {
295
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
296
+ "read_key": "dec_patches",
297
+ "write_key": "vae_latents_reconst",
298
+ "per_sample_op": {
299
+ "_target_": "videoflextok.model.utils.dict_ops.channels_last_to_first",
300
+ "_partial_": true
301
+ }
302
+ }
303
+ }
304
+ },
305
+ "flow_matching_noise_module": {
306
+ "_target_": "videoflextok.flow_matching.noise_modules.MinRFNoiseModule",
307
+ "clean_images_read_key": "vae_latents",
308
+ "noised_images_write_key": "vae_latents_noised",
309
+ "noise_write_key": "flow_noise",
310
+ "timesteps_write_key": "timesteps",
311
+ "sigmas_write_key": "sigmas",
312
+ "ln": false,
313
+ "stratisfied": false,
314
+ "mode_scale": 0.25
315
+ },
316
+ "pipeline": {
317
+ "_target_": "videoflextok.flow_matching.pipelines.VideoMinRFPipeline",
318
+ "_partial_": true,
319
+ "target_sizes_read_key": null,
320
+ "latents_read_key": "enc_registers_quant",
321
+ "timesteps_read_key": "timesteps",
322
+ "noised_videos_read_key": "vae_latents_noised",
323
+ "reconst_write_key": "vae_latents_reconst",
324
+ "out_channels": 16
325
+ }
326
+ },
327
+ "chunk_size": 17,
328
+ "overlap_size_frames": 0,
329
+ "vae_video_sizes": [
330
+ 5,
331
+ 16,
332
+ 16
333
+ ],
334
+ "video_preprocess_args": {
335
+ "size": 128,
336
+ "chunk_size": 17,
337
+ "overlap_size_frames": 0
338
+ }
339
+ },
340
+ "hub_vae": {
341
+ "repo": "EPFL-VILAB/videoflextok_d18_d18_k600",
342
+ "revision": "b1e468168ba7a1869206d3fc74e369c095895722"
343
+ }
344
+ }
tokenizers/uco3d-semantok/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d26ee492afe0ea4b1be767f23cc059817c2f340e6878a7dc6bbf32221e870e14
3
+ size 1804312076
tokenizers/uco3d-videoflextok/config.json ADDED
@@ -0,0 +1,344 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "semantok-tokenizer/1",
3
+ "semantok": {
4
+ "name": "uco3d-videoflextok",
5
+ "arm": "videoflextok",
6
+ "dataset": "uco3d",
7
+ "tokenizer_step": 100000,
8
+ "dino_input": false,
9
+ "global_token": false,
10
+ "dino_frame_agg": null,
11
+ "global_token_mode": null
12
+ },
13
+ "videoflextok": {
14
+ "video_flex_tok": {
15
+ "_target_": "videoflextok.resamplers.video_wrappers.VideoFlowMatchingResampler",
16
+ "vae": {
17
+ "_target_": "videoflextok.compressors.vidtok.VidTokVAE",
18
+ "tokenizer_id": "vidtok_kl_causal_488_16chn",
19
+ "images_read_key": "rgb",
20
+ "vae_latents_read_key": "vae_latents_reconst",
21
+ "vae_latents_write_key": "vae_latents",
22
+ "images_reconst_write_key": "rgb_reconst",
23
+ "force_vae_encode": false
24
+ },
25
+ "encoder": {
26
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
27
+ "module_dict": {
28
+ "enc_channels_to_last": {
29
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
30
+ "read_key": "vae_latents",
31
+ "write_key": "vae_latents_bthwc",
32
+ "per_sample_op": {
33
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
34
+ "_partial_": true
35
+ }
36
+ },
37
+ "enc_patch_emb": {
38
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
39
+ "input_tensor_list_read_key": "vae_latents_bthwc",
40
+ "patches_list_write_key": "enc_vae_latents_patched",
41
+ "n_patches_write_key": "enc_n_patches",
42
+ "channels_in": 16,
43
+ "dim": 1152,
44
+ "patch_sizes": [
45
+ 1,
46
+ 1,
47
+ 1
48
+ ],
49
+ "flatten_patches": false
50
+ },
51
+ "enc_posemb_module": {
52
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
53
+ "read_key": "enc_vae_latents_patched",
54
+ "write_key": "enc_vae_latents_patched",
55
+ "dim": 1152,
56
+ "max_sizes": [
57
+ 16,
58
+ 16,
59
+ 16
60
+ ],
61
+ "posemb_type": "learnable_sum",
62
+ "posemb_scaling": "absolute"
63
+ },
64
+ "enc_register_module": {
65
+ "_target_": "videoflextok.model.preprocessors.registers.RegistersTemporal2D",
66
+ "input_tensor_list_read_key": "enc_vae_latents_patched",
67
+ "register_sizes_read_write_key": "enc_register_sizes",
68
+ "registers_write_key": "enc_registers",
69
+ "dim": 1152,
70
+ "t_max": 5,
71
+ "n_min": 256,
72
+ "n_max": 256,
73
+ "size_sampling_mode": "uniform",
74
+ "ordering_mode": "nested"
75
+ },
76
+ "enc_seq_packer": {
77
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
78
+ "input_list_read_keys": [
79
+ "enc_vae_latents_patched",
80
+ "enc_registers"
81
+ ],
82
+ "packed_seq_write_key": "enc_packed_seq",
83
+ "block_mask_write_key": "enc_block_mask",
84
+ "inner_packed_shapes_write_key": "enc_ps_inner",
85
+ "outer_packed_shapes_write_key": "enc_ps_outer",
86
+ "mask_mode": "time_block_causal_last",
87
+ "pad_to_multiple": 128
88
+ },
89
+ "enc_transformer": {
90
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
91
+ "input_seq_read_key": "enc_packed_seq",
92
+ "output_seq_write_key": "enc_packed_seq",
93
+ "dim": 1152,
94
+ "depth": 18,
95
+ "block_mask_read_key": "enc_block_mask",
96
+ "use_act_checkpoint": false
97
+ },
98
+ "enc_unpacker": {
99
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
100
+ "packed_seq_read_key": "enc_packed_seq",
101
+ "inner_seq_write_keys": [
102
+ "enc_vae_latents_patched",
103
+ "enc_registers"
104
+ ],
105
+ "inner_packed_shapes_read_key": "enc_ps_inner",
106
+ "outer_packed_shapes_read_key": "enc_ps_outer"
107
+ },
108
+ "enc_to_latents": {
109
+ "_target_": "videoflextok.model.postprocessors.heads.LinearHead",
110
+ "read_key": "enc_registers",
111
+ "write_key": "enc_registers",
112
+ "dim": 1152,
113
+ "dim_out": 6,
114
+ "use_mup_readout": false,
115
+ "weight_init_style": "zero",
116
+ "dtype_override": null
117
+ }
118
+ }
119
+ },
120
+ "regularizer": {
121
+ "_target_": "videoflextok.regularizers.quantize_fsq.FSQ",
122
+ "latents_read_key": "enc_registers",
123
+ "quants_write_key": "enc_registers_quant",
124
+ "tokens_write_key": "tokens",
125
+ "levels": [
126
+ 8,
127
+ 8,
128
+ 8,
129
+ 5,
130
+ 5,
131
+ 5
132
+ ],
133
+ "drop_quant_p": 0.0,
134
+ "packed_call": false
135
+ },
136
+ "decoder": {
137
+ "_target_": "videoflextok.model.utils.wrappers.SequentialModuleDictWrapper",
138
+ "module_dict": {
139
+ "dec_from_latents": {
140
+ "_target_": "videoflextok.model.preprocessors.linear.LinearLayer",
141
+ "read_key": "enc_registers_quant",
142
+ "write_key": "dec_registers_proj",
143
+ "dim_in": 6,
144
+ "dim": 1152
145
+ },
146
+ "dec_registers_posemb_module": {
147
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
148
+ "read_key": "dec_registers_proj",
149
+ "write_key": "dec_registers_proj",
150
+ "dim": 1152,
151
+ "max_sizes": [
152
+ 256,
153
+ 256
154
+ ],
155
+ "posemb_type": "learnable_sum",
156
+ "posemb_scaling": "absolute"
157
+ },
158
+ "dec_nested_dropout": {
159
+ "_target_": "videoflextok.model.preprocessors.token_dropout.MaskedNestedDropout",
160
+ "read_write_key": "dec_registers_proj",
161
+ "dim": 1152,
162
+ "size_sampling_mode": "pow2"
163
+ },
164
+ "dec_latent_dropout": {
165
+ "_target_": "videoflextok.model.preprocessors.nullcond.LearnedNullCond",
166
+ "read_write_key": "dec_registers_proj",
167
+ "dim": 1152,
168
+ "dropout_prob": 0.2
169
+ },
170
+ "dec_noise_channels_to_last": {
171
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
172
+ "read_key": "vae_latents_noised",
173
+ "write_key": "vae_latents_noised_bthwc",
174
+ "per_sample_op": {
175
+ "_target_": "videoflextok.model.utils.dict_ops.channels_first_to_last",
176
+ "_partial_": true
177
+ }
178
+ },
179
+ "dec_noise_patch_emb": {
180
+ "_target_": "videoflextok.model.preprocessors.patching.PatchEmbedder",
181
+ "input_tensor_list_read_key": "vae_latents_noised_bthwc",
182
+ "patches_list_write_key": "vae_latents_noised_patched",
183
+ "n_patches_write_key": "dec_n_patches",
184
+ "channels_in": 16,
185
+ "dim": 1152,
186
+ "patch_sizes": [
187
+ 1,
188
+ 1,
189
+ 1
190
+ ],
191
+ "flatten_patches": false
192
+ },
193
+ "dec_patches_posemb_module": {
194
+ "_target_": "videoflextok.model.utils.posembs.PositionalEmbeddingAdder",
195
+ "read_key": "vae_latents_noised_patched",
196
+ "write_key": "dec_patches",
197
+ "dim": 1152,
198
+ "max_sizes": [
199
+ 16,
200
+ 16,
201
+ 16
202
+ ],
203
+ "posemb_type": "learnable_sum",
204
+ "posemb_scaling": "absolute"
205
+ },
206
+ "dec_seq_packer": {
207
+ "_target_": "videoflextok.model.preprocessors.flex_seq_packing.BlockWiseSequenceInterleavePacker",
208
+ "input_list_read_keys": [
209
+ "dec_registers_proj",
210
+ "dec_patches"
211
+ ],
212
+ "packed_seq_write_key": "dec_packed_seq",
213
+ "block_mask_write_key": "dec_block_mask",
214
+ "inner_packed_shapes_write_key": "dec_ps_inner",
215
+ "outer_packed_shapes_write_key": "dec_ps_outer",
216
+ "emb_packing_fn_write_key": "emb_packing_fn",
217
+ "mask_mode": "time_block_causal",
218
+ "pad_to_multiple": 128,
219
+ "per_subseq_embs": true,
220
+ "repa_block_mask_write_key": "dec_repa_block_mask",
221
+ "repa_mask_mode": "time_block_causal",
222
+ "repa_patch_index": 1
223
+ },
224
+ "dec_time_embedder": {
225
+ "_target_": "videoflextok.model.preprocessors.time_embedding.TimestepEmbedder",
226
+ "timesteps_read_key": "timesteps",
227
+ "time_embedding_write_key": "dec_temb",
228
+ "dim": 1152,
229
+ "frequency_embedding_size": 256,
230
+ "max_timestep": 1000.0
231
+ },
232
+ "dec_transformer": {
233
+ "_target_": "videoflextok.model.trunks.transformers.FlexTransformer",
234
+ "input_seq_read_key": "dec_packed_seq",
235
+ "output_seq_write_key": "dec_packed_seq",
236
+ "dim": 1152,
237
+ "depth": 18,
238
+ "block_mask_read_key": "dec_block_mask",
239
+ "adaLN_emb_read_key": "dec_temb",
240
+ "adaLN_packing_fn_read_key": "emb_packing_fn",
241
+ "adaLN_expansion": 2,
242
+ "use_act_checkpoint": false,
243
+ "intermediate_layer_write_key": "dec_packed_seq_repa_layer",
244
+ "intermediate_layers": [
245
+ 1
246
+ ]
247
+ },
248
+ "dec_unpacker": {
249
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
250
+ "packed_seq_read_key": "dec_packed_seq",
251
+ "inner_seq_write_keys": [
252
+ "dec_registers_proj",
253
+ "dec_patches"
254
+ ],
255
+ "inner_packed_shapes_read_key": "dec_ps_inner",
256
+ "outer_packed_shapes_read_key": "dec_ps_outer"
257
+ },
258
+ "dec_repa_unpacker": {
259
+ "_target_": "videoflextok.model.postprocessors.seq_unpacking.SequenceInterleavedUnpacker",
260
+ "packed_seq_read_key": "dec_packed_seq_repa_layer",
261
+ "inner_seq_write_keys": [
262
+ "dec_registers_repa_layer",
263
+ "dec_patches_repa_layer"
264
+ ],
265
+ "inner_packed_shapes_read_key": "dec_ps_inner",
266
+ "outer_packed_shapes_read_key": "dec_ps_outer"
267
+ },
268
+ "dec_repa_proj": {
269
+ "_target_": "videoflextok.model.postprocessors.heads.TransformerHead",
270
+ "read_key": "dec_patches_repa_layer",
271
+ "write_key": "dec_repa_pred",
272
+ "dim": 1152,
273
+ "dim_out": 1024,
274
+ "depth": 2,
275
+ "use_mup_readout": false,
276
+ "weight_init_style": "xavier",
277
+ "block_mask_read_key": "dec_repa_block_mask"
278
+ },
279
+ "dec_to_patches": {
280
+ "_target_": "videoflextok.model.postprocessors.heads.ToPatchesLinearHead",
281
+ "read_key": "dec_patches",
282
+ "write_key": "dec_patches",
283
+ "dim": 1152,
284
+ "channels_out": 16,
285
+ "patch_sizes": [
286
+ 1,
287
+ 1,
288
+ 1
289
+ ],
290
+ "use_mup_readout": false,
291
+ "weight_init_style": "zero",
292
+ "adaLN_emb_read_key": "dec_temb"
293
+ },
294
+ "dec_channels_to_first": {
295
+ "_target_": "videoflextok.model.utils.dict_ops.PerSampleOp",
296
+ "read_key": "dec_patches",
297
+ "write_key": "vae_latents_reconst",
298
+ "per_sample_op": {
299
+ "_target_": "videoflextok.model.utils.dict_ops.channels_last_to_first",
300
+ "_partial_": true
301
+ }
302
+ }
303
+ }
304
+ },
305
+ "flow_matching_noise_module": {
306
+ "_target_": "videoflextok.flow_matching.noise_modules.MinRFNoiseModule",
307
+ "clean_images_read_key": "vae_latents",
308
+ "noised_images_write_key": "vae_latents_noised",
309
+ "noise_write_key": "flow_noise",
310
+ "timesteps_write_key": "timesteps",
311
+ "sigmas_write_key": "sigmas",
312
+ "ln": false,
313
+ "stratisfied": false,
314
+ "mode_scale": 0.25
315
+ },
316
+ "pipeline": {
317
+ "_target_": "videoflextok.flow_matching.pipelines.VideoMinRFPipeline",
318
+ "_partial_": true,
319
+ "target_sizes_read_key": null,
320
+ "latents_read_key": "enc_registers_quant",
321
+ "timesteps_read_key": "timesteps",
322
+ "noised_videos_read_key": "vae_latents_noised",
323
+ "reconst_write_key": "vae_latents_reconst",
324
+ "out_channels": 16
325
+ }
326
+ },
327
+ "chunk_size": 17,
328
+ "overlap_size_frames": 0,
329
+ "vae_video_sizes": [
330
+ 5,
331
+ 16,
332
+ 16
333
+ ],
334
+ "video_preprocess_args": {
335
+ "size": 128,
336
+ "chunk_size": 17,
337
+ "overlap_size_frames": 0
338
+ }
339
+ },
340
+ "hub_vae": {
341
+ "repo": "EPFL-VILAB/videoflextok_d18_d18_k600",
342
+ "revision": "b1e468168ba7a1869206d3fc74e369c095895722"
343
+ }
344
+ }
tokenizers/uco3d-videoflextok/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50a3d430d5c8c80c60775e8bf71e742bf8f3078fc140075b2f7d6b946a7f04af
3
+ size 1799590964