File size: 11,145 Bytes
a66f0e1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 | {
"loss_args": {
"losses": "img_distill,retrieval",
"retrieval_loss_weight": 0.5,
"retrieval_queue_size": 8192,
"retrieval_loss_type": "sigmoid",
"retrieval_scale": 100.0,
"retrieval_bias": -15.0,
"retrieval_causal_positive_base": 0.2,
"distill_loss_weight": 1.0,
"temporal_memory_loss_weight": 1.0,
"temporal_memory_temperature": 0.05,
"temporal_memory_gamma": 0.8,
"order_loss_weight": 0.5,
"order_temperature": 0.07,
"order_num_negatives": 1,
"order_num_frames_per_video": 8,
"order_ema_decay": 0.0,
"order_save_prob": 0.0,
"order_save_dir": "order_debug_samples",
"order_save_max_samples": 200,
"temporal_state_loss_weight": 1.0,
"temporal_state_alpha": 1.0,
"temporal_state_stable_weight": 1.0,
"temporal_state_delta_weight": 1.0,
"temporal_state_teacher_chunk_size": 32,
"temporal_state_use_projection": true,
"pred_loss_weight": 1.0,
"history_decode_loss_weight": 1.0,
"img_distill_loss_weight": 1.0,
"img_distill_cos_weight": 1.0,
"img_distill_map_weight": 0.01,
"img_distill_retrieval_kl_weight": 1.0,
"img_distill_queue_size": 8192,
"img_distill_teacher_temperature": 0.05,
"img_distill_student_temperature": 0.1,
"img_distill_map_teacher_temperature": 0.07,
"img_distill_map_student_temperature": 0.07,
"img_distill_num_layers": [
"-1"
]
},
"model_args": {
"model_name_or_path": "encoder_only",
"dataset_type": "image_video_webdataset",
"vision_tower": "gatedeltanet_siglip_2cross_history",
"pretrain_encoder": "/home/pubw/proj/encoder/playground/models/siglip2-base-patch16-224",
"mm_projector_type": "state_proj",
"vision_checkpoint_path": null,
"use_video_boundaries": true,
"video_cut": false,
"num_temporal_layers": 4,
"temporal_layer_indices": [
-1,
-2,
-3,
-4
],
"conv_kernel_size": 4,
"state_expansion_factor": 1,
"temporal_state_init_mode": "learn",
"temporal_mlp_ratio": "2",
"image_token_index": 151655,
"video_token_index": 151656,
"vision_token_index": 151654,
"mm_hidden_size": null,
"collator_type": "video_instruction",
"target_frames_per_batch": 0,
"state_select_layer": -2,
"frames_per_stage": 16,
"max_stages": null,
"min_stages": 1,
"max_visual_stages": 16,
"model_mode": "encoder",
"vision_minimal_forward": false,
"frozen_spatial_chunk_size": 128,
"sparse_temporal_patch_ratio": 1.0
},
"data_args": {
"dataset_use": null,
"data_path": "./data/train.jsonl",
"image_data_path": "/home/pubw/proj/encoder/playground/images/cc12m_imagenet21k_recap_256_20m_wds/{00000..00045}.tar",
"video_data_path": "/home/pubw/proj/encoder/playground/videos/internvid-2m/shards_fps2_max32_jpgzip/data-{000001..000446}.tar",
"image_mix_ratio": 0.5,
"mixed_image_batch_size": 64,
"mixed_video_batch_size": 10,
"mixed_image_num_workers": 2,
"mixed_video_num_workers": 4,
"mixed_video_decode_threads": 1,
"mixed_video_oversample_factor": 1.0,
"mixed_video_target_frames": 180,
"mixed_min_frames_per_video": 8,
"mixed_max_image_repeats": 2,
"mixed_image_jitter_px": 0,
"mixed_image_jitter_brightness": 0.0,
"mixed_image_jitter_contrast": 0.0,
"mixed_image_jitter_noise_std": 0.0,
"step_mixed_image_steps": 1,
"step_mixed_video_steps": 4,
"video_root": null,
"video_metadata_cache_path": null,
"image_static_frames": 2,
"allowed_image_grids": null,
"num_frames": 4,
"fps": 2.0,
"eval_data_path": null,
"image_folder": "./data/images",
"image_min_pixels": 3136,
"image_max_pixels": 12845056,
"image_resized_width": null,
"image_resized_height": null,
"video_min_pixels": 100352,
"video_max_pixels": 602112,
"video_resized_width": null,
"video_resized_height": null,
"max_frames": 24
},
"training_args": {
"output_dir": "./output/siglip_1234_vid30_vf160_20000_sigmoid_scale100_bias015_ImgDistill4_nocut_causalretrieval",
"overwrite_output_dir": false,
"do_train": false,
"do_eval": false,
"do_predict": false,
"eval_strategy": "no",
"prediction_loss_only": false,
"per_device_train_batch_size": 1,
"per_device_eval_batch_size": 8,
"per_gpu_train_batch_size": null,
"per_gpu_eval_batch_size": null,
"gradient_accumulation_steps": 5,
"eval_accumulation_steps": null,
"eval_delay": 0,
"torch_empty_cache_steps": null,
"learning_rate": 2e-05,
"weight_decay": 0.05,
"adam_beta1": 0.9,
"adam_beta2": 0.95,
"adam_epsilon": 1e-08,
"max_grad_norm": 1.0,
"num_train_epochs": 3.0,
"max_steps": 20000,
"lr_scheduler_type": "cosine",
"lr_scheduler_kwargs": {},
"warmup_ratio": 0.03,
"warmup_steps": 0,
"log_level": "passive",
"log_level_replica": "warning",
"log_on_each_node": true,
"logging_dir": "./output/siglip_1234_vid30_vf160_20000_sigmoid_scale100_bias015_ImgDistill4_nocut_causalretrieval/runs/Aug11_13-43-12_612d7ecd0b4e",
"logging_strategy": "steps",
"logging_first_step": false,
"logging_steps": 1.0,
"logging_nan_inf_filter": true,
"save_strategy": "steps",
"save_steps": 1000,
"save_total_limit": 3,
"save_safetensors": true,
"save_on_each_node": false,
"save_only_model": false,
"restore_callback_states_from_checkpoint": false,
"no_cuda": false,
"use_cpu": false,
"use_mps_device": false,
"seed": 42,
"data_seed": null,
"jit_mode_eval": false,
"bf16": true,
"fp16": false,
"fp16_opt_level": "O1",
"half_precision_backend": "auto",
"bf16_full_eval": false,
"fp16_full_eval": false,
"tf32": true,
"local_rank": 1,
"ddp_backend": null,
"tpu_num_cores": null,
"tpu_metrics_debug": false,
"debug": [],
"dataloader_drop_last": true,
"eval_steps": null,
"dataloader_num_workers": 2,
"dataloader_prefetch_factor": 2,
"past_index": -1,
"run_name": "siglip_1234_vid30_vf160_20000_sigmoid_scale100_bias015_ImgDistill4_nocut_causalretrieval",
"disable_tqdm": false,
"remove_unused_columns": false,
"label_names": null,
"load_best_model_at_end": false,
"metric_for_best_model": null,
"greater_is_better": null,
"ignore_data_skip": true,
"fsdp": [],
"fsdp_min_num_params": 0,
"fsdp_config": {
"min_num_params": 0,
"xla": false,
"xla_fsdp_v2": false,
"xla_fsdp_grad_ckpt": false
},
"fsdp_transformer_layer_cls_to_wrap": null,
"accelerator_config": {
"split_batches": false,
"dispatch_batches": null,
"even_batches": true,
"use_seedable_sampler": true,
"non_blocking": false,
"gradient_accumulation_kwargs": null
},
"parallelism_config": null,
"deepspeed": "scripts/zero1.json",
"label_smoothing_factor": 0.0,
"optim": "adamw_torch",
"optim_args": null,
"adafactor": false,
"group_by_length": false,
"length_column_name": "length",
"report_to": [
"wandb"
],
"project": "huggingface",
"trackio_space_id": "trackio",
"ddp_find_unused_parameters": false,
"ddp_bucket_cap_mb": null,
"ddp_broadcast_buffers": null,
"dataloader_pin_memory": true,
"dataloader_persistent_workers": false,
"skip_memory_metrics": true,
"use_legacy_prediction_loop": false,
"push_to_hub": false,
"resume_from_checkpoint": "False",
"hub_model_id": null,
"hub_strategy": "every_save",
"hub_token": "<HUB_TOKEN>",
"hub_private_repo": null,
"hub_always_push": false,
"hub_revision": null,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": null,
"include_inputs_for_metrics": false,
"include_for_metrics": [],
"eval_do_concat_batches": true,
"fp16_backend": "auto",
"push_to_hub_model_id": null,
"push_to_hub_organization": null,
"push_to_hub_token": "<PUSH_TO_HUB_TOKEN>",
"mp_parameters": "",
"auto_find_batch_size": false,
"full_determinism": false,
"torchdynamo": null,
"ray_scope": "last",
"ddp_timeout": 1800,
"torch_compile": false,
"torch_compile_backend": null,
"torch_compile_mode": null,
"include_tokens_per_second": false,
"include_num_input_tokens_seen": "no",
"neftune_noise_alpha": null,
"optim_target_modules": null,
"batch_eval_metrics": false,
"eval_on_start": false,
"use_liger_kernel": true,
"liger_kernel_config": null,
"eval_use_gather_object": false,
"average_tokens_across_devices": true,
"cache_dir": null,
"double_quant": true,
"quant_type": "nf4",
"bits": 16,
"trainable_layers": 0,
"frozen_text_encoder": false,
"lora_enable": false,
"lora_rank": 64,
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_weight_path": "",
"lora_bias": "none",
"use_lora": false,
"tune_llm": false,
"tune_connector": false,
"tune_vision_temporal": true,
"tune_vision_other": false,
"tune_vision_spatial_pair_t": false,
"tune_pooling_head": false,
"max_train_samples": 20000000,
"llm_lr": 0.0,
"connector_lr": 0.0,
"vision_temporal_lr": 0.0005,
"vision_other_lr": 0.0,
"vision_spatial_lora_lr": 0.0005,
"vision_spatial_pair_t_lr": null,
"pooling_head_lr": 5e-06,
"vision_spatial_lora_enable": false,
"vision_spatial_lora_rank": 32,
"vision_spatial_lora_alpha": 32,
"vision_spatial_lora_dropout": 0.0,
"llm_sequence_packing": false,
"max_packed_tokens": 0,
"packing_buffer_size": 512,
"packing_prefetch_workers": 4,
"cross_rank_packing_balance": true,
"grad_norm_skip_threshold": 0.0,
"drop_token": "<DROP_TOKEN>",
"downsample": false,
"debug_cuda_sync_timing": false,
"log_rank_balance": true
}
} |