root commited on
Commit ·
5387d19
0
Parent(s):
Upload fine-tuned Phi-4 multimodal instruct model
Browse files- .gitattributes +3 -0
- added_tokens.json +12 -0
- args.json +327 -0
- checksums_machine1.md5 +0 -0
- config.json +221 -0
- configuration_phi4mm.py +235 -0
- generation_config.json +10 -0
- logging.jsonl +0 -0
- merges.txt +0 -0
- model-00001-of-00003.safetensors +3 -0
- model-00002-of-00003.safetensors +3 -0
- model-00003-of-00003.safetensors +3 -0
- model.safetensors.index.json +0 -0
- modeling_phi4mm.py +0 -0
- optimizer.pt +3 -0
- preprocessor_config.json +0 -0
- processing_phi4mm.py +733 -0
- processor_config.json +6 -0
- rng_state.pth +0 -0
- scheduler.pt +3 -0
- special_tokens_map.json +30 -0
- speech_conformer_encoder.py +0 -0
- tokenizer_config.json +130 -0
- trainer_state.json +2596 -0
- training_args.bin +3 -0
- vision_siglip_navit.py +1717 -0
- vocab.json +0 -0
.gitattributes
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
added_tokens.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<|/tool_call|>": 200026,
|
| 3 |
+
"<|/tool|>": 200024,
|
| 4 |
+
"<|assistant|>": 200019,
|
| 5 |
+
"<|end|>": 200020,
|
| 6 |
+
"<|system|>": 200022,
|
| 7 |
+
"<|tag|>": 200028,
|
| 8 |
+
"<|tool_call|>": 200025,
|
| 9 |
+
"<|tool_response|>": 200027,
|
| 10 |
+
"<|tool|>": 200023,
|
| 11 |
+
"<|user|>": 200021
|
| 12 |
+
}
|
args.json
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "LLM-Research/Phi-4-multimodal-instruct",
|
| 3 |
+
"model_type": "phi4_multimodal",
|
| 4 |
+
"model_revision": null,
|
| 5 |
+
"task_type": "causal_lm",
|
| 6 |
+
"torch_dtype": "bfloat16",
|
| 7 |
+
"attn_impl": null,
|
| 8 |
+
"num_labels": null,
|
| 9 |
+
"rope_scaling": null,
|
| 10 |
+
"device_map": null,
|
| 11 |
+
"max_memory": {},
|
| 12 |
+
"local_repo_path": null,
|
| 13 |
+
"template": "phi4_multimodal",
|
| 14 |
+
"system": null,
|
| 15 |
+
"max_length": 2048,
|
| 16 |
+
"truncation_strategy": "delete",
|
| 17 |
+
"max_pixels": null,
|
| 18 |
+
"tools_prompt": "react_en",
|
| 19 |
+
"norm_bbox": null,
|
| 20 |
+
"response_prefix": null,
|
| 21 |
+
"padding_side": "right",
|
| 22 |
+
"loss_scale": "default",
|
| 23 |
+
"sequence_parallel_size": 1,
|
| 24 |
+
"use_chat_template": true,
|
| 25 |
+
"template_backend": "swift",
|
| 26 |
+
"dataset": [
|
| 27 |
+
"AI-ModelScope/LaTeX_OCR:human_handwrite#20000"
|
| 28 |
+
],
|
| 29 |
+
"val_dataset": [],
|
| 30 |
+
"split_dataset_ratio": 0.01,
|
| 31 |
+
"data_seed": 42,
|
| 32 |
+
"dataset_num_proc": 1,
|
| 33 |
+
"streaming": false,
|
| 34 |
+
"enable_cache": false,
|
| 35 |
+
"download_mode": "reuse_dataset_if_exists",
|
| 36 |
+
"columns": {},
|
| 37 |
+
"strict": false,
|
| 38 |
+
"remove_unused_columns": true,
|
| 39 |
+
"model_name": [
|
| 40 |
+
null,
|
| 41 |
+
null
|
| 42 |
+
],
|
| 43 |
+
"model_author": [
|
| 44 |
+
null,
|
| 45 |
+
null
|
| 46 |
+
],
|
| 47 |
+
"custom_dataset_info": [],
|
| 48 |
+
"quant_method": null,
|
| 49 |
+
"quant_bits": null,
|
| 50 |
+
"hqq_axis": null,
|
| 51 |
+
"bnb_4bit_compute_dtype": "bfloat16",
|
| 52 |
+
"bnb_4bit_quant_type": "nf4",
|
| 53 |
+
"bnb_4bit_use_double_quant": true,
|
| 54 |
+
"bnb_4bit_quant_storage": null,
|
| 55 |
+
"max_new_tokens": 64,
|
| 56 |
+
"temperature": 0.0,
|
| 57 |
+
"top_k": null,
|
| 58 |
+
"top_p": null,
|
| 59 |
+
"repetition_penalty": null,
|
| 60 |
+
"num_beams": 1,
|
| 61 |
+
"stream": false,
|
| 62 |
+
"stop_words": [],
|
| 63 |
+
"logprobs": false,
|
| 64 |
+
"top_logprobs": null,
|
| 65 |
+
"ckpt_dir": null,
|
| 66 |
+
"load_dataset_config": null,
|
| 67 |
+
"lora_modules": [],
|
| 68 |
+
"tuner_backend": "peft",
|
| 69 |
+
"train_type": "dummy",
|
| 70 |
+
"adapters": [],
|
| 71 |
+
"external_plugins": [],
|
| 72 |
+
"seed": 42,
|
| 73 |
+
"model_kwargs": {},
|
| 74 |
+
"load_args": false,
|
| 75 |
+
"load_data_args": false,
|
| 76 |
+
"use_hf": false,
|
| 77 |
+
"hub_token": null,
|
| 78 |
+
"custom_register_path": [],
|
| 79 |
+
"ignore_args_error": false,
|
| 80 |
+
"use_swift_lora": false,
|
| 81 |
+
"output_dir": "/root/autodl-tmp/ms-swift/output/v3-20250320-230237",
|
| 82 |
+
"overwrite_output_dir": false,
|
| 83 |
+
"do_train": false,
|
| 84 |
+
"do_eval": false,
|
| 85 |
+
"do_predict": false,
|
| 86 |
+
"eval_strategy": "steps",
|
| 87 |
+
"prediction_loss_only": false,
|
| 88 |
+
"per_device_train_batch_size": 1,
|
| 89 |
+
"per_device_eval_batch_size": 1,
|
| 90 |
+
"per_gpu_train_batch_size": null,
|
| 91 |
+
"per_gpu_eval_batch_size": null,
|
| 92 |
+
"gradient_accumulation_steps": 16,
|
| 93 |
+
"eval_accumulation_steps": null,
|
| 94 |
+
"eval_delay": 0,
|
| 95 |
+
"torch_empty_cache_steps": null,
|
| 96 |
+
"learning_rate": 0.0001,
|
| 97 |
+
"weight_decay": 0.1,
|
| 98 |
+
"adam_beta1": 0.9,
|
| 99 |
+
"adam_beta2": 0.95,
|
| 100 |
+
"adam_epsilon": 1e-08,
|
| 101 |
+
"max_grad_norm": 1.0,
|
| 102 |
+
"num_train_epochs": 1.0,
|
| 103 |
+
"max_steps": -1,
|
| 104 |
+
"lr_scheduler_type": "cosine",
|
| 105 |
+
"lr_scheduler_kwargs": null,
|
| 106 |
+
"warmup_ratio": 0.05,
|
| 107 |
+
"warmup_steps": 0,
|
| 108 |
+
"log_level": "passive",
|
| 109 |
+
"log_level_replica": "warning",
|
| 110 |
+
"log_on_each_node": true,
|
| 111 |
+
"logging_dir": "/root/autodl-tmp/ms-swift/output/v3-20250320-230237/runs",
|
| 112 |
+
"logging_strategy": "steps",
|
| 113 |
+
"logging_first_step": true,
|
| 114 |
+
"logging_steps": 5,
|
| 115 |
+
"logging_nan_inf_filter": true,
|
| 116 |
+
"save_strategy": "steps",
|
| 117 |
+
"save_steps": 200.0,
|
| 118 |
+
"save_total_limit": 5,
|
| 119 |
+
"save_safetensors": true,
|
| 120 |
+
"save_on_each_node": false,
|
| 121 |
+
"save_only_model": false,
|
| 122 |
+
"restore_callback_states_from_checkpoint": false,
|
| 123 |
+
"no_cuda": false,
|
| 124 |
+
"use_cpu": false,
|
| 125 |
+
"use_mps_device": false,
|
| 126 |
+
"jit_mode_eval": false,
|
| 127 |
+
"use_ipex": false,
|
| 128 |
+
"bf16": true,
|
| 129 |
+
"fp16": false,
|
| 130 |
+
"fp16_opt_level": "O1",
|
| 131 |
+
"half_precision_backend": "auto",
|
| 132 |
+
"bf16_full_eval": false,
|
| 133 |
+
"fp16_full_eval": false,
|
| 134 |
+
"tf32": null,
|
| 135 |
+
"local_rank": -1,
|
| 136 |
+
"ddp_backend": null,
|
| 137 |
+
"tpu_num_cores": null,
|
| 138 |
+
"tpu_metrics_debug": false,
|
| 139 |
+
"debug": null,
|
| 140 |
+
"dataloader_drop_last": false,
|
| 141 |
+
"eval_steps": 200.0,
|
| 142 |
+
"dataloader_num_workers": 4,
|
| 143 |
+
"dataloader_prefetch_factor": null,
|
| 144 |
+
"past_index": -1,
|
| 145 |
+
"run_name": null,
|
| 146 |
+
"disable_tqdm": null,
|
| 147 |
+
"label_names": null,
|
| 148 |
+
"load_best_model_at_end": false,
|
| 149 |
+
"metric_for_best_model": "loss",
|
| 150 |
+
"greater_is_better": false,
|
| 151 |
+
"ignore_data_skip": false,
|
| 152 |
+
"fsdp": "",
|
| 153 |
+
"fsdp_min_num_params": 0,
|
| 154 |
+
"fsdp_config": null,
|
| 155 |
+
"fsdp_transformer_layer_cls_to_wrap": null,
|
| 156 |
+
"accelerator_config": {
|
| 157 |
+
"dispatch_batches": false
|
| 158 |
+
},
|
| 159 |
+
"deepspeed": null,
|
| 160 |
+
"label_smoothing_factor": 0.0,
|
| 161 |
+
"optim": "adamw_torch",
|
| 162 |
+
"optim_args": null,
|
| 163 |
+
"adafactor": false,
|
| 164 |
+
"group_by_length": false,
|
| 165 |
+
"length_column_name": "length",
|
| 166 |
+
"report_to": [
|
| 167 |
+
"tensorboard"
|
| 168 |
+
],
|
| 169 |
+
"ddp_find_unused_parameters": null,
|
| 170 |
+
"ddp_bucket_cap_mb": null,
|
| 171 |
+
"ddp_broadcast_buffers": null,
|
| 172 |
+
"dataloader_pin_memory": true,
|
| 173 |
+
"dataloader_persistent_workers": false,
|
| 174 |
+
"skip_memory_metrics": true,
|
| 175 |
+
"use_legacy_prediction_loop": false,
|
| 176 |
+
"push_to_hub": false,
|
| 177 |
+
"resume_from_checkpoint": null,
|
| 178 |
+
"hub_model_id": null,
|
| 179 |
+
"hub_strategy": "every_save",
|
| 180 |
+
"hub_private_repo": null,
|
| 181 |
+
"hub_always_push": false,
|
| 182 |
+
"gradient_checkpointing": true,
|
| 183 |
+
"gradient_checkpointing_kwargs": null,
|
| 184 |
+
"include_inputs_for_metrics": false,
|
| 185 |
+
"include_for_metrics": [],
|
| 186 |
+
"eval_do_concat_batches": true,
|
| 187 |
+
"fp16_backend": "auto",
|
| 188 |
+
"evaluation_strategy": "steps",
|
| 189 |
+
"push_to_hub_model_id": null,
|
| 190 |
+
"push_to_hub_organization": null,
|
| 191 |
+
"push_to_hub_token": null,
|
| 192 |
+
"mp_parameters": "",
|
| 193 |
+
"auto_find_batch_size": false,
|
| 194 |
+
"full_determinism": false,
|
| 195 |
+
"torchdynamo": null,
|
| 196 |
+
"ray_scope": "last",
|
| 197 |
+
"ddp_timeout": 1800,
|
| 198 |
+
"torch_compile": false,
|
| 199 |
+
"torch_compile_backend": null,
|
| 200 |
+
"torch_compile_mode": null,
|
| 201 |
+
"dispatch_batches": null,
|
| 202 |
+
"split_batches": null,
|
| 203 |
+
"include_tokens_per_second": false,
|
| 204 |
+
"include_num_input_tokens_seen": false,
|
| 205 |
+
"neftune_noise_alpha": null,
|
| 206 |
+
"optim_target_modules": null,
|
| 207 |
+
"batch_eval_metrics": false,
|
| 208 |
+
"eval_on_start": false,
|
| 209 |
+
"use_liger_kernel": false,
|
| 210 |
+
"eval_use_gather_object": false,
|
| 211 |
+
"average_tokens_across_devices": false,
|
| 212 |
+
"sortish_sampler": false,
|
| 213 |
+
"predict_with_generate": false,
|
| 214 |
+
"generation_max_length": null,
|
| 215 |
+
"generation_num_beams": null,
|
| 216 |
+
"generation_config": null,
|
| 217 |
+
"check_model": true,
|
| 218 |
+
"acc_strategy": "token",
|
| 219 |
+
"train_sampler_random": true,
|
| 220 |
+
"metric_warmup_step": 0,
|
| 221 |
+
"fsdp_num": 1,
|
| 222 |
+
"acc_steps": 1,
|
| 223 |
+
"eval_use_evalscope": false,
|
| 224 |
+
"eval_datasets": [],
|
| 225 |
+
"eval_limit": null,
|
| 226 |
+
"eval_datasets_args": null,
|
| 227 |
+
"eval_generation_config": null,
|
| 228 |
+
"freeze_parameters": [
|
| 229 |
+
"model.embed_tokens_extend.image_embed.img_processor",
|
| 230 |
+
"model.embed_tokens_extend.audio_embed.encoder",
|
| 231 |
+
"model.embed_tokens_extend.image_embed.img_projection",
|
| 232 |
+
"model.embed_tokens_extend.audio_embed.audio_projection"
|
| 233 |
+
],
|
| 234 |
+
"freeze_parameters_ratio": 0.0,
|
| 235 |
+
"trainable_parameters": [],
|
| 236 |
+
"freeze_llm": false,
|
| 237 |
+
"freeze_vit": true,
|
| 238 |
+
"freeze_aligner": true,
|
| 239 |
+
"target_modules": [
|
| 240 |
+
"all-linear"
|
| 241 |
+
],
|
| 242 |
+
"target_regex": null,
|
| 243 |
+
"modules_to_save": [],
|
| 244 |
+
"lora_rank": 8,
|
| 245 |
+
"lora_alpha": 32,
|
| 246 |
+
"lora_dropout": 0.05,
|
| 247 |
+
"lora_bias": "none",
|
| 248 |
+
"lora_dtype": null,
|
| 249 |
+
"lorap_lr_ratio": null,
|
| 250 |
+
"use_rslora": false,
|
| 251 |
+
"use_dora": false,
|
| 252 |
+
"lora_ga_batch_size": 2,
|
| 253 |
+
"lora_ga_iters": 2,
|
| 254 |
+
"lora_ga_max_length": 1024,
|
| 255 |
+
"lora_ga_direction": "ArB2r",
|
| 256 |
+
"lora_ga_scale": "stable",
|
| 257 |
+
"lora_ga_stable_gamma": 16,
|
| 258 |
+
"init_weights": true,
|
| 259 |
+
"fourier_n_frequency": 2000,
|
| 260 |
+
"fourier_scaling": 300.0,
|
| 261 |
+
"boft_block_size": 4,
|
| 262 |
+
"boft_block_num": 0,
|
| 263 |
+
"boft_n_butterfly_factor": 1,
|
| 264 |
+
"boft_dropout": 0.0,
|
| 265 |
+
"vera_rank": 256,
|
| 266 |
+
"vera_projection_prng_key": 0,
|
| 267 |
+
"vera_dropout": 0.0,
|
| 268 |
+
"vera_d_initial": 0.1,
|
| 269 |
+
"adapter_act": "gelu",
|
| 270 |
+
"adapter_length": 128,
|
| 271 |
+
"use_galore": false,
|
| 272 |
+
"galore_target_modules": null,
|
| 273 |
+
"galore_rank": 128,
|
| 274 |
+
"galore_update_proj_gap": 50,
|
| 275 |
+
"galore_scale": 1.0,
|
| 276 |
+
"galore_proj_type": "std",
|
| 277 |
+
"galore_optim_per_parameter": false,
|
| 278 |
+
"galore_with_embedding": false,
|
| 279 |
+
"galore_quantization": false,
|
| 280 |
+
"galore_proj_quant": false,
|
| 281 |
+
"galore_proj_bits": 4,
|
| 282 |
+
"galore_proj_group_size": 256,
|
| 283 |
+
"galore_cos_threshold": 0.4,
|
| 284 |
+
"galore_gamma_proj": 2,
|
| 285 |
+
"galore_queue_size": 5,
|
| 286 |
+
"adalora_target_r": 8,
|
| 287 |
+
"adalora_init_r": 12,
|
| 288 |
+
"adalora_tinit": 0,
|
| 289 |
+
"adalora_tfinal": 0,
|
| 290 |
+
"adalora_deltaT": 1,
|
| 291 |
+
"adalora_beta1": 0.85,
|
| 292 |
+
"adalora_beta2": 0.85,
|
| 293 |
+
"adalora_orth_reg_weight": 0.5,
|
| 294 |
+
"llamapro_num_new_blocks": 4,
|
| 295 |
+
"llamapro_num_groups": null,
|
| 296 |
+
"lisa_activated_layers": 0,
|
| 297 |
+
"lisa_step_interval": 20,
|
| 298 |
+
"reft_layer_key": null,
|
| 299 |
+
"reft_layers": null,
|
| 300 |
+
"reft_rank": 4,
|
| 301 |
+
"reft_intervention_type": "LoreftIntervention",
|
| 302 |
+
"reft_args": null,
|
| 303 |
+
"use_liger": false,
|
| 304 |
+
"swanlab_token": null,
|
| 305 |
+
"swanlab_project": null,
|
| 306 |
+
"swanlab_workspace": null,
|
| 307 |
+
"swanlab_exp_name": null,
|
| 308 |
+
"swanlab_mode": "cloud",
|
| 309 |
+
"add_version": true,
|
| 310 |
+
"resume_only_model": false,
|
| 311 |
+
"create_checkpoint_symlink": false,
|
| 312 |
+
"packing": false,
|
| 313 |
+
"lazy_tokenize": true,
|
| 314 |
+
"loss_type": null,
|
| 315 |
+
"optimizer": null,
|
| 316 |
+
"metric": null,
|
| 317 |
+
"zero_hpz_partition_size": null,
|
| 318 |
+
"rank": -1,
|
| 319 |
+
"global_world_size": 1,
|
| 320 |
+
"local_world_size": 1,
|
| 321 |
+
"model_suffix": "Phi-4-multimodal-instruct",
|
| 322 |
+
"model_info": "ModelInfo(model_type='phi4_multimodal', model_dir='/root/autodl-tmp/.cache/modelscope/models/LLM-Research/Phi-4-multimodal-instruct', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling={'long_factor': [1, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77], 'short_factor': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'type': 'longrope'}, config=None, task_type='causal_lm', num_labels=None)",
|
| 323 |
+
"model_meta": "ModelMeta(model_type='phi4_multimodal', model_groups=[ModelGroup(models=[Model(ms_model_id='LLM-Research/Phi-4-multimodal-instruct', hf_model_id='microsoft/Phi-4-multimodal-instruct', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='phi4_multimodal', get_function=<function get_model_tokenizer_phi4_multimodal at 0x7f09dbe314e0>, model_arch='phi4_multimodal', architectures=['Phi4MMForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=['*.zip', '*.gguf', '*.pth', '*.pt', 'consolidated*', 'onnx/*', '*.safetensors.md', '*.msgpack', '*.onnx', '*.ot', '*.h5', '*.bin', '*.safetensors'], requires=['transformers>=4.36,<4.49', 'backoff', 'soundfile'], tags=[])",
|
| 324 |
+
"model_dir": "/root/autodl-tmp/.cache/modelscope/models/LLM-Research/Phi-4-multimodal-instruct",
|
| 325 |
+
"hub": "<class 'swift.hub.hub.MSHub'>",
|
| 326 |
+
"training_args": "Seq2SeqTrainingArguments(output_dir='/root/autodl-tmp/ms-swift/output/v3-20250320-230237', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=0.0001, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/root/autodl-tmp/ms-swift/output/v3-20250320-230237/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=200, save_total_limit=5, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=200, dataloader_num_workers=4, dataloader_prefetch_factor=None, past_index=-1, run_name='/root/autodl-tmp/ms-swift/output/v3-20250320-230237', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', evaluation_strategy='steps', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, split_batches=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, check_model=True, acc_strategy='token', train_sampler_random=True, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='dummy', optimizer=None, local_repo_path=None, galore_config=None)"
|
| 327 |
+
}
|
checksums_machine1.md5
ADDED
|
File without changes
|
config.json
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "/root/autodl-tmp/.cache/modelscope/models/LLM-Research/Phi-4-multimodal-instruct",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"Phi4MMForCausalLM"
|
| 5 |
+
],
|
| 6 |
+
"attention_bias": false,
|
| 7 |
+
"attention_dropout": 0.0,
|
| 8 |
+
"audio_processor": {
|
| 9 |
+
"config": {
|
| 10 |
+
"activation": "swish",
|
| 11 |
+
"activation_checkpointing": {
|
| 12 |
+
"interval": 1,
|
| 13 |
+
"module": "transformer",
|
| 14 |
+
"offload": false
|
| 15 |
+
},
|
| 16 |
+
"attention_dim": 1024,
|
| 17 |
+
"attention_heads": 16,
|
| 18 |
+
"batch_norm": false,
|
| 19 |
+
"bias_in_glu": true,
|
| 20 |
+
"causal": true,
|
| 21 |
+
"chunk_size": -1,
|
| 22 |
+
"cnn_layer_norm": true,
|
| 23 |
+
"conv_activation": "swish",
|
| 24 |
+
"conv_glu_type": "swish",
|
| 25 |
+
"depthwise_multiplier": 1,
|
| 26 |
+
"depthwise_seperable_out_channel": 1024,
|
| 27 |
+
"dropout_rate": 0.0,
|
| 28 |
+
"encoder_embedding_config": {
|
| 29 |
+
"input_size": 80
|
| 30 |
+
},
|
| 31 |
+
"ext_pw_kernel_size": 1,
|
| 32 |
+
"ext_pw_out_channel": 1024,
|
| 33 |
+
"input_layer": "nemo_conv",
|
| 34 |
+
"input_size": 80,
|
| 35 |
+
"kernel_size": 3,
|
| 36 |
+
"left_chunk": 18,
|
| 37 |
+
"linear_units": 1536,
|
| 38 |
+
"nemo_conv_settings": {
|
| 39 |
+
"conv_channels": 1024
|
| 40 |
+
},
|
| 41 |
+
"num_blocks": 24,
|
| 42 |
+
"relative_attention_bias_args": {
|
| 43 |
+
"t5_bias_max_distance": 500,
|
| 44 |
+
"type": "t5"
|
| 45 |
+
},
|
| 46 |
+
"time_reduction": 8
|
| 47 |
+
},
|
| 48 |
+
"name": "cascades"
|
| 49 |
+
},
|
| 50 |
+
"auto_map": {
|
| 51 |
+
"AutoConfig": "configuration_phi4mm.Phi4MMConfig",
|
| 52 |
+
"AutoModelForCausalLM": "modeling_phi4mm.Phi4MMForCausalLM",
|
| 53 |
+
"AutoTokenizer": "Xenova/gpt-4o"
|
| 54 |
+
},
|
| 55 |
+
"bos_token_id": 199999,
|
| 56 |
+
"embd_layer": {
|
| 57 |
+
"audio_embd_layer": {
|
| 58 |
+
"compression_rate": 8,
|
| 59 |
+
"downsample_rate": 1,
|
| 60 |
+
"embedding_cls": "audio",
|
| 61 |
+
"enable_gradient_checkpointing": true,
|
| 62 |
+
"projection_cls": "mlp",
|
| 63 |
+
"use_conv_downsample": false,
|
| 64 |
+
"use_qformer": false
|
| 65 |
+
},
|
| 66 |
+
"embedding_cls": "image_audio",
|
| 67 |
+
"image_embd_layer": {
|
| 68 |
+
"crop_size": 448,
|
| 69 |
+
"embedding_cls": "tune_image",
|
| 70 |
+
"enable_gradient_checkpointing": true,
|
| 71 |
+
"hd_transform_order": "sub_glb",
|
| 72 |
+
"image_token_compression_cls": "avg_pool_2d",
|
| 73 |
+
"projection_cls": "mlp",
|
| 74 |
+
"use_hd_transform": true,
|
| 75 |
+
"with_learnable_separator": true
|
| 76 |
+
}
|
| 77 |
+
},
|
| 78 |
+
"embd_pdrop": 0.0,
|
| 79 |
+
"eos_token_id": 199999,
|
| 80 |
+
"full_attn_mod": 1,
|
| 81 |
+
"hidden_act": "silu",
|
| 82 |
+
"hidden_size": 3072,
|
| 83 |
+
"img_processor": null,
|
| 84 |
+
"initializer_range": 0.02,
|
| 85 |
+
"intermediate_size": 8192,
|
| 86 |
+
"interpolate_factor": 1,
|
| 87 |
+
"lm_head_bias": false,
|
| 88 |
+
"max_position_embeddings": 131072,
|
| 89 |
+
"mlp_bias": false,
|
| 90 |
+
"model_type": "phi4mm",
|
| 91 |
+
"num_attention_heads": 24,
|
| 92 |
+
"num_hidden_layers": 32,
|
| 93 |
+
"num_key_value_heads": 8,
|
| 94 |
+
"original_max_position_embeddings": 4096,
|
| 95 |
+
"pad_token_id": 199999,
|
| 96 |
+
"partial_rotary_factor": 0.75,
|
| 97 |
+
"resid_pdrop": 0.0,
|
| 98 |
+
"rms_norm_eps": 1e-05,
|
| 99 |
+
"rope_scaling": {
|
| 100 |
+
"long_factor": [
|
| 101 |
+
1,
|
| 102 |
+
1.118320672,
|
| 103 |
+
1.250641126,
|
| 104 |
+
1.398617824,
|
| 105 |
+
1.564103225,
|
| 106 |
+
1.74916897,
|
| 107 |
+
1.956131817,
|
| 108 |
+
2.187582649,
|
| 109 |
+
2.446418898,
|
| 110 |
+
2.735880826,
|
| 111 |
+
3.059592084,
|
| 112 |
+
3.421605075,
|
| 113 |
+
3.826451687,
|
| 114 |
+
4.279200023,
|
| 115 |
+
4.785517845,
|
| 116 |
+
5.351743533,
|
| 117 |
+
5.984965424,
|
| 118 |
+
6.693110555,
|
| 119 |
+
7.485043894,
|
| 120 |
+
8.370679318,
|
| 121 |
+
9.36110372,
|
| 122 |
+
10.4687158,
|
| 123 |
+
11.70738129,
|
| 124 |
+
13.09260651,
|
| 125 |
+
14.64173252,
|
| 126 |
+
16.37415215,
|
| 127 |
+
18.31155283,
|
| 128 |
+
20.47818807,
|
| 129 |
+
22.90118105,
|
| 130 |
+
25.61086418,
|
| 131 |
+
28.64115884,
|
| 132 |
+
32.03,
|
| 133 |
+
32.1,
|
| 134 |
+
32.13,
|
| 135 |
+
32.23,
|
| 136 |
+
32.6,
|
| 137 |
+
32.61,
|
| 138 |
+
32.64,
|
| 139 |
+
32.66,
|
| 140 |
+
32.7,
|
| 141 |
+
32.71,
|
| 142 |
+
32.93,
|
| 143 |
+
32.97,
|
| 144 |
+
33.28,
|
| 145 |
+
33.49,
|
| 146 |
+
33.5,
|
| 147 |
+
44.16,
|
| 148 |
+
47.77
|
| 149 |
+
],
|
| 150 |
+
"short_factor": [
|
| 151 |
+
1.0,
|
| 152 |
+
1.0,
|
| 153 |
+
1.0,
|
| 154 |
+
1.0,
|
| 155 |
+
1.0,
|
| 156 |
+
1.0,
|
| 157 |
+
1.0,
|
| 158 |
+
1.0,
|
| 159 |
+
1.0,
|
| 160 |
+
1.0,
|
| 161 |
+
1.0,
|
| 162 |
+
1.0,
|
| 163 |
+
1.0,
|
| 164 |
+
1.0,
|
| 165 |
+
1.0,
|
| 166 |
+
1.0,
|
| 167 |
+
1.0,
|
| 168 |
+
1.0,
|
| 169 |
+
1.0,
|
| 170 |
+
1.0,
|
| 171 |
+
1.0,
|
| 172 |
+
1.0,
|
| 173 |
+
1.0,
|
| 174 |
+
1.0,
|
| 175 |
+
1.0,
|
| 176 |
+
1.0,
|
| 177 |
+
1.0,
|
| 178 |
+
1.0,
|
| 179 |
+
1.0,
|
| 180 |
+
1.0,
|
| 181 |
+
1.0,
|
| 182 |
+
1.0,
|
| 183 |
+
1.0,
|
| 184 |
+
1.0,
|
| 185 |
+
1.0,
|
| 186 |
+
1.0,
|
| 187 |
+
1.0,
|
| 188 |
+
1.0,
|
| 189 |
+
1.0,
|
| 190 |
+
1.0,
|
| 191 |
+
1.0,
|
| 192 |
+
1.0,
|
| 193 |
+
1.0,
|
| 194 |
+
1.0,
|
| 195 |
+
1.0,
|
| 196 |
+
1.0,
|
| 197 |
+
1.0,
|
| 198 |
+
1.0
|
| 199 |
+
],
|
| 200 |
+
"type": "longrope"
|
| 201 |
+
},
|
| 202 |
+
"rope_theta": 10000.0,
|
| 203 |
+
"sliding_window": 262144,
|
| 204 |
+
"speech_lora": {
|
| 205 |
+
"dp": 0.01,
|
| 206 |
+
"layer": "((layers.*self_attn\\.(qkv|o)_proj)|(layers.*mlp\\.(gate_up|down)_proj))",
|
| 207 |
+
"lora_alpha": 640,
|
| 208 |
+
"r": 320
|
| 209 |
+
},
|
| 210 |
+
"tie_word_embeddings": true,
|
| 211 |
+
"torch_dtype": "bfloat16",
|
| 212 |
+
"transformers_version": "4.49.0",
|
| 213 |
+
"use_cache": false,
|
| 214 |
+
"vision_lora": {
|
| 215 |
+
"dp": 0.0,
|
| 216 |
+
"layer": "layers.*((self_attn\\.(qkv_proj|o_proj))|(mlp\\.(gate_up|down)_proj))",
|
| 217 |
+
"lora_alpha": 512,
|
| 218 |
+
"r": 256
|
| 219 |
+
},
|
| 220 |
+
"vocab_size": 200064
|
| 221 |
+
}
|
configuration_phi4mm.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
""" Phi-4-MM model configuration"""
|
| 17 |
+
|
| 18 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 19 |
+
from transformers.utils import logging
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
logger = logging.get_logger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class Phi4MMConfig(PretrainedConfig):
|
| 26 |
+
r"""
|
| 27 |
+
This is the configuration class to store the configuration of a [`Phi4MMModel`]. It is used to instantiate a Phi-4-MM
|
| 28 |
+
model according to the specified arguments, defining the model architecture.
|
| 29 |
+
|
| 30 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 31 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
vocab_size (`int`, *optional*, defaults to 200064):
|
| 35 |
+
Vocabulary size of the Phi-4-MM model. Defines the number of different tokens that can be represented by the
|
| 36 |
+
`inputs_ids` passed when calling [`Phi4MMModel`].
|
| 37 |
+
hidden_size (`int`, *optional*, defaults to 3072):
|
| 38 |
+
Dimension of the hidden representations.
|
| 39 |
+
intermediate_size (`int`, *optional*, defaults to 8192):
|
| 40 |
+
Dimension of the MLP representations.
|
| 41 |
+
num_hidden_layers (`int`, *optional*, defaults to 32):
|
| 42 |
+
Number of hidden layers in the Transformer decoder.
|
| 43 |
+
num_attention_heads (`int`, *optional*, defaults to 32):
|
| 44 |
+
Number of attention heads for each attention layer in the Transformer decoder.
|
| 45 |
+
num_key_value_heads (`int`, *optional*):
|
| 46 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
| 47 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
| 48 |
+
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
| 49 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
| 50 |
+
by meanpooling all the original heads within that group. For more details checkout [this
|
| 51 |
+
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
|
| 52 |
+
`num_attention_heads`.
|
| 53 |
+
resid_pdrop (`float`, *optional*, defaults to 0.0):
|
| 54 |
+
Dropout probability for mlp outputs.
|
| 55 |
+
embd_pdrop (`int`, *optional*, defaults to 0.0):
|
| 56 |
+
The dropout ratio for the embeddings.
|
| 57 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 58 |
+
The dropout ratio after computing the attention scores.
|
| 59 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
| 60 |
+
The non-linear activation function (function or string) in the decoder.
|
| 61 |
+
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
| 62 |
+
The maximum sequence length that this model might ever be used with.
|
| 63 |
+
original_max_position_embeddings (`int`, *optional*, defaults to 4096):
|
| 64 |
+
The maximum sequence length that this model was trained with. This is used to determine the size of the
|
| 65 |
+
original RoPE embeddings when using long scaling.
|
| 66 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 67 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 68 |
+
rms_norm_eps (`float`, *optional*, defaults to 1e-05):
|
| 69 |
+
The epsilon value used for the RMSNorm.
|
| 70 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
| 71 |
+
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
| 72 |
+
relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
|
| 73 |
+
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
| 74 |
+
Whether to tie weight embeddings
|
| 75 |
+
rope_theta (`float`, *optional*, defaults to 10000.0):
|
| 76 |
+
The base period of the RoPE embeddings.
|
| 77 |
+
rope_scaling (`dict`, *optional*):
|
| 78 |
+
The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
|
| 79 |
+
contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and
|
| 80 |
+
the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
|
| 81 |
+
divided by the number of attention heads divided by 2.
|
| 82 |
+
partial_rotary_factor (`float`, *optional*, defaults to 0.5):
|
| 83 |
+
Percentage of the query and keys which will have rotary embedding.
|
| 84 |
+
bos_token_id (`int`, *optional*, defaults to 199999):
|
| 85 |
+
The id of the "beginning-of-sequence" token.
|
| 86 |
+
eos_token_id (`int`, *optional*, defaults to 199999):
|
| 87 |
+
The id of the "end-of-sequence" token.
|
| 88 |
+
pad_token_id (`int`, *optional*, defaults to 199999):
|
| 89 |
+
The id of the padding token.
|
| 90 |
+
sliding_window (`int`, *optional*):
|
| 91 |
+
Sliding window attention window size. If `None`, no sliding window is applied.
|
| 92 |
+
|
| 93 |
+
Example:
|
| 94 |
+
|
| 95 |
+
```python
|
| 96 |
+
>>> from transformers import Phi4MMModel, Phi4MMConfig
|
| 97 |
+
|
| 98 |
+
>>> # Initializing a Phi-4-MM style configuration
|
| 99 |
+
>>> configuration = Phi4MMConfig.from_pretrained("TBA")
|
| 100 |
+
|
| 101 |
+
>>> # Initializing a model from the configuration
|
| 102 |
+
>>> model = Phi4MMModel(configuration)
|
| 103 |
+
|
| 104 |
+
>>> # Accessing the model configuration
|
| 105 |
+
>>> configuration = model.config
|
| 106 |
+
```"""
|
| 107 |
+
|
| 108 |
+
model_type = "phi4mm"
|
| 109 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 110 |
+
|
| 111 |
+
def __init__(
|
| 112 |
+
self,
|
| 113 |
+
vocab_size=200064,
|
| 114 |
+
hidden_size=3072,
|
| 115 |
+
intermediate_size=8192,
|
| 116 |
+
num_hidden_layers=32,
|
| 117 |
+
num_attention_heads=32,
|
| 118 |
+
num_key_value_heads=None,
|
| 119 |
+
resid_pdrop=0.0,
|
| 120 |
+
embd_pdrop=0.0,
|
| 121 |
+
attention_dropout=0.0,
|
| 122 |
+
hidden_act="silu",
|
| 123 |
+
max_position_embeddings=4096,
|
| 124 |
+
original_max_position_embeddings=4096,
|
| 125 |
+
initializer_range=0.02,
|
| 126 |
+
rms_norm_eps=1e-5,
|
| 127 |
+
use_cache=True,
|
| 128 |
+
tie_word_embeddings=False,
|
| 129 |
+
rope_theta=10000.0,
|
| 130 |
+
rope_scaling=None,
|
| 131 |
+
partial_rotary_factor=1,
|
| 132 |
+
bos_token_id=199999,
|
| 133 |
+
eos_token_id=199999,
|
| 134 |
+
pad_token_id=199999,
|
| 135 |
+
sliding_window=None,
|
| 136 |
+
embd_layer: str = "default",
|
| 137 |
+
img_processor=None,
|
| 138 |
+
audio_processor=None,
|
| 139 |
+
vision_lora=None,
|
| 140 |
+
speech_lora=None,
|
| 141 |
+
**kwargs,
|
| 142 |
+
):
|
| 143 |
+
self.embd_layer = embd_layer
|
| 144 |
+
self.img_processor = img_processor
|
| 145 |
+
self.audio_processor = audio_processor
|
| 146 |
+
self.vision_lora = vision_lora
|
| 147 |
+
self.speech_lora = speech_lora
|
| 148 |
+
|
| 149 |
+
self.vocab_size = vocab_size
|
| 150 |
+
self.hidden_size = hidden_size
|
| 151 |
+
self.intermediate_size = intermediate_size
|
| 152 |
+
self.num_hidden_layers = num_hidden_layers
|
| 153 |
+
self.num_attention_heads = num_attention_heads
|
| 154 |
+
|
| 155 |
+
if num_key_value_heads is None:
|
| 156 |
+
num_key_value_heads = num_attention_heads
|
| 157 |
+
|
| 158 |
+
self.num_key_value_heads = num_key_value_heads
|
| 159 |
+
self.resid_pdrop = resid_pdrop
|
| 160 |
+
self.embd_pdrop = embd_pdrop
|
| 161 |
+
self.attention_dropout = attention_dropout
|
| 162 |
+
self.hidden_act = hidden_act
|
| 163 |
+
self.max_position_embeddings = max_position_embeddings
|
| 164 |
+
self.original_max_position_embeddings = original_max_position_embeddings
|
| 165 |
+
self.initializer_range = initializer_range
|
| 166 |
+
self.rms_norm_eps = rms_norm_eps
|
| 167 |
+
self.use_cache = use_cache
|
| 168 |
+
self.rope_theta = rope_theta
|
| 169 |
+
self.rope_scaling = rope_scaling
|
| 170 |
+
self.partial_rotary_factor = partial_rotary_factor
|
| 171 |
+
self._rope_scaling_adjustment()
|
| 172 |
+
self._rope_scaling_validation()
|
| 173 |
+
self.sliding_window = sliding_window
|
| 174 |
+
|
| 175 |
+
super().__init__(
|
| 176 |
+
bos_token_id=bos_token_id,
|
| 177 |
+
eos_token_id=eos_token_id,
|
| 178 |
+
pad_token_id=pad_token_id,
|
| 179 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 180 |
+
**kwargs,
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
def _rope_scaling_adjustment(self):
|
| 184 |
+
"""
|
| 185 |
+
Adjust the `type` of the `rope_scaling` configuration for backward compatibility.
|
| 186 |
+
"""
|
| 187 |
+
if self.rope_scaling is None:
|
| 188 |
+
return
|
| 189 |
+
|
| 190 |
+
rope_scaling_type = self.rope_scaling.get("type", None)
|
| 191 |
+
|
| 192 |
+
# For backward compatibility if previous version used "su" or "yarn"
|
| 193 |
+
if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]:
|
| 194 |
+
self.rope_scaling["type"] = "longrope"
|
| 195 |
+
|
| 196 |
+
def _rope_scaling_validation(self):
|
| 197 |
+
"""
|
| 198 |
+
Validate the `rope_scaling` configuration.
|
| 199 |
+
"""
|
| 200 |
+
if self.rope_scaling is None:
|
| 201 |
+
return
|
| 202 |
+
|
| 203 |
+
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
|
| 204 |
+
raise ValueError(
|
| 205 |
+
"`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
|
| 206 |
+
f"got {self.rope_scaling}"
|
| 207 |
+
)
|
| 208 |
+
rope_scaling_type = self.rope_scaling.get("type", None)
|
| 209 |
+
rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
|
| 210 |
+
rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
|
| 211 |
+
if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
|
| 212 |
+
raise ValueError(f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}")
|
| 213 |
+
if not (
|
| 214 |
+
isinstance(rope_scaling_short_factor, list)
|
| 215 |
+
and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
|
| 216 |
+
):
|
| 217 |
+
raise ValueError(
|
| 218 |
+
f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
|
| 219 |
+
)
|
| 220 |
+
rotary_ndims = int(self.hidden_size // self.num_attention_heads * self.partial_rotary_factor)
|
| 221 |
+
if not len(rope_scaling_short_factor) == rotary_ndims // 2:
|
| 222 |
+
raise ValueError(
|
| 223 |
+
f"`rope_scaling`'s short_factor field must have length {rotary_ndims // 2}, got {len(rope_scaling_short_factor)}"
|
| 224 |
+
)
|
| 225 |
+
if not (
|
| 226 |
+
isinstance(rope_scaling_long_factor, list)
|
| 227 |
+
and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
|
| 228 |
+
):
|
| 229 |
+
raise ValueError(
|
| 230 |
+
f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
|
| 231 |
+
)
|
| 232 |
+
if not len(rope_scaling_long_factor) == rotary_ndims // 2:
|
| 233 |
+
raise ValueError(
|
| 234 |
+
f"`rope_scaling`'s long_factor field must have length {rotary_ndims // 2}, got {len(rope_scaling_long_factor)}"
|
| 235 |
+
)
|
generation_config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 199999,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
200020,
|
| 6 |
+
199999
|
| 7 |
+
],
|
| 8 |
+
"pad_token_id": 199999,
|
| 9 |
+
"transformers_version": "4.49.0"
|
| 10 |
+
}
|
logging.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model-00001-of-00003.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c42bf917c915218210afd0ea49fab890dc57b28395dbe4a0319dff02736abe36
|
| 3 |
+
size 4998420448
|
model-00002-of-00003.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3cc55197af454e74c5b8f706fdc9afa3b55fd23dc1fc53ddae13a162465a11c0
|
| 3 |
+
size 4983891952
|
model-00003-of-00003.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f128d4354ed726fbb96ec98756a2079f778d0fbd79e593cf313394d4f28d0ee8
|
| 3 |
+
size 1905111704
|
model.safetensors.index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
modeling_phi4mm.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
optimizer.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:da037573bd8e574791095b372a73515024438bb0de284cbcd38f68474ad7d234
|
| 3 |
+
size 2953007482
|
preprocessor_config.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
processing_phi4mm.py
ADDED
|
@@ -0,0 +1,733 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
"""
|
| 16 |
+
Processor class for Phi4MM
|
| 17 |
+
"""
|
| 18 |
+
import re
|
| 19 |
+
from typing import List, Optional, Tuple, Union
|
| 20 |
+
import math
|
| 21 |
+
from enum import Enum
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
import scipy
|
| 25 |
+
import torch
|
| 26 |
+
import torchvision
|
| 27 |
+
|
| 28 |
+
from transformers import AutoFeatureExtractor, AutoImageProcessor
|
| 29 |
+
from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor
|
| 30 |
+
from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
|
| 31 |
+
from transformers.image_utils import (
|
| 32 |
+
ImageInput,
|
| 33 |
+
make_list_of_images,
|
| 34 |
+
valid_images,
|
| 35 |
+
)
|
| 36 |
+
from transformers.processing_utils import ProcessorMixin
|
| 37 |
+
from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
|
| 38 |
+
from transformers.utils import TensorType, logging
|
| 39 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
logger = logging.get_logger(__name__)
|
| 43 |
+
|
| 44 |
+
# Special tokens
|
| 45 |
+
_COMPATIBLE_IMAGE_SPECIAL_TOKEN_PATTERN = r'<\|image_\d+\|>' # For backward compatibility
|
| 46 |
+
_COMPATIBLE_AUDIO_SPECIAL_TOKEN_PATTERN = r'<\|audio_\d+\|>' # For backward compatibility
|
| 47 |
+
_IMAGE_SPECIAL_TOKEN = '<|endoftext10|>'
|
| 48 |
+
_AUDIO_SPECIAL_TOKEN = '<|endoftext11|>'
|
| 49 |
+
_IMAGE_SPECIAL_TOKEN_ID = 200010 # '<|endoftext10|>', or we can better name it (in `tokenizer_config.json`)
|
| 50 |
+
_AUDIO_SPECIAL_TOKEN_ID = 200011 # '<|endoftext11|>'
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class InputMode(Enum):
|
| 54 |
+
LANGUAGE = 0
|
| 55 |
+
VISION = 1
|
| 56 |
+
SPEECH = 2
|
| 57 |
+
VISION_SPEECH = 3
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class Phi4MMImageProcessor(BaseImageProcessor):
|
| 61 |
+
r"""
|
| 62 |
+
Constructs a Phi4MM image processor.
|
| 63 |
+
"""
|
| 64 |
+
model_input_names = ["input_image_embeds", "image_sizes", "image_attention_mask"]
|
| 65 |
+
|
| 66 |
+
def __init__(
|
| 67 |
+
self,
|
| 68 |
+
dynamic_hd,
|
| 69 |
+
**kwargs,
|
| 70 |
+
) -> None:
|
| 71 |
+
super().__init__(**kwargs)
|
| 72 |
+
self.dynamic_hd = dynamic_hd
|
| 73 |
+
|
| 74 |
+
def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):
|
| 75 |
+
best_ratio_diff = float('inf')
|
| 76 |
+
best_ratio = (1, 1)
|
| 77 |
+
area = width * height
|
| 78 |
+
for ratio in target_ratios:
|
| 79 |
+
target_aspect_ratio = ratio[0] / ratio[1]
|
| 80 |
+
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
| 81 |
+
if ratio_diff < best_ratio_diff:
|
| 82 |
+
best_ratio_diff = ratio_diff
|
| 83 |
+
best_ratio = ratio
|
| 84 |
+
elif ratio_diff == best_ratio_diff:
|
| 85 |
+
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
| 86 |
+
best_ratio = ratio
|
| 87 |
+
return best_ratio
|
| 88 |
+
|
| 89 |
+
def dynamic_preprocess(self, image, min_num=1, max_num=12, image_size=384, mask_size=27, use_thumbnail=True):
|
| 90 |
+
orig_width, orig_height = image.size
|
| 91 |
+
|
| 92 |
+
w_crop_num = math.ceil(orig_width/float(image_size))
|
| 93 |
+
h_crop_num = math.ceil(orig_height/float(image_size))
|
| 94 |
+
if w_crop_num * h_crop_num > max_num:
|
| 95 |
+
|
| 96 |
+
aspect_ratio = orig_width / orig_height
|
| 97 |
+
|
| 98 |
+
# calculate the existing image aspect ratio
|
| 99 |
+
target_ratios = set(
|
| 100 |
+
(i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
|
| 101 |
+
i * j <= max_num and i * j >= min_num)
|
| 102 |
+
target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
| 103 |
+
|
| 104 |
+
# find the closest aspect ratio to the target
|
| 105 |
+
target_aspect_ratio = self.find_closest_aspect_ratio(
|
| 106 |
+
aspect_ratio, target_ratios, orig_width, orig_height, image_size)
|
| 107 |
+
|
| 108 |
+
# calculate the target width and height
|
| 109 |
+
target_width = image_size * target_aspect_ratio[0]
|
| 110 |
+
target_height = image_size * target_aspect_ratio[1]
|
| 111 |
+
else:
|
| 112 |
+
target_width = image_size * w_crop_num
|
| 113 |
+
target_height = image_size * h_crop_num
|
| 114 |
+
target_aspect_ratio = (w_crop_num, h_crop_num)
|
| 115 |
+
|
| 116 |
+
# Calculate the ratio
|
| 117 |
+
ratio_width = target_width / orig_width
|
| 118 |
+
ratio_height = target_height / orig_height
|
| 119 |
+
if ratio_width < ratio_height:
|
| 120 |
+
new_size = (target_width, int(orig_height * ratio_width))
|
| 121 |
+
padding_width = 0
|
| 122 |
+
padding_height = target_height - int(orig_height * ratio_width)
|
| 123 |
+
else:
|
| 124 |
+
new_size = (int(orig_width * ratio_height), target_height)
|
| 125 |
+
padding_width = target_width - int(orig_width * ratio_height)
|
| 126 |
+
padding_height = 0
|
| 127 |
+
|
| 128 |
+
attention_mask = torch.ones((int(mask_size*target_aspect_ratio[1]), int(mask_size*target_aspect_ratio[0])))
|
| 129 |
+
if padding_width >= 14:
|
| 130 |
+
attention_mask[:, -math.floor(padding_width/14):] = 0
|
| 131 |
+
if padding_height >= 14:
|
| 132 |
+
attention_mask[-math.floor(padding_height/14):,:] = 0
|
| 133 |
+
assert attention_mask.sum() > 0
|
| 134 |
+
|
| 135 |
+
if min(new_size[1], target_height) < 10 or min(new_size[0], target_width) < 10:
|
| 136 |
+
raise ValueError(f'the aspect ratio is very extreme {new_size}')
|
| 137 |
+
|
| 138 |
+
image = torchvision.transforms.functional.resize(image, [new_size[1], new_size[0]],)
|
| 139 |
+
|
| 140 |
+
resized_img = torchvision.transforms.functional.pad(image, [0, 0, padding_width, padding_height], fill=[255,255,255])
|
| 141 |
+
|
| 142 |
+
return resized_img, attention_mask
|
| 143 |
+
|
| 144 |
+
def pad_to_max_num_crops(self, images, max_crops=5):
|
| 145 |
+
"""
|
| 146 |
+
images: B x 3 x H x W, B<=max_crops
|
| 147 |
+
"""
|
| 148 |
+
B, _, H, W = images.shape
|
| 149 |
+
if B < max_crops:
|
| 150 |
+
pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)
|
| 151 |
+
images = torch.cat([images, pad], dim=0)
|
| 152 |
+
return images
|
| 153 |
+
|
| 154 |
+
def pad_mask_to_max_num_crops(self, masks, max_crops=5):
|
| 155 |
+
B, H, W = masks.shape
|
| 156 |
+
if B < max_crops:
|
| 157 |
+
pad = torch.ones(max_crops - B, H, W, dtype=masks.dtype, device=masks.device)
|
| 158 |
+
masks = torch.cat([masks, pad], dim=0)
|
| 159 |
+
return masks
|
| 160 |
+
|
| 161 |
+
def preprocess(
|
| 162 |
+
self,
|
| 163 |
+
images: ImageInput,
|
| 164 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 165 |
+
):
|
| 166 |
+
"""
|
| 167 |
+
Args:
|
| 168 |
+
images (`ImageInput`):
|
| 169 |
+
Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
|
| 170 |
+
passing in images with pixel values between 0 and 1, set `do_rescale=False`.
|
| 171 |
+
return_tensors (`str` or `TensorType`, *optional*):
|
| 172 |
+
The type of tensors to return. Can be one of:
|
| 173 |
+
- Unset: Return a list of `np.ndarray`.
|
| 174 |
+
- `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
|
| 175 |
+
- `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
|
| 176 |
+
- `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
|
| 177 |
+
- `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
|
| 178 |
+
"""
|
| 179 |
+
images = make_list_of_images(images)
|
| 180 |
+
|
| 181 |
+
if not valid_images(images):
|
| 182 |
+
raise ValueError(
|
| 183 |
+
"Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
|
| 184 |
+
"torch.Tensor, tf.Tensor or jax.ndarray."
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
# Basic settings.
|
| 188 |
+
img_processor = torchvision.transforms.Compose([
|
| 189 |
+
torchvision.transforms.ToTensor(),
|
| 190 |
+
torchvision.transforms.Normalize(
|
| 191 |
+
(0.5, 0.5, 0.5),
|
| 192 |
+
(0.5, 0.5, 0.5)
|
| 193 |
+
),
|
| 194 |
+
])
|
| 195 |
+
dyhd_base_resolution = 448
|
| 196 |
+
|
| 197 |
+
# Dynamic HD
|
| 198 |
+
base_resolution = dyhd_base_resolution
|
| 199 |
+
images = [image.convert('RGB') for image in images]
|
| 200 |
+
# cover 384 and 448 resolution
|
| 201 |
+
mask_resolution = base_resolution // 14
|
| 202 |
+
elems, image_attention_masks = [], []
|
| 203 |
+
for im in images:
|
| 204 |
+
elem, attention_mask = self.dynamic_preprocess(im, max_num=self.dynamic_hd, image_size=base_resolution, mask_size=mask_resolution)
|
| 205 |
+
elems.append(elem)
|
| 206 |
+
image_attention_masks.append(attention_mask)
|
| 207 |
+
hd_images = [img_processor(im) for im in elems]
|
| 208 |
+
global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(base_resolution, base_resolution), mode='bicubic',).to(im.dtype) for im in hd_images]
|
| 209 |
+
shapes = [[im.size(1), im.size(2)] for im in hd_images]
|
| 210 |
+
mask_shapes = [[mask.size(0), mask.size(1)] for mask in image_attention_masks]
|
| 211 |
+
global_attention_mask = [torch.ones((1, mask_resolution, mask_resolution)) for _ in hd_images]
|
| 212 |
+
hd_images_reshape = [im.reshape(1, 3,
|
| 213 |
+
h//base_resolution,
|
| 214 |
+
base_resolution,
|
| 215 |
+
w//base_resolution,
|
| 216 |
+
base_resolution
|
| 217 |
+
).permute(0,2,4,1,3,5).reshape(-1, 3, base_resolution, base_resolution).contiguous() for im, (h, w) in zip(hd_images, shapes)]
|
| 218 |
+
attention_masks_reshape = [mask.reshape(1,
|
| 219 |
+
h//mask_resolution,
|
| 220 |
+
mask_resolution,
|
| 221 |
+
w//mask_resolution,
|
| 222 |
+
mask_resolution
|
| 223 |
+
).permute(0,1,3,2,4).reshape(-1, mask_resolution, mask_resolution).contiguous() for mask, (h, w) in zip(image_attention_masks, mask_shapes)]
|
| 224 |
+
downsample_attention_masks = [mask[:,0::2,0::2].reshape(1,
|
| 225 |
+
h//mask_resolution,
|
| 226 |
+
w//mask_resolution,
|
| 227 |
+
mask_resolution//2+mask_resolution%2,
|
| 228 |
+
mask_resolution//2+mask_resolution%2
|
| 229 |
+
).permute(0,1,3,2,4) for mask, (h,w) in zip(attention_masks_reshape, mask_shapes)]
|
| 230 |
+
downsample_attention_masks = [mask.reshape(mask.size(1)*mask.size(2), mask.size(3)*mask.size(4))for mask in downsample_attention_masks]
|
| 231 |
+
num_img_tokens = [256 + 1 + int(mask.sum().item()) + int(mask[:,0].sum().item()) + 16 for mask in downsample_attention_masks]
|
| 232 |
+
|
| 233 |
+
hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]
|
| 234 |
+
hd_masks_reshape = [torch.cat([_global_mask] + [_mask], dim=0) for _global_mask, _mask in zip(global_attention_mask, attention_masks_reshape)]
|
| 235 |
+
max_crops = max([img.size(0) for img in hd_images_reshape])
|
| 236 |
+
image_transformed = [self.pad_to_max_num_crops(im, max_crops) for im in hd_images_reshape]
|
| 237 |
+
image_transformed = torch.stack(image_transformed, dim=0)
|
| 238 |
+
mask_transformed = [self.pad_mask_to_max_num_crops(mask, max_crops) for mask in hd_masks_reshape]
|
| 239 |
+
mask_transformed = torch.stack(mask_transformed, dim=0)
|
| 240 |
+
|
| 241 |
+
returned_input_image_embeds = image_transformed
|
| 242 |
+
returned_image_sizes = torch.tensor(shapes, dtype=torch.long)
|
| 243 |
+
returned_image_attention_mask = mask_transformed
|
| 244 |
+
returned_num_img_tokens = num_img_tokens
|
| 245 |
+
|
| 246 |
+
data = {
|
| 247 |
+
"input_image_embeds": returned_input_image_embeds,
|
| 248 |
+
"image_sizes": returned_image_sizes,
|
| 249 |
+
"image_attention_mask": returned_image_attention_mask,
|
| 250 |
+
"num_img_tokens": returned_num_img_tokens,
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
return BatchFeature(data=data, tensor_type=return_tensors)
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
AudioInput = Tuple[Union[np.ndarray, torch.Tensor], int]
|
| 257 |
+
AudioInputs = List[AudioInput]
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def speechlib_mel(sample_rate, n_fft, n_mels, fmin=None, fmax=None):
|
| 261 |
+
"""Create a Mel filter-bank the same as SpeechLib FbankFC.
|
| 262 |
+
|
| 263 |
+
Args:
|
| 264 |
+
sample_rate (int): Sample rate in Hz. number > 0 [scalar]
|
| 265 |
+
n_fft (int): FFT size. int > 0 [scalar]
|
| 266 |
+
n_mel (int): Mel filter size. int > 0 [scalar]
|
| 267 |
+
fmin (float): lowest frequency (in Hz). If None use 0.0.
|
| 268 |
+
float >= 0 [scalar]
|
| 269 |
+
fmax: highest frequency (in Hz). If None use sample_rate / 2.
|
| 270 |
+
float >= 0 [scalar]
|
| 271 |
+
|
| 272 |
+
Returns
|
| 273 |
+
out (numpy.ndarray): Mel transform matrix
|
| 274 |
+
[shape=(n_mels, 1 + n_fft/2)]
|
| 275 |
+
"""
|
| 276 |
+
|
| 277 |
+
bank_width = int(n_fft // 2 + 1)
|
| 278 |
+
if fmax is None:
|
| 279 |
+
fmax = sample_rate / 2
|
| 280 |
+
if fmin is None:
|
| 281 |
+
fmin = 0
|
| 282 |
+
assert fmin >= 0, "fmin cannot be negtive"
|
| 283 |
+
assert fmin < fmax <= sample_rate / 2, "fmax must be between (fmin, samplerate / 2]"
|
| 284 |
+
|
| 285 |
+
def mel(f):
|
| 286 |
+
return 1127.0 * np.log(1.0 + f / 700.0)
|
| 287 |
+
|
| 288 |
+
def bin2mel(fft_bin):
|
| 289 |
+
return 1127.0 * np.log(1.0 + fft_bin * sample_rate / (n_fft * 700.0))
|
| 290 |
+
|
| 291 |
+
def f2bin(f):
|
| 292 |
+
return int((f * n_fft / sample_rate) + 0.5)
|
| 293 |
+
|
| 294 |
+
# Spec 1: FFT bin range [f2bin(fmin) + 1, f2bin(fmax) - 1]
|
| 295 |
+
klo = f2bin(fmin) + 1
|
| 296 |
+
khi = f2bin(fmax)
|
| 297 |
+
|
| 298 |
+
khi = max(khi, klo)
|
| 299 |
+
|
| 300 |
+
# Spec 2: SpeechLib uses trianges in Mel space
|
| 301 |
+
mlo = mel(fmin)
|
| 302 |
+
mhi = mel(fmax)
|
| 303 |
+
m_centers = np.linspace(mlo, mhi, n_mels + 2)
|
| 304 |
+
ms = (mhi - mlo) / (n_mels + 1)
|
| 305 |
+
|
| 306 |
+
matrix = np.zeros((n_mels, bank_width), dtype=np.float32)
|
| 307 |
+
for m in range(0, n_mels):
|
| 308 |
+
left = m_centers[m]
|
| 309 |
+
center = m_centers[m + 1]
|
| 310 |
+
right = m_centers[m + 2]
|
| 311 |
+
for fft_bin in range(klo, khi):
|
| 312 |
+
mbin = bin2mel(fft_bin)
|
| 313 |
+
if left < mbin < right:
|
| 314 |
+
matrix[m, fft_bin] = 1.0 - abs(center - mbin) / ms
|
| 315 |
+
|
| 316 |
+
return matrix
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
class Phi4MMAudioFeatureExtractor(SequenceFeatureExtractor):
|
| 320 |
+
model_input_names = ["input_audio_embeds", "audio_embed_sizes", "audio_attention_mask"]
|
| 321 |
+
|
| 322 |
+
def __init__(self, audio_compression_rate, audio_downsample_rate, audio_feat_stride, **kwargs):
|
| 323 |
+
feature_size = 80
|
| 324 |
+
sampling_rate = 16000
|
| 325 |
+
padding_value = 0.0
|
| 326 |
+
super().__init__(feature_size, sampling_rate, padding_value, **kwargs)
|
| 327 |
+
|
| 328 |
+
self.compression_rate = audio_compression_rate
|
| 329 |
+
self.qformer_compression_rate = audio_downsample_rate
|
| 330 |
+
self.feat_stride = audio_feat_stride
|
| 331 |
+
|
| 332 |
+
self._eightk_method = "fillzero"
|
| 333 |
+
self._mel = speechlib_mel(16000, 512, 80, fmin=None, fmax=7690).T
|
| 334 |
+
|
| 335 |
+
self._hamming400 = np.hamming(400) # for 16k audio
|
| 336 |
+
self._hamming200 = np.hamming(200) # for 8k audio
|
| 337 |
+
|
| 338 |
+
def duration_to_frames(self, duration):
|
| 339 |
+
"""duration in s, estimated frames"""
|
| 340 |
+
frame_rate = 10
|
| 341 |
+
|
| 342 |
+
num_frames = duration * 1000 // frame_rate
|
| 343 |
+
return num_frames
|
| 344 |
+
|
| 345 |
+
def __call__(
|
| 346 |
+
self,
|
| 347 |
+
audios: List[AudioInput],
|
| 348 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 349 |
+
):
|
| 350 |
+
# Ref: https://github.com/huggingface/transformers/blob/v4.47.0/src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py#L161
|
| 351 |
+
returned_input_audio_embeds = []
|
| 352 |
+
returned_audio_embed_sizes = []
|
| 353 |
+
audio_frames_list = []
|
| 354 |
+
|
| 355 |
+
for audio_data, sample_rate in audios:
|
| 356 |
+
audio_embeds = self._extract_features(audio_data, sample_rate)
|
| 357 |
+
audio_frames = len(audio_embeds) * self.feat_stride
|
| 358 |
+
audio_embed_size = self._compute_audio_embed_size(audio_frames)
|
| 359 |
+
|
| 360 |
+
returned_input_audio_embeds.append(torch.tensor(audio_embeds))
|
| 361 |
+
returned_audio_embed_sizes.append(torch.tensor(audio_embed_size).long())
|
| 362 |
+
audio_frames_list.append(audio_frames)
|
| 363 |
+
|
| 364 |
+
returned_input_audio_embeds = pad_sequence(
|
| 365 |
+
returned_input_audio_embeds, batch_first=True
|
| 366 |
+
)
|
| 367 |
+
returned_audio_embed_sizes = torch.stack(returned_audio_embed_sizes, dim=0)
|
| 368 |
+
audio_frames = torch.tensor(audio_frames_list)
|
| 369 |
+
returned_audio_attention_mask = torch.arange(0, audio_frames.max()).unsqueeze(0) < audio_frames.unsqueeze(1) if len(audios) > 1 else None
|
| 370 |
+
|
| 371 |
+
data = {
|
| 372 |
+
"input_audio_embeds": returned_input_audio_embeds,
|
| 373 |
+
"audio_embed_sizes": returned_audio_embed_sizes,
|
| 374 |
+
}
|
| 375 |
+
if returned_audio_attention_mask is not None:
|
| 376 |
+
data["audio_attention_mask"] = returned_audio_attention_mask
|
| 377 |
+
|
| 378 |
+
return BatchFeature(data=data, tensor_type=return_tensors)
|
| 379 |
+
|
| 380 |
+
def _extract_spectrogram(self, wav, fs):
|
| 381 |
+
"""Extract spectrogram features from waveform.
|
| 382 |
+
Args:
|
| 383 |
+
wav (1D array): waveform of the input
|
| 384 |
+
fs (int): sampling rate of the waveform, 16000 or 8000.
|
| 385 |
+
If fs=8000, the waveform will be resampled to 16000Hz.
|
| 386 |
+
Output:
|
| 387 |
+
log_fbank (2D array): a TxD matrix of log Mel filterbank features.
|
| 388 |
+
D=80, and T is the number of frames.
|
| 389 |
+
"""
|
| 390 |
+
if wav.ndim > 1:
|
| 391 |
+
wav = np.squeeze(wav)
|
| 392 |
+
|
| 393 |
+
# by default, we extract the mean if stereo
|
| 394 |
+
if len(wav.shape) == 2:
|
| 395 |
+
wav = wav.mean(1)
|
| 396 |
+
|
| 397 |
+
# Resample to 16000 or 8000 if needed
|
| 398 |
+
if fs > 16000:
|
| 399 |
+
wav = scipy.signal.resample_poly(wav, 1, fs // 16000)
|
| 400 |
+
fs = 16000
|
| 401 |
+
elif 8000 < fs < 16000:
|
| 402 |
+
wav = scipy.signal.resample_poly(wav, 1, fs // 8000)
|
| 403 |
+
fs = 8000
|
| 404 |
+
elif fs < 8000:
|
| 405 |
+
raise RuntimeError(f"Unsupported sample rate {fs}")
|
| 406 |
+
|
| 407 |
+
if fs == 8000:
|
| 408 |
+
if self._eightk_method == "resample":
|
| 409 |
+
# Input audio is 8 kHz. Convert to 16 kHz before feature
|
| 410 |
+
# extraction
|
| 411 |
+
wav = scipy.signal.resample_poly(wav, 2, 1)
|
| 412 |
+
fs = 16000
|
| 413 |
+
# Do nothing here for fillzero method
|
| 414 |
+
elif fs != 16000:
|
| 415 |
+
# Input audio is not a supported sample rate.
|
| 416 |
+
raise RuntimeError(f"Input data using an unsupported sample rate: {fs}")
|
| 417 |
+
|
| 418 |
+
preemphasis = 0.97
|
| 419 |
+
|
| 420 |
+
if fs == 8000:
|
| 421 |
+
n_fft = 256
|
| 422 |
+
win_length = 200
|
| 423 |
+
hop_length = 80
|
| 424 |
+
fft_window = self._hamming200
|
| 425 |
+
elif fs == 16000:
|
| 426 |
+
n_fft = 512
|
| 427 |
+
win_length = 400
|
| 428 |
+
hop_length = 160
|
| 429 |
+
fft_window = self._hamming400
|
| 430 |
+
|
| 431 |
+
# Spec 1: SpeechLib cut remaining sample insufficient for a hop
|
| 432 |
+
n_batch = (wav.shape[0] - win_length) // hop_length + 1
|
| 433 |
+
# Here we don't use stride_tricks since the input array may not satisfy
|
| 434 |
+
# memory layout requirement and we need writeable output
|
| 435 |
+
# Here we only use list of views before copy to desination
|
| 436 |
+
# so it is more efficient than broadcasting
|
| 437 |
+
y_frames = np.array(
|
| 438 |
+
[wav[_stride : _stride + win_length] for _stride in range(0, hop_length * n_batch, hop_length)],
|
| 439 |
+
dtype=np.float32,
|
| 440 |
+
)
|
| 441 |
+
|
| 442 |
+
# Spec 2: SpeechLib applies preemphasis within each batch
|
| 443 |
+
y_frames_prev = np.roll(y_frames, 1, axis=1)
|
| 444 |
+
y_frames_prev[:, 0] = y_frames_prev[:, 1]
|
| 445 |
+
y_frames = (y_frames - preemphasis * y_frames_prev) * 32768
|
| 446 |
+
|
| 447 |
+
S = np.fft.rfft(fft_window * y_frames, n=n_fft, axis=1).astype(np.complex64)
|
| 448 |
+
|
| 449 |
+
if fs == 8000:
|
| 450 |
+
# Need to pad the output to look like 16 kHz data but with zeros in
|
| 451 |
+
# the 4 to 8 kHz bins.
|
| 452 |
+
frames, bins = S.shape
|
| 453 |
+
padarray = np.zeros((frames, bins))
|
| 454 |
+
S = np.concatenate((S[:, 0:-1], padarray), axis=1) # Nyquist bin gets set to zero
|
| 455 |
+
|
| 456 |
+
spec = np.abs(S).astype(np.float32)
|
| 457 |
+
return spec
|
| 458 |
+
|
| 459 |
+
def _extract_features(self, wav, fs):
|
| 460 |
+
"""Extract log filterbank features from waveform.
|
| 461 |
+
Args:
|
| 462 |
+
wav (1D array): waveform of the input
|
| 463 |
+
fs (int): sampling rate of the waveform, 16000 or 8000.
|
| 464 |
+
If fs=8000, the waveform will be resampled to 16000Hz.
|
| 465 |
+
Output:
|
| 466 |
+
log_fbank (2D array): a TxD matrix of log Mel filterbank features.
|
| 467 |
+
D=80, and T is the number of frames.
|
| 468 |
+
"""
|
| 469 |
+
spec = self._extract_spectrogram(wav, fs)
|
| 470 |
+
spec_power = spec**2
|
| 471 |
+
|
| 472 |
+
fbank_power = np.clip(spec_power.dot(self._mel), 1.0, None)
|
| 473 |
+
log_fbank = np.log(fbank_power).astype(np.float32)
|
| 474 |
+
|
| 475 |
+
return log_fbank
|
| 476 |
+
|
| 477 |
+
def _compute_audio_embed_size(self, audio_frames):
|
| 478 |
+
integer = audio_frames // self.compression_rate
|
| 479 |
+
remainder = audio_frames % self.compression_rate
|
| 480 |
+
|
| 481 |
+
result = integer if remainder == 0 else integer + 1
|
| 482 |
+
|
| 483 |
+
integer = result // self.qformer_compression_rate
|
| 484 |
+
remainder = result % self.qformer_compression_rate
|
| 485 |
+
result = integer if remainder == 0 else integer + 1 # qformer compression
|
| 486 |
+
|
| 487 |
+
return result
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
class Phi4MMProcessor(ProcessorMixin):
|
| 491 |
+
r"""
|
| 492 |
+
Constructs a Phi4MM processor which raps an image processor, a audio processor, and a GPT tokenizer into a single processor.
|
| 493 |
+
|
| 494 |
+
[`Phi4MMProcessor`] offers all the functionalities of [`Phi4MMImageProcessor`] and [`GPT2Tokenizer`]. See the
|
| 495 |
+
[`~Phi4MMProcessor.__call__`] and [`~Phi4MMProcessor.decode`] for more information.
|
| 496 |
+
|
| 497 |
+
Args:
|
| 498 |
+
image_processor ([`Phi4MMImageProcessor`], *optional*):
|
| 499 |
+
The image processor is a required input.
|
| 500 |
+
tokenizer ([`GPT2Tokenizer`], *optional*):
|
| 501 |
+
The tokenizer is a required input.
|
| 502 |
+
"""
|
| 503 |
+
|
| 504 |
+
attributes = ["image_processor", "audio_processor", "tokenizer"]
|
| 505 |
+
tokenizer_class = "GPT2TokenizerFast"
|
| 506 |
+
image_processor_class = "AutoImageProcessor" # Phi4MMImageProcessor will be registered later
|
| 507 |
+
audio_processor_class = "AutoFeatureExtractor" # Phi4MMAudioFeatureExtractor will be registered later
|
| 508 |
+
|
| 509 |
+
def __init__(self, image_processor, audio_processor, tokenizer):
|
| 510 |
+
self.image_processor = image_processor
|
| 511 |
+
self.audio_processor = audio_processor
|
| 512 |
+
self.tokenizer = tokenizer
|
| 513 |
+
|
| 514 |
+
def __call__(
|
| 515 |
+
self,
|
| 516 |
+
text: Union[TextInput, List[TextInput]],
|
| 517 |
+
images: Optional[ImageInput] = None,
|
| 518 |
+
audios: Optional[AudioInputs] = None,
|
| 519 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 520 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 521 |
+
max_length=None,
|
| 522 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 523 |
+
) -> BatchFeature:
|
| 524 |
+
"""
|
| 525 |
+
Main method to prepare for the model one or several sequences(s) and image(s). This method forards the `text`
|
| 526 |
+
and `kwargs` arguments to GPT2Tokenizer's [`~GPT2Tokenizer.__call__`] if `text` is not `None` to encode
|
| 527 |
+
the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
|
| 528 |
+
Phi4MMImageProcessor's [`~Phi4MMImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
|
| 529 |
+
of the above two methods for more information.
|
| 530 |
+
|
| 531 |
+
Args:
|
| 532 |
+
text (`str`, `List[str]`, `List[List[str]]`):
|
| 533 |
+
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
|
| 534 |
+
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
|
| 535 |
+
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
|
| 536 |
+
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
|
| 537 |
+
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
|
| 538 |
+
tensor. Both channels-first and channels-last formats are supported.
|
| 539 |
+
padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
|
| 540 |
+
Select a strategy to pad the returned sequences (according to the model's padding side and padding
|
| 541 |
+
index) among:
|
| 542 |
+
- `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
|
| 543 |
+
sequence if provided).
|
| 544 |
+
- `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
|
| 545 |
+
acceptable input length for the model if that argument is not provided.
|
| 546 |
+
- `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
|
| 547 |
+
lengths).
|
| 548 |
+
max_length (`int`, *optional*):
|
| 549 |
+
Maximum length of the returned list and optionally padding length (see above).
|
| 550 |
+
truncation (`bool`, *optional*):
|
| 551 |
+
Activates truncation to cut input sequences longer than `max_length` to `max_length`.
|
| 552 |
+
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
| 553 |
+
If set, will return tensors of a particular framework. Acceptable values are:
|
| 554 |
+
|
| 555 |
+
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
| 556 |
+
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
| 557 |
+
- `'np'`: Return NumPy `np.ndarray` objects.
|
| 558 |
+
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
| 559 |
+
|
| 560 |
+
Returns:
|
| 561 |
+
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
| 562 |
+
|
| 563 |
+
- **input_ids** -- List of token ids to be fed to a model.
|
| 564 |
+
- **input_image_embeds** -- Pixel values to be fed to a model.
|
| 565 |
+
- **image_sizes** -- List of tuples specifying the size of each image in `input_image_embeds`.
|
| 566 |
+
- **image_attention_mask** -- List of attention masks for each image in `input_image_embeds`.
|
| 567 |
+
- **input_audio_embeds** -- Audio embeddings to be fed to a model.
|
| 568 |
+
- **audio_embed_sizes** -- List of integers specifying the size of each audio in `input_audio_embeds`.
|
| 569 |
+
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model.
|
| 570 |
+
"""
|
| 571 |
+
image_inputs = self.image_processor(images, return_tensors=return_tensors) if images is not None else {}
|
| 572 |
+
audio_inputs = self.audio_processor(audios, return_tensors=return_tensors) if audios is not None else {}
|
| 573 |
+
inputs = self._convert_images_audios_text_to_inputs(
|
| 574 |
+
image_inputs,
|
| 575 |
+
audio_inputs,
|
| 576 |
+
text,
|
| 577 |
+
padding=padding,
|
| 578 |
+
truncation=truncation,
|
| 579 |
+
max_length=max_length,
|
| 580 |
+
return_tensors=return_tensors,
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
# idenfity the input mode
|
| 584 |
+
if len(image_inputs) > 0 and len(audio_inputs) > 0:
|
| 585 |
+
input_mode = InputMode.VISION_SPEECH
|
| 586 |
+
elif len(image_inputs) > 0:
|
| 587 |
+
input_mode = InputMode.VISION
|
| 588 |
+
elif len(audio_inputs) > 0:
|
| 589 |
+
input_mode = InputMode.SPEECH
|
| 590 |
+
else:
|
| 591 |
+
input_mode = InputMode.LANGUAGE
|
| 592 |
+
inputs["input_mode"] = torch.tensor([input_mode.value], dtype=torch.long)
|
| 593 |
+
|
| 594 |
+
return inputs
|
| 595 |
+
|
| 596 |
+
@property
|
| 597 |
+
def special_image_token_id(self):
|
| 598 |
+
return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
|
| 599 |
+
|
| 600 |
+
def get_special_image_token_id(self):
|
| 601 |
+
return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
|
| 602 |
+
|
| 603 |
+
@property
|
| 604 |
+
def chat_template(self):
|
| 605 |
+
return self.tokenizer.chat_template
|
| 606 |
+
|
| 607 |
+
def _convert_images_audios_text_to_inputs(
|
| 608 |
+
self, images, audios, text, padding=False, truncation=None, max_length=None, return_tensors=None
|
| 609 |
+
):
|
| 610 |
+
# prepare image id to image input ids
|
| 611 |
+
if len(images) > 0:
|
| 612 |
+
input_image_embeds = images["input_image_embeds"]
|
| 613 |
+
image_sizes = images["image_sizes"]
|
| 614 |
+
image_attention_mask = images["image_attention_mask"]
|
| 615 |
+
num_img_tokens = images['num_img_tokens']
|
| 616 |
+
else:
|
| 617 |
+
input_image_embeds = torch.tensor([])
|
| 618 |
+
image_sizes = torch.tensor([])
|
| 619 |
+
image_attention_mask = torch.tensor([])
|
| 620 |
+
num_img_tokens = []
|
| 621 |
+
|
| 622 |
+
# prepare audio id to audio input ids
|
| 623 |
+
if len(audios) > 0:
|
| 624 |
+
input_audio_embeds = audios["input_audio_embeds"]
|
| 625 |
+
audio_embed_sizes = audios["audio_embed_sizes"]
|
| 626 |
+
audio_attention_mask = audios.get("audio_attention_mask", None)
|
| 627 |
+
else:
|
| 628 |
+
input_audio_embeds = torch.tensor([])
|
| 629 |
+
audio_embed_sizes = torch.tensor([])
|
| 630 |
+
audio_attention_mask = None
|
| 631 |
+
|
| 632 |
+
# Replace certain special tokens for compatibility
|
| 633 |
+
# Ref: https://stackoverflow.com/questions/11475885/python-replace-regex
|
| 634 |
+
if isinstance(text, str):
|
| 635 |
+
text = [text]
|
| 636 |
+
assert isinstance(text, list)
|
| 637 |
+
processed_text = [re.sub(_COMPATIBLE_IMAGE_SPECIAL_TOKEN_PATTERN, _IMAGE_SPECIAL_TOKEN, t) for t in text]
|
| 638 |
+
processed_text = [re.sub(_COMPATIBLE_AUDIO_SPECIAL_TOKEN_PATTERN, _AUDIO_SPECIAL_TOKEN, t) for t in processed_text]
|
| 639 |
+
|
| 640 |
+
input_ids_list = [self.tokenizer(t).input_ids for t in processed_text]
|
| 641 |
+
|
| 642 |
+
img_cnt, audio_cnt = 0, 0 # only needed for later assertion
|
| 643 |
+
image_token_count_iter = iter(num_img_tokens)
|
| 644 |
+
audio_embed_size_iter = iter(audio_embed_sizes.tolist())
|
| 645 |
+
new_input_ids_list = []
|
| 646 |
+
for input_ids in input_ids_list:
|
| 647 |
+
i = 0
|
| 648 |
+
while i < len(input_ids):
|
| 649 |
+
token_id = input_ids[i]
|
| 650 |
+
if token_id == _AUDIO_SPECIAL_TOKEN_ID:
|
| 651 |
+
token_count = next(audio_embed_size_iter)
|
| 652 |
+
audio_cnt += 1
|
| 653 |
+
elif token_id == _IMAGE_SPECIAL_TOKEN_ID:
|
| 654 |
+
token_count = next(image_token_count_iter)
|
| 655 |
+
img_cnt += 1
|
| 656 |
+
else:
|
| 657 |
+
i += 1
|
| 658 |
+
continue
|
| 659 |
+
tokens = [token_id] * token_count
|
| 660 |
+
input_ids = input_ids[:i] + tokens + input_ids[i + 1:]
|
| 661 |
+
i += token_count
|
| 662 |
+
input_ids = torch.tensor(input_ids, dtype=torch.long)
|
| 663 |
+
new_input_ids_list.append(input_ids)
|
| 664 |
+
lengths = torch.tensor([len(input_ids) for input_ids in new_input_ids_list])
|
| 665 |
+
max_len = lengths.max()
|
| 666 |
+
input_ids = input_ids.new_full((len(new_input_ids_list), max_len), self.tokenizer.pad_token_id)
|
| 667 |
+
# batched inference requires left padding
|
| 668 |
+
for i in range(len(new_input_ids_list)):
|
| 669 |
+
input_ids[i, max_len - len(new_input_ids_list[i]):] = new_input_ids_list[i]
|
| 670 |
+
|
| 671 |
+
# If the below assertion fails, it might be that input pure-text
|
| 672 |
+
# messages contain image/audio special tokens literally
|
| 673 |
+
# (<|endoftext10|>, <|endoftext11|>).
|
| 674 |
+
assert (
|
| 675 |
+
img_cnt == len(num_img_tokens)
|
| 676 |
+
), (
|
| 677 |
+
f"Number of image tokens in prompt_token_ids ({img_cnt}) "
|
| 678 |
+
f"does not match number of images ({len(num_img_tokens)})"
|
| 679 |
+
)
|
| 680 |
+
assert (
|
| 681 |
+
audio_cnt == len(audio_embed_sizes)
|
| 682 |
+
), (
|
| 683 |
+
f"Number of audio tokens in prompt_token_ids ({audio_cnt}) "
|
| 684 |
+
f"does not match number of audios ({len(audio_embed_sizes)})"
|
| 685 |
+
)
|
| 686 |
+
|
| 687 |
+
# prepare attention mask
|
| 688 |
+
seq_range = torch.arange(max_len - 1, -1, -1)
|
| 689 |
+
attention_mask = seq_range.unsqueeze(0) < lengths.unsqueeze(1)
|
| 690 |
+
|
| 691 |
+
# prepare batch feature
|
| 692 |
+
data = {
|
| 693 |
+
"input_ids": input_ids,
|
| 694 |
+
"input_image_embeds": input_image_embeds,
|
| 695 |
+
"image_sizes": image_sizes,
|
| 696 |
+
"image_attention_mask": image_attention_mask,
|
| 697 |
+
"input_audio_embeds": input_audio_embeds,
|
| 698 |
+
"audio_embed_sizes": audio_embed_sizes,
|
| 699 |
+
"audio_attention_mask": audio_attention_mask,
|
| 700 |
+
"attention_mask": attention_mask,
|
| 701 |
+
}
|
| 702 |
+
|
| 703 |
+
return BatchFeature(
|
| 704 |
+
data=data
|
| 705 |
+
)
|
| 706 |
+
|
| 707 |
+
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
|
| 708 |
+
def batch_decode(self, *args, **kwargs):
|
| 709 |
+
"""
|
| 710 |
+
This method forwards all its arguments to GPT2Tokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
|
| 711 |
+
refer to the docstring of this method for more information.
|
| 712 |
+
"""
|
| 713 |
+
return self.tokenizer.batch_decode(*args, **kwargs)
|
| 714 |
+
|
| 715 |
+
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
|
| 716 |
+
def decode(self, *args, **kwargs):
|
| 717 |
+
"""
|
| 718 |
+
This method forwards all its arguments to GPT2Tokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
|
| 719 |
+
the docstring of this method for more information.
|
| 720 |
+
"""
|
| 721 |
+
return self.tokenizer.decode(*args, **kwargs)
|
| 722 |
+
|
| 723 |
+
@property
|
| 724 |
+
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
|
| 725 |
+
def model_input_names(self):
|
| 726 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 727 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 728 |
+
audio_processor_input_names = self.audio_processor.model_input_names
|
| 729 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names + audio_processor_input_names))
|
| 730 |
+
|
| 731 |
+
|
| 732 |
+
AutoImageProcessor.register("Phi4MMImageProcessor", Phi4MMImageProcessor)
|
| 733 |
+
AutoFeatureExtractor.register("Phi4MMAudioFeatureExtractor", Phi4MMAudioFeatureExtractor)
|
processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_phi4mm.Phi4MMProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "Phi4MMProcessor"
|
| 6 |
+
}
|
rng_state.pth
ADDED
|
Binary file (14.2 kB). View file
|
|
|
scheduler.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7cd4986fadccddc15534264cc66123d1355f09868477aae79556812d62c342ed
|
| 3 |
+
size 1064
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<|endoftext|>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
speech_conformer_encoder.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"added_tokens_decoder": {
|
| 4 |
+
"199999": {
|
| 5 |
+
"content": "<|endoftext|>",
|
| 6 |
+
"lstrip": false,
|
| 7 |
+
"normalized": false,
|
| 8 |
+
"rstrip": false,
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"special": true
|
| 11 |
+
},
|
| 12 |
+
"200010": {
|
| 13 |
+
"content": "<|endoftext10|>",
|
| 14 |
+
"lstrip": false,
|
| 15 |
+
"normalized": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"single_word": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
"200011": {
|
| 21 |
+
"content": "<|endoftext11|>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": false,
|
| 25 |
+
"single_word": false,
|
| 26 |
+
"special": true
|
| 27 |
+
},
|
| 28 |
+
"200018": {
|
| 29 |
+
"content": "<|endofprompt|>",
|
| 30 |
+
"lstrip": false,
|
| 31 |
+
"normalized": false,
|
| 32 |
+
"rstrip": false,
|
| 33 |
+
"single_word": false,
|
| 34 |
+
"special": true
|
| 35 |
+
},
|
| 36 |
+
"200019": {
|
| 37 |
+
"content": "<|assistant|>",
|
| 38 |
+
"lstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"rstrip": true,
|
| 41 |
+
"single_word": false,
|
| 42 |
+
"special": true
|
| 43 |
+
},
|
| 44 |
+
"200020": {
|
| 45 |
+
"content": "<|end|>",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": true,
|
| 49 |
+
"single_word": false,
|
| 50 |
+
"special": true
|
| 51 |
+
},
|
| 52 |
+
"200021": {
|
| 53 |
+
"content": "<|user|>",
|
| 54 |
+
"lstrip": false,
|
| 55 |
+
"normalized": false,
|
| 56 |
+
"rstrip": true,
|
| 57 |
+
"single_word": false,
|
| 58 |
+
"special": true
|
| 59 |
+
},
|
| 60 |
+
"200022": {
|
| 61 |
+
"content": "<|system|>",
|
| 62 |
+
"lstrip": false,
|
| 63 |
+
"normalized": false,
|
| 64 |
+
"rstrip": true,
|
| 65 |
+
"single_word": false,
|
| 66 |
+
"special": true
|
| 67 |
+
},
|
| 68 |
+
"200023": {
|
| 69 |
+
"content": "<|tool|>",
|
| 70 |
+
"lstrip": false,
|
| 71 |
+
"normalized": false,
|
| 72 |
+
"rstrip": true,
|
| 73 |
+
"single_word": false,
|
| 74 |
+
"special": false
|
| 75 |
+
},
|
| 76 |
+
"200024": {
|
| 77 |
+
"content": "<|/tool|>",
|
| 78 |
+
"lstrip": false,
|
| 79 |
+
"normalized": false,
|
| 80 |
+
"rstrip": true,
|
| 81 |
+
"single_word": false,
|
| 82 |
+
"special": false
|
| 83 |
+
},
|
| 84 |
+
"200025": {
|
| 85 |
+
"content": "<|tool_call|>",
|
| 86 |
+
"lstrip": false,
|
| 87 |
+
"normalized": false,
|
| 88 |
+
"rstrip": true,
|
| 89 |
+
"single_word": false,
|
| 90 |
+
"special": false
|
| 91 |
+
},
|
| 92 |
+
"200026": {
|
| 93 |
+
"content": "<|/tool_call|>",
|
| 94 |
+
"lstrip": false,
|
| 95 |
+
"normalized": false,
|
| 96 |
+
"rstrip": true,
|
| 97 |
+
"single_word": false,
|
| 98 |
+
"special": false
|
| 99 |
+
},
|
| 100 |
+
"200027": {
|
| 101 |
+
"content": "<|tool_response|>",
|
| 102 |
+
"lstrip": false,
|
| 103 |
+
"normalized": false,
|
| 104 |
+
"rstrip": true,
|
| 105 |
+
"single_word": false,
|
| 106 |
+
"special": false
|
| 107 |
+
},
|
| 108 |
+
"200028": {
|
| 109 |
+
"content": "<|tag|>",
|
| 110 |
+
"lstrip": false,
|
| 111 |
+
"normalized": false,
|
| 112 |
+
"rstrip": true,
|
| 113 |
+
"single_word": false,
|
| 114 |
+
"special": true
|
| 115 |
+
}
|
| 116 |
+
},
|
| 117 |
+
"auto_map": {
|
| 118 |
+
"AutoProcessor": "processing_phi4mm.Phi4MMProcessor"
|
| 119 |
+
},
|
| 120 |
+
"bos_token": "<|endoftext|>",
|
| 121 |
+
"chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",
|
| 122 |
+
"clean_up_tokenization_spaces": false,
|
| 123 |
+
"eos_token": "<|endoftext|>",
|
| 124 |
+
"extra_special_tokens": {},
|
| 125 |
+
"model_max_length": 131072,
|
| 126 |
+
"pad_token": "<|endoftext|>",
|
| 127 |
+
"processor_class": "Phi4MMProcessor",
|
| 128 |
+
"tokenizer_class": "GPT2Tokenizer",
|
| 129 |
+
"unk_token": "<|endoftext|>"
|
| 130 |
+
}
|
trainer_state.json
ADDED
|
@@ -0,0 +1,2596 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_metric": 0.01646235,
|
| 3 |
+
"best_model_checkpoint": "/root/autodl-tmp/ms-swift/output/v3-20250320-230237/checkpoint-200",
|
| 4 |
+
"epoch": 0.9998499024365838,
|
| 5 |
+
"eval_steps": 200,
|
| 6 |
+
"global_step": 1249,
|
| 7 |
+
"is_hyper_param_search": false,
|
| 8 |
+
"is_local_process_zero": true,
|
| 9 |
+
"is_world_process_zero": true,
|
| 10 |
+
"log_history": [
|
| 11 |
+
{
|
| 12 |
+
"epoch": 0.0008005203382198429,
|
| 13 |
+
"grad_norm": 8.173952102661133,
|
| 14 |
+
"learning_rate": 1.5873015873015873e-06,
|
| 15 |
+
"loss": 0.44585809111595154,
|
| 16 |
+
"memory(GiB)": 19.93,
|
| 17 |
+
"step": 1,
|
| 18 |
+
"token_acc": 0.9618138424821002,
|
| 19 |
+
"train_speed(iter/s)": 0.117269
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 0.004002601691099215,
|
| 23 |
+
"grad_norm": 2.755690574645996,
|
| 24 |
+
"learning_rate": 7.936507936507936e-06,
|
| 25 |
+
"loss": 0.22828824818134308,
|
| 26 |
+
"memory(GiB)": 19.93,
|
| 27 |
+
"step": 5,
|
| 28 |
+
"token_acc": 0.9664602683178535,
|
| 29 |
+
"train_speed(iter/s)": 0.140285
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"epoch": 0.00800520338219843,
|
| 33 |
+
"grad_norm": 1.0583274364471436,
|
| 34 |
+
"learning_rate": 1.5873015873015872e-05,
|
| 35 |
+
"loss": 0.1148987889289856,
|
| 36 |
+
"memory(GiB)": 21.87,
|
| 37 |
+
"step": 10,
|
| 38 |
+
"token_acc": 0.9737647573240052,
|
| 39 |
+
"train_speed(iter/s)": 0.143044
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"epoch": 0.012007805073297644,
|
| 43 |
+
"grad_norm": 0.47796979546546936,
|
| 44 |
+
"learning_rate": 2.380952380952381e-05,
|
| 45 |
+
"loss": 0.041056069731712344,
|
| 46 |
+
"memory(GiB)": 21.87,
|
| 47 |
+
"step": 15,
|
| 48 |
+
"token_acc": 0.9885004599816007,
|
| 49 |
+
"train_speed(iter/s)": 0.144308
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"epoch": 0.01601040676439686,
|
| 53 |
+
"grad_norm": 0.6110995411872864,
|
| 54 |
+
"learning_rate": 3.1746031746031745e-05,
|
| 55 |
+
"loss": 0.029762446880340576,
|
| 56 |
+
"memory(GiB)": 21.87,
|
| 57 |
+
"step": 20,
|
| 58 |
+
"token_acc": 0.9895644283121597,
|
| 59 |
+
"train_speed(iter/s)": 0.144678
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"epoch": 0.020013008455496074,
|
| 63 |
+
"grad_norm": 0.4749644994735718,
|
| 64 |
+
"learning_rate": 3.968253968253968e-05,
|
| 65 |
+
"loss": 0.03434528708457947,
|
| 66 |
+
"memory(GiB)": 21.87,
|
| 67 |
+
"step": 25,
|
| 68 |
+
"token_acc": 0.9889553612517257,
|
| 69 |
+
"train_speed(iter/s)": 0.145044
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"epoch": 0.024015610146595288,
|
| 73 |
+
"grad_norm": 0.2586625814437866,
|
| 74 |
+
"learning_rate": 4.761904761904762e-05,
|
| 75 |
+
"loss": 0.022176292538642884,
|
| 76 |
+
"memory(GiB)": 21.87,
|
| 77 |
+
"step": 30,
|
| 78 |
+
"token_acc": 0.9934762348555451,
|
| 79 |
+
"train_speed(iter/s)": 0.145327
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"epoch": 0.028018211837694502,
|
| 83 |
+
"grad_norm": 0.8793371915817261,
|
| 84 |
+
"learning_rate": 5.555555555555556e-05,
|
| 85 |
+
"loss": 0.045995473861694336,
|
| 86 |
+
"memory(GiB)": 21.87,
|
| 87 |
+
"step": 35,
|
| 88 |
+
"token_acc": 0.9864982578397212,
|
| 89 |
+
"train_speed(iter/s)": 0.145153
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"epoch": 0.03202081352879372,
|
| 93 |
+
"grad_norm": 0.6064431667327881,
|
| 94 |
+
"learning_rate": 6.349206349206349e-05,
|
| 95 |
+
"loss": 0.02957049906253815,
|
| 96 |
+
"memory(GiB)": 21.87,
|
| 97 |
+
"step": 40,
|
| 98 |
+
"token_acc": 0.9923534409515717,
|
| 99 |
+
"train_speed(iter/s)": 0.144907
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"epoch": 0.03602341521989293,
|
| 103 |
+
"grad_norm": 0.8034117221832275,
|
| 104 |
+
"learning_rate": 7.142857142857143e-05,
|
| 105 |
+
"loss": 0.03551977276802063,
|
| 106 |
+
"memory(GiB)": 21.87,
|
| 107 |
+
"step": 45,
|
| 108 |
+
"token_acc": 0.990418118466899,
|
| 109 |
+
"train_speed(iter/s)": 0.144666
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"epoch": 0.04002601691099215,
|
| 113 |
+
"grad_norm": 0.8057456612586975,
|
| 114 |
+
"learning_rate": 7.936507936507937e-05,
|
| 115 |
+
"loss": 0.02472989559173584,
|
| 116 |
+
"memory(GiB)": 21.87,
|
| 117 |
+
"step": 50,
|
| 118 |
+
"token_acc": 0.9941715237302248,
|
| 119 |
+
"train_speed(iter/s)": 0.144617
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"epoch": 0.04402861860209136,
|
| 123 |
+
"grad_norm": 0.23828290402889252,
|
| 124 |
+
"learning_rate": 8.730158730158731e-05,
|
| 125 |
+
"loss": 0.037221938371658325,
|
| 126 |
+
"memory(GiB)": 21.87,
|
| 127 |
+
"step": 55,
|
| 128 |
+
"token_acc": 0.9872785097682871,
|
| 129 |
+
"train_speed(iter/s)": 0.144496
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"epoch": 0.048031220293190575,
|
| 133 |
+
"grad_norm": 0.9102380871772766,
|
| 134 |
+
"learning_rate": 9.523809523809524e-05,
|
| 135 |
+
"loss": 0.04597596824169159,
|
| 136 |
+
"memory(GiB)": 21.87,
|
| 137 |
+
"step": 60,
|
| 138 |
+
"token_acc": 0.9875776397515528,
|
| 139 |
+
"train_speed(iter/s)": 0.144413
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"epoch": 0.05203382198428979,
|
| 143 |
+
"grad_norm": 1.7829787731170654,
|
| 144 |
+
"learning_rate": 9.999929833574692e-05,
|
| 145 |
+
"loss": 0.03467706441879272,
|
| 146 |
+
"memory(GiB)": 21.87,
|
| 147 |
+
"step": 65,
|
| 148 |
+
"token_acc": 0.9929971988795518,
|
| 149 |
+
"train_speed(iter/s)": 0.144223
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"epoch": 0.056036423675389004,
|
| 153 |
+
"grad_norm": 0.9495298266410828,
|
| 154 |
+
"learning_rate": 9.99914048390634e-05,
|
| 155 |
+
"loss": 0.037359020113945006,
|
| 156 |
+
"memory(GiB)": 21.87,
|
| 157 |
+
"step": 70,
|
| 158 |
+
"token_acc": 0.9889958734525447,
|
| 159 |
+
"train_speed(iter/s)": 0.144151
|
| 160 |
+
},
|
| 161 |
+
{
|
| 162 |
+
"epoch": 0.06003902536648822,
|
| 163 |
+
"grad_norm": 0.3380143344402313,
|
| 164 |
+
"learning_rate": 9.997474215462472e-05,
|
| 165 |
+
"loss": 0.03453690111637116,
|
| 166 |
+
"memory(GiB)": 21.87,
|
| 167 |
+
"step": 75,
|
| 168 |
+
"token_acc": 0.9895090222408729,
|
| 169 |
+
"train_speed(iter/s)": 0.144062
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"epoch": 0.06404162705758744,
|
| 173 |
+
"grad_norm": 0.3318343758583069,
|
| 174 |
+
"learning_rate": 9.994931320529753e-05,
|
| 175 |
+
"loss": 0.04619415700435638,
|
| 176 |
+
"memory(GiB)": 21.87,
|
| 177 |
+
"step": 80,
|
| 178 |
+
"token_acc": 0.9849411764705882,
|
| 179 |
+
"train_speed(iter/s)": 0.14396
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"epoch": 0.06804422874868665,
|
| 183 |
+
"grad_norm": 0.2660139799118042,
|
| 184 |
+
"learning_rate": 9.991512245167323e-05,
|
| 185 |
+
"loss": 0.026491838693618774,
|
| 186 |
+
"memory(GiB)": 21.87,
|
| 187 |
+
"step": 85,
|
| 188 |
+
"token_acc": 0.9905482041587902,
|
| 189 |
+
"train_speed(iter/s)": 0.143912
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"epoch": 0.07204683043978587,
|
| 193 |
+
"grad_norm": 1.473689079284668,
|
| 194 |
+
"learning_rate": 9.987217589128557e-05,
|
| 195 |
+
"loss": 0.07692851424217224,
|
| 196 |
+
"memory(GiB)": 21.87,
|
| 197 |
+
"step": 90,
|
| 198 |
+
"token_acc": 0.9860262008733625,
|
| 199 |
+
"train_speed(iter/s)": 0.143875
|
| 200 |
+
},
|
| 201 |
+
{
|
| 202 |
+
"epoch": 0.07604943213088508,
|
| 203 |
+
"grad_norm": 0.916643500328064,
|
| 204 |
+
"learning_rate": 9.982048105755859e-05,
|
| 205 |
+
"loss": 0.040849074721336365,
|
| 206 |
+
"memory(GiB)": 21.87,
|
| 207 |
+
"step": 95,
|
| 208 |
+
"token_acc": 0.9864864864864865,
|
| 209 |
+
"train_speed(iter/s)": 0.143834
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"epoch": 0.0800520338219843,
|
| 213 |
+
"grad_norm": 0.3694117069244385,
|
| 214 |
+
"learning_rate": 9.976004701848521e-05,
|
| 215 |
+
"loss": 0.02239110916852951,
|
| 216 |
+
"memory(GiB)": 21.87,
|
| 217 |
+
"step": 100,
|
| 218 |
+
"token_acc": 0.9964726631393298,
|
| 219 |
+
"train_speed(iter/s)": 0.14367
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"epoch": 0.08405463551308351,
|
| 223 |
+
"grad_norm": 0.8251020312309265,
|
| 224 |
+
"learning_rate": 9.969088437503642e-05,
|
| 225 |
+
"loss": 0.043125709891319274,
|
| 226 |
+
"memory(GiB)": 21.87,
|
| 227 |
+
"step": 105,
|
| 228 |
+
"token_acc": 0.9852765618782332,
|
| 229 |
+
"train_speed(iter/s)": 0.143584
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"epoch": 0.08805723720418272,
|
| 233 |
+
"grad_norm": 0.7774543166160583,
|
| 234 |
+
"learning_rate": 9.961300525930193e-05,
|
| 235 |
+
"loss": 0.041256973147392274,
|
| 236 |
+
"memory(GiB)": 21.87,
|
| 237 |
+
"step": 110,
|
| 238 |
+
"token_acc": 0.9878758486905916,
|
| 239 |
+
"train_speed(iter/s)": 0.143528
|
| 240 |
+
},
|
| 241 |
+
{
|
| 242 |
+
"epoch": 0.09205983889528194,
|
| 243 |
+
"grad_norm": 0.4797303080558777,
|
| 244 |
+
"learning_rate": 9.952642333236186e-05,
|
| 245 |
+
"loss": 0.03012470304965973,
|
| 246 |
+
"memory(GiB)": 21.87,
|
| 247 |
+
"step": 115,
|
| 248 |
+
"token_acc": 0.9928603302097277,
|
| 249 |
+
"train_speed(iter/s)": 0.143472
|
| 250 |
+
},
|
| 251 |
+
{
|
| 252 |
+
"epoch": 0.09606244058638115,
|
| 253 |
+
"grad_norm": 1.3094687461853027,
|
| 254 |
+
"learning_rate": 9.94311537818905e-05,
|
| 255 |
+
"loss": 0.10325220823287964,
|
| 256 |
+
"memory(GiB)": 21.87,
|
| 257 |
+
"step": 120,
|
| 258 |
+
"token_acc": 0.9766244057052298,
|
| 259 |
+
"train_speed(iter/s)": 0.143438
|
| 260 |
+
},
|
| 261 |
+
{
|
| 262 |
+
"epoch": 0.10006504227748036,
|
| 263 |
+
"grad_norm": 0.3123111128807068,
|
| 264 |
+
"learning_rate": 9.932721331949215e-05,
|
| 265 |
+
"loss": 0.03137581646442413,
|
| 266 |
+
"memory(GiB)": 21.87,
|
| 267 |
+
"step": 125,
|
| 268 |
+
"token_acc": 0.993103448275862,
|
| 269 |
+
"train_speed(iter/s)": 0.143374
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"epoch": 0.10406764396857958,
|
| 273 |
+
"grad_norm": 0.7394413352012634,
|
| 274 |
+
"learning_rate": 9.921462017776963e-05,
|
| 275 |
+
"loss": 0.07408838868141174,
|
| 276 |
+
"memory(GiB)": 21.87,
|
| 277 |
+
"step": 130,
|
| 278 |
+
"token_acc": 0.9848484848484849,
|
| 279 |
+
"train_speed(iter/s)": 0.14333
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"epoch": 0.10807024565967879,
|
| 283 |
+
"grad_norm": 0.26981663703918457,
|
| 284 |
+
"learning_rate": 9.909339410712612e-05,
|
| 285 |
+
"loss": 0.02576894164085388,
|
| 286 |
+
"memory(GiB)": 21.87,
|
| 287 |
+
"step": 135,
|
| 288 |
+
"token_acc": 0.9928507596067918,
|
| 289 |
+
"train_speed(iter/s)": 0.143377
|
| 290 |
+
},
|
| 291 |
+
{
|
| 292 |
+
"epoch": 0.11207284735077801,
|
| 293 |
+
"grad_norm": 0.9989132881164551,
|
| 294 |
+
"learning_rate": 9.896355637230057e-05,
|
| 295 |
+
"loss": 0.01885353773832321,
|
| 296 |
+
"memory(GiB)": 21.87,
|
| 297 |
+
"step": 140,
|
| 298 |
+
"token_acc": 0.9942579505300353,
|
| 299 |
+
"train_speed(iter/s)": 0.143344
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"epoch": 0.11607544904187722,
|
| 303 |
+
"grad_norm": 1.0498735904693604,
|
| 304 |
+
"learning_rate": 9.882512974863762e-05,
|
| 305 |
+
"loss": 0.01057654172182083,
|
| 306 |
+
"memory(GiB)": 21.87,
|
| 307 |
+
"step": 145,
|
| 308 |
+
"token_acc": 0.9967667436489608,
|
| 309 |
+
"train_speed(iter/s)": 0.143373
|
| 310 |
+
},
|
| 311 |
+
{
|
| 312 |
+
"epoch": 0.12007805073297644,
|
| 313 |
+
"grad_norm": 0.8206981420516968,
|
| 314 |
+
"learning_rate": 9.867813851809247e-05,
|
| 315 |
+
"loss": 0.032333412766456605,
|
| 316 |
+
"memory(GiB)": 21.87,
|
| 317 |
+
"step": 150,
|
| 318 |
+
"token_acc": 0.9908015768725361,
|
| 319 |
+
"train_speed(iter/s)": 0.14338
|
| 320 |
+
},
|
| 321 |
+
{
|
| 322 |
+
"epoch": 0.12408065242407565,
|
| 323 |
+
"grad_norm": 0.7933772802352905,
|
| 324 |
+
"learning_rate": 9.852260846497153e-05,
|
| 325 |
+
"loss": 0.021973614394664765,
|
| 326 |
+
"memory(GiB)": 21.87,
|
| 327 |
+
"step": 155,
|
| 328 |
+
"token_acc": 0.9939167056621432,
|
| 329 |
+
"train_speed(iter/s)": 0.143407
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"epoch": 0.12808325411517488,
|
| 333 |
+
"grad_norm": 0.3294403553009033,
|
| 334 |
+
"learning_rate": 9.835856687140941e-05,
|
| 335 |
+
"loss": 0.05915788412094116,
|
| 336 |
+
"memory(GiB)": 21.87,
|
| 337 |
+
"step": 160,
|
| 338 |
+
"token_acc": 0.985632183908046,
|
| 339 |
+
"train_speed(iter/s)": 0.143446
|
| 340 |
+
},
|
| 341 |
+
{
|
| 342 |
+
"epoch": 0.13208585580627408,
|
| 343 |
+
"grad_norm": 1.490613341331482,
|
| 344 |
+
"learning_rate": 9.818604251258335e-05,
|
| 345 |
+
"loss": 0.0674498736858368,
|
| 346 |
+
"memory(GiB)": 21.87,
|
| 347 |
+
"step": 165,
|
| 348 |
+
"token_acc": 0.9815212720240654,
|
| 349 |
+
"train_speed(iter/s)": 0.143479
|
| 350 |
+
},
|
| 351 |
+
{
|
| 352 |
+
"epoch": 0.1360884574973733,
|
| 353 |
+
"grad_norm": 0.08009309321641922,
|
| 354 |
+
"learning_rate": 9.800506565166559e-05,
|
| 355 |
+
"loss": 0.015468017756938934,
|
| 356 |
+
"memory(GiB)": 21.87,
|
| 357 |
+
"step": 170,
|
| 358 |
+
"token_acc": 0.9956597222222222,
|
| 359 |
+
"train_speed(iter/s)": 0.143452
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"epoch": 0.1400910591884725,
|
| 363 |
+
"grad_norm": 0.21452507376670837,
|
| 364 |
+
"learning_rate": 9.781566803451475e-05,
|
| 365 |
+
"loss": 0.025325089693069458,
|
| 366 |
+
"memory(GiB)": 21.87,
|
| 367 |
+
"step": 175,
|
| 368 |
+
"token_acc": 0.9945833333333334,
|
| 369 |
+
"train_speed(iter/s)": 0.143445
|
| 370 |
+
},
|
| 371 |
+
{
|
| 372 |
+
"epoch": 0.14409366087957173,
|
| 373 |
+
"grad_norm": 0.7213698029518127,
|
| 374 |
+
"learning_rate": 9.761788288410732e-05,
|
| 375 |
+
"loss": 0.04960155189037323,
|
| 376 |
+
"memory(GiB)": 21.87,
|
| 377 |
+
"step": 180,
|
| 378 |
+
"token_acc": 0.988500663423264,
|
| 379 |
+
"train_speed(iter/s)": 0.143399
|
| 380 |
+
},
|
| 381 |
+
{
|
| 382 |
+
"epoch": 0.14809626257067093,
|
| 383 |
+
"grad_norm": 0.36322739720344543,
|
| 384 |
+
"learning_rate": 9.741174489470972e-05,
|
| 385 |
+
"loss": 0.019899863004684448,
|
| 386 |
+
"memory(GiB)": 21.87,
|
| 387 |
+
"step": 185,
|
| 388 |
+
"token_acc": 0.9958256029684601,
|
| 389 |
+
"train_speed(iter/s)": 0.143413
|
| 390 |
+
},
|
| 391 |
+
{
|
| 392 |
+
"epoch": 0.15209886426177016,
|
| 393 |
+
"grad_norm": 0.1350119411945343,
|
| 394 |
+
"learning_rate": 9.719729022579253e-05,
|
| 395 |
+
"loss": 0.018680891394615172,
|
| 396 |
+
"memory(GiB)": 21.87,
|
| 397 |
+
"step": 190,
|
| 398 |
+
"token_acc": 0.9951853635050554,
|
| 399 |
+
"train_speed(iter/s)": 0.143454
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"epoch": 0.15610146595286936,
|
| 403 |
+
"grad_norm": 0.5075913667678833,
|
| 404 |
+
"learning_rate": 9.697455649568761e-05,
|
| 405 |
+
"loss": 0.03849280476570129,
|
| 406 |
+
"memory(GiB)": 21.87,
|
| 407 |
+
"step": 195,
|
| 408 |
+
"token_acc": 0.9893143365983972,
|
| 409 |
+
"train_speed(iter/s)": 0.14348
|
| 410 |
+
},
|
| 411 |
+
{
|
| 412 |
+
"epoch": 0.1601040676439686,
|
| 413 |
+
"grad_norm": 0.5222131013870239,
|
| 414 |
+
"learning_rate": 9.674358277498926e-05,
|
| 415 |
+
"loss": 0.01666124016046524,
|
| 416 |
+
"memory(GiB)": 21.87,
|
| 417 |
+
"step": 200,
|
| 418 |
+
"token_acc": 0.995366079703429,
|
| 419 |
+
"train_speed(iter/s)": 0.143495
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"epoch": 0.1601040676439686,
|
| 423 |
+
"eval_loss": 0.016462353989481926,
|
| 424 |
+
"eval_runtime": 2.5148,
|
| 425 |
+
"eval_samples_per_second": 5.169,
|
| 426 |
+
"eval_steps_per_second": 5.169,
|
| 427 |
+
"eval_token_acc": 0.9974683544303797,
|
| 428 |
+
"step": 200
|
| 429 |
+
},
|
| 430 |
+
{
|
| 431 |
+
"epoch": 0.1641066693350678,
|
| 432 |
+
"grad_norm": 0.5556055903434753,
|
| 433 |
+
"learning_rate": 9.650440957970081e-05,
|
| 434 |
+
"loss": 0.029794663190841675,
|
| 435 |
+
"memory(GiB)": 21.87,
|
| 436 |
+
"step": 205,
|
| 437 |
+
"token_acc": 0.9902302861130495,
|
| 438 |
+
"train_speed(iter/s)": 0.141737
|
| 439 |
+
},
|
| 440 |
+
{
|
| 441 |
+
"epoch": 0.16810927102616702,
|
| 442 |
+
"grad_norm": 0.6666015982627869,
|
| 443 |
+
"learning_rate": 9.625707886412745e-05,
|
| 444 |
+
"loss": 0.03334639370441437,
|
| 445 |
+
"memory(GiB)": 21.87,
|
| 446 |
+
"step": 210,
|
| 447 |
+
"token_acc": 0.9917748917748918,
|
| 448 |
+
"train_speed(iter/s)": 0.141728
|
| 449 |
+
},
|
| 450 |
+
{
|
| 451 |
+
"epoch": 0.17211187271726622,
|
| 452 |
+
"grad_norm": 0.6179186701774597,
|
| 453 |
+
"learning_rate": 9.600163401351688e-05,
|
| 454 |
+
"loss": 0.030014854669570924,
|
| 455 |
+
"memory(GiB)": 21.87,
|
| 456 |
+
"step": 215,
|
| 457 |
+
"token_acc": 0.9909871244635193,
|
| 458 |
+
"train_speed(iter/s)": 0.141767
|
| 459 |
+
},
|
| 460 |
+
{
|
| 461 |
+
"epoch": 0.17611447440836545,
|
| 462 |
+
"grad_norm": 0.22975869476795197,
|
| 463 |
+
"learning_rate": 9.573811983644895e-05,
|
| 464 |
+
"loss": 0.013575367629528046,
|
| 465 |
+
"memory(GiB)": 21.87,
|
| 466 |
+
"step": 220,
|
| 467 |
+
"token_acc": 0.9966297544535387,
|
| 468 |
+
"train_speed(iter/s)": 0.14176
|
| 469 |
+
},
|
| 470 |
+
{
|
| 471 |
+
"epoch": 0.18011707609946465,
|
| 472 |
+
"grad_norm": 0.8102268576622009,
|
| 473 |
+
"learning_rate": 9.546658255697564e-05,
|
| 474 |
+
"loss": 0.023204405605793,
|
| 475 |
+
"memory(GiB)": 21.87,
|
| 476 |
+
"step": 225,
|
| 477 |
+
"token_acc": 0.9945627548708654,
|
| 478 |
+
"train_speed(iter/s)": 0.141823
|
| 479 |
+
},
|
| 480 |
+
{
|
| 481 |
+
"epoch": 0.18411967779056387,
|
| 482 |
+
"grad_norm": 0.328828364610672,
|
| 483 |
+
"learning_rate": 9.518706980651266e-05,
|
| 484 |
+
"loss": 0.030618682503700256,
|
| 485 |
+
"memory(GiB)": 21.87,
|
| 486 |
+
"step": 230,
|
| 487 |
+
"token_acc": 0.990447242726878,
|
| 488 |
+
"train_speed(iter/s)": 0.141869
|
| 489 |
+
},
|
| 490 |
+
{
|
| 491 |
+
"epoch": 0.18812227948166307,
|
| 492 |
+
"grad_norm": 0.2663295865058899,
|
| 493 |
+
"learning_rate": 9.489963061548428e-05,
|
| 494 |
+
"loss": 0.022738894820213316,
|
| 495 |
+
"memory(GiB)": 21.87,
|
| 496 |
+
"step": 235,
|
| 497 |
+
"token_acc": 0.9934725848563969,
|
| 498 |
+
"train_speed(iter/s)": 0.141905
|
| 499 |
+
},
|
| 500 |
+
{
|
| 501 |
+
"epoch": 0.1921248811727623,
|
| 502 |
+
"grad_norm": 0.5860876441001892,
|
| 503 |
+
"learning_rate": 9.460431540472268e-05,
|
| 504 |
+
"loss": 0.046448147296905516,
|
| 505 |
+
"memory(GiB)": 21.87,
|
| 506 |
+
"step": 240,
|
| 507 |
+
"token_acc": 0.9884615384615385,
|
| 508 |
+
"train_speed(iter/s)": 0.141925
|
| 509 |
+
},
|
| 510 |
+
{
|
| 511 |
+
"epoch": 0.1961274828638615,
|
| 512 |
+
"grad_norm": 0.518479585647583,
|
| 513 |
+
"learning_rate": 9.430117597662347e-05,
|
| 514 |
+
"loss": 0.014139848947525024,
|
| 515 |
+
"memory(GiB)": 21.87,
|
| 516 |
+
"step": 245,
|
| 517 |
+
"token_acc": 0.9968971631205674,
|
| 518 |
+
"train_speed(iter/s)": 0.141977
|
| 519 |
+
},
|
| 520 |
+
{
|
| 521 |
+
"epoch": 0.20013008455496073,
|
| 522 |
+
"grad_norm": 0.8027576804161072,
|
| 523 |
+
"learning_rate": 9.399026550605879e-05,
|
| 524 |
+
"loss": 0.01970670074224472,
|
| 525 |
+
"memory(GiB)": 21.87,
|
| 526 |
+
"step": 250,
|
| 527 |
+
"token_acc": 0.9951627088830255,
|
| 528 |
+
"train_speed(iter/s)": 0.142039
|
| 529 |
+
},
|
| 530 |
+
{
|
| 531 |
+
"epoch": 0.20413268624605993,
|
| 532 |
+
"grad_norm": 0.6967362761497498,
|
| 533 |
+
"learning_rate": 9.367163853104975e-05,
|
| 534 |
+
"loss": 0.046951514482498166,
|
| 535 |
+
"memory(GiB)": 21.87,
|
| 536 |
+
"step": 255,
|
| 537 |
+
"token_acc": 0.986117136659436,
|
| 538 |
+
"train_speed(iter/s)": 0.14209
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"epoch": 0.20813528793715916,
|
| 542 |
+
"grad_norm": 0.3053382635116577,
|
| 543 |
+
"learning_rate": 9.334535094319963e-05,
|
| 544 |
+
"loss": 0.01998966634273529,
|
| 545 |
+
"memory(GiB)": 21.87,
|
| 546 |
+
"step": 260,
|
| 547 |
+
"token_acc": 0.9940284795590262,
|
| 548 |
+
"train_speed(iter/s)": 0.1421
|
| 549 |
+
},
|
| 550 |
+
{
|
| 551 |
+
"epoch": 0.21213788962825836,
|
| 552 |
+
"grad_norm": 0.6150457859039307,
|
| 553 |
+
"learning_rate": 9.301145997788978e-05,
|
| 554 |
+
"loss": 0.03244458734989166,
|
| 555 |
+
"memory(GiB)": 21.87,
|
| 556 |
+
"step": 265,
|
| 557 |
+
"token_acc": 0.9909173478655767,
|
| 558 |
+
"train_speed(iter/s)": 0.142149
|
| 559 |
+
},
|
| 560 |
+
{
|
| 561 |
+
"epoch": 0.21614049131935759,
|
| 562 |
+
"grad_norm": 0.29773569107055664,
|
| 563 |
+
"learning_rate": 9.267002420423972e-05,
|
| 564 |
+
"loss": 0.014630559086799621,
|
| 565 |
+
"memory(GiB)": 21.87,
|
| 566 |
+
"step": 270,
|
| 567 |
+
"token_acc": 0.9933582399335824,
|
| 568 |
+
"train_speed(iter/s)": 0.142159
|
| 569 |
+
},
|
| 570 |
+
{
|
| 571 |
+
"epoch": 0.2201430930104568,
|
| 572 |
+
"grad_norm": 0.33867722749710083,
|
| 573 |
+
"learning_rate": 9.232110351483327e-05,
|
| 574 |
+
"loss": 0.019022081792354584,
|
| 575 |
+
"memory(GiB)": 21.87,
|
| 576 |
+
"step": 275,
|
| 577 |
+
"token_acc": 0.9932553956834532,
|
| 578 |
+
"train_speed(iter/s)": 0.142116
|
| 579 |
+
},
|
| 580 |
+
{
|
| 581 |
+
"epoch": 0.22414569470155601,
|
| 582 |
+
"grad_norm": 1.6940982341766357,
|
| 583 |
+
"learning_rate": 9.196475911521263e-05,
|
| 584 |
+
"loss": 0.033858922123908994,
|
| 585 |
+
"memory(GiB)": 21.87,
|
| 586 |
+
"step": 280,
|
| 587 |
+
"token_acc": 0.9900690846286702,
|
| 588 |
+
"train_speed(iter/s)": 0.142057
|
| 589 |
+
},
|
| 590 |
+
{
|
| 591 |
+
"epoch": 0.22814829639265521,
|
| 592 |
+
"grad_norm": 0.5266643166542053,
|
| 593 |
+
"learning_rate": 9.160105351314199e-05,
|
| 594 |
+
"loss": 0.024715608358383177,
|
| 595 |
+
"memory(GiB)": 21.87,
|
| 596 |
+
"step": 285,
|
| 597 |
+
"token_acc": 0.9898538961038961,
|
| 598 |
+
"train_speed(iter/s)": 0.142047
|
| 599 |
+
},
|
| 600 |
+
{
|
| 601 |
+
"epoch": 0.23215089808375444,
|
| 602 |
+
"grad_norm": 0.37173131108283997,
|
| 603 |
+
"learning_rate": 9.123005050764283e-05,
|
| 604 |
+
"loss": 0.018766339123249053,
|
| 605 |
+
"memory(GiB)": 21.87,
|
| 606 |
+
"step": 290,
|
| 607 |
+
"token_acc": 0.9941747572815534,
|
| 608 |
+
"train_speed(iter/s)": 0.142008
|
| 609 |
+
},
|
| 610 |
+
{
|
| 611 |
+
"epoch": 0.23615349977485364,
|
| 612 |
+
"grad_norm": 0.5264220833778381,
|
| 613 |
+
"learning_rate": 9.085181517780273e-05,
|
| 614 |
+
"loss": 0.03025185763835907,
|
| 615 |
+
"memory(GiB)": 21.87,
|
| 616 |
+
"step": 295,
|
| 617 |
+
"token_acc": 0.9920782851817335,
|
| 618 |
+
"train_speed(iter/s)": 0.141984
|
| 619 |
+
},
|
| 620 |
+
{
|
| 621 |
+
"epoch": 0.24015610146595287,
|
| 622 |
+
"grad_norm": 0.8184806704521179,
|
| 623 |
+
"learning_rate": 9.046641387135954e-05,
|
| 624 |
+
"loss": 0.01774435639381409,
|
| 625 |
+
"memory(GiB)": 21.87,
|
| 626 |
+
"step": 300,
|
| 627 |
+
"token_acc": 0.9955643173977329,
|
| 628 |
+
"train_speed(iter/s)": 0.141983
|
| 629 |
+
},
|
| 630 |
+
{
|
| 631 |
+
"epoch": 0.24415870315705207,
|
| 632 |
+
"grad_norm": 0.6606205105781555,
|
| 633 |
+
"learning_rate": 9.007391419306308e-05,
|
| 634 |
+
"loss": 0.021882972121238707,
|
| 635 |
+
"memory(GiB)": 21.87,
|
| 636 |
+
"step": 305,
|
| 637 |
+
"token_acc": 0.9945230488361478,
|
| 638 |
+
"train_speed(iter/s)": 0.141955
|
| 639 |
+
},
|
| 640 |
+
{
|
| 641 |
+
"epoch": 0.2481613048481513,
|
| 642 |
+
"grad_norm": 0.7625671625137329,
|
| 643 |
+
"learning_rate": 8.967438499281632e-05,
|
| 644 |
+
"loss": 0.010945966094732284,
|
| 645 |
+
"memory(GiB)": 21.87,
|
| 646 |
+
"step": 310,
|
| 647 |
+
"token_acc": 0.9954934655250113,
|
| 648 |
+
"train_speed(iter/s)": 0.141742
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"epoch": 0.2521639065392505,
|
| 652 |
+
"grad_norm": 0.483966588973999,
|
| 653 |
+
"learning_rate": 8.926789635359817e-05,
|
| 654 |
+
"loss": 0.012021870911121368,
|
| 655 |
+
"memory(GiB)": 21.87,
|
| 656 |
+
"step": 315,
|
| 657 |
+
"token_acc": 0.9961750956226094,
|
| 658 |
+
"train_speed(iter/s)": 0.141717
|
| 659 |
+
},
|
| 660 |
+
{
|
| 661 |
+
"epoch": 0.25616650823034975,
|
| 662 |
+
"grad_norm": 0.4055178761482239,
|
| 663 |
+
"learning_rate": 8.885451957916993e-05,
|
| 664 |
+
"loss": 0.009706312417984008,
|
| 665 |
+
"memory(GiB)": 21.87,
|
| 666 |
+
"step": 320,
|
| 667 |
+
"token_acc": 0.9957264957264957,
|
| 668 |
+
"train_speed(iter/s)": 0.141657
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"epoch": 0.2601691099214489,
|
| 672 |
+
"grad_norm": 0.7214549779891968,
|
| 673 |
+
"learning_rate": 8.84343271815676e-05,
|
| 674 |
+
"loss": 0.017278000712394714,
|
| 675 |
+
"memory(GiB)": 21.87,
|
| 676 |
+
"step": 325,
|
| 677 |
+
"token_acc": 0.994367417677643,
|
| 678 |
+
"train_speed(iter/s)": 0.141657
|
| 679 |
+
},
|
| 680 |
+
{
|
| 681 |
+
"epoch": 0.26417171161254815,
|
| 682 |
+
"grad_norm": 0.08001359552145004,
|
| 683 |
+
"learning_rate": 8.800739286838234e-05,
|
| 684 |
+
"loss": 0.01176077127456665,
|
| 685 |
+
"memory(GiB)": 21.87,
|
| 686 |
+
"step": 330,
|
| 687 |
+
"token_acc": 0.9960612691466083,
|
| 688 |
+
"train_speed(iter/s)": 0.141654
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"epoch": 0.2681743133036474,
|
| 692 |
+
"grad_norm": 0.29422757029533386,
|
| 693 |
+
"learning_rate": 8.757379152983103e-05,
|
| 694 |
+
"loss": 0.022312307357788087,
|
| 695 |
+
"memory(GiB)": 21.87,
|
| 696 |
+
"step": 335,
|
| 697 |
+
"token_acc": 0.9949302915082383,
|
| 698 |
+
"train_speed(iter/s)": 0.141643
|
| 699 |
+
},
|
| 700 |
+
{
|
| 701 |
+
"epoch": 0.2721769149947466,
|
| 702 |
+
"grad_norm": 0.6405854821205139,
|
| 703 |
+
"learning_rate": 8.713359922561957e-05,
|
| 704 |
+
"loss": 0.010415679961442947,
|
| 705 |
+
"memory(GiB)": 21.87,
|
| 706 |
+
"step": 340,
|
| 707 |
+
"token_acc": 0.9969982847341338,
|
| 708 |
+
"train_speed(iter/s)": 0.141638
|
| 709 |
+
},
|
| 710 |
+
{
|
| 711 |
+
"epoch": 0.2761795166858458,
|
| 712 |
+
"grad_norm": 0.6182126998901367,
|
| 713 |
+
"learning_rate": 8.668689317160086e-05,
|
| 714 |
+
"loss": 0.014076076447963715,
|
| 715 |
+
"memory(GiB)": 21.87,
|
| 716 |
+
"step": 345,
|
| 717 |
+
"token_acc": 0.9967243799719232,
|
| 718 |
+
"train_speed(iter/s)": 0.141698
|
| 719 |
+
},
|
| 720 |
+
{
|
| 721 |
+
"epoch": 0.280182118376945,
|
| 722 |
+
"grad_norm": 0.037689343094825745,
|
| 723 |
+
"learning_rate": 8.62337517262302e-05,
|
| 724 |
+
"loss": 0.009472431987524033,
|
| 725 |
+
"memory(GiB)": 21.87,
|
| 726 |
+
"step": 350,
|
| 727 |
+
"token_acc": 0.9972439136426274,
|
| 728 |
+
"train_speed(iter/s)": 0.141706
|
| 729 |
+
},
|
| 730 |
+
{
|
| 731 |
+
"epoch": 0.28418472006804424,
|
| 732 |
+
"grad_norm": 0.9789212942123413,
|
| 733 |
+
"learning_rate": 8.577425437681994e-05,
|
| 734 |
+
"loss": 0.01110573261976242,
|
| 735 |
+
"memory(GiB)": 21.87,
|
| 736 |
+
"step": 355,
|
| 737 |
+
"token_acc": 0.9966901117087298,
|
| 738 |
+
"train_speed(iter/s)": 0.141732
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"epoch": 0.28818732175914347,
|
| 742 |
+
"grad_norm": 0.5865841507911682,
|
| 743 |
+
"learning_rate": 8.530848172559651e-05,
|
| 744 |
+
"loss": 0.007035895437002182,
|
| 745 |
+
"memory(GiB)": 21.87,
|
| 746 |
+
"step": 360,
|
| 747 |
+
"token_acc": 0.9965679965679966,
|
| 748 |
+
"train_speed(iter/s)": 0.141755
|
| 749 |
+
},
|
| 750 |
+
{
|
| 751 |
+
"epoch": 0.29218992345024264,
|
| 752 |
+
"grad_norm": 0.12133053690195084,
|
| 753 |
+
"learning_rate": 8.483651547556149e-05,
|
| 754 |
+
"loss": 0.006775320321321487,
|
| 755 |
+
"memory(GiB)": 21.87,
|
| 756 |
+
"step": 365,
|
| 757 |
+
"token_acc": 0.9981524249422633,
|
| 758 |
+
"train_speed(iter/s)": 0.141784
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"epoch": 0.29619252514134187,
|
| 762 |
+
"grad_norm": 0.023800989612936974,
|
| 763 |
+
"learning_rate": 8.435843841615987e-05,
|
| 764 |
+
"loss": 0.005875982716679573,
|
| 765 |
+
"memory(GiB)": 21.87,
|
| 766 |
+
"step": 370,
|
| 767 |
+
"token_acc": 0.9983381803074366,
|
| 768 |
+
"train_speed(iter/s)": 0.141828
|
| 769 |
+
},
|
| 770 |
+
{
|
| 771 |
+
"epoch": 0.3001951268324411,
|
| 772 |
+
"grad_norm": 0.0036720961797982454,
|
| 773 |
+
"learning_rate": 8.387433440875758e-05,
|
| 774 |
+
"loss": 0.0028623411431908607,
|
| 775 |
+
"memory(GiB)": 21.87,
|
| 776 |
+
"step": 375,
|
| 777 |
+
"token_acc": 0.9987644151565074,
|
| 778 |
+
"train_speed(iter/s)": 0.141855
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"epoch": 0.3041977285235403,
|
| 782 |
+
"grad_norm": 0.4227054715156555,
|
| 783 |
+
"learning_rate": 8.338428837193111e-05,
|
| 784 |
+
"loss": 0.008840169757604599,
|
| 785 |
+
"memory(GiB)": 21.87,
|
| 786 |
+
"step": 380,
|
| 787 |
+
"token_acc": 0.9979140592407175,
|
| 788 |
+
"train_speed(iter/s)": 0.141877
|
| 789 |
+
},
|
| 790 |
+
{
|
| 791 |
+
"epoch": 0.3082003302146395,
|
| 792 |
+
"grad_norm": 0.45253515243530273,
|
| 793 |
+
"learning_rate": 8.28883862665715e-05,
|
| 794 |
+
"loss": 0.017132483422756195,
|
| 795 |
+
"memory(GiB)": 21.87,
|
| 796 |
+
"step": 385,
|
| 797 |
+
"token_acc": 0.9968397291196388,
|
| 798 |
+
"train_speed(iter/s)": 0.141875
|
| 799 |
+
},
|
| 800 |
+
{
|
| 801 |
+
"epoch": 0.3122029319057387,
|
| 802 |
+
"grad_norm": 0.8797569274902344,
|
| 803 |
+
"learning_rate": 8.238671508080569e-05,
|
| 804 |
+
"loss": 0.0167108491063118,
|
| 805 |
+
"memory(GiB)": 21.87,
|
| 806 |
+
"step": 390,
|
| 807 |
+
"token_acc": 0.9946917109024092,
|
| 808 |
+
"train_speed(iter/s)": 0.141925
|
| 809 |
+
},
|
| 810 |
+
{
|
| 811 |
+
"epoch": 0.31620553359683795,
|
| 812 |
+
"grad_norm": 1.025416374206543,
|
| 813 |
+
"learning_rate": 8.187936281473758e-05,
|
| 814 |
+
"loss": 0.015887443721294404,
|
| 815 |
+
"memory(GiB)": 21.87,
|
| 816 |
+
"step": 395,
|
| 817 |
+
"token_acc": 0.9954065227377125,
|
| 818 |
+
"train_speed(iter/s)": 0.141929
|
| 819 |
+
},
|
| 820 |
+
{
|
| 821 |
+
"epoch": 0.3202081352879372,
|
| 822 |
+
"grad_norm": 0.560427188873291,
|
| 823 |
+
"learning_rate": 8.136641846501156e-05,
|
| 824 |
+
"loss": 0.029094675183296205,
|
| 825 |
+
"memory(GiB)": 21.87,
|
| 826 |
+
"step": 400,
|
| 827 |
+
"token_acc": 0.9940366972477064,
|
| 828 |
+
"train_speed(iter/s)": 0.141962
|
| 829 |
+
},
|
| 830 |
+
{
|
| 831 |
+
"epoch": 0.3202081352879372,
|
| 832 |
+
"eval_loss": 0.041564591228961945,
|
| 833 |
+
"eval_runtime": 2.5083,
|
| 834 |
+
"eval_samples_per_second": 5.183,
|
| 835 |
+
"eval_steps_per_second": 5.183,
|
| 836 |
+
"eval_token_acc": 0.9848101265822785,
|
| 837 |
+
"step": 400
|
| 838 |
+
},
|
| 839 |
+
{
|
| 840 |
+
"epoch": 0.32421073697903635,
|
| 841 |
+
"grad_norm": 0.8376790881156921,
|
| 842 |
+
"learning_rate": 8.084797200920127e-05,
|
| 843 |
+
"loss": 0.026179602742195128,
|
| 844 |
+
"memory(GiB)": 21.87,
|
| 845 |
+
"step": 405,
|
| 846 |
+
"token_acc": 0.9920666402221341,
|
| 847 |
+
"train_speed(iter/s)": 0.141119
|
| 848 |
+
},
|
| 849 |
+
{
|
| 850 |
+
"epoch": 0.3282133386701356,
|
| 851 |
+
"grad_norm": 0.7372497320175171,
|
| 852 |
+
"learning_rate": 8.032411439002631e-05,
|
| 853 |
+
"loss": 0.0161885142326355,
|
| 854 |
+
"memory(GiB)": 23.34,
|
| 855 |
+
"step": 410,
|
| 856 |
+
"token_acc": 0.9958734525447043,
|
| 857 |
+
"train_speed(iter/s)": 0.141144
|
| 858 |
+
},
|
| 859 |
+
{
|
| 860 |
+
"epoch": 0.3322159403612348,
|
| 861 |
+
"grad_norm": 0.11186628043651581,
|
| 862 |
+
"learning_rate": 7.979493749939955e-05,
|
| 863 |
+
"loss": 0.016891977190971373,
|
| 864 |
+
"memory(GiB)": 23.34,
|
| 865 |
+
"step": 415,
|
| 866 |
+
"token_acc": 0.9947552447552448,
|
| 867 |
+
"train_speed(iter/s)": 0.141168
|
| 868 |
+
},
|
| 869 |
+
{
|
| 870 |
+
"epoch": 0.33621854205233404,
|
| 871 |
+
"grad_norm": 0.5271028876304626,
|
| 872 |
+
"learning_rate": 7.926053416230815e-05,
|
| 873 |
+
"loss": 0.018610107898712158,
|
| 874 |
+
"memory(GiB)": 23.34,
|
| 875 |
+
"step": 420,
|
| 876 |
+
"token_acc": 0.9946140035906643,
|
| 877 |
+
"train_speed(iter/s)": 0.141204
|
| 878 |
+
},
|
| 879 |
+
{
|
| 880 |
+
"epoch": 0.3402211437434332,
|
| 881 |
+
"grad_norm": 1.3814899921417236,
|
| 882 |
+
"learning_rate": 7.872099812053058e-05,
|
| 883 |
+
"loss": 0.020191578567028044,
|
| 884 |
+
"memory(GiB)": 23.34,
|
| 885 |
+
"step": 425,
|
| 886 |
+
"token_acc": 0.9910674408217954,
|
| 887 |
+
"train_speed(iter/s)": 0.141243
|
| 888 |
+
},
|
| 889 |
+
{
|
| 890 |
+
"epoch": 0.34422374543453244,
|
| 891 |
+
"grad_norm": 0.009208492934703827,
|
| 892 |
+
"learning_rate": 7.817642401619316e-05,
|
| 893 |
+
"loss": 0.013885082304477691,
|
| 894 |
+
"memory(GiB)": 23.34,
|
| 895 |
+
"step": 430,
|
| 896 |
+
"token_acc": 0.9960852544584602,
|
| 897 |
+
"train_speed(iter/s)": 0.141251
|
| 898 |
+
},
|
| 899 |
+
{
|
| 900 |
+
"epoch": 0.34822634712563166,
|
| 901 |
+
"grad_norm": 0.35401901602745056,
|
| 902 |
+
"learning_rate": 7.762690737516846e-05,
|
| 903 |
+
"loss": 0.014876057207584382,
|
| 904 |
+
"memory(GiB)": 23.34,
|
| 905 |
+
"step": 435,
|
| 906 |
+
"token_acc": 0.9968268359020852,
|
| 907 |
+
"train_speed(iter/s)": 0.141276
|
| 908 |
+
},
|
| 909 |
+
{
|
| 910 |
+
"epoch": 0.3522289488167309,
|
| 911 |
+
"grad_norm": 0.17650920152664185,
|
| 912 |
+
"learning_rate": 7.707254459031869e-05,
|
| 913 |
+
"loss": 0.04629639387130737,
|
| 914 |
+
"memory(GiB)": 23.34,
|
| 915 |
+
"step": 440,
|
| 916 |
+
"token_acc": 0.9875751503006012,
|
| 917 |
+
"train_speed(iter/s)": 0.141304
|
| 918 |
+
},
|
| 919 |
+
{
|
| 920 |
+
"epoch": 0.35623155050783006,
|
| 921 |
+
"grad_norm": 0.2500922381877899,
|
| 922 |
+
"learning_rate": 7.651343290458714e-05,
|
| 923 |
+
"loss": 0.009627890586853028,
|
| 924 |
+
"memory(GiB)": 23.34,
|
| 925 |
+
"step": 445,
|
| 926 |
+
"token_acc": 0.9979175343606831,
|
| 927 |
+
"train_speed(iter/s)": 0.141338
|
| 928 |
+
},
|
| 929 |
+
{
|
| 930 |
+
"epoch": 0.3602341521989293,
|
| 931 |
+
"grad_norm": 0.052707232534885406,
|
| 932 |
+
"learning_rate": 7.594967039394034e-05,
|
| 933 |
+
"loss": 0.013120321929454804,
|
| 934 |
+
"memory(GiB)": 23.34,
|
| 935 |
+
"step": 450,
|
| 936 |
+
"token_acc": 0.9959695476936856,
|
| 937 |
+
"train_speed(iter/s)": 0.141368
|
| 938 |
+
},
|
| 939 |
+
{
|
| 940 |
+
"epoch": 0.3642367538900285,
|
| 941 |
+
"grad_norm": 0.08748254925012589,
|
| 942 |
+
"learning_rate": 7.538135595016423e-05,
|
| 943 |
+
"loss": 0.009847868978977204,
|
| 944 |
+
"memory(GiB)": 23.34,
|
| 945 |
+
"step": 455,
|
| 946 |
+
"token_acc": 0.9968999114260407,
|
| 947 |
+
"train_speed(iter/s)": 0.141362
|
| 948 |
+
},
|
| 949 |
+
{
|
| 950 |
+
"epoch": 0.36823935558112775,
|
| 951 |
+
"grad_norm": 0.12798438966274261,
|
| 952 |
+
"learning_rate": 7.480858926351708e-05,
|
| 953 |
+
"loss": 0.009454815834760665,
|
| 954 |
+
"memory(GiB)": 23.34,
|
| 955 |
+
"step": 460,
|
| 956 |
+
"token_acc": 0.9971777986829727,
|
| 957 |
+
"train_speed(iter/s)": 0.141363
|
| 958 |
+
},
|
| 959 |
+
{
|
| 960 |
+
"epoch": 0.3722419572722269,
|
| 961 |
+
"grad_norm": 0.1612037569284439,
|
| 962 |
+
"learning_rate": 7.423147080524254e-05,
|
| 963 |
+
"loss": 0.005590154975652695,
|
| 964 |
+
"memory(GiB)": 23.34,
|
| 965 |
+
"step": 465,
|
| 966 |
+
"token_acc": 0.9991902834008097,
|
| 967 |
+
"train_speed(iter/s)": 0.141376
|
| 968 |
+
},
|
| 969 |
+
{
|
| 970 |
+
"epoch": 0.37624455896332615,
|
| 971 |
+
"grad_norm": 0.04762154072523117,
|
| 972 |
+
"learning_rate": 7.365010180994554e-05,
|
| 973 |
+
"loss": 0.004805747047066688,
|
| 974 |
+
"memory(GiB)": 23.34,
|
| 975 |
+
"step": 470,
|
| 976 |
+
"token_acc": 0.9986443741527339,
|
| 977 |
+
"train_speed(iter/s)": 0.141384
|
| 978 |
+
},
|
| 979 |
+
{
|
| 980 |
+
"epoch": 0.3802471606544254,
|
| 981 |
+
"grad_norm": 0.013279656879603863,
|
| 982 |
+
"learning_rate": 7.306458425783426e-05,
|
| 983 |
+
"loss": 0.0074897401034832,
|
| 984 |
+
"memory(GiB)": 23.34,
|
| 985 |
+
"step": 475,
|
| 986 |
+
"token_acc": 0.9972739663789186,
|
| 987 |
+
"train_speed(iter/s)": 0.141377
|
| 988 |
+
},
|
| 989 |
+
{
|
| 990 |
+
"epoch": 0.3842497623455246,
|
| 991 |
+
"grad_norm": 0.013300618156790733,
|
| 992 |
+
"learning_rate": 7.247502085683151e-05,
|
| 993 |
+
"loss": 0.0009232103824615478,
|
| 994 |
+
"memory(GiB)": 23.34,
|
| 995 |
+
"step": 480,
|
| 996 |
+
"token_acc": 1.0,
|
| 997 |
+
"train_speed(iter/s)": 0.141369
|
| 998 |
+
},
|
| 999 |
+
{
|
| 1000 |
+
"epoch": 0.38825236403662383,
|
| 1001 |
+
"grad_norm": 0.07290496677160263,
|
| 1002 |
+
"learning_rate": 7.188151502455826e-05,
|
| 1003 |
+
"loss": 0.008077611029148103,
|
| 1004 |
+
"memory(GiB)": 23.34,
|
| 1005 |
+
"step": 485,
|
| 1006 |
+
"token_acc": 0.9980685659101883,
|
| 1007 |
+
"train_speed(iter/s)": 0.141352
|
| 1008 |
+
},
|
| 1009 |
+
{
|
| 1010 |
+
"epoch": 0.392254965727723,
|
| 1011 |
+
"grad_norm": 0.03351164609193802,
|
| 1012 |
+
"learning_rate": 7.128417087019273e-05,
|
| 1013 |
+
"loss": 0.013489419221878051,
|
| 1014 |
+
"memory(GiB)": 23.34,
|
| 1015 |
+
"step": 490,
|
| 1016 |
+
"token_acc": 0.997764863656683,
|
| 1017 |
+
"train_speed(iter/s)": 0.141365
|
| 1018 |
+
},
|
| 1019 |
+
{
|
| 1020 |
+
"epoch": 0.39625756741882223,
|
| 1021 |
+
"grad_norm": 1.0084375143051147,
|
| 1022 |
+
"learning_rate": 7.068309317620827e-05,
|
| 1023 |
+
"loss": 0.008754704892635346,
|
| 1024 |
+
"memory(GiB)": 23.34,
|
| 1025 |
+
"step": 495,
|
| 1026 |
+
"token_acc": 0.9957191780821918,
|
| 1027 |
+
"train_speed(iter/s)": 0.141391
|
| 1028 |
+
},
|
| 1029 |
+
{
|
| 1030 |
+
"epoch": 0.40026016910992146,
|
| 1031 |
+
"grad_norm": 0.540347158908844,
|
| 1032 |
+
"learning_rate": 7.007838737999297e-05,
|
| 1033 |
+
"loss": 0.005627882480621338,
|
| 1034 |
+
"memory(GiB)": 23.34,
|
| 1035 |
+
"step": 500,
|
| 1036 |
+
"token_acc": 0.9990757855822551,
|
| 1037 |
+
"train_speed(iter/s)": 0.141412
|
| 1038 |
+
},
|
| 1039 |
+
{
|
| 1040 |
+
"epoch": 0.4042627708010207,
|
| 1041 |
+
"grad_norm": 0.07603906095027924,
|
| 1042 |
+
"learning_rate": 6.947015955535452e-05,
|
| 1043 |
+
"loss": 0.01039251834154129,
|
| 1044 |
+
"memory(GiB)": 23.34,
|
| 1045 |
+
"step": 505,
|
| 1046 |
+
"token_acc": 0.9981965734896303,
|
| 1047 |
+
"train_speed(iter/s)": 0.141442
|
| 1048 |
+
},
|
| 1049 |
+
{
|
| 1050 |
+
"epoch": 0.40826537249211986,
|
| 1051 |
+
"grad_norm": 0.5115349292755127,
|
| 1052 |
+
"learning_rate": 6.885851639391344e-05,
|
| 1053 |
+
"loss": 0.016630645096302032,
|
| 1054 |
+
"memory(GiB)": 23.34,
|
| 1055 |
+
"step": 510,
|
| 1056 |
+
"token_acc": 0.9940499801666005,
|
| 1057 |
+
"train_speed(iter/s)": 0.141458
|
| 1058 |
+
},
|
| 1059 |
+
{
|
| 1060 |
+
"epoch": 0.4122679741832191,
|
| 1061 |
+
"grad_norm": 0.024469200521707535,
|
| 1062 |
+
"learning_rate": 6.824356518638775e-05,
|
| 1063 |
+
"loss": 0.006416448950767517,
|
| 1064 |
+
"memory(GiB)": 23.34,
|
| 1065 |
+
"step": 515,
|
| 1066 |
+
"token_acc": 0.998220640569395,
|
| 1067 |
+
"train_speed(iter/s)": 0.141496
|
| 1068 |
+
},
|
| 1069 |
+
{
|
| 1070 |
+
"epoch": 0.4162705758743183,
|
| 1071 |
+
"grad_norm": 0.053554967045784,
|
| 1072 |
+
"learning_rate": 6.762541380377279e-05,
|
| 1073 |
+
"loss": 0.006349309533834458,
|
| 1074 |
+
"memory(GiB)": 23.34,
|
| 1075 |
+
"step": 520,
|
| 1076 |
+
"token_acc": 0.9978354978354979,
|
| 1077 |
+
"train_speed(iter/s)": 0.141518
|
| 1078 |
+
},
|
| 1079 |
+
{
|
| 1080 |
+
"epoch": 0.42027317756541754,
|
| 1081 |
+
"grad_norm": 0.41779887676239014,
|
| 1082 |
+
"learning_rate": 6.700417067841918e-05,
|
| 1083 |
+
"loss": 0.003070826083421707,
|
| 1084 |
+
"memory(GiB)": 23.34,
|
| 1085 |
+
"step": 525,
|
| 1086 |
+
"token_acc": 0.9991311902693311,
|
| 1087 |
+
"train_speed(iter/s)": 0.141532
|
| 1088 |
+
},
|
| 1089 |
+
{
|
| 1090 |
+
"epoch": 0.4242757792565167,
|
| 1091 |
+
"grad_norm": 0.04191483184695244,
|
| 1092 |
+
"learning_rate": 6.637994478501214e-05,
|
| 1093 |
+
"loss": 0.003617442026734352,
|
| 1094 |
+
"memory(GiB)": 23.34,
|
| 1095 |
+
"step": 530,
|
| 1096 |
+
"token_acc": 0.9981834695731153,
|
| 1097 |
+
"train_speed(iter/s)": 0.14153
|
| 1098 |
+
},
|
| 1099 |
+
{
|
| 1100 |
+
"epoch": 0.42827838094761594,
|
| 1101 |
+
"grad_norm": 0.010227917693555355,
|
| 1102 |
+
"learning_rate": 6.575284562145593e-05,
|
| 1103 |
+
"loss": 0.0006208265665918588,
|
| 1104 |
+
"memory(GiB)": 23.34,
|
| 1105 |
+
"step": 535,
|
| 1106 |
+
"token_acc": 1.0,
|
| 1107 |
+
"train_speed(iter/s)": 0.141546
|
| 1108 |
+
},
|
| 1109 |
+
{
|
| 1110 |
+
"epoch": 0.43228098263871517,
|
| 1111 |
+
"grad_norm": 0.018105624243617058,
|
| 1112 |
+
"learning_rate": 6.512298318966639e-05,
|
| 1113 |
+
"loss": 0.009379193931818009,
|
| 1114 |
+
"memory(GiB)": 23.34,
|
| 1115 |
+
"step": 540,
|
| 1116 |
+
"token_acc": 0.9978822532825075,
|
| 1117 |
+
"train_speed(iter/s)": 0.141547
|
| 1118 |
+
},
|
| 1119 |
+
{
|
| 1120 |
+
"epoch": 0.4362835843298144,
|
| 1121 |
+
"grad_norm": 0.01871226727962494,
|
| 1122 |
+
"learning_rate": 6.449046797627499e-05,
|
| 1123 |
+
"loss": 0.008023256063461303,
|
| 1124 |
+
"memory(GiB)": 23.34,
|
| 1125 |
+
"step": 545,
|
| 1126 |
+
"token_acc": 0.9982174688057041,
|
| 1127 |
+
"train_speed(iter/s)": 0.141554
|
| 1128 |
+
},
|
| 1129 |
+
{
|
| 1130 |
+
"epoch": 0.4402861860209136,
|
| 1131 |
+
"grad_norm": 0.09551273286342621,
|
| 1132 |
+
"learning_rate": 6.385541093324799e-05,
|
| 1133 |
+
"loss": 0.011187158524990082,
|
| 1134 |
+
"memory(GiB)": 23.34,
|
| 1135 |
+
"step": 550,
|
| 1136 |
+
"token_acc": 0.9947575360419397,
|
| 1137 |
+
"train_speed(iter/s)": 0.141564
|
| 1138 |
+
},
|
| 1139 |
+
{
|
| 1140 |
+
"epoch": 0.4442887877120128,
|
| 1141 |
+
"grad_norm": 0.36312055587768555,
|
| 1142 |
+
"learning_rate": 6.321792345842402e-05,
|
| 1143 |
+
"loss": 0.01205039769411087,
|
| 1144 |
+
"memory(GiB)": 23.34,
|
| 1145 |
+
"step": 555,
|
| 1146 |
+
"token_acc": 0.9963570127504554,
|
| 1147 |
+
"train_speed(iter/s)": 0.141543
|
| 1148 |
+
},
|
| 1149 |
+
{
|
| 1150 |
+
"epoch": 0.44829138940311203,
|
| 1151 |
+
"grad_norm": 2.144325017929077,
|
| 1152 |
+
"learning_rate": 6.257811737597311e-05,
|
| 1153 |
+
"loss": 0.014584457874298096,
|
| 1154 |
+
"memory(GiB)": 23.34,
|
| 1155 |
+
"step": 560,
|
| 1156 |
+
"token_acc": 0.9947064485081809,
|
| 1157 |
+
"train_speed(iter/s)": 0.141538
|
| 1158 |
+
},
|
| 1159 |
+
{
|
| 1160 |
+
"epoch": 0.45229399109421126,
|
| 1161 |
+
"grad_norm": 0.028064021840691566,
|
| 1162 |
+
"learning_rate": 6.193610491678141e-05,
|
| 1163 |
+
"loss": 0.011823876947164535,
|
| 1164 |
+
"memory(GiB)": 23.34,
|
| 1165 |
+
"step": 565,
|
| 1166 |
+
"token_acc": 0.997893850042123,
|
| 1167 |
+
"train_speed(iter/s)": 0.141541
|
| 1168 |
+
},
|
| 1169 |
+
{
|
| 1170 |
+
"epoch": 0.45629659278531043,
|
| 1171 |
+
"grad_norm": 0.03831484913825989,
|
| 1172 |
+
"learning_rate": 6.129199869876424e-05,
|
| 1173 |
+
"loss": 0.013196608424186707,
|
| 1174 |
+
"memory(GiB)": 23.34,
|
| 1175 |
+
"step": 570,
|
| 1176 |
+
"token_acc": 0.9971223021582734,
|
| 1177 |
+
"train_speed(iter/s)": 0.141531
|
| 1178 |
+
},
|
| 1179 |
+
{
|
| 1180 |
+
"epoch": 0.46029919447640966,
|
| 1181 |
+
"grad_norm": 0.5108913779258728,
|
| 1182 |
+
"learning_rate": 6.06459117071113e-05,
|
| 1183 |
+
"loss": 0.01156686395406723,
|
| 1184 |
+
"memory(GiB)": 23.34,
|
| 1185 |
+
"step": 575,
|
| 1186 |
+
"token_acc": 0.9943529411764706,
|
| 1187 |
+
"train_speed(iter/s)": 0.141513
|
| 1188 |
+
},
|
| 1189 |
+
{
|
| 1190 |
+
"epoch": 0.4643017961675089,
|
| 1191 |
+
"grad_norm": 2.259899616241455,
|
| 1192 |
+
"learning_rate": 5.9997957274467555e-05,
|
| 1193 |
+
"loss": 0.01473301202058792,
|
| 1194 |
+
"memory(GiB)": 23.34,
|
| 1195 |
+
"step": 580,
|
| 1196 |
+
"token_acc": 0.9941096511101042,
|
| 1197 |
+
"train_speed(iter/s)": 0.141503
|
| 1198 |
+
},
|
| 1199 |
+
{
|
| 1200 |
+
"epoch": 0.4683043978586081,
|
| 1201 |
+
"grad_norm": 0.013200435787439346,
|
| 1202 |
+
"learning_rate": 5.934824906105309e-05,
|
| 1203 |
+
"loss": 0.004953300207853317,
|
| 1204 |
+
"memory(GiB)": 23.34,
|
| 1205 |
+
"step": 585,
|
| 1206 |
+
"token_acc": 0.9972350230414746,
|
| 1207 |
+
"train_speed(iter/s)": 0.141486
|
| 1208 |
+
},
|
| 1209 |
+
{
|
| 1210 |
+
"epoch": 0.4723069995497073,
|
| 1211 |
+
"grad_norm": 0.23236890137195587,
|
| 1212 |
+
"learning_rate": 5.869690103472554e-05,
|
| 1213 |
+
"loss": 0.0051562488079071045,
|
| 1214 |
+
"memory(GiB)": 23.34,
|
| 1215 |
+
"step": 590,
|
| 1216 |
+
"token_acc": 0.9977220956719818,
|
| 1217 |
+
"train_speed(iter/s)": 0.141483
|
| 1218 |
+
},
|
| 1219 |
+
{
|
| 1220 |
+
"epoch": 0.4763096012408065,
|
| 1221 |
+
"grad_norm": 0.9093765020370483,
|
| 1222 |
+
"learning_rate": 5.8044027450988546e-05,
|
| 1223 |
+
"loss": 0.013899902999401092,
|
| 1224 |
+
"memory(GiB)": 23.34,
|
| 1225 |
+
"step": 595,
|
| 1226 |
+
"token_acc": 0.9960560911481157,
|
| 1227 |
+
"train_speed(iter/s)": 0.14152
|
| 1228 |
+
},
|
| 1229 |
+
{
|
| 1230 |
+
"epoch": 0.48031220293190574,
|
| 1231 |
+
"grad_norm": 0.03756490349769592,
|
| 1232 |
+
"learning_rate": 5.73897428329497e-05,
|
| 1233 |
+
"loss": 0.0008009488694369793,
|
| 1234 |
+
"memory(GiB)": 23.34,
|
| 1235 |
+
"step": 600,
|
| 1236 |
+
"token_acc": 0.9995269631031221,
|
| 1237 |
+
"train_speed(iter/s)": 0.14153
|
| 1238 |
+
},
|
| 1239 |
+
{
|
| 1240 |
+
"epoch": 0.48031220293190574,
|
| 1241 |
+
"eval_loss": 0.05568527430295944,
|
| 1242 |
+
"eval_runtime": 2.5017,
|
| 1243 |
+
"eval_samples_per_second": 5.196,
|
| 1244 |
+
"eval_steps_per_second": 5.196,
|
| 1245 |
+
"eval_token_acc": 0.9848101265822785,
|
| 1246 |
+
"step": 600
|
| 1247 |
+
},
|
| 1248 |
+
{
|
| 1249 |
+
"epoch": 0.48431480462300497,
|
| 1250 |
+
"grad_norm": 1.8104074001312256,
|
| 1251 |
+
"learning_rate": 5.673416195123167e-05,
|
| 1252 |
+
"loss": 0.0030754044651985168,
|
| 1253 |
+
"memory(GiB)": 23.34,
|
| 1254 |
+
"step": 605,
|
| 1255 |
+
"token_acc": 0.9966911764705882,
|
| 1256 |
+
"train_speed(iter/s)": 0.140693
|
| 1257 |
+
},
|
| 1258 |
+
{
|
| 1259 |
+
"epoch": 0.48831740631410414,
|
| 1260 |
+
"grad_norm": 0.0047276695258915424,
|
| 1261 |
+
"learning_rate": 5.607739980383979e-05,
|
| 1262 |
+
"loss": 0.004061650484800339,
|
| 1263 |
+
"memory(GiB)": 23.34,
|
| 1264 |
+
"step": 610,
|
| 1265 |
+
"token_acc": 0.9977973568281938,
|
| 1266 |
+
"train_speed(iter/s)": 0.140707
|
| 1267 |
+
},
|
| 1268 |
+
{
|
| 1269 |
+
"epoch": 0.49232000800520337,
|
| 1270 |
+
"grad_norm": 0.07247687876224518,
|
| 1271 |
+
"learning_rate": 5.5419571595989825e-05,
|
| 1272 |
+
"loss": 0.013193260133266448,
|
| 1273 |
+
"memory(GiB)": 23.34,
|
| 1274 |
+
"step": 615,
|
| 1275 |
+
"token_acc": 0.9974662162162162,
|
| 1276 |
+
"train_speed(iter/s)": 0.140724
|
| 1277 |
+
},
|
| 1278 |
+
{
|
| 1279 |
+
"epoch": 0.4963226096963026,
|
| 1280 |
+
"grad_norm": 0.07320598512887955,
|
| 1281 |
+
"learning_rate": 5.476079271989939e-05,
|
| 1282 |
+
"loss": 0.0059086386114358905,
|
| 1283 |
+
"memory(GiB)": 23.34,
|
| 1284 |
+
"step": 620,
|
| 1285 |
+
"token_acc": 0.9975114060555786,
|
| 1286 |
+
"train_speed(iter/s)": 0.140739
|
| 1287 |
+
},
|
| 1288 |
+
{
|
| 1289 |
+
"epoch": 0.5003252113874018,
|
| 1290 |
+
"grad_norm": 0.04662081226706505,
|
| 1291 |
+
"learning_rate": 5.41011787345465e-05,
|
| 1292 |
+
"loss": 0.003809797018766403,
|
| 1293 |
+
"memory(GiB)": 23.34,
|
| 1294 |
+
"step": 625,
|
| 1295 |
+
"token_acc": 0.9987293519695044,
|
| 1296 |
+
"train_speed(iter/s)": 0.140748
|
| 1297 |
+
},
|
| 1298 |
+
{
|
| 1299 |
+
"epoch": 0.504327813078501,
|
| 1300 |
+
"grad_norm": 0.21870079636573792,
|
| 1301 |
+
"learning_rate": 5.344084534539896e-05,
|
| 1302 |
+
"loss": 0.029582920670509338,
|
| 1303 |
+
"memory(GiB)": 23.34,
|
| 1304 |
+
"step": 630,
|
| 1305 |
+
"token_acc": 0.995949594959496,
|
| 1306 |
+
"train_speed(iter/s)": 0.140737
|
| 1307 |
+
},
|
| 1308 |
+
{
|
| 1309 |
+
"epoch": 0.5083304147696003,
|
| 1310 |
+
"grad_norm": 0.5850868225097656,
|
| 1311 |
+
"learning_rate": 5.2779908384118025e-05,
|
| 1312 |
+
"loss": 0.00632546991109848,
|
| 1313 |
+
"memory(GiB)": 23.34,
|
| 1314 |
+
"step": 635,
|
| 1315 |
+
"token_acc": 0.9982578397212544,
|
| 1316 |
+
"train_speed(iter/s)": 0.140745
|
| 1317 |
+
},
|
| 1318 |
+
{
|
| 1319 |
+
"epoch": 0.5123330164606995,
|
| 1320 |
+
"grad_norm": 0.12139564752578735,
|
| 1321 |
+
"learning_rate": 5.211848378823988e-05,
|
| 1322 |
+
"loss": 0.00618828721344471,
|
| 1323 |
+
"memory(GiB)": 23.34,
|
| 1324 |
+
"step": 640,
|
| 1325 |
+
"token_acc": 0.997793468667255,
|
| 1326 |
+
"train_speed(iter/s)": 0.140762
|
| 1327 |
+
},
|
| 1328 |
+
{
|
| 1329 |
+
"epoch": 0.5163356181517986,
|
| 1330 |
+
"grad_norm": 0.009014817886054516,
|
| 1331 |
+
"learning_rate": 5.145668758083862e-05,
|
| 1332 |
+
"loss": 0.003024919517338276,
|
| 1333 |
+
"memory(GiB)": 23.34,
|
| 1334 |
+
"step": 645,
|
| 1335 |
+
"token_acc": 0.9987780040733197,
|
| 1336 |
+
"train_speed(iter/s)": 0.140786
|
| 1337 |
+
},
|
| 1338 |
+
{
|
| 1339 |
+
"epoch": 0.5203382198428979,
|
| 1340 |
+
"grad_norm": 0.004454301670193672,
|
| 1341 |
+
"learning_rate": 5.079463585017416e-05,
|
| 1342 |
+
"loss": 0.006423936784267425,
|
| 1343 |
+
"memory(GiB)": 23.34,
|
| 1344 |
+
"step": 650,
|
| 1345 |
+
"token_acc": 0.9971228935470613,
|
| 1346 |
+
"train_speed(iter/s)": 0.140803
|
| 1347 |
+
},
|
| 1348 |
+
{
|
| 1349 |
+
"epoch": 0.5243408215339971,
|
| 1350 |
+
"grad_norm": 0.025101356208324432,
|
| 1351 |
+
"learning_rate": 5.013244472932872e-05,
|
| 1352 |
+
"loss": 0.006796837598085403,
|
| 1353 |
+
"memory(GiB)": 23.34,
|
| 1354 |
+
"step": 655,
|
| 1355 |
+
"token_acc": 0.997852233676976,
|
| 1356 |
+
"train_speed(iter/s)": 0.140826
|
| 1357 |
+
},
|
| 1358 |
+
{
|
| 1359 |
+
"epoch": 0.5283434232250963,
|
| 1360 |
+
"grad_norm": 0.07231304794549942,
|
| 1361 |
+
"learning_rate": 4.947023037583553e-05,
|
| 1362 |
+
"loss": 0.0074545606970787045,
|
| 1363 |
+
"memory(GiB)": 23.34,
|
| 1364 |
+
"step": 660,
|
| 1365 |
+
"token_acc": 0.9977924944812362,
|
| 1366 |
+
"train_speed(iter/s)": 0.140838
|
| 1367 |
+
},
|
| 1368 |
+
{
|
| 1369 |
+
"epoch": 0.5323460249161955,
|
| 1370 |
+
"grad_norm": 0.0038576293736696243,
|
| 1371 |
+
"learning_rate": 4.880810895130318e-05,
|
| 1372 |
+
"loss": 0.0033898912370204926,
|
| 1373 |
+
"memory(GiB)": 23.34,
|
| 1374 |
+
"step": 665,
|
| 1375 |
+
"token_acc": 0.9987618654560462,
|
| 1376 |
+
"train_speed(iter/s)": 0.140851
|
| 1377 |
+
},
|
| 1378 |
+
{
|
| 1379 |
+
"epoch": 0.5363486266072948,
|
| 1380 |
+
"grad_norm": 0.12802980840206146,
|
| 1381 |
+
"learning_rate": 4.814619660103915e-05,
|
| 1382 |
+
"loss": 0.006313430517911911,
|
| 1383 |
+
"memory(GiB)": 23.34,
|
| 1384 |
+
"step": 670,
|
| 1385 |
+
"token_acc": 0.9981351981351981,
|
| 1386 |
+
"train_speed(iter/s)": 0.140863
|
| 1387 |
+
},
|
| 1388 |
+
{
|
| 1389 |
+
"epoch": 0.540351228298394,
|
| 1390 |
+
"grad_norm": 0.07677621394395828,
|
| 1391 |
+
"learning_rate": 4.748460943367643e-05,
|
| 1392 |
+
"loss": 0.004006941616535187,
|
| 1393 |
+
"memory(GiB)": 23.34,
|
| 1394 |
+
"step": 675,
|
| 1395 |
+
"token_acc": 0.9991521831284442,
|
| 1396 |
+
"train_speed(iter/s)": 0.140884
|
| 1397 |
+
},
|
| 1398 |
+
{
|
| 1399 |
+
"epoch": 0.5443538299894932,
|
| 1400 |
+
"grad_norm": 0.005235038232058287,
|
| 1401 |
+
"learning_rate": 4.682346350080643e-05,
|
| 1402 |
+
"loss": 0.007923303544521332,
|
| 1403 |
+
"memory(GiB)": 23.34,
|
| 1404 |
+
"step": 680,
|
| 1405 |
+
"token_acc": 0.9982078853046595,
|
| 1406 |
+
"train_speed(iter/s)": 0.14092
|
| 1407 |
+
},
|
| 1408 |
+
{
|
| 1409 |
+
"epoch": 0.5483564316805923,
|
| 1410 |
+
"grad_norm": 0.03792576864361763,
|
| 1411 |
+
"learning_rate": 4.6162874776621836e-05,
|
| 1412 |
+
"loss": 0.0012888189405202866,
|
| 1413 |
+
"memory(GiB)": 23.34,
|
| 1414 |
+
"step": 685,
|
| 1415 |
+
"token_acc": 0.9995760915642221,
|
| 1416 |
+
"train_speed(iter/s)": 0.140924
|
| 1417 |
+
},
|
| 1418 |
+
{
|
| 1419 |
+
"epoch": 0.5523590333716916,
|
| 1420 |
+
"grad_norm": 0.5021899342536926,
|
| 1421 |
+
"learning_rate": 4.550295913757322e-05,
|
| 1422 |
+
"loss": 0.0020086320117115976,
|
| 1423 |
+
"memory(GiB)": 23.34,
|
| 1424 |
+
"step": 690,
|
| 1425 |
+
"token_acc": 0.999124343257443,
|
| 1426 |
+
"train_speed(iter/s)": 0.140962
|
| 1427 |
+
},
|
| 1428 |
+
{
|
| 1429 |
+
"epoch": 0.5563616350627908,
|
| 1430 |
+
"grad_norm": 0.018016846850514412,
|
| 1431 |
+
"learning_rate": 4.484383234204266e-05,
|
| 1432 |
+
"loss": 0.004791406914591789,
|
| 1433 |
+
"memory(GiB)": 23.34,
|
| 1434 |
+
"step": 695,
|
| 1435 |
+
"token_acc": 0.9974337040205303,
|
| 1436 |
+
"train_speed(iter/s)": 0.140991
|
| 1437 |
+
},
|
| 1438 |
+
{
|
| 1439 |
+
"epoch": 0.56036423675389,
|
| 1440 |
+
"grad_norm": 0.0007839211029931903,
|
| 1441 |
+
"learning_rate": 4.418561001003807e-05,
|
| 1442 |
+
"loss": 0.0029749471694231035,
|
| 1443 |
+
"memory(GiB)": 23.34,
|
| 1444 |
+
"step": 700,
|
| 1445 |
+
"token_acc": 0.9986553115194979,
|
| 1446 |
+
"train_speed(iter/s)": 0.141023
|
| 1447 |
+
},
|
| 1448 |
+
{
|
| 1449 |
+
"epoch": 0.5643668384449892,
|
| 1450 |
+
"grad_norm": 0.18348391354084015,
|
| 1451 |
+
"learning_rate": 4.3528407602911906e-05,
|
| 1452 |
+
"loss": 0.007216111570596695,
|
| 1453 |
+
"memory(GiB)": 23.34,
|
| 1454 |
+
"step": 705,
|
| 1455 |
+
"token_acc": 0.997289972899729,
|
| 1456 |
+
"train_speed(iter/s)": 0.141033
|
| 1457 |
+
},
|
| 1458 |
+
{
|
| 1459 |
+
"epoch": 0.5683694401360885,
|
| 1460 |
+
"grad_norm": 0.15839047729969025,
|
| 1461 |
+
"learning_rate": 4.2872340403107664e-05,
|
| 1462 |
+
"loss": 0.0020360426977276804,
|
| 1463 |
+
"memory(GiB)": 23.34,
|
| 1464 |
+
"step": 710,
|
| 1465 |
+
"token_acc": 0.999059708509638,
|
| 1466 |
+
"train_speed(iter/s)": 0.141109
|
| 1467 |
+
},
|
| 1468 |
+
{
|
| 1469 |
+
"epoch": 0.5723720418271877,
|
| 1470 |
+
"grad_norm": 0.024005182087421417,
|
| 1471 |
+
"learning_rate": 4.221752349393758e-05,
|
| 1472 |
+
"loss": 0.003486247733235359,
|
| 1473 |
+
"memory(GiB)": 23.34,
|
| 1474 |
+
"step": 715,
|
| 1475 |
+
"token_acc": 0.9982773471145564,
|
| 1476 |
+
"train_speed(iter/s)": 0.141198
|
| 1477 |
+
},
|
| 1478 |
+
{
|
| 1479 |
+
"epoch": 0.5763746435182869,
|
| 1480 |
+
"grad_norm": 0.004409647546708584,
|
| 1481 |
+
"learning_rate": 4.15640717393956e-05,
|
| 1482 |
+
"loss": 0.00223893728107214,
|
| 1483 |
+
"memory(GiB)": 23.34,
|
| 1484 |
+
"step": 720,
|
| 1485 |
+
"token_acc": 0.9986807387862797,
|
| 1486 |
+
"train_speed(iter/s)": 0.141304
|
| 1487 |
+
},
|
| 1488 |
+
{
|
| 1489 |
+
"epoch": 0.580377245209386,
|
| 1490 |
+
"grad_norm": 0.31079161167144775,
|
| 1491 |
+
"learning_rate": 4.091209976400858e-05,
|
| 1492 |
+
"loss": 0.008539219200611115,
|
| 1493 |
+
"memory(GiB)": 23.34,
|
| 1494 |
+
"step": 725,
|
| 1495 |
+
"token_acc": 0.99822695035461,
|
| 1496 |
+
"train_speed(iter/s)": 0.141405
|
| 1497 |
+
},
|
| 1498 |
+
{
|
| 1499 |
+
"epoch": 0.5843798469004853,
|
| 1500 |
+
"grad_norm": 0.014985853806138039,
|
| 1501 |
+
"learning_rate": 4.026172193272941e-05,
|
| 1502 |
+
"loss": 0.004056619852781296,
|
| 1503 |
+
"memory(GiB)": 23.34,
|
| 1504 |
+
"step": 730,
|
| 1505 |
+
"token_acc": 0.9977836879432624,
|
| 1506 |
+
"train_speed(iter/s)": 0.141508
|
| 1507 |
+
},
|
| 1508 |
+
{
|
| 1509 |
+
"epoch": 0.5883824485915845,
|
| 1510 |
+
"grad_norm": 1.119271159172058,
|
| 1511 |
+
"learning_rate": 3.9613052330876064e-05,
|
| 1512 |
+
"loss": 0.002341046743094921,
|
| 1513 |
+
"memory(GiB)": 23.34,
|
| 1514 |
+
"step": 735,
|
| 1515 |
+
"token_acc": 0.9991095280498664,
|
| 1516 |
+
"train_speed(iter/s)": 0.141603
|
| 1517 |
+
},
|
| 1518 |
+
{
|
| 1519 |
+
"epoch": 0.5923850502826837,
|
| 1520 |
+
"grad_norm": 0.005460113286972046,
|
| 1521 |
+
"learning_rate": 3.896620474411926e-05,
|
| 1522 |
+
"loss": 0.00231602918356657,
|
| 1523 |
+
"memory(GiB)": 23.34,
|
| 1524 |
+
"step": 740,
|
| 1525 |
+
"token_acc": 0.9986462093862816,
|
| 1526 |
+
"train_speed(iter/s)": 0.141702
|
| 1527 |
+
},
|
| 1528 |
+
{
|
| 1529 |
+
"epoch": 0.596387651973783,
|
| 1530 |
+
"grad_norm": 0.11408213526010513,
|
| 1531 |
+
"learning_rate": 3.8321292638522885e-05,
|
| 1532 |
+
"loss": 0.002146661654114723,
|
| 1533 |
+
"memory(GiB)": 23.34,
|
| 1534 |
+
"step": 745,
|
| 1535 |
+
"token_acc": 0.9986000933271115,
|
| 1536 |
+
"train_speed(iter/s)": 0.1418
|
| 1537 |
+
},
|
| 1538 |
+
{
|
| 1539 |
+
"epoch": 0.6003902536648822,
|
| 1540 |
+
"grad_norm": 0.5729314684867859,
|
| 1541 |
+
"learning_rate": 3.767842914064058e-05,
|
| 1542 |
+
"loss": 0.010377025604248047,
|
| 1543 |
+
"memory(GiB)": 23.34,
|
| 1544 |
+
"step": 750,
|
| 1545 |
+
"token_acc": 0.9973404255319149,
|
| 1546 |
+
"train_speed(iter/s)": 0.141896
|
| 1547 |
+
},
|
| 1548 |
+
{
|
| 1549 |
+
"epoch": 0.6043928553559814,
|
| 1550 |
+
"grad_norm": 0.16427081823349,
|
| 1551 |
+
"learning_rate": 3.703772701767167e-05,
|
| 1552 |
+
"loss": 0.00588022954761982,
|
| 1553 |
+
"memory(GiB)": 23.34,
|
| 1554 |
+
"step": 755,
|
| 1555 |
+
"token_acc": 0.9974619289340102,
|
| 1556 |
+
"train_speed(iter/s)": 0.141987
|
| 1557 |
+
},
|
| 1558 |
+
{
|
| 1559 |
+
"epoch": 0.6083954570470806,
|
| 1560 |
+
"grad_norm": 0.04225505143404007,
|
| 1561 |
+
"learning_rate": 3.6399298657680226e-05,
|
| 1562 |
+
"loss": 0.003077872656285763,
|
| 1563 |
+
"memory(GiB)": 23.34,
|
| 1564 |
+
"step": 760,
|
| 1565 |
+
"token_acc": 0.999124343257443,
|
| 1566 |
+
"train_speed(iter/s)": 0.142049
|
| 1567 |
+
},
|
| 1568 |
+
{
|
| 1569 |
+
"epoch": 0.6123980587381798,
|
| 1570 |
+
"grad_norm": 0.4019262492656708,
|
| 1571 |
+
"learning_rate": 3.576325604988071e-05,
|
| 1572 |
+
"loss": 0.006566619127988815,
|
| 1573 |
+
"memory(GiB)": 23.34,
|
| 1574 |
+
"step": 765,
|
| 1575 |
+
"token_acc": 0.998278088678433,
|
| 1576 |
+
"train_speed(iter/s)": 0.142142
|
| 1577 |
+
},
|
| 1578 |
+
{
|
| 1579 |
+
"epoch": 0.616400660429279,
|
| 1580 |
+
"grad_norm": 0.009038143791258335,
|
| 1581 |
+
"learning_rate": 3.5129710764993474e-05,
|
| 1582 |
+
"loss": 0.0020761976018548014,
|
| 1583 |
+
"memory(GiB)": 23.34,
|
| 1584 |
+
"step": 770,
|
| 1585 |
+
"token_acc": 0.9990974729241877,
|
| 1586 |
+
"train_speed(iter/s)": 0.142241
|
| 1587 |
+
},
|
| 1588 |
+
{
|
| 1589 |
+
"epoch": 0.6204032621203782,
|
| 1590 |
+
"grad_norm": 0.0399525947868824,
|
| 1591 |
+
"learning_rate": 3.4498773935673626e-05,
|
| 1592 |
+
"loss": 0.003108718246221542,
|
| 1593 |
+
"memory(GiB)": 23.34,
|
| 1594 |
+
"step": 775,
|
| 1595 |
+
"token_acc": 0.9986559139784946,
|
| 1596 |
+
"train_speed(iter/s)": 0.142344
|
| 1597 |
+
},
|
| 1598 |
+
{
|
| 1599 |
+
"epoch": 0.6244058638114774,
|
| 1600 |
+
"grad_norm": 0.025889351963996887,
|
| 1601 |
+
"learning_rate": 3.387055623701695e-05,
|
| 1602 |
+
"loss": 0.001817745715379715,
|
| 1603 |
+
"memory(GiB)": 23.34,
|
| 1604 |
+
"step": 780,
|
| 1605 |
+
"token_acc": 0.999168744804655,
|
| 1606 |
+
"train_speed(iter/s)": 0.142436
|
| 1607 |
+
},
|
| 1608 |
+
{
|
| 1609 |
+
"epoch": 0.6284084655025767,
|
| 1610 |
+
"grad_norm": 0.025395702570676804,
|
| 1611 |
+
"learning_rate": 3.324516786714589e-05,
|
| 1612 |
+
"loss": 0.0022980201989412307,
|
| 1613 |
+
"memory(GiB)": 23.34,
|
| 1614 |
+
"step": 785,
|
| 1615 |
+
"token_acc": 0.9986136783733827,
|
| 1616 |
+
"train_speed(iter/s)": 0.142519
|
| 1617 |
+
},
|
| 1618 |
+
{
|
| 1619 |
+
"epoch": 0.6324110671936759,
|
| 1620 |
+
"grad_norm": 0.005791814997792244,
|
| 1621 |
+
"learning_rate": 3.262271852787919e-05,
|
| 1622 |
+
"loss": 0.006876673549413681,
|
| 1623 |
+
"memory(GiB)": 23.34,
|
| 1624 |
+
"step": 790,
|
| 1625 |
+
"token_acc": 0.9974937343358395,
|
| 1626 |
+
"train_speed(iter/s)": 0.142604
|
| 1627 |
+
},
|
| 1628 |
+
{
|
| 1629 |
+
"epoch": 0.6364136688847751,
|
| 1630 |
+
"grad_norm": 0.03827682510018349,
|
| 1631 |
+
"learning_rate": 3.200331740548887e-05,
|
| 1632 |
+
"loss": 0.00113014979287982,
|
| 1633 |
+
"memory(GiB)": 23.34,
|
| 1634 |
+
"step": 795,
|
| 1635 |
+
"token_acc": 0.9990123456790123,
|
| 1636 |
+
"train_speed(iter/s)": 0.142686
|
| 1637 |
+
},
|
| 1638 |
+
{
|
| 1639 |
+
"epoch": 0.6404162705758744,
|
| 1640 |
+
"grad_norm": 0.011377369053661823,
|
| 1641 |
+
"learning_rate": 3.138707315154722e-05,
|
| 1642 |
+
"loss": 0.0018325556069612504,
|
| 1643 |
+
"memory(GiB)": 23.34,
|
| 1644 |
+
"step": 800,
|
| 1645 |
+
"token_acc": 0.9990892531876139,
|
| 1646 |
+
"train_speed(iter/s)": 0.142776
|
| 1647 |
+
},
|
| 1648 |
+
{
|
| 1649 |
+
"epoch": 0.6404162705758744,
|
| 1650 |
+
"eval_loss": 0.04980294406414032,
|
| 1651 |
+
"eval_runtime": 2.2639,
|
| 1652 |
+
"eval_samples_per_second": 5.742,
|
| 1653 |
+
"eval_steps_per_second": 5.742,
|
| 1654 |
+
"eval_token_acc": 0.9898734177215189,
|
| 1655 |
+
"step": 800
|
| 1656 |
+
},
|
| 1657 |
+
{
|
| 1658 |
+
"epoch": 0.6444188722669736,
|
| 1659 |
+
"grad_norm": 0.020515426993370056,
|
| 1660 |
+
"learning_rate": 3.077409386386789e-05,
|
| 1661 |
+
"loss": 0.0018859829753637314,
|
| 1662 |
+
"memory(GiB)": 23.34,
|
| 1663 |
+
"step": 805,
|
| 1664 |
+
"token_acc": 0.9977494373593399,
|
| 1665 |
+
"train_speed(iter/s)": 0.142465
|
| 1666 |
+
},
|
| 1667 |
+
{
|
| 1668 |
+
"epoch": 0.6484214739580727,
|
| 1669 |
+
"grad_norm": 0.002434853697195649,
|
| 1670 |
+
"learning_rate": 3.016448706754411e-05,
|
| 1671 |
+
"loss": 0.0008121992461383343,
|
| 1672 |
+
"memory(GiB)": 23.34,
|
| 1673 |
+
"step": 810,
|
| 1674 |
+
"token_acc": 0.9995007488766849,
|
| 1675 |
+
"train_speed(iter/s)": 0.142545
|
| 1676 |
+
},
|
| 1677 |
+
{
|
| 1678 |
+
"epoch": 0.6524240756491719,
|
| 1679 |
+
"grad_norm": 0.760565459728241,
|
| 1680 |
+
"learning_rate": 2.9558359696087152e-05,
|
| 1681 |
+
"loss": 0.0013134488835930824,
|
| 1682 |
+
"memory(GiB)": 23.34,
|
| 1683 |
+
"step": 815,
|
| 1684 |
+
"token_acc": 0.9995608256477821,
|
| 1685 |
+
"train_speed(iter/s)": 0.142636
|
| 1686 |
+
},
|
| 1687 |
+
{
|
| 1688 |
+
"epoch": 0.6564266773402712,
|
| 1689 |
+
"grad_norm": 0.04273993894457817,
|
| 1690 |
+
"learning_rate": 2.895581807266876e-05,
|
| 1691 |
+
"loss": 0.006279489398002625,
|
| 1692 |
+
"memory(GiB)": 23.34,
|
| 1693 |
+
"step": 820,
|
| 1694 |
+
"token_acc": 0.9982927870251814,
|
| 1695 |
+
"train_speed(iter/s)": 0.142718
|
| 1696 |
+
},
|
| 1697 |
+
{
|
| 1698 |
+
"epoch": 0.6604292790313704,
|
| 1699 |
+
"grad_norm": 0.0202541034668684,
|
| 1700 |
+
"learning_rate": 2.8356967891470663e-05,
|
| 1701 |
+
"loss": 0.0004955935757607222,
|
| 1702 |
+
"memory(GiB)": 23.34,
|
| 1703 |
+
"step": 825,
|
| 1704 |
+
"token_acc": 1.0,
|
| 1705 |
+
"train_speed(iter/s)": 0.1428
|
| 1706 |
+
},
|
| 1707 |
+
{
|
| 1708 |
+
"epoch": 0.6644318807224696,
|
| 1709 |
+
"grad_norm": 0.002425926737487316,
|
| 1710 |
+
"learning_rate": 2.7761914199144233e-05,
|
| 1711 |
+
"loss": 0.0028613714501261713,
|
| 1712 |
+
"memory(GiB)": 23.34,
|
| 1713 |
+
"step": 830,
|
| 1714 |
+
"token_acc": 0.9995810640971932,
|
| 1715 |
+
"train_speed(iter/s)": 0.142886
|
| 1716 |
+
},
|
| 1717 |
+
{
|
| 1718 |
+
"epoch": 0.6684344824135688,
|
| 1719 |
+
"grad_norm": 0.032539863139390945,
|
| 1720 |
+
"learning_rate": 2.717076137638388e-05,
|
| 1721 |
+
"loss": 0.002701476775109768,
|
| 1722 |
+
"memory(GiB)": 23.34,
|
| 1723 |
+
"step": 835,
|
| 1724 |
+
"token_acc": 0.9982646420824295,
|
| 1725 |
+
"train_speed(iter/s)": 0.142968
|
| 1726 |
+
},
|
| 1727 |
+
{
|
| 1728 |
+
"epoch": 0.6724370841046681,
|
| 1729 |
+
"grad_norm": 0.02689575031399727,
|
| 1730 |
+
"learning_rate": 2.6583613119617267e-05,
|
| 1731 |
+
"loss": 0.00220328476279974,
|
| 1732 |
+
"memory(GiB)": 23.34,
|
| 1733 |
+
"step": 840,
|
| 1734 |
+
"token_acc": 0.9991471215351813,
|
| 1735 |
+
"train_speed(iter/s)": 0.143047
|
| 1736 |
+
},
|
| 1737 |
+
{
|
| 1738 |
+
"epoch": 0.6764396857957673,
|
| 1739 |
+
"grad_norm": 0.0010226444574072957,
|
| 1740 |
+
"learning_rate": 2.600057242281545e-05,
|
| 1741 |
+
"loss": 0.003987661376595497,
|
| 1742 |
+
"memory(GiB)": 23.34,
|
| 1743 |
+
"step": 845,
|
| 1744 |
+
"token_acc": 0.997254004576659,
|
| 1745 |
+
"train_speed(iter/s)": 0.143106
|
| 1746 |
+
},
|
| 1747 |
+
{
|
| 1748 |
+
"epoch": 0.6804422874868664,
|
| 1749 |
+
"grad_norm": 0.11139519512653351,
|
| 1750 |
+
"learning_rate": 2.542174155942635e-05,
|
| 1751 |
+
"loss": 0.001826801523566246,
|
| 1752 |
+
"memory(GiB)": 23.34,
|
| 1753 |
+
"step": 850,
|
| 1754 |
+
"token_acc": 0.9990954319312528,
|
| 1755 |
+
"train_speed(iter/s)": 0.143119
|
| 1756 |
+
},
|
| 1757 |
+
{
|
| 1758 |
+
"epoch": 0.6844448891779656,
|
| 1759 |
+
"grad_norm": 0.030525051057338715,
|
| 1760 |
+
"learning_rate": 2.484722206443455e-05,
|
| 1761 |
+
"loss": 0.0009514863602817058,
|
| 1762 |
+
"memory(GiB)": 23.34,
|
| 1763 |
+
"step": 855,
|
| 1764 |
+
"token_acc": 0.9991456642460487,
|
| 1765 |
+
"train_speed(iter/s)": 0.143118
|
| 1766 |
+
},
|
| 1767 |
+
{
|
| 1768 |
+
"epoch": 0.6884474908690649,
|
| 1769 |
+
"grad_norm": 0.0555870346724987,
|
| 1770 |
+
"learning_rate": 2.427711471655065e-05,
|
| 1771 |
+
"loss": 0.0025129817426204682,
|
| 1772 |
+
"memory(GiB)": 23.34,
|
| 1773 |
+
"step": 860,
|
| 1774 |
+
"token_acc": 0.9977718360071302,
|
| 1775 |
+
"train_speed(iter/s)": 0.143187
|
| 1776 |
+
},
|
| 1777 |
+
{
|
| 1778 |
+
"epoch": 0.6924500925601641,
|
| 1779 |
+
"grad_norm": 0.0004600695101544261,
|
| 1780 |
+
"learning_rate": 2.3711519520533266e-05,
|
| 1781 |
+
"loss": 0.0017839709296822547,
|
| 1782 |
+
"memory(GiB)": 23.34,
|
| 1783 |
+
"step": 865,
|
| 1784 |
+
"token_acc": 0.9991423670668954,
|
| 1785 |
+
"train_speed(iter/s)": 0.143258
|
| 1786 |
+
},
|
| 1787 |
+
{
|
| 1788 |
+
"epoch": 0.6964526942512633,
|
| 1789 |
+
"grad_norm": 0.00028400577139109373,
|
| 1790 |
+
"learning_rate": 2.3150535689646823e-05,
|
| 1791 |
+
"loss": 0.0022279206663370133,
|
| 1792 |
+
"memory(GiB)": 23.34,
|
| 1793 |
+
"step": 870,
|
| 1794 |
+
"token_acc": 0.9982616253802694,
|
| 1795 |
+
"train_speed(iter/s)": 0.143327
|
| 1796 |
+
},
|
| 1797 |
+
{
|
| 1798 |
+
"epoch": 0.7004552959423626,
|
| 1799 |
+
"grad_norm": 0.000644134299363941,
|
| 1800 |
+
"learning_rate": 2.259426162825819e-05,
|
| 1801 |
+
"loss": 0.0009273622184991836,
|
| 1802 |
+
"memory(GiB)": 23.34,
|
| 1803 |
+
"step": 875,
|
| 1804 |
+
"token_acc": 0.9990176817288802,
|
| 1805 |
+
"train_speed(iter/s)": 0.143396
|
| 1806 |
+
},
|
| 1807 |
+
{
|
| 1808 |
+
"epoch": 0.7044578976334618,
|
| 1809 |
+
"grad_norm": 0.022241495549678802,
|
| 1810 |
+
"learning_rate": 2.2042794914575062e-05,
|
| 1811 |
+
"loss": 0.0011893250979483128,
|
| 1812 |
+
"memory(GiB)": 23.34,
|
| 1813 |
+
"step": 880,
|
| 1814 |
+
"token_acc": 0.9987720016373312,
|
| 1815 |
+
"train_speed(iter/s)": 0.143466
|
| 1816 |
+
},
|
| 1817 |
+
{
|
| 1818 |
+
"epoch": 0.708460499324561,
|
| 1819 |
+
"grad_norm": 0.01767512410879135,
|
| 1820 |
+
"learning_rate": 2.1496232283529605e-05,
|
| 1821 |
+
"loss": 0.007832873612642288,
|
| 1822 |
+
"memory(GiB)": 23.34,
|
| 1823 |
+
"step": 885,
|
| 1824 |
+
"token_acc": 0.9983505154639175,
|
| 1825 |
+
"train_speed(iter/s)": 0.14354
|
| 1826 |
+
},
|
| 1827 |
+
{
|
| 1828 |
+
"epoch": 0.7124631010156601,
|
| 1829 |
+
"grad_norm": 0.013221798464655876,
|
| 1830 |
+
"learning_rate": 2.0954669609809613e-05,
|
| 1831 |
+
"loss": 0.0015020850114524364,
|
| 1832 |
+
"memory(GiB)": 23.34,
|
| 1833 |
+
"step": 890,
|
| 1834 |
+
"token_acc": 0.9990838295923041,
|
| 1835 |
+
"train_speed(iter/s)": 0.143612
|
| 1836 |
+
},
|
| 1837 |
+
{
|
| 1838 |
+
"epoch": 0.7164657027067594,
|
| 1839 |
+
"grad_norm": 0.00896095298230648,
|
| 1840 |
+
"learning_rate": 2.0418201891040778e-05,
|
| 1841 |
+
"loss": 0.0009665372781455517,
|
| 1842 |
+
"memory(GiB)": 23.34,
|
| 1843 |
+
"step": 895,
|
| 1844 |
+
"token_acc": 1.0,
|
| 1845 |
+
"train_speed(iter/s)": 0.143667
|
| 1846 |
+
},
|
| 1847 |
+
{
|
| 1848 |
+
"epoch": 0.7204683043978586,
|
| 1849 |
+
"grad_norm": 0.0018415815429762006,
|
| 1850 |
+
"learning_rate": 1.9886923231122918e-05,
|
| 1851 |
+
"loss": 0.0029326347634196282,
|
| 1852 |
+
"memory(GiB)": 23.34,
|
| 1853 |
+
"step": 900,
|
| 1854 |
+
"token_acc": 0.9978194505015264,
|
| 1855 |
+
"train_speed(iter/s)": 0.14373
|
| 1856 |
+
},
|
| 1857 |
+
{
|
| 1858 |
+
"epoch": 0.7244709060889578,
|
| 1859 |
+
"grad_norm": 0.0017369468696415424,
|
| 1860 |
+
"learning_rate": 1.9360926823722713e-05,
|
| 1861 |
+
"loss": 0.001794830709695816,
|
| 1862 |
+
"memory(GiB)": 23.34,
|
| 1863 |
+
"step": 905,
|
| 1864 |
+
"token_acc": 0.998697351280938,
|
| 1865 |
+
"train_speed(iter/s)": 0.143805
|
| 1866 |
+
},
|
| 1867 |
+
{
|
| 1868 |
+
"epoch": 0.728473507780057,
|
| 1869 |
+
"grad_norm": 0.017949458211660385,
|
| 1870 |
+
"learning_rate": 1.8840304935926217e-05,
|
| 1871 |
+
"loss": 0.011546848714351654,
|
| 1872 |
+
"memory(GiB)": 23.34,
|
| 1873 |
+
"step": 910,
|
| 1874 |
+
"token_acc": 0.9986813186813187,
|
| 1875 |
+
"train_speed(iter/s)": 0.143872
|
| 1876 |
+
},
|
| 1877 |
+
{
|
| 1878 |
+
"epoch": 0.7324761094711563,
|
| 1879 |
+
"grad_norm": 0.011381692253053188,
|
| 1880 |
+
"learning_rate": 1.8325148892054093e-05,
|
| 1881 |
+
"loss": 0.0010601507499814033,
|
| 1882 |
+
"memory(GiB)": 23.34,
|
| 1883 |
+
"step": 915,
|
| 1884 |
+
"token_acc": 0.9995446265938069,
|
| 1885 |
+
"train_speed(iter/s)": 0.143899
|
| 1886 |
+
},
|
| 1887 |
+
{
|
| 1888 |
+
"epoch": 0.7364787111622555,
|
| 1889 |
+
"grad_norm": 0.11789742857217789,
|
| 1890 |
+
"learning_rate": 1.7815549057641917e-05,
|
| 1891 |
+
"loss": 0.002404318377375603,
|
| 1892 |
+
"memory(GiB)": 23.34,
|
| 1893 |
+
"step": 920,
|
| 1894 |
+
"token_acc": 0.9987336428872943,
|
| 1895 |
+
"train_speed(iter/s)": 0.1439
|
| 1896 |
+
},
|
| 1897 |
+
{
|
| 1898 |
+
"epoch": 0.7404813128533547,
|
| 1899 |
+
"grad_norm": 0.009267464280128479,
|
| 1900 |
+
"learning_rate": 1.731159482358877e-05,
|
| 1901 |
+
"loss": 0.0014395067468285561,
|
| 1902 |
+
"memory(GiB)": 23.34,
|
| 1903 |
+
"step": 925,
|
| 1904 |
+
"token_acc": 0.9991177767975298,
|
| 1905 |
+
"train_speed(iter/s)": 0.143904
|
| 1906 |
+
},
|
| 1907 |
+
{
|
| 1908 |
+
"epoch": 0.7444839145444538,
|
| 1909 |
+
"grad_norm": 0.014848796650767326,
|
| 1910 |
+
"learning_rate": 1.6813374590477014e-05,
|
| 1911 |
+
"loss": 0.003967106714844704,
|
| 1912 |
+
"memory(GiB)": 23.34,
|
| 1913 |
+
"step": 930,
|
| 1914 |
+
"token_acc": 0.9968225147526101,
|
| 1915 |
+
"train_speed(iter/s)": 0.143909
|
| 1916 |
+
},
|
| 1917 |
+
{
|
| 1918 |
+
"epoch": 0.7484865162355531,
|
| 1919 |
+
"grad_norm": 0.006364669185131788,
|
| 1920 |
+
"learning_rate": 1.6320975753065453e-05,
|
| 1921 |
+
"loss": 0.0028301239013671874,
|
| 1922 |
+
"memory(GiB)": 23.34,
|
| 1923 |
+
"step": 935,
|
| 1924 |
+
"token_acc": 0.9986136783733827,
|
| 1925 |
+
"train_speed(iter/s)": 0.143907
|
| 1926 |
+
},
|
| 1927 |
+
{
|
| 1928 |
+
"epoch": 0.7524891179266523,
|
| 1929 |
+
"grad_norm": 0.0007381622563116252,
|
| 1930 |
+
"learning_rate": 1.583448468495904e-05,
|
| 1931 |
+
"loss": 0.006910444051027298,
|
| 1932 |
+
"memory(GiB)": 23.34,
|
| 1933 |
+
"step": 940,
|
| 1934 |
+
"token_acc": 0.9991146525011066,
|
| 1935 |
+
"train_speed(iter/s)": 0.143892
|
| 1936 |
+
},
|
| 1937 |
+
{
|
| 1938 |
+
"epoch": 0.7564917196177515,
|
| 1939 |
+
"grad_norm": 0.002728588180616498,
|
| 1940 |
+
"learning_rate": 1.535398672345793e-05,
|
| 1941 |
+
"loss": 0.00154438940808177,
|
| 1942 |
+
"memory(GiB)": 23.34,
|
| 1943 |
+
"step": 945,
|
| 1944 |
+
"token_acc": 0.998547215496368,
|
| 1945 |
+
"train_speed(iter/s)": 0.143881
|
| 1946 |
+
},
|
| 1947 |
+
{
|
| 1948 |
+
"epoch": 0.7604943213088508,
|
| 1949 |
+
"grad_norm": 0.19096170365810394,
|
| 1950 |
+
"learning_rate": 1.4879566154587976e-05,
|
| 1951 |
+
"loss": 0.0021604156121611596,
|
| 1952 |
+
"memory(GiB)": 23.34,
|
| 1953 |
+
"step": 950,
|
| 1954 |
+
"token_acc": 0.9987765089722676,
|
| 1955 |
+
"train_speed(iter/s)": 0.143862
|
| 1956 |
+
},
|
| 1957 |
+
{
|
| 1958 |
+
"epoch": 0.76449692299995,
|
| 1959 |
+
"grad_norm": 0.01451029907912016,
|
| 1960 |
+
"learning_rate": 1.4411306198315777e-05,
|
| 1961 |
+
"loss": 0.0033991307020187376,
|
| 1962 |
+
"memory(GiB)": 23.34,
|
| 1963 |
+
"step": 955,
|
| 1964 |
+
"token_acc": 0.9978559176672385,
|
| 1965 |
+
"train_speed(iter/s)": 0.143852
|
| 1966 |
+
},
|
| 1967 |
+
{
|
| 1968 |
+
"epoch": 0.7684995246910492,
|
| 1969 |
+
"grad_norm": 0.15964210033416748,
|
| 1970 |
+
"learning_rate": 1.3949288993950893e-05,
|
| 1971 |
+
"loss": 0.0070294119417667385,
|
| 1972 |
+
"memory(GiB)": 23.34,
|
| 1973 |
+
"step": 960,
|
| 1974 |
+
"token_acc": 0.9976325757575758,
|
| 1975 |
+
"train_speed(iter/s)": 0.143836
|
| 1976 |
+
},
|
| 1977 |
+
{
|
| 1978 |
+
"epoch": 0.7725021263821484,
|
| 1979 |
+
"grad_norm": 0.035655975341796875,
|
| 1980 |
+
"learning_rate": 1.3493595585737273e-05,
|
| 1981 |
+
"loss": 0.001282591000199318,
|
| 1982 |
+
"memory(GiB)": 23.34,
|
| 1983 |
+
"step": 965,
|
| 1984 |
+
"token_acc": 0.9990605918271489,
|
| 1985 |
+
"train_speed(iter/s)": 0.143821
|
| 1986 |
+
},
|
| 1987 |
+
{
|
| 1988 |
+
"epoch": 0.7765047280732477,
|
| 1989 |
+
"grad_norm": 0.34104758501052856,
|
| 1990 |
+
"learning_rate": 1.3044305908637105e-05,
|
| 1991 |
+
"loss": 0.008086320012807846,
|
| 1992 |
+
"memory(GiB)": 23.34,
|
| 1993 |
+
"step": 970,
|
| 1994 |
+
"token_acc": 0.9965679965679966,
|
| 1995 |
+
"train_speed(iter/s)": 0.1438
|
| 1996 |
+
},
|
| 1997 |
+
{
|
| 1998 |
+
"epoch": 0.7805073297643468,
|
| 1999 |
+
"grad_norm": 0.0753471627831459,
|
| 2000 |
+
"learning_rate": 1.2601498774309112e-05,
|
| 2001 |
+
"loss": 0.0025702202692627907,
|
| 2002 |
+
"memory(GiB)": 23.34,
|
| 2003 |
+
"step": 975,
|
| 2004 |
+
"token_acc": 0.9986772486772487,
|
| 2005 |
+
"train_speed(iter/s)": 0.143781
|
| 2006 |
+
},
|
| 2007 |
+
{
|
| 2008 |
+
"epoch": 0.784509931455446,
|
| 2009 |
+
"grad_norm": 0.010594204999506474,
|
| 2010 |
+
"learning_rate": 1.216525185728375e-05,
|
| 2011 |
+
"loss": 0.0020033037289977074,
|
| 2012 |
+
"memory(GiB)": 23.34,
|
| 2013 |
+
"step": 980,
|
| 2014 |
+
"token_acc": 0.9982316534040672,
|
| 2015 |
+
"train_speed(iter/s)": 0.143762
|
| 2016 |
+
},
|
| 2017 |
+
{
|
| 2018 |
+
"epoch": 0.7885125331465452,
|
| 2019 |
+
"grad_norm": 0.02695535495877266,
|
| 2020 |
+
"learning_rate": 1.1735641681338167e-05,
|
| 2021 |
+
"loss": 0.0015294927172362805,
|
| 2022 |
+
"memory(GiB)": 23.34,
|
| 2023 |
+
"step": 985,
|
| 2024 |
+
"token_acc": 0.998592210229939,
|
| 2025 |
+
"train_speed(iter/s)": 0.143757
|
| 2026 |
+
},
|
| 2027 |
+
{
|
| 2028 |
+
"epoch": 0.7925151348376445,
|
| 2029 |
+
"grad_norm": 0.058420345187187195,
|
| 2030 |
+
"learning_rate": 1.1312743606072845e-05,
|
| 2031 |
+
"loss": 0.0025140175595879553,
|
| 2032 |
+
"memory(GiB)": 23.34,
|
| 2033 |
+
"step": 990,
|
| 2034 |
+
"token_acc": 0.9978165938864629,
|
| 2035 |
+
"train_speed(iter/s)": 0.143737
|
| 2036 |
+
},
|
| 2037 |
+
{
|
| 2038 |
+
"epoch": 0.7965177365287437,
|
| 2039 |
+
"grad_norm": 0.009234153665602207,
|
| 2040 |
+
"learning_rate": 1.0896631813692377e-05,
|
| 2041 |
+
"loss": 0.0019039958715438843,
|
| 2042 |
+
"memory(GiB)": 23.34,
|
| 2043 |
+
"step": 995,
|
| 2044 |
+
"token_acc": 0.9986743261157756,
|
| 2045 |
+
"train_speed(iter/s)": 0.143728
|
| 2046 |
+
},
|
| 2047 |
+
{
|
| 2048 |
+
"epoch": 0.8005203382198429,
|
| 2049 |
+
"grad_norm": 0.020775901153683662,
|
| 2050 |
+
"learning_rate": 1.0487379295992971e-05,
|
| 2051 |
+
"loss": 0.0015889856964349746,
|
| 2052 |
+
"memory(GiB)": 23.34,
|
| 2053 |
+
"step": 1000,
|
| 2054 |
+
"token_acc": 0.998598785614199,
|
| 2055 |
+
"train_speed(iter/s)": 0.143728
|
| 2056 |
+
},
|
| 2057 |
+
{
|
| 2058 |
+
"epoch": 0.8005203382198429,
|
| 2059 |
+
"eval_loss": 0.03210083395242691,
|
| 2060 |
+
"eval_runtime": 2.4701,
|
| 2061 |
+
"eval_samples_per_second": 5.263,
|
| 2062 |
+
"eval_steps_per_second": 5.263,
|
| 2063 |
+
"eval_token_acc": 0.9924050632911392,
|
| 2064 |
+
"step": 1000
|
| 2065 |
+
},
|
| 2066 |
+
{
|
| 2067 |
+
"epoch": 0.8045229399109421,
|
| 2068 |
+
"grad_norm": 0.011230681091547012,
|
| 2069 |
+
"learning_rate": 1.0085057841558699e-05,
|
| 2070 |
+
"loss": 0.0014883812516927719,
|
| 2071 |
+
"memory(GiB)": 23.91,
|
| 2072 |
+
"step": 1005,
|
| 2073 |
+
"token_acc": 0.9978331527627302,
|
| 2074 |
+
"train_speed(iter/s)": 0.143355
|
| 2075 |
+
},
|
| 2076 |
+
{
|
| 2077 |
+
"epoch": 0.8085255416020414,
|
| 2078 |
+
"grad_norm": 0.011859547346830368,
|
| 2079 |
+
"learning_rate": 9.689738023168644e-06,
|
| 2080 |
+
"loss": 0.0020302459597587585,
|
| 2081 |
+
"memory(GiB)": 23.91,
|
| 2082 |
+
"step": 1010,
|
| 2083 |
+
"token_acc": 0.9977252047315741,
|
| 2084 |
+
"train_speed(iter/s)": 0.14335
|
| 2085 |
+
},
|
| 2086 |
+
{
|
| 2087 |
+
"epoch": 0.8125281432931405,
|
| 2088 |
+
"grad_norm": 0.0017468325095251203,
|
| 2089 |
+
"learning_rate": 9.301489185417578e-06,
|
| 2090 |
+
"loss": 0.002153720334172249,
|
| 2091 |
+
"memory(GiB)": 25.85,
|
| 2092 |
+
"step": 1015,
|
| 2093 |
+
"token_acc": 0.9990850869167429,
|
| 2094 |
+
"train_speed(iter/s)": 0.14332
|
| 2095 |
+
},
|
| 2096 |
+
{
|
| 2097 |
+
"epoch": 0.8165307449842397,
|
| 2098 |
+
"grad_norm": 0.0005466677830554545,
|
| 2099 |
+
"learning_rate": 8.920379432551967e-06,
|
| 2100 |
+
"loss": 0.00105759184807539,
|
| 2101 |
+
"memory(GiB)": 25.85,
|
| 2102 |
+
"step": 1020,
|
| 2103 |
+
"token_acc": 0.9995276334435522,
|
| 2104 |
+
"train_speed(iter/s)": 0.143296
|
| 2105 |
+
},
|
| 2106 |
+
{
|
| 2107 |
+
"epoch": 0.820533346675339,
|
| 2108 |
+
"grad_norm": 0.0006127957021817565,
|
| 2109 |
+
"learning_rate": 8.546475616523397e-06,
|
| 2110 |
+
"loss": 0.0005460224114358425,
|
| 2111 |
+
"memory(GiB)": 25.85,
|
| 2112 |
+
"step": 1025,
|
| 2113 |
+
"token_acc": 1.0,
|
| 2114 |
+
"train_speed(iter/s)": 0.143284
|
| 2115 |
+
},
|
| 2116 |
+
{
|
| 2117 |
+
"epoch": 0.8245359483664382,
|
| 2118 |
+
"grad_norm": 0.009868661873042583,
|
| 2119 |
+
"learning_rate": 8.179843325261943e-06,
|
| 2120 |
+
"loss": 0.0012353837490081787,
|
| 2121 |
+
"memory(GiB)": 25.85,
|
| 2122 |
+
"step": 1030,
|
| 2123 |
+
"token_acc": 0.9995471014492754,
|
| 2124 |
+
"train_speed(iter/s)": 0.14328
|
| 2125 |
+
},
|
| 2126 |
+
{
|
| 2127 |
+
"epoch": 0.8285385500575374,
|
| 2128 |
+
"grad_norm": 0.010744879953563213,
|
| 2129 |
+
"learning_rate": 7.820546871171142e-06,
|
| 2130 |
+
"loss": 0.0025576120242476463,
|
| 2131 |
+
"memory(GiB)": 25.85,
|
| 2132 |
+
"step": 1035,
|
| 2133 |
+
"token_acc": 0.9982245894363071,
|
| 2134 |
+
"train_speed(iter/s)": 0.143262
|
| 2135 |
+
},
|
| 2136 |
+
{
|
| 2137 |
+
"epoch": 0.8325411517486366,
|
| 2138 |
+
"grad_norm": 0.009720387868583202,
|
| 2139 |
+
"learning_rate": 7.468649279846568e-06,
|
| 2140 |
+
"loss": 0.002662136405706406,
|
| 2141 |
+
"memory(GiB)": 25.85,
|
| 2142 |
+
"step": 1040,
|
| 2143 |
+
"token_acc": 0.9986059479553904,
|
| 2144 |
+
"train_speed(iter/s)": 0.143257
|
| 2145 |
+
},
|
| 2146 |
+
{
|
| 2147 |
+
"epoch": 0.8365437534397359,
|
| 2148 |
+
"grad_norm": 0.008323716931045055,
|
| 2149 |
+
"learning_rate": 7.124212279020409e-06,
|
| 2150 |
+
"loss": 0.0020709151402115824,
|
| 2151 |
+
"memory(GiB)": 25.85,
|
| 2152 |
+
"step": 1045,
|
| 2153 |
+
"token_acc": 0.998343685300207,
|
| 2154 |
+
"train_speed(iter/s)": 0.143254
|
| 2155 |
+
},
|
| 2156 |
+
{
|
| 2157 |
+
"epoch": 0.8405463551308351,
|
| 2158 |
+
"grad_norm": 0.0004628105671145022,
|
| 2159 |
+
"learning_rate": 6.787296287733497e-06,
|
| 2160 |
+
"loss": 0.0013332159258425235,
|
| 2161 |
+
"memory(GiB)": 25.85,
|
| 2162 |
+
"step": 1050,
|
| 2163 |
+
"token_acc": 0.9986474301172227,
|
| 2164 |
+
"train_speed(iter/s)": 0.143248
|
| 2165 |
+
},
|
| 2166 |
+
{
|
| 2167 |
+
"epoch": 0.8445489568219342,
|
| 2168 |
+
"grad_norm": 0.011886617168784142,
|
| 2169 |
+
"learning_rate": 6.457960405736968e-06,
|
| 2170 |
+
"loss": 0.0009525774046778678,
|
| 2171 |
+
"memory(GiB)": 25.85,
|
| 2172 |
+
"step": 1055,
|
| 2173 |
+
"token_acc": 0.9995479204339964,
|
| 2174 |
+
"train_speed(iter/s)": 0.143232
|
| 2175 |
+
},
|
| 2176 |
+
{
|
| 2177 |
+
"epoch": 0.8485515585130334,
|
| 2178 |
+
"grad_norm": 0.009538613259792328,
|
| 2179 |
+
"learning_rate": 6.13626240312537e-06,
|
| 2180 |
+
"loss": 0.0010814583860337735,
|
| 2181 |
+
"memory(GiB)": 25.85,
|
| 2182 |
+
"step": 1060,
|
| 2183 |
+
"token_acc": 0.9991738950846758,
|
| 2184 |
+
"train_speed(iter/s)": 0.143228
|
| 2185 |
+
},
|
| 2186 |
+
{
|
| 2187 |
+
"epoch": 0.8525541602041327,
|
| 2188 |
+
"grad_norm": 0.01372472383081913,
|
| 2189 |
+
"learning_rate": 5.822258710202955e-06,
|
| 2190 |
+
"loss": 0.0011989958584308624,
|
| 2191 |
+
"memory(GiB)": 25.85,
|
| 2192 |
+
"step": 1065,
|
| 2193 |
+
"token_acc": 0.9987179487179487,
|
| 2194 |
+
"train_speed(iter/s)": 0.143214
|
| 2195 |
+
},
|
| 2196 |
+
{
|
| 2197 |
+
"epoch": 0.8565567618952319,
|
| 2198 |
+
"grad_norm": 0.006767522543668747,
|
| 2199 |
+
"learning_rate": 5.51600440758504e-06,
|
| 2200 |
+
"loss": 0.0011768504977226258,
|
| 2201 |
+
"memory(GiB)": 25.85,
|
| 2202 |
+
"step": 1070,
|
| 2203 |
+
"token_acc": 0.9991158267020336,
|
| 2204 |
+
"train_speed(iter/s)": 0.14321
|
| 2205 |
+
},
|
| 2206 |
+
{
|
| 2207 |
+
"epoch": 0.8605593635863311,
|
| 2208 |
+
"grad_norm": 0.0013376051792874932,
|
| 2209 |
+
"learning_rate": 5.217553216536098e-06,
|
| 2210 |
+
"loss": 0.0017566321417689323,
|
| 2211 |
+
"memory(GiB)": 25.85,
|
| 2212 |
+
"step": 1075,
|
| 2213 |
+
"token_acc": 0.9991177767975298,
|
| 2214 |
+
"train_speed(iter/s)": 0.143206
|
| 2215 |
+
},
|
| 2216 |
+
{
|
| 2217 |
+
"epoch": 0.8645619652774303,
|
| 2218 |
+
"grad_norm": 0.0017687728395685554,
|
| 2219 |
+
"learning_rate": 4.926957489546285e-06,
|
| 2220 |
+
"loss": 0.0014382892288267613,
|
| 2221 |
+
"memory(GiB)": 25.85,
|
| 2222 |
+
"step": 1080,
|
| 2223 |
+
"token_acc": 0.9995748299319728,
|
| 2224 |
+
"train_speed(iter/s)": 0.143208
|
| 2225 |
+
},
|
| 2226 |
+
{
|
| 2227 |
+
"epoch": 0.8685645669685296,
|
| 2228 |
+
"grad_norm": 0.01269242912530899,
|
| 2229 |
+
"learning_rate": 4.644268201148105e-06,
|
| 2230 |
+
"loss": 0.0014995370991528034,
|
| 2231 |
+
"memory(GiB)": 25.85,
|
| 2232 |
+
"step": 1085,
|
| 2233 |
+
"token_acc": 0.9987315010570824,
|
| 2234 |
+
"train_speed(iter/s)": 0.143195
|
| 2235 |
+
},
|
| 2236 |
+
{
|
| 2237 |
+
"epoch": 0.8725671686596288,
|
| 2238 |
+
"grad_norm": 0.009657932445406914,
|
| 2239 |
+
"learning_rate": 4.369534938974762e-06,
|
| 2240 |
+
"loss": 0.002657635323703289,
|
| 2241 |
+
"memory(GiB)": 25.85,
|
| 2242 |
+
"step": 1090,
|
| 2243 |
+
"token_acc": 0.998223011994669,
|
| 2244 |
+
"train_speed(iter/s)": 0.14318
|
| 2245 |
+
},
|
| 2246 |
+
{
|
| 2247 |
+
"epoch": 0.876569770350728,
|
| 2248 |
+
"grad_norm": 0.001976604573428631,
|
| 2249 |
+
"learning_rate": 4.10280589506179e-06,
|
| 2250 |
+
"loss": 0.0008614543825387954,
|
| 2251 |
+
"memory(GiB)": 25.85,
|
| 2252 |
+
"step": 1095,
|
| 2253 |
+
"token_acc": 1.0,
|
| 2254 |
+
"train_speed(iter/s)": 0.14317
|
| 2255 |
+
},
|
| 2256 |
+
{
|
| 2257 |
+
"epoch": 0.8805723720418271,
|
| 2258 |
+
"grad_norm": 0.006187944207340479,
|
| 2259 |
+
"learning_rate": 3.8441278573935145e-06,
|
| 2260 |
+
"loss": 0.00155127365142107,
|
| 2261 |
+
"memory(GiB)": 25.85,
|
| 2262 |
+
"step": 1100,
|
| 2263 |
+
"token_acc": 0.9991386735572783,
|
| 2264 |
+
"train_speed(iter/s)": 0.143152
|
| 2265 |
+
},
|
| 2266 |
+
{
|
| 2267 |
+
"epoch": 0.8845749737329264,
|
| 2268 |
+
"grad_norm": 0.008681927807629108,
|
| 2269 |
+
"learning_rate": 3.5935462016957742e-06,
|
| 2270 |
+
"loss": 0.00024302804376929998,
|
| 2271 |
+
"memory(GiB)": 25.85,
|
| 2272 |
+
"step": 1105,
|
| 2273 |
+
"token_acc": 1.0,
|
| 2274 |
+
"train_speed(iter/s)": 0.143139
|
| 2275 |
+
},
|
| 2276 |
+
{
|
| 2277 |
+
"epoch": 0.8885775754240256,
|
| 2278 |
+
"grad_norm": 0.00021675020980183035,
|
| 2279 |
+
"learning_rate": 3.3511048834763968e-06,
|
| 2280 |
+
"loss": 0.0006325814872980117,
|
| 2281 |
+
"memory(GiB)": 25.85,
|
| 2282 |
+
"step": 1110,
|
| 2283 |
+
"token_acc": 0.9995719178082192,
|
| 2284 |
+
"train_speed(iter/s)": 0.143119
|
| 2285 |
+
},
|
| 2286 |
+
{
|
| 2287 |
+
"epoch": 0.8925801771151248,
|
| 2288 |
+
"grad_norm": 0.0002966501924674958,
|
| 2289 |
+
"learning_rate": 3.116846430314796e-06,
|
| 2290 |
+
"loss": 0.0013758828863501549,
|
| 2291 |
+
"memory(GiB)": 25.85,
|
| 2292 |
+
"step": 1115,
|
| 2293 |
+
"token_acc": 0.9995669120831528,
|
| 2294 |
+
"train_speed(iter/s)": 0.143097
|
| 2295 |
+
},
|
| 2296 |
+
{
|
| 2297 |
+
"epoch": 0.8965827788062241,
|
| 2298 |
+
"grad_norm": 0.002682023448869586,
|
| 2299 |
+
"learning_rate": 2.8908119344020266e-06,
|
| 2300 |
+
"loss": 0.00017717975424602628,
|
| 2301 |
+
"memory(GiB)": 25.85,
|
| 2302 |
+
"step": 1120,
|
| 2303 |
+
"token_acc": 1.0,
|
| 2304 |
+
"train_speed(iter/s)": 0.143082
|
| 2305 |
+
},
|
| 2306 |
+
{
|
| 2307 |
+
"epoch": 0.9005853804973233,
|
| 2308 |
+
"grad_norm": 0.018612444400787354,
|
| 2309 |
+
"learning_rate": 2.673041045332664e-06,
|
| 2310 |
+
"loss": 0.0019644765183329583,
|
| 2311 |
+
"memory(GiB)": 25.85,
|
| 2312 |
+
"step": 1125,
|
| 2313 |
+
"token_acc": 0.9986474301172227,
|
| 2314 |
+
"train_speed(iter/s)": 0.143072
|
| 2315 |
+
},
|
| 2316 |
+
{
|
| 2317 |
+
"epoch": 0.9045879821884225,
|
| 2318 |
+
"grad_norm": 0.013840870000422001,
|
| 2319 |
+
"learning_rate": 2.463571963149691e-06,
|
| 2320 |
+
"loss": 0.0015004985965788365,
|
| 2321 |
+
"memory(GiB)": 25.85,
|
| 2322 |
+
"step": 1130,
|
| 2323 |
+
"token_acc": 0.9990762124711317,
|
| 2324 |
+
"train_speed(iter/s)": 0.143056
|
| 2325 |
+
},
|
| 2326 |
+
{
|
| 2327 |
+
"epoch": 0.9085905838795217,
|
| 2328 |
+
"grad_norm": 0.0075715817511081696,
|
| 2329 |
+
"learning_rate": 2.262441431643697e-06,
|
| 2330 |
+
"loss": 0.0006865543778985739,
|
| 2331 |
+
"memory(GiB)": 25.85,
|
| 2332 |
+
"step": 1135,
|
| 2333 |
+
"token_acc": 1.0,
|
| 2334 |
+
"train_speed(iter/s)": 0.143041
|
| 2335 |
+
},
|
| 2336 |
+
{
|
| 2337 |
+
"epoch": 0.9125931855706209,
|
| 2338 |
+
"grad_norm": 0.006183127872645855,
|
| 2339 |
+
"learning_rate": 2.069684731907495e-06,
|
| 2340 |
+
"loss": 0.0031918030232191085,
|
| 2341 |
+
"memory(GiB)": 25.85,
|
| 2342 |
+
"step": 1140,
|
| 2343 |
+
"token_acc": 0.9977158519872088,
|
| 2344 |
+
"train_speed(iter/s)": 0.14304
|
| 2345 |
+
},
|
| 2346 |
+
{
|
| 2347 |
+
"epoch": 0.9165957872617201,
|
| 2348 |
+
"grad_norm": 0.00900453981012106,
|
| 2349 |
+
"learning_rate": 1.8853356761473319e-06,
|
| 2350 |
+
"loss": 0.00178102720528841,
|
| 2351 |
+
"memory(GiB)": 25.85,
|
| 2352 |
+
"step": 1145,
|
| 2353 |
+
"token_acc": 0.9990714948932219,
|
| 2354 |
+
"train_speed(iter/s)": 0.143042
|
| 2355 |
+
},
|
| 2356 |
+
{
|
| 2357 |
+
"epoch": 0.9205983889528193,
|
| 2358 |
+
"grad_norm": 0.015010940842330456,
|
| 2359 |
+
"learning_rate": 1.7094266017517624e-06,
|
| 2360 |
+
"loss": 0.003154096379876137,
|
| 2361 |
+
"memory(GiB)": 25.85,
|
| 2362 |
+
"step": 1150,
|
| 2363 |
+
"token_acc": 0.9983471074380166,
|
| 2364 |
+
"train_speed(iter/s)": 0.143034
|
| 2365 |
+
},
|
| 2366 |
+
{
|
| 2367 |
+
"epoch": 0.9246009906439185,
|
| 2368 |
+
"grad_norm": 0.019525939598679543,
|
| 2369 |
+
"learning_rate": 1.541988365619207e-06,
|
| 2370 |
+
"loss": 0.001761198788881302,
|
| 2371 |
+
"memory(GiB)": 25.85,
|
| 2372 |
+
"step": 1155,
|
| 2373 |
+
"token_acc": 0.9987107864202837,
|
| 2374 |
+
"train_speed(iter/s)": 0.143039
|
| 2375 |
+
},
|
| 2376 |
+
{
|
| 2377 |
+
"epoch": 0.9286035923350178,
|
| 2378 |
+
"grad_norm": 0.0006654814933426678,
|
| 2379 |
+
"learning_rate": 1.3830503387452342e-06,
|
| 2380 |
+
"loss": 0.0008027138188481331,
|
| 2381 |
+
"memory(GiB)": 25.85,
|
| 2382 |
+
"step": 1160,
|
| 2383 |
+
"token_acc": 0.9995276334435522,
|
| 2384 |
+
"train_speed(iter/s)": 0.143015
|
| 2385 |
+
},
|
| 2386 |
+
{
|
| 2387 |
+
"epoch": 0.932606194026117,
|
| 2388 |
+
"grad_norm": 0.014265005476772785,
|
| 2389 |
+
"learning_rate": 1.2326404010704784e-06,
|
| 2390 |
+
"loss": 0.002566094323992729,
|
| 2391 |
+
"memory(GiB)": 25.85,
|
| 2392 |
+
"step": 1165,
|
| 2393 |
+
"token_acc": 0.9981842941443486,
|
| 2394 |
+
"train_speed(iter/s)": 0.143005
|
| 2395 |
+
},
|
| 2396 |
+
{
|
| 2397 |
+
"epoch": 0.9366087957172162,
|
| 2398 |
+
"grad_norm": 0.009172003716230392,
|
| 2399 |
+
"learning_rate": 1.0907849365901113e-06,
|
| 2400 |
+
"loss": 0.0019826874136924743,
|
| 2401 |
+
"memory(GiB)": 25.85,
|
| 2402 |
+
"step": 1170,
|
| 2403 |
+
"token_acc": 0.99859616284511,
|
| 2404 |
+
"train_speed(iter/s)": 0.142993
|
| 2405 |
+
},
|
| 2406 |
+
{
|
| 2407 |
+
"epoch": 0.9406113974083155,
|
| 2408 |
+
"grad_norm": 0.009388328529894352,
|
| 2409 |
+
"learning_rate": 9.575088287257118e-07,
|
| 2410 |
+
"loss": 0.001204074639827013,
|
| 2411 |
+
"memory(GiB)": 25.85,
|
| 2412 |
+
"step": 1175,
|
| 2413 |
+
"token_acc": 0.9995265151515151,
|
| 2414 |
+
"train_speed(iter/s)": 0.142981
|
| 2415 |
+
},
|
| 2416 |
+
{
|
| 2417 |
+
"epoch": 0.9446139990994146,
|
| 2418 |
+
"grad_norm": 0.003953286446630955,
|
| 2419 |
+
"learning_rate": 8.32835455960379e-07,
|
| 2420 |
+
"loss": 0.0009021904319524765,
|
| 2421 |
+
"memory(GiB)": 25.85,
|
| 2422 |
+
"step": 1180,
|
| 2423 |
+
"token_acc": 0.9996033320111067,
|
| 2424 |
+
"train_speed(iter/s)": 0.14297
|
| 2425 |
+
},
|
| 2426 |
+
{
|
| 2427 |
+
"epoch": 0.9486166007905138,
|
| 2428 |
+
"grad_norm": 0.007500405889004469,
|
| 2429 |
+
"learning_rate": 7.167866877378193e-07,
|
| 2430 |
+
"loss": 0.001672871969640255,
|
| 2431 |
+
"memory(GiB)": 25.85,
|
| 2432 |
+
"step": 1185,
|
| 2433 |
+
"token_acc": 0.9990880072959416,
|
| 2434 |
+
"train_speed(iter/s)": 0.142956
|
| 2435 |
+
},
|
| 2436 |
+
{
|
| 2437 |
+
"epoch": 0.952619202481613,
|
| 2438 |
+
"grad_norm": 0.009868037886917591,
|
| 2439 |
+
"learning_rate": 6.09382880626136e-07,
|
| 2440 |
+
"loss": 0.0016309935599565506,
|
| 2441 |
+
"memory(GiB)": 25.85,
|
| 2442 |
+
"step": 1190,
|
| 2443 |
+
"token_acc": 0.9986400725294651,
|
| 2444 |
+
"train_speed(iter/s)": 0.142944
|
| 2445 |
+
},
|
| 2446 |
+
{
|
| 2447 |
+
"epoch": 0.9566218041727123,
|
| 2448 |
+
"grad_norm": 0.0009460118017159402,
|
| 2449 |
+
"learning_rate": 5.106428747470137e-07,
|
| 2450 |
+
"loss": 0.0006910431198775768,
|
| 2451 |
+
"memory(GiB)": 25.85,
|
| 2452 |
+
"step": 1195,
|
| 2453 |
+
"token_acc": 0.9995711835334476,
|
| 2454 |
+
"train_speed(iter/s)": 0.142945
|
| 2455 |
+
},
|
| 2456 |
+
{
|
| 2457 |
+
"epoch": 0.9606244058638115,
|
| 2458 |
+
"grad_norm": 0.011792745441198349,
|
| 2459 |
+
"learning_rate": 4.2058399047088525e-07,
|
| 2460 |
+
"loss": 0.0024824630469083785,
|
| 2461 |
+
"memory(GiB)": 25.85,
|
| 2462 |
+
"step": 1200,
|
| 2463 |
+
"token_acc": 0.9978345604157643,
|
| 2464 |
+
"train_speed(iter/s)": 0.142938
|
| 2465 |
+
},
|
| 2466 |
+
{
|
| 2467 |
+
"epoch": 0.9606244058638115,
|
| 2468 |
+
"eval_loss": 0.03278211131691933,
|
| 2469 |
+
"eval_runtime": 2.4817,
|
| 2470 |
+
"eval_samples_per_second": 5.238,
|
| 2471 |
+
"eval_steps_per_second": 5.238,
|
| 2472 |
+
"eval_token_acc": 0.9924050632911392,
|
| 2473 |
+
"step": 1200
|
| 2474 |
+
},
|
| 2475 |
+
{
|
| 2476 |
+
"epoch": 0.9646270075549107,
|
| 2477 |
+
"grad_norm": 0.010857181623578072,
|
| 2478 |
+
"learning_rate": 3.392220253787159e-07,
|
| 2479 |
+
"loss": 0.0015170341357588768,
|
| 2480 |
+
"memory(GiB)": 25.85,
|
| 2481 |
+
"step": 1205,
|
| 2482 |
+
"token_acc": 0.9978738483345145,
|
| 2483 |
+
"train_speed(iter/s)": 0.142628
|
| 2484 |
+
},
|
| 2485 |
+
{
|
| 2486 |
+
"epoch": 0.9686296092460099,
|
| 2487 |
+
"grad_norm": 0.019318712875247,
|
| 2488 |
+
"learning_rate": 2.6657125149086583e-07,
|
| 2489 |
+
"loss": 0.002496711164712906,
|
| 2490 |
+
"memory(GiB)": 25.85,
|
| 2491 |
+
"step": 1210,
|
| 2492 |
+
"token_acc": 0.9990421455938697,
|
| 2493 |
+
"train_speed(iter/s)": 0.142618
|
| 2494 |
+
},
|
| 2495 |
+
{
|
| 2496 |
+
"epoch": 0.9726322109371092,
|
| 2497 |
+
"grad_norm": 0.011089876294136047,
|
| 2498 |
+
"learning_rate": 2.026444127636029e-07,
|
| 2499 |
+
"loss": 0.0011911606416106224,
|
| 2500 |
+
"memory(GiB)": 25.85,
|
| 2501 |
+
"step": 1215,
|
| 2502 |
+
"token_acc": 0.9995529727313366,
|
| 2503 |
+
"train_speed(iter/s)": 0.142608
|
| 2504 |
+
},
|
| 2505 |
+
{
|
| 2506 |
+
"epoch": 0.9766348126282083,
|
| 2507 |
+
"grad_norm": 0.0003616033645812422,
|
| 2508 |
+
"learning_rate": 1.4745272285361933e-07,
|
| 2509 |
+
"loss": 0.0015979666262865067,
|
| 2510 |
+
"memory(GiB)": 25.85,
|
| 2511 |
+
"step": 1220,
|
| 2512 |
+
"token_acc": 0.9995408631772268,
|
| 2513 |
+
"train_speed(iter/s)": 0.142602
|
| 2514 |
+
},
|
| 2515 |
+
{
|
| 2516 |
+
"epoch": 0.9806374143193075,
|
| 2517 |
+
"grad_norm": 0.003614463610574603,
|
| 2518 |
+
"learning_rate": 1.0100586315099914e-07,
|
| 2519 |
+
"loss": 0.0017183024436235427,
|
| 2520 |
+
"memory(GiB)": 25.85,
|
| 2521 |
+
"step": 1225,
|
| 2522 |
+
"token_acc": 0.9987041036717063,
|
| 2523 |
+
"train_speed(iter/s)": 0.142591
|
| 2524 |
+
},
|
| 2525 |
+
{
|
| 2526 |
+
"epoch": 0.9846400160104067,
|
| 2527 |
+
"grad_norm": 0.0009111692197620869,
|
| 2528 |
+
"learning_rate": 6.331198108098235e-08,
|
| 2529 |
+
"loss": 0.0013875597156584262,
|
| 2530 |
+
"memory(GiB)": 25.85,
|
| 2531 |
+
"step": 1230,
|
| 2532 |
+
"token_acc": 0.9991304347826087,
|
| 2533 |
+
"train_speed(iter/s)": 0.142583
|
| 2534 |
+
},
|
| 2535 |
+
{
|
| 2536 |
+
"epoch": 0.988642617701506,
|
| 2537 |
+
"grad_norm": 0.0003311476903036237,
|
| 2538 |
+
"learning_rate": 3.4377688674774866e-08,
|
| 2539 |
+
"loss": 0.0013471558690071107,
|
| 2540 |
+
"memory(GiB)": 25.85,
|
| 2541 |
+
"step": 1235,
|
| 2542 |
+
"token_acc": 0.9995719178082192,
|
| 2543 |
+
"train_speed(iter/s)": 0.142578
|
| 2544 |
+
},
|
| 2545 |
+
{
|
| 2546 |
+
"epoch": 0.9926452193926052,
|
| 2547 |
+
"grad_norm": 0.01064266450703144,
|
| 2548 |
+
"learning_rate": 1.4208061409726192e-08,
|
| 2549 |
+
"loss": 0.0028278909623622893,
|
| 2550 |
+
"memory(GiB)": 25.85,
|
| 2551 |
+
"step": 1240,
|
| 2552 |
+
"token_acc": 0.9986059479553904,
|
| 2553 |
+
"train_speed(iter/s)": 0.142575
|
| 2554 |
+
},
|
| 2555 |
+
{
|
| 2556 |
+
"epoch": 0.9966478210837044,
|
| 2557 |
+
"grad_norm": 0.018854791298508644,
|
| 2558 |
+
"learning_rate": 2.8066373190027695e-09,
|
| 2559 |
+
"loss": 0.001478036679327488,
|
| 2560 |
+
"memory(GiB)": 25.85,
|
| 2561 |
+
"step": 1245,
|
| 2562 |
+
"token_acc": 0.99907019990702,
|
| 2563 |
+
"train_speed(iter/s)": 0.142561
|
| 2564 |
+
},
|
| 2565 |
+
{
|
| 2566 |
+
"epoch": 0.9998499024365838,
|
| 2567 |
+
"eval_loss": 0.033838555216789246,
|
| 2568 |
+
"eval_runtime": 2.4598,
|
| 2569 |
+
"eval_samples_per_second": 5.285,
|
| 2570 |
+
"eval_steps_per_second": 5.285,
|
| 2571 |
+
"eval_token_acc": 0.9924050632911392,
|
| 2572 |
+
"step": 1249
|
| 2573 |
+
}
|
| 2574 |
+
],
|
| 2575 |
+
"logging_steps": 5,
|
| 2576 |
+
"max_steps": 1249,
|
| 2577 |
+
"num_input_tokens_seen": 0,
|
| 2578 |
+
"num_train_epochs": 1,
|
| 2579 |
+
"save_steps": 200,
|
| 2580 |
+
"stateful_callbacks": {
|
| 2581 |
+
"TrainerControl": {
|
| 2582 |
+
"args": {
|
| 2583 |
+
"should_epoch_stop": false,
|
| 2584 |
+
"should_evaluate": false,
|
| 2585 |
+
"should_log": false,
|
| 2586 |
+
"should_save": true,
|
| 2587 |
+
"should_training_stop": true
|
| 2588 |
+
},
|
| 2589 |
+
"attributes": {}
|
| 2590 |
+
}
|
| 2591 |
+
},
|
| 2592 |
+
"total_flos": 3.840263911903768e+17,
|
| 2593 |
+
"train_batch_size": 1,
|
| 2594 |
+
"trial_name": null,
|
| 2595 |
+
"trial_params": null
|
| 2596 |
+
}
|
training_args.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dee42a9bb2705d57589cb47f173efbdf1d554490da9909d0152cde695f0702f1
|
| 3 |
+
size 5880
|
vision_siglip_navit.py
ADDED
|
@@ -0,0 +1,1717 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
""" Siglip model configuration"""
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
from typing import Union
|
| 19 |
+
|
| 20 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 21 |
+
from transformers.utils import logging
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
logger = logging.get_logger(__name__)
|
| 25 |
+
|
| 26 |
+
SIGLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
| 27 |
+
"google/siglip-base-patch16-224": "https://huggingface.co/google/siglip-base-patch16-224/resolve/main/config.json",
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class SiglipTextConfig(PretrainedConfig):
|
| 32 |
+
r"""
|
| 33 |
+
This is the configuration class to store the configuration of a [`SiglipTextModel`]. It is used to instantiate a
|
| 34 |
+
Siglip text encoder according to the specified arguments, defining the model architecture. Instantiating a
|
| 35 |
+
configuration with the defaults will yield a similar configuration to that of the text encoder of the Siglip
|
| 36 |
+
[google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
|
| 37 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 38 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 39 |
+
Args:
|
| 40 |
+
vocab_size (`int`, *optional*, defaults to 32000):
|
| 41 |
+
Vocabulary size of the Siglip text model. Defines the number of different tokens that can be represented by
|
| 42 |
+
the `inputs_ids` passed when calling [`SiglipModel`].
|
| 43 |
+
hidden_size (`int`, *optional*, defaults to 768):
|
| 44 |
+
Dimensionality of the encoder layers and the pooler layer.
|
| 45 |
+
intermediate_size (`int`, *optional*, defaults to 3072):
|
| 46 |
+
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
| 47 |
+
num_hidden_layers (`int`, *optional*, defaults to 12):
|
| 48 |
+
Number of hidden layers in the Transformer encoder.
|
| 49 |
+
num_attention_heads (`int`, *optional*, defaults to 12):
|
| 50 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
| 51 |
+
max_position_embeddings (`int`, *optional*, defaults to 64):
|
| 52 |
+
The maximum sequence length that this model might ever be used with. Typically set this to something large
|
| 53 |
+
just in case (e.g., 512 or 1024 or 2048).
|
| 54 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
|
| 55 |
+
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
| 56 |
+
`"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
|
| 57 |
+
layer_norm_eps (`float`, *optional*, defaults to 1e-06):
|
| 58 |
+
The epsilon used by the layer normalization layers.
|
| 59 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 60 |
+
The dropout ratio for the attention probabilities.
|
| 61 |
+
pad_token_id (`int`, *optional*, defaults to 1):
|
| 62 |
+
The id of the padding token in the vocabulary.
|
| 63 |
+
bos_token_id (`int`, *optional*, defaults to 49406):
|
| 64 |
+
The id of the beginning-of-sequence token in the vocabulary.
|
| 65 |
+
eos_token_id (`int`, *optional*, defaults to 49407):
|
| 66 |
+
The id of the end-of-sequence token in the vocabulary.
|
| 67 |
+
Example:
|
| 68 |
+
```python
|
| 69 |
+
>>> from transformers import SiglipTextConfig, SiglipTextModel
|
| 70 |
+
>>> # Initializing a SiglipTextConfig with google/siglip-base-patch16-224 style configuration
|
| 71 |
+
>>> configuration = SiglipTextConfig()
|
| 72 |
+
>>> # Initializing a SiglipTextModel (with random weights) from the google/siglip-base-patch16-224 style configuration
|
| 73 |
+
>>> model = SiglipTextModel(configuration)
|
| 74 |
+
>>> # Accessing the model configuration
|
| 75 |
+
>>> configuration = model.config
|
| 76 |
+
```"""
|
| 77 |
+
|
| 78 |
+
model_type = "siglip_text_model"
|
| 79 |
+
|
| 80 |
+
def __init__(
|
| 81 |
+
self,
|
| 82 |
+
vocab_size=32000,
|
| 83 |
+
hidden_size=768,
|
| 84 |
+
intermediate_size=3072,
|
| 85 |
+
num_hidden_layers=12,
|
| 86 |
+
num_attention_heads=12,
|
| 87 |
+
max_position_embeddings=64,
|
| 88 |
+
hidden_act="gelu_pytorch_tanh",
|
| 89 |
+
layer_norm_eps=1e-6,
|
| 90 |
+
attention_dropout=0.0,
|
| 91 |
+
# This differs from `CLIPTokenizer`'s default and from openai/siglip
|
| 92 |
+
# See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
|
| 93 |
+
pad_token_id=1,
|
| 94 |
+
bos_token_id=49406,
|
| 95 |
+
eos_token_id=49407,
|
| 96 |
+
_flash_attn_2_enabled=True,
|
| 97 |
+
**kwargs,
|
| 98 |
+
):
|
| 99 |
+
super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
| 100 |
+
|
| 101 |
+
self.vocab_size = vocab_size
|
| 102 |
+
self.hidden_size = hidden_size
|
| 103 |
+
self.intermediate_size = intermediate_size
|
| 104 |
+
self.num_hidden_layers = num_hidden_layers
|
| 105 |
+
self.num_attention_heads = num_attention_heads
|
| 106 |
+
self.max_position_embeddings = max_position_embeddings
|
| 107 |
+
self.layer_norm_eps = layer_norm_eps
|
| 108 |
+
self.hidden_act = hidden_act
|
| 109 |
+
self.attention_dropout = attention_dropout
|
| 110 |
+
self._flash_attn_2_enabled = _flash_attn_2_enabled
|
| 111 |
+
|
| 112 |
+
@classmethod
|
| 113 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
|
| 114 |
+
cls._set_token_in_kwargs(kwargs)
|
| 115 |
+
|
| 116 |
+
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
| 117 |
+
|
| 118 |
+
# get the text config dict if we are loading from SiglipConfig
|
| 119 |
+
if config_dict.get("model_type") == "siglip":
|
| 120 |
+
config_dict = config_dict["text_config"]
|
| 121 |
+
|
| 122 |
+
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
|
| 123 |
+
logger.warning(
|
| 124 |
+
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
|
| 125 |
+
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
return cls.from_dict(config_dict, **kwargs)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class SiglipVisionConfig(PretrainedConfig):
|
| 132 |
+
r"""
|
| 133 |
+
This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
|
| 134 |
+
Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
|
| 135 |
+
configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
|
| 136 |
+
[google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
|
| 137 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 138 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 139 |
+
Args:
|
| 140 |
+
hidden_size (`int`, *optional*, defaults to 768):
|
| 141 |
+
Dimensionality of the encoder layers and the pooler layer.
|
| 142 |
+
intermediate_size (`int`, *optional*, defaults to 3072):
|
| 143 |
+
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
| 144 |
+
num_hidden_layers (`int`, *optional*, defaults to 12):
|
| 145 |
+
Number of hidden layers in the Transformer encoder.
|
| 146 |
+
num_attention_heads (`int`, *optional*, defaults to 12):
|
| 147 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
| 148 |
+
num_channels (`int`, *optional*, defaults to 3):
|
| 149 |
+
Number of channels in the input images.
|
| 150 |
+
image_size (`int`, *optional*, defaults to 224):
|
| 151 |
+
The size (resolution) of each image.
|
| 152 |
+
patch_size (`int`, *optional*, defaults to 16):
|
| 153 |
+
The size (resolution) of each patch.
|
| 154 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
|
| 155 |
+
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
| 156 |
+
`"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
|
| 157 |
+
layer_norm_eps (`float`, *optional*, defaults to 1e-06):
|
| 158 |
+
The epsilon used by the layer normalization layers.
|
| 159 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 160 |
+
The dropout ratio for the attention probabilities.
|
| 161 |
+
Example:
|
| 162 |
+
```python
|
| 163 |
+
>>> from transformers import SiglipVisionConfig, SiglipVisionModel
|
| 164 |
+
>>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
|
| 165 |
+
>>> configuration = SiglipVisionConfig()
|
| 166 |
+
>>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
|
| 167 |
+
>>> model = SiglipVisionModel(configuration)
|
| 168 |
+
>>> # Accessing the model configuration
|
| 169 |
+
>>> configuration = model.config
|
| 170 |
+
```"""
|
| 171 |
+
|
| 172 |
+
model_type = "siglip_vision_model"
|
| 173 |
+
|
| 174 |
+
def __init__(
|
| 175 |
+
self,
|
| 176 |
+
hidden_size=768,
|
| 177 |
+
intermediate_size=3072,
|
| 178 |
+
num_hidden_layers=12,
|
| 179 |
+
num_attention_heads=12,
|
| 180 |
+
num_channels=3,
|
| 181 |
+
image_size=224,
|
| 182 |
+
patch_size=16,
|
| 183 |
+
hidden_act="gelu_pytorch_tanh",
|
| 184 |
+
layer_norm_eps=1e-6,
|
| 185 |
+
attention_dropout=0.0,
|
| 186 |
+
_flash_attn_2_enabled=True,
|
| 187 |
+
**kwargs,
|
| 188 |
+
):
|
| 189 |
+
super().__init__(**kwargs)
|
| 190 |
+
|
| 191 |
+
self.hidden_size = hidden_size
|
| 192 |
+
self.intermediate_size = intermediate_size
|
| 193 |
+
self.num_hidden_layers = num_hidden_layers
|
| 194 |
+
self.num_attention_heads = num_attention_heads
|
| 195 |
+
self.num_channels = num_channels
|
| 196 |
+
self.patch_size = patch_size
|
| 197 |
+
self.image_size = image_size
|
| 198 |
+
self.attention_dropout = attention_dropout
|
| 199 |
+
self.layer_norm_eps = layer_norm_eps
|
| 200 |
+
self.hidden_act = hidden_act
|
| 201 |
+
self._flash_attn_2_enabled = _flash_attn_2_enabled
|
| 202 |
+
|
| 203 |
+
@classmethod
|
| 204 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
|
| 205 |
+
cls._set_token_in_kwargs(kwargs)
|
| 206 |
+
|
| 207 |
+
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
| 208 |
+
|
| 209 |
+
# get the vision config dict if we are loading from SiglipConfig
|
| 210 |
+
if config_dict.get("model_type") == "siglip":
|
| 211 |
+
config_dict = config_dict["vision_config"]
|
| 212 |
+
|
| 213 |
+
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
|
| 214 |
+
logger.warning(
|
| 215 |
+
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
|
| 216 |
+
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
return cls.from_dict(config_dict, **kwargs)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class SiglipConfig(PretrainedConfig):
|
| 223 |
+
r"""
|
| 224 |
+
[`SiglipConfig`] is the configuration class to store the configuration of a [`SiglipModel`]. It is used to
|
| 225 |
+
instantiate a Siglip model according to the specified arguments, defining the text model and vision model configs.
|
| 226 |
+
Instantiating a configuration with the defaults will yield a similar configuration to that of the Siglip
|
| 227 |
+
[google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
|
| 228 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 229 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 230 |
+
Args:
|
| 231 |
+
text_config (`dict`, *optional*):
|
| 232 |
+
Dictionary of configuration options used to initialize [`SiglipTextConfig`].
|
| 233 |
+
vision_config (`dict`, *optional*):
|
| 234 |
+
Dictionary of configuration options used to initialize [`SiglipVisionConfig`].
|
| 235 |
+
kwargs (*optional*):
|
| 236 |
+
Dictionary of keyword arguments.
|
| 237 |
+
Example:
|
| 238 |
+
```python
|
| 239 |
+
>>> from transformers import SiglipConfig, SiglipModel
|
| 240 |
+
>>> # Initializing a SiglipConfig with google/siglip-base-patch16-224 style configuration
|
| 241 |
+
>>> configuration = SiglipConfig()
|
| 242 |
+
>>> # Initializing a SiglipModel (with random weights) from the google/siglip-base-patch16-224 style configuration
|
| 243 |
+
>>> model = SiglipModel(configuration)
|
| 244 |
+
>>> # Accessing the model configuration
|
| 245 |
+
>>> configuration = model.config
|
| 246 |
+
>>> # We can also initialize a SiglipConfig from a SiglipTextConfig and a SiglipVisionConfig
|
| 247 |
+
>>> from transformers import SiglipTextConfig, SiglipVisionConfig
|
| 248 |
+
>>> # Initializing a SiglipText and SiglipVision configuration
|
| 249 |
+
>>> config_text = SiglipTextConfig()
|
| 250 |
+
>>> config_vision = SiglipVisionConfig()
|
| 251 |
+
>>> config = SiglipConfig.from_text_vision_configs(config_text, config_vision)
|
| 252 |
+
```"""
|
| 253 |
+
|
| 254 |
+
model_type = "siglip"
|
| 255 |
+
|
| 256 |
+
def __init__(self, text_config=None, vision_config=None, **kwargs):
|
| 257 |
+
super().__init__(**kwargs)
|
| 258 |
+
|
| 259 |
+
if text_config is None:
|
| 260 |
+
text_config = {}
|
| 261 |
+
logger.info("`text_config` is `None`. Initializing the `SiglipTextConfig` with default values.")
|
| 262 |
+
|
| 263 |
+
if vision_config is None:
|
| 264 |
+
vision_config = {}
|
| 265 |
+
logger.info("`vision_config` is `None`. initializing the `SiglipVisionConfig` with default values.")
|
| 266 |
+
|
| 267 |
+
self.text_config = SiglipTextConfig(**text_config)
|
| 268 |
+
self.vision_config = SiglipVisionConfig(**vision_config)
|
| 269 |
+
|
| 270 |
+
self.initializer_factor = 1.0
|
| 271 |
+
|
| 272 |
+
@classmethod
|
| 273 |
+
def from_text_vision_configs(cls, text_config: SiglipTextConfig, vision_config: SiglipVisionConfig, **kwargs):
|
| 274 |
+
r"""
|
| 275 |
+
Instantiate a [`SiglipConfig`] (or a derived class) from siglip text model configuration and siglip vision
|
| 276 |
+
model configuration.
|
| 277 |
+
Returns:
|
| 278 |
+
[`SiglipConfig`]: An instance of a configuration object
|
| 279 |
+
"""
|
| 280 |
+
|
| 281 |
+
return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
|
| 282 |
+
|
| 283 |
+
# coding=utf-8
|
| 284 |
+
# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
|
| 285 |
+
#
|
| 286 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 287 |
+
# you may not use this file except in compliance with the License.
|
| 288 |
+
# You may obtain a copy of the License at
|
| 289 |
+
#
|
| 290 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 291 |
+
#
|
| 292 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 293 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 294 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 295 |
+
# See the License for the specific language governing permissions and
|
| 296 |
+
# limitations under the License.
|
| 297 |
+
""" PyTorch Siglip model."""
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
import math
|
| 301 |
+
import warnings
|
| 302 |
+
from dataclasses import dataclass
|
| 303 |
+
from typing import Any, Optional, Tuple, Union
|
| 304 |
+
|
| 305 |
+
import numpy as np
|
| 306 |
+
import torch
|
| 307 |
+
import torch.nn.functional as F
|
| 308 |
+
import torch.utils.checkpoint
|
| 309 |
+
from torch import nn
|
| 310 |
+
from torch.nn.init import _calculate_fan_in_and_fan_out
|
| 311 |
+
|
| 312 |
+
from transformers.activations import ACT2FN
|
| 313 |
+
from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
|
| 314 |
+
from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
|
| 315 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 316 |
+
from transformers.utils import (
|
| 317 |
+
ModelOutput,
|
| 318 |
+
add_start_docstrings,
|
| 319 |
+
add_start_docstrings_to_model_forward,
|
| 320 |
+
is_flash_attn_2_available,
|
| 321 |
+
logging,
|
| 322 |
+
replace_return_docstrings,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
logger = logging.get_logger(__name__)
|
| 326 |
+
|
| 327 |
+
_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
|
| 328 |
+
|
| 329 |
+
SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
| 330 |
+
"google/siglip-base-patch16-224",
|
| 331 |
+
# See all SigLIP models at https://huggingface.co/models?filter=siglip
|
| 332 |
+
]
|
| 333 |
+
|
| 334 |
+
if is_flash_attn_2_available():
|
| 335 |
+
from flash_attn import flash_attn_func, flash_attn_varlen_func
|
| 336 |
+
from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
# Copied from transformers.models.llama.modeling_llama._get_unpad_data
|
| 340 |
+
def _get_unpad_data(attention_mask):
|
| 341 |
+
seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
|
| 342 |
+
indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
|
| 343 |
+
max_seqlen_in_batch = seqlens_in_batch.max().item()
|
| 344 |
+
cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
|
| 345 |
+
return (
|
| 346 |
+
indices,
|
| 347 |
+
cu_seqlens,
|
| 348 |
+
max_seqlen_in_batch,
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def _trunc_normal_(tensor, mean, std, a, b):
|
| 353 |
+
# Cut & paste from PyTorch official master until it's in a few official releases - RW
|
| 354 |
+
# Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
|
| 355 |
+
def norm_cdf(x):
|
| 356 |
+
# Computes standard normal cumulative distribution function
|
| 357 |
+
return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
|
| 358 |
+
|
| 359 |
+
if (mean < a - 2 * std) or (mean > b + 2 * std):
|
| 360 |
+
warnings.warn(
|
| 361 |
+
"mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
|
| 362 |
+
"The distribution of values may be incorrect.",
|
| 363 |
+
stacklevel=2,
|
| 364 |
+
)
|
| 365 |
+
|
| 366 |
+
# Values are generated by using a truncated uniform distribution and
|
| 367 |
+
# then using the inverse CDF for the normal distribution.
|
| 368 |
+
# Get upper and lower cdf values
|
| 369 |
+
l = norm_cdf((a - mean) / std)
|
| 370 |
+
u = norm_cdf((b - mean) / std)
|
| 371 |
+
|
| 372 |
+
# Uniformly fill tensor with values from [l, u], then translate to
|
| 373 |
+
# [2l-1, 2u-1].
|
| 374 |
+
tensor.uniform_(2 * l - 1, 2 * u - 1)
|
| 375 |
+
|
| 376 |
+
# Use inverse cdf transform for normal distribution to get truncated
|
| 377 |
+
# standard normal
|
| 378 |
+
if tensor.dtype in [torch.float16, torch.bfloat16]:
|
| 379 |
+
# The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
|
| 380 |
+
og_dtype = tensor.dtype
|
| 381 |
+
tensor = tensor.to(torch.float32)
|
| 382 |
+
tensor.erfinv_()
|
| 383 |
+
tensor = tensor.to(og_dtype)
|
| 384 |
+
else:
|
| 385 |
+
tensor.erfinv_()
|
| 386 |
+
|
| 387 |
+
# Transform to proper mean, std
|
| 388 |
+
tensor.mul_(std * math.sqrt(2.0))
|
| 389 |
+
tensor.add_(mean)
|
| 390 |
+
|
| 391 |
+
# Clamp to ensure it's in the proper range
|
| 392 |
+
if tensor.dtype == torch.float16:
|
| 393 |
+
# The `clamp_` op is not (yet?) defined in float16+cpu
|
| 394 |
+
tensor = tensor.to(torch.float32)
|
| 395 |
+
tensor.clamp_(min=a, max=b)
|
| 396 |
+
tensor = tensor.to(torch.float16)
|
| 397 |
+
else:
|
| 398 |
+
tensor.clamp_(min=a, max=b)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def trunc_normal_tf_(
|
| 402 |
+
tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
|
| 403 |
+
) -> torch.Tensor:
|
| 404 |
+
"""Fills the input Tensor with values drawn from a truncated
|
| 405 |
+
normal distribution. The values are effectively drawn from the
|
| 406 |
+
normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
|
| 407 |
+
with values outside :math:`[a, b]` redrawn until they are within
|
| 408 |
+
the bounds. The method used for generating the random values works
|
| 409 |
+
best when :math:`a \\leq \text{mean} \\leq b`.
|
| 410 |
+
NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
|
| 411 |
+
bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
|
| 412 |
+
and the result is subsquently scaled and shifted by the mean and std args.
|
| 413 |
+
Args:
|
| 414 |
+
tensor: an n-dimensional `torch.Tensor`
|
| 415 |
+
mean: the mean of the normal distribution
|
| 416 |
+
std: the standard deviation of the normal distribution
|
| 417 |
+
a: the minimum cutoff value
|
| 418 |
+
b: the maximum cutoff value
|
| 419 |
+
"""
|
| 420 |
+
with torch.no_grad():
|
| 421 |
+
_trunc_normal_(tensor, 0, 1.0, a, b)
|
| 422 |
+
tensor.mul_(std).add_(mean)
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
|
| 426 |
+
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
|
| 427 |
+
if mode == "fan_in":
|
| 428 |
+
denom = fan_in
|
| 429 |
+
elif mode == "fan_out":
|
| 430 |
+
denom = fan_out
|
| 431 |
+
elif mode == "fan_avg":
|
| 432 |
+
denom = (fan_in + fan_out) / 2
|
| 433 |
+
|
| 434 |
+
variance = scale / denom
|
| 435 |
+
|
| 436 |
+
if distribution == "truncated_normal":
|
| 437 |
+
# constant is stddev of standard normal truncated to (-2, 2)
|
| 438 |
+
trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
|
| 439 |
+
elif distribution == "normal":
|
| 440 |
+
with torch.no_grad():
|
| 441 |
+
tensor.normal_(std=math.sqrt(variance))
|
| 442 |
+
elif distribution == "uniform":
|
| 443 |
+
bound = math.sqrt(3 * variance)
|
| 444 |
+
with torch.no_grad():
|
| 445 |
+
tensor.uniform_(-bound, bound)
|
| 446 |
+
else:
|
| 447 |
+
raise ValueError(f"invalid distribution {distribution}")
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def lecun_normal_(tensor):
|
| 451 |
+
variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def default_flax_embed_init(tensor):
|
| 455 |
+
variance_scaling_(tensor, mode="fan_in", distribution="normal")
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
@dataclass
|
| 459 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->Siglip
|
| 460 |
+
class SiglipVisionModelOutput(ModelOutput):
|
| 461 |
+
"""
|
| 462 |
+
Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
|
| 463 |
+
Args:
|
| 464 |
+
image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
|
| 465 |
+
The image embeddings obtained by applying the projection layer to the pooler_output.
|
| 466 |
+
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 467 |
+
Sequence of hidden-states at the output of the last layer of the model.
|
| 468 |
+
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| 469 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 470 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 471 |
+
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| 472 |
+
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 473 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 474 |
+
sequence_length)`.
|
| 475 |
+
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
|
| 476 |
+
heads.
|
| 477 |
+
"""
|
| 478 |
+
|
| 479 |
+
image_embeds: Optional[torch.FloatTensor] = None
|
| 480 |
+
last_hidden_state: torch.FloatTensor = None
|
| 481 |
+
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
| 482 |
+
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
@dataclass
|
| 486 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPTextModelOutput with CLIP->Siglip
|
| 487 |
+
class SiglipTextModelOutput(ModelOutput):
|
| 488 |
+
"""
|
| 489 |
+
Base class for text model's outputs that also contains a pooling of the last hidden states.
|
| 490 |
+
Args:
|
| 491 |
+
text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
|
| 492 |
+
The text embeddings obtained by applying the projection layer to the pooler_output.
|
| 493 |
+
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 494 |
+
Sequence of hidden-states at the output of the last layer of the model.
|
| 495 |
+
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
|
| 496 |
+
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
|
| 497 |
+
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
|
| 498 |
+
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
|
| 499 |
+
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
|
| 500 |
+
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
|
| 501 |
+
sequence_length)`.
|
| 502 |
+
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
|
| 503 |
+
heads.
|
| 504 |
+
"""
|
| 505 |
+
|
| 506 |
+
text_embeds: Optional[torch.FloatTensor] = None
|
| 507 |
+
last_hidden_state: torch.FloatTensor = None
|
| 508 |
+
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
| 509 |
+
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
@dataclass
|
| 513 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPOutput with CLIP->Siglip
|
| 514 |
+
class SiglipOutput(ModelOutput):
|
| 515 |
+
"""
|
| 516 |
+
Args:
|
| 517 |
+
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
|
| 518 |
+
Contrastive loss for image-text similarity.
|
| 519 |
+
logits_per_image:(`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
|
| 520 |
+
The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
|
| 521 |
+
similarity scores.
|
| 522 |
+
logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
|
| 523 |
+
The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
|
| 524 |
+
similarity scores.
|
| 525 |
+
text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
|
| 526 |
+
The text embeddings obtained by applying the projection layer to the pooled output of [`SiglipTextModel`].
|
| 527 |
+
image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
|
| 528 |
+
The image embeddings obtained by applying the projection layer to the pooled output of [`SiglipVisionModel`].
|
| 529 |
+
text_model_output(`BaseModelOutputWithPooling`):
|
| 530 |
+
The output of the [`SiglipTextModel`].
|
| 531 |
+
vision_model_output(`BaseModelOutputWithPooling`):
|
| 532 |
+
The output of the [`SiglipVisionModel`].
|
| 533 |
+
"""
|
| 534 |
+
|
| 535 |
+
loss: Optional[torch.FloatTensor] = None
|
| 536 |
+
logits_per_image: torch.FloatTensor = None
|
| 537 |
+
logits_per_text: torch.FloatTensor = None
|
| 538 |
+
text_embeds: torch.FloatTensor = None
|
| 539 |
+
image_embeds: torch.FloatTensor = None
|
| 540 |
+
text_model_output: BaseModelOutputWithPooling = None
|
| 541 |
+
vision_model_output: BaseModelOutputWithPooling = None
|
| 542 |
+
|
| 543 |
+
def to_tuple(self) -> Tuple[Any]:
|
| 544 |
+
return tuple(
|
| 545 |
+
self[k] if k not in ["text_model_output", "vision_model_output"] else getattr(self, k).to_tuple()
|
| 546 |
+
for k in self.keys()
|
| 547 |
+
)
|
| 548 |
+
|
| 549 |
+
|
| 550 |
+
class SiglipVisionEmbeddings(nn.Module):
|
| 551 |
+
def __init__(self, config: SiglipVisionConfig):
|
| 552 |
+
super().__init__()
|
| 553 |
+
self.config = config
|
| 554 |
+
self.embed_dim = config.hidden_size
|
| 555 |
+
self.image_size = config.image_size
|
| 556 |
+
self.patch_size = config.patch_size
|
| 557 |
+
|
| 558 |
+
self.patch_embedding = nn.Conv2d(
|
| 559 |
+
in_channels=config.num_channels,
|
| 560 |
+
out_channels=self.embed_dim,
|
| 561 |
+
kernel_size=self.patch_size,
|
| 562 |
+
stride=self.patch_size,
|
| 563 |
+
padding="valid",
|
| 564 |
+
)
|
| 565 |
+
|
| 566 |
+
self.num_patches_per_side = self.image_size // self.patch_size
|
| 567 |
+
self.num_patches = self.num_patches_per_side**2
|
| 568 |
+
self.num_positions = self.num_patches
|
| 569 |
+
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
|
| 570 |
+
|
| 571 |
+
def forward(self, pixel_values: torch.FloatTensor, patch_attention_mask: torch.BoolTensor) -> torch.Tensor:
|
| 572 |
+
batch_size = pixel_values.size(0)
|
| 573 |
+
|
| 574 |
+
patch_embeds = self.patch_embedding(pixel_values)
|
| 575 |
+
embeddings = patch_embeds.flatten(2).transpose(1, 2)
|
| 576 |
+
|
| 577 |
+
max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
|
| 578 |
+
max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, max_im_w // self.patch_size
|
| 579 |
+
boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side)
|
| 580 |
+
position_ids = torch.full(
|
| 581 |
+
size=(
|
| 582 |
+
batch_size,
|
| 583 |
+
max_nb_patches_h * max_nb_patches_w,
|
| 584 |
+
),
|
| 585 |
+
fill_value=0,
|
| 586 |
+
)
|
| 587 |
+
|
| 588 |
+
for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
|
| 589 |
+
nb_patches_h = p_attn_mask[:, 0].sum()
|
| 590 |
+
nb_patches_w = p_attn_mask[0].sum()
|
| 591 |
+
|
| 592 |
+
fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
|
| 593 |
+
fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
|
| 594 |
+
|
| 595 |
+
bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True)
|
| 596 |
+
bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True)
|
| 597 |
+
|
| 598 |
+
pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten()
|
| 599 |
+
position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
|
| 600 |
+
|
| 601 |
+
position_ids = position_ids.to(self.position_embedding.weight.device)
|
| 602 |
+
|
| 603 |
+
embeddings = embeddings + self.position_embedding(position_ids)
|
| 604 |
+
return embeddings
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPTextEmbeddings with CLIP->Siglip
|
| 608 |
+
class SiglipTextEmbeddings(nn.Module):
|
| 609 |
+
def __init__(self, config: SiglipTextConfig):
|
| 610 |
+
super().__init__()
|
| 611 |
+
embed_dim = config.hidden_size
|
| 612 |
+
|
| 613 |
+
self.token_embedding = nn.Embedding(config.vocab_size, embed_dim)
|
| 614 |
+
self.position_embedding = nn.Embedding(config.max_position_embeddings, embed_dim)
|
| 615 |
+
|
| 616 |
+
# position_ids (1, len position emb) is contiguous in memory and exported when serialized
|
| 617 |
+
self.register_buffer(
|
| 618 |
+
"position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False
|
| 619 |
+
)
|
| 620 |
+
|
| 621 |
+
def forward(
|
| 622 |
+
self,
|
| 623 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 624 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 625 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 626 |
+
) -> torch.Tensor:
|
| 627 |
+
seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]
|
| 628 |
+
|
| 629 |
+
if position_ids is None:
|
| 630 |
+
position_ids = self.position_ids[:, :seq_length]
|
| 631 |
+
|
| 632 |
+
if inputs_embeds is None:
|
| 633 |
+
inputs_embeds = self.token_embedding(input_ids)
|
| 634 |
+
|
| 635 |
+
position_embeddings = self.position_embedding(position_ids)
|
| 636 |
+
embeddings = inputs_embeds + position_embeddings
|
| 637 |
+
|
| 638 |
+
return embeddings
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
class SiglipAttention(nn.Module):
|
| 642 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 643 |
+
|
| 644 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
|
| 645 |
+
def __init__(self, config):
|
| 646 |
+
super().__init__()
|
| 647 |
+
self.config = config
|
| 648 |
+
self.embed_dim = config.hidden_size
|
| 649 |
+
self.num_heads = config.num_attention_heads
|
| 650 |
+
self.head_dim = self.embed_dim // self.num_heads
|
| 651 |
+
if self.head_dim * self.num_heads != self.embed_dim:
|
| 652 |
+
raise ValueError(
|
| 653 |
+
f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
|
| 654 |
+
f" {self.num_heads})."
|
| 655 |
+
)
|
| 656 |
+
self.scale = self.head_dim**-0.5
|
| 657 |
+
self.dropout = config.attention_dropout
|
| 658 |
+
|
| 659 |
+
self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 660 |
+
self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 661 |
+
self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 662 |
+
self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 663 |
+
|
| 664 |
+
def forward(
|
| 665 |
+
self,
|
| 666 |
+
hidden_states: torch.Tensor,
|
| 667 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 668 |
+
output_attentions: Optional[bool] = False,
|
| 669 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 670 |
+
"""Input shape: Batch x Time x Channel"""
|
| 671 |
+
|
| 672 |
+
batch_size, q_len, _ = hidden_states.size()
|
| 673 |
+
|
| 674 |
+
query_states = self.q_proj(hidden_states)
|
| 675 |
+
key_states = self.k_proj(hidden_states)
|
| 676 |
+
value_states = self.v_proj(hidden_states)
|
| 677 |
+
|
| 678 |
+
query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 679 |
+
key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 680 |
+
value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 681 |
+
|
| 682 |
+
k_v_seq_len = key_states.shape[-2]
|
| 683 |
+
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
|
| 684 |
+
|
| 685 |
+
if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
|
| 686 |
+
raise ValueError(
|
| 687 |
+
f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
|
| 688 |
+
f" {attn_weights.size()}"
|
| 689 |
+
)
|
| 690 |
+
|
| 691 |
+
if attention_mask is not None:
|
| 692 |
+
if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
|
| 693 |
+
raise ValueError(
|
| 694 |
+
f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
|
| 695 |
+
)
|
| 696 |
+
attn_weights = attn_weights + attention_mask
|
| 697 |
+
|
| 698 |
+
# upcast attention to fp32
|
| 699 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
|
| 700 |
+
attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
|
| 701 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
| 702 |
+
|
| 703 |
+
if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
|
| 704 |
+
raise ValueError(
|
| 705 |
+
f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
|
| 706 |
+
f" {attn_output.size()}"
|
| 707 |
+
)
|
| 708 |
+
|
| 709 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 710 |
+
attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
|
| 711 |
+
|
| 712 |
+
attn_output = self.out_proj(attn_output)
|
| 713 |
+
|
| 714 |
+
return attn_output, attn_weights
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
class SiglipFlashAttention2(SiglipAttention):
|
| 718 |
+
"""
|
| 719 |
+
Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays
|
| 720 |
+
untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
|
| 721 |
+
flash attention and deal with padding tokens in case the input contains any of them.
|
| 722 |
+
"""
|
| 723 |
+
|
| 724 |
+
def __init__(self, *args, **kwargs):
|
| 725 |
+
super().__init__(*args, **kwargs)
|
| 726 |
+
self.is_causal = False # Hack to make sure we don't use a causal mask
|
| 727 |
+
|
| 728 |
+
def forward(
|
| 729 |
+
self,
|
| 730 |
+
hidden_states: torch.Tensor,
|
| 731 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 732 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 733 |
+
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
| 734 |
+
output_attentions: bool = False,
|
| 735 |
+
use_cache: bool = False,
|
| 736 |
+
**kwargs,
|
| 737 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 738 |
+
output_attentions = False
|
| 739 |
+
|
| 740 |
+
bsz, q_len, _ = hidden_states.size()
|
| 741 |
+
|
| 742 |
+
query_states = self.q_proj(hidden_states)
|
| 743 |
+
key_states = self.k_proj(hidden_states)
|
| 744 |
+
value_states = self.v_proj(hidden_states)
|
| 745 |
+
|
| 746 |
+
# Flash attention requires the input to have the shape
|
| 747 |
+
# batch_size x seq_length x head_dim x hidden_dim
|
| 748 |
+
# therefore we just need to keep the original shape
|
| 749 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 750 |
+
key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 751 |
+
value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 752 |
+
|
| 753 |
+
kv_seq_len = key_states.shape[-2]
|
| 754 |
+
if past_key_value is not None:
|
| 755 |
+
kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
|
| 756 |
+
# cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
|
| 757 |
+
# query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
| 758 |
+
|
| 759 |
+
# if past_key_value is not None:
|
| 760 |
+
# cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
|
| 761 |
+
# key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 762 |
+
|
| 763 |
+
# TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
|
| 764 |
+
# to be able to avoid many of these transpose/reshape/view.
|
| 765 |
+
query_states = query_states.transpose(1, 2)
|
| 766 |
+
key_states = key_states.transpose(1, 2)
|
| 767 |
+
value_states = value_states.transpose(1, 2)
|
| 768 |
+
|
| 769 |
+
dropout_rate = self.dropout if self.training else 0.0
|
| 770 |
+
|
| 771 |
+
# In PEFT, usually we cast the layer norms in float32 for training stability reasons
|
| 772 |
+
# therefore the input hidden states gets silently casted in float32. Hence, we need
|
| 773 |
+
# cast them back in the correct dtype just to be sure everything works as expected.
|
| 774 |
+
# This might slowdown training & inference so it is recommended to not cast the LayerNorms
|
| 775 |
+
# in fp32. (LlamaRMSNorm handles it correctly)
|
| 776 |
+
|
| 777 |
+
input_dtype = query_states.dtype
|
| 778 |
+
if input_dtype == torch.float32:
|
| 779 |
+
if torch.is_autocast_enabled():
|
| 780 |
+
target_dtype = torch.get_autocast_gpu_dtype()
|
| 781 |
+
# Handle the case where the model is quantized
|
| 782 |
+
elif hasattr(self.config, "_pre_quantization_dtype"):
|
| 783 |
+
target_dtype = self.config._pre_quantization_dtype
|
| 784 |
+
else:
|
| 785 |
+
target_dtype = self.q_proj.weight.dtype
|
| 786 |
+
|
| 787 |
+
logger.warning_once(
|
| 788 |
+
"The input hidden states seems to be silently casted in float32, this might be related to the fact"
|
| 789 |
+
" you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
|
| 790 |
+
f" {target_dtype}."
|
| 791 |
+
)
|
| 792 |
+
|
| 793 |
+
query_states = query_states.to(target_dtype)
|
| 794 |
+
key_states = key_states.to(target_dtype)
|
| 795 |
+
value_states = value_states.to(target_dtype)
|
| 796 |
+
|
| 797 |
+
attn_output = self._flash_attention_forward(
|
| 798 |
+
query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
|
| 799 |
+
)
|
| 800 |
+
|
| 801 |
+
attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
|
| 802 |
+
attn_output = self.out_proj(attn_output)
|
| 803 |
+
|
| 804 |
+
if not output_attentions:
|
| 805 |
+
attn_weights = None
|
| 806 |
+
|
| 807 |
+
return attn_output, attn_weights
|
| 808 |
+
|
| 809 |
+
def _flash_attention_forward(
|
| 810 |
+
self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
|
| 811 |
+
):
|
| 812 |
+
"""
|
| 813 |
+
Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
|
| 814 |
+
first unpad the input, then computes the attention scores and pad the final attention scores.
|
| 815 |
+
Args:
|
| 816 |
+
query_states (`torch.Tensor`):
|
| 817 |
+
Input query states to be passed to Flash Attention API
|
| 818 |
+
key_states (`torch.Tensor`):
|
| 819 |
+
Input key states to be passed to Flash Attention API
|
| 820 |
+
value_states (`torch.Tensor`):
|
| 821 |
+
Input value states to be passed to Flash Attention API
|
| 822 |
+
attention_mask (`torch.Tensor`):
|
| 823 |
+
The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
|
| 824 |
+
position of padding tokens and 1 for the position of non-padding tokens.
|
| 825 |
+
dropout (`int`, *optional*):
|
| 826 |
+
Attention dropout
|
| 827 |
+
softmax_scale (`float`, *optional*):
|
| 828 |
+
The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
|
| 829 |
+
"""
|
| 830 |
+
|
| 831 |
+
# TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
|
| 832 |
+
causal = self.is_causal and query_length != 1
|
| 833 |
+
|
| 834 |
+
# Contains at least one padding token in the sequence
|
| 835 |
+
if attention_mask is not None:
|
| 836 |
+
batch_size = query_states.shape[0]
|
| 837 |
+
query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
|
| 838 |
+
query_states, key_states, value_states, attention_mask, query_length
|
| 839 |
+
)
|
| 840 |
+
|
| 841 |
+
cu_seqlens_q, cu_seqlens_k = cu_seq_lens
|
| 842 |
+
max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
|
| 843 |
+
|
| 844 |
+
attn_output_unpad = flash_attn_varlen_func(
|
| 845 |
+
query_states,
|
| 846 |
+
key_states,
|
| 847 |
+
value_states,
|
| 848 |
+
cu_seqlens_q=cu_seqlens_q,
|
| 849 |
+
cu_seqlens_k=cu_seqlens_k,
|
| 850 |
+
max_seqlen_q=max_seqlen_in_batch_q,
|
| 851 |
+
max_seqlen_k=max_seqlen_in_batch_k,
|
| 852 |
+
dropout_p=dropout,
|
| 853 |
+
softmax_scale=softmax_scale,
|
| 854 |
+
causal=causal,
|
| 855 |
+
)
|
| 856 |
+
|
| 857 |
+
attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
|
| 858 |
+
else:
|
| 859 |
+
attn_output = flash_attn_func(
|
| 860 |
+
query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
|
| 861 |
+
)
|
| 862 |
+
|
| 863 |
+
return attn_output
|
| 864 |
+
|
| 865 |
+
def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
|
| 866 |
+
indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
|
| 867 |
+
batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
|
| 868 |
+
|
| 869 |
+
key_layer = index_first_axis(
|
| 870 |
+
key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
|
| 871 |
+
)
|
| 872 |
+
value_layer = index_first_axis(
|
| 873 |
+
value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
|
| 874 |
+
)
|
| 875 |
+
if query_length == kv_seq_len:
|
| 876 |
+
query_layer = index_first_axis(
|
| 877 |
+
query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
|
| 878 |
+
)
|
| 879 |
+
cu_seqlens_q = cu_seqlens_k
|
| 880 |
+
max_seqlen_in_batch_q = max_seqlen_in_batch_k
|
| 881 |
+
indices_q = indices_k
|
| 882 |
+
elif query_length == 1:
|
| 883 |
+
max_seqlen_in_batch_q = 1
|
| 884 |
+
cu_seqlens_q = torch.arange(
|
| 885 |
+
batch_size + 1, dtype=torch.int32, device=query_layer.device
|
| 886 |
+
) # There is a memcpy here, that is very bad.
|
| 887 |
+
indices_q = cu_seqlens_q[:-1]
|
| 888 |
+
query_layer = query_layer.squeeze(1)
|
| 889 |
+
else:
|
| 890 |
+
# The -q_len: slice assumes left padding.
|
| 891 |
+
attention_mask = attention_mask[:, -query_length:]
|
| 892 |
+
query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
|
| 893 |
+
|
| 894 |
+
return (
|
| 895 |
+
query_layer,
|
| 896 |
+
key_layer,
|
| 897 |
+
value_layer,
|
| 898 |
+
indices_q,
|
| 899 |
+
(cu_seqlens_q, cu_seqlens_k),
|
| 900 |
+
(max_seqlen_in_batch_q, max_seqlen_in_batch_k),
|
| 901 |
+
)
|
| 902 |
+
|
| 903 |
+
|
| 904 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
|
| 905 |
+
class SiglipMLP(nn.Module):
|
| 906 |
+
def __init__(self, config):
|
| 907 |
+
super().__init__()
|
| 908 |
+
self.config = config
|
| 909 |
+
self.activation_fn = ACT2FN[config.hidden_act]
|
| 910 |
+
self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
| 911 |
+
self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
| 912 |
+
|
| 913 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 914 |
+
hidden_states = self.fc1(hidden_states)
|
| 915 |
+
hidden_states = self.activation_fn(hidden_states)
|
| 916 |
+
hidden_states = self.fc2(hidden_states)
|
| 917 |
+
return hidden_states
|
| 918 |
+
|
| 919 |
+
|
| 920 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
|
| 921 |
+
class SiglipEncoderLayer(nn.Module):
|
| 922 |
+
def __init__(self, config: SiglipConfig):
|
| 923 |
+
super().__init__()
|
| 924 |
+
self.embed_dim = config.hidden_size
|
| 925 |
+
self.self_attn = (
|
| 926 |
+
SiglipAttention(config)
|
| 927 |
+
if not getattr(config, "_flash_attn_2_enabled", False)
|
| 928 |
+
else SiglipFlashAttention2(config)
|
| 929 |
+
)
|
| 930 |
+
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 931 |
+
self.mlp = SiglipMLP(config)
|
| 932 |
+
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 933 |
+
|
| 934 |
+
def forward(
|
| 935 |
+
self,
|
| 936 |
+
hidden_states: torch.Tensor,
|
| 937 |
+
attention_mask: torch.Tensor,
|
| 938 |
+
output_attentions: Optional[bool] = False,
|
| 939 |
+
) -> Tuple[torch.FloatTensor]:
|
| 940 |
+
"""
|
| 941 |
+
Args:
|
| 942 |
+
hidden_states (`torch.FloatTensor`):
|
| 943 |
+
Input to the layer of shape `(batch, seq_len, embed_dim)`.
|
| 944 |
+
attention_mask (`torch.FloatTensor`):
|
| 945 |
+
Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
|
| 946 |
+
output_attentions (`bool`, *optional*, defaults to `False`):
|
| 947 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 948 |
+
returned tensors for more detail.
|
| 949 |
+
"""
|
| 950 |
+
residual = hidden_states
|
| 951 |
+
|
| 952 |
+
hidden_states = self.layer_norm1(hidden_states)
|
| 953 |
+
hidden_states, attn_weights = self.self_attn(
|
| 954 |
+
hidden_states=hidden_states,
|
| 955 |
+
attention_mask=attention_mask,
|
| 956 |
+
output_attentions=output_attentions,
|
| 957 |
+
)
|
| 958 |
+
hidden_states = residual + hidden_states
|
| 959 |
+
|
| 960 |
+
residual = hidden_states
|
| 961 |
+
hidden_states = self.layer_norm2(hidden_states)
|
| 962 |
+
hidden_states = self.mlp(hidden_states)
|
| 963 |
+
hidden_states = residual + hidden_states
|
| 964 |
+
|
| 965 |
+
outputs = (hidden_states,)
|
| 966 |
+
|
| 967 |
+
if output_attentions:
|
| 968 |
+
outputs += (attn_weights,)
|
| 969 |
+
|
| 970 |
+
return outputs
|
| 971 |
+
|
| 972 |
+
|
| 973 |
+
class SiglipPreTrainedModel(PreTrainedModel):
|
| 974 |
+
"""
|
| 975 |
+
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
| 976 |
+
models.
|
| 977 |
+
"""
|
| 978 |
+
|
| 979 |
+
config_class = SiglipConfig
|
| 980 |
+
base_model_prefix = "siglip"
|
| 981 |
+
supports_gradient_checkpointing = True
|
| 982 |
+
|
| 983 |
+
def _init_weights(self, module):
|
| 984 |
+
"""Initialize the weights"""
|
| 985 |
+
|
| 986 |
+
if isinstance(module, SiglipVisionEmbeddings):
|
| 987 |
+
width = (
|
| 988 |
+
self.config.vision_config.hidden_size
|
| 989 |
+
if isinstance(self.config, SiglipConfig)
|
| 990 |
+
else self.config.hidden_size
|
| 991 |
+
)
|
| 992 |
+
nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
|
| 993 |
+
elif isinstance(module, nn.Embedding):
|
| 994 |
+
default_flax_embed_init(module.weight)
|
| 995 |
+
elif isinstance(module, SiglipAttention):
|
| 996 |
+
nn.init.normal_(module.q_proj.weight)
|
| 997 |
+
nn.init.normal_(module.k_proj.weight)
|
| 998 |
+
nn.init.normal_(module.v_proj.weight)
|
| 999 |
+
nn.init.normal_(module.out_proj.weight)
|
| 1000 |
+
nn.init.zeros_(module.q_proj.bias)
|
| 1001 |
+
nn.init.zeros_(module.k_proj.bias)
|
| 1002 |
+
nn.init.zeros_(module.v_proj.bias)
|
| 1003 |
+
nn.init.zeros_(module.out_proj.bias)
|
| 1004 |
+
elif isinstance(module, SiglipMLP):
|
| 1005 |
+
nn.init.normal_(module.fc1.weight)
|
| 1006 |
+
nn.init.normal_(module.fc2.weight)
|
| 1007 |
+
nn.init.normal_(module.fc1.bias, std=1e-6)
|
| 1008 |
+
nn.init.normal_(module.fc2.bias, std=1e-6)
|
| 1009 |
+
elif isinstance(module, SiglipMultiheadAttentionPoolingHead):
|
| 1010 |
+
nn.init.normal_(module.probe.data)
|
| 1011 |
+
nn.init.normal_(module.attention.in_proj_weight.data)
|
| 1012 |
+
nn.init.zeros_(module.attention.in_proj_bias.data)
|
| 1013 |
+
elif isinstance(module, SiglipModel):
|
| 1014 |
+
logit_scale_init = torch.tensor(0.0)
|
| 1015 |
+
module.logit_scale.data.fill_(logit_scale_init)
|
| 1016 |
+
module.logit_bias.data.zero_()
|
| 1017 |
+
elif isinstance(module, (nn.Linear, nn.Conv2d)):
|
| 1018 |
+
lecun_normal_(module.weight)
|
| 1019 |
+
if module.bias is not None:
|
| 1020 |
+
nn.init.zeros_(module.bias)
|
| 1021 |
+
elif isinstance(module, nn.LayerNorm):
|
| 1022 |
+
module.bias.data.zero_()
|
| 1023 |
+
module.weight.data.fill_(1.0)
|
| 1024 |
+
|
| 1025 |
+
|
| 1026 |
+
SIGLIP_START_DOCSTRING = r"""
|
| 1027 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
| 1028 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
| 1029 |
+
etc.)
|
| 1030 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
| 1031 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
| 1032 |
+
and behavior.
|
| 1033 |
+
Parameters:
|
| 1034 |
+
config ([`SiglipConfig`]): Model configuration class with all the parameters of the model.
|
| 1035 |
+
Initializing with a config file does not load the weights associated with the model, only the
|
| 1036 |
+
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
| 1037 |
+
"""
|
| 1038 |
+
|
| 1039 |
+
SIGLIP_TEXT_INPUTS_DOCSTRING = r"""
|
| 1040 |
+
Args:
|
| 1041 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
| 1042 |
+
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
| 1043 |
+
it.
|
| 1044 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 1045 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
| 1046 |
+
[What are input IDs?](../glossary#input-ids)
|
| 1047 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1048 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 1049 |
+
- 1 for tokens that are **not masked**,
|
| 1050 |
+
- 0 for tokens that are **masked**.
|
| 1051 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 1052 |
+
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1053 |
+
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
| 1054 |
+
config.max_position_embeddings - 1]`.
|
| 1055 |
+
[What are position IDs?](../glossary#position-ids)
|
| 1056 |
+
output_attentions (`bool`, *optional*):
|
| 1057 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 1058 |
+
tensors for more detail.
|
| 1059 |
+
output_hidden_states (`bool`, *optional*):
|
| 1060 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 1061 |
+
more detail.
|
| 1062 |
+
return_dict (`bool`, *optional*):
|
| 1063 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 1064 |
+
"""
|
| 1065 |
+
|
| 1066 |
+
SIGLIP_VISION_INPUTS_DOCSTRING = r"""
|
| 1067 |
+
Args:
|
| 1068 |
+
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
|
| 1069 |
+
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
|
| 1070 |
+
[`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
|
| 1071 |
+
output_attentions (`bool`, *optional*):
|
| 1072 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 1073 |
+
tensors for more detail.
|
| 1074 |
+
output_hidden_states (`bool`, *optional*):
|
| 1075 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 1076 |
+
more detail.
|
| 1077 |
+
return_dict (`bool`, *optional*):
|
| 1078 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 1079 |
+
"""
|
| 1080 |
+
|
| 1081 |
+
SIGLIP_INPUTS_DOCSTRING = r"""
|
| 1082 |
+
Args:
|
| 1083 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
| 1084 |
+
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
| 1085 |
+
it.
|
| 1086 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 1087 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
| 1088 |
+
[What are input IDs?](../glossary#input-ids)
|
| 1089 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1090 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 1091 |
+
- 1 for tokens that are **not masked**,
|
| 1092 |
+
- 0 for tokens that are **masked**.
|
| 1093 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 1094 |
+
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1095 |
+
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
| 1096 |
+
config.max_position_embeddings - 1]`.
|
| 1097 |
+
[What are position IDs?](../glossary#position-ids)
|
| 1098 |
+
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
|
| 1099 |
+
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
|
| 1100 |
+
[`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
|
| 1101 |
+
return_loss (`bool`, *optional*):
|
| 1102 |
+
Whether or not to return the contrastive loss.
|
| 1103 |
+
output_attentions (`bool`, *optional*):
|
| 1104 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 1105 |
+
tensors for more detail.
|
| 1106 |
+
output_hidden_states (`bool`, *optional*):
|
| 1107 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 1108 |
+
more detail.
|
| 1109 |
+
return_dict (`bool`, *optional*):
|
| 1110 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 1111 |
+
"""
|
| 1112 |
+
|
| 1113 |
+
|
| 1114 |
+
# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
|
| 1115 |
+
class SiglipEncoder(nn.Module):
|
| 1116 |
+
"""
|
| 1117 |
+
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
| 1118 |
+
[`SiglipEncoderLayer`].
|
| 1119 |
+
Args:
|
| 1120 |
+
config: SiglipConfig
|
| 1121 |
+
"""
|
| 1122 |
+
|
| 1123 |
+
def __init__(self, config: SiglipConfig):
|
| 1124 |
+
super().__init__()
|
| 1125 |
+
self.config = config
|
| 1126 |
+
self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
|
| 1127 |
+
self.gradient_checkpointing = False
|
| 1128 |
+
|
| 1129 |
+
# Ignore copy
|
| 1130 |
+
def forward(
|
| 1131 |
+
self,
|
| 1132 |
+
inputs_embeds,
|
| 1133 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1134 |
+
output_attentions: Optional[bool] = None,
|
| 1135 |
+
output_hidden_states: Optional[bool] = None,
|
| 1136 |
+
return_dict: Optional[bool] = None,
|
| 1137 |
+
) -> Union[Tuple, BaseModelOutput]:
|
| 1138 |
+
r"""
|
| 1139 |
+
Args:
|
| 1140 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 1141 |
+
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
|
| 1142 |
+
This is useful if you want more control over how to convert `input_ids` indices into associated vectors
|
| 1143 |
+
than the model's internal embedding lookup matrix.
|
| 1144 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1145 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 1146 |
+
- 1 for tokens that are **not masked**,
|
| 1147 |
+
- 0 for tokens that are **masked**.
|
| 1148 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 1149 |
+
output_attentions (`bool`, *optional*):
|
| 1150 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 1151 |
+
returned tensors for more detail.
|
| 1152 |
+
output_hidden_states (`bool`, *optional*):
|
| 1153 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
|
| 1154 |
+
for more detail.
|
| 1155 |
+
return_dict (`bool`, *optional*):
|
| 1156 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 1157 |
+
"""
|
| 1158 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1159 |
+
output_hidden_states = (
|
| 1160 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1161 |
+
)
|
| 1162 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1163 |
+
|
| 1164 |
+
encoder_states = () if output_hidden_states else None
|
| 1165 |
+
all_attentions = () if output_attentions else None
|
| 1166 |
+
|
| 1167 |
+
hidden_states = inputs_embeds
|
| 1168 |
+
for encoder_layer in self.layers:
|
| 1169 |
+
if output_hidden_states:
|
| 1170 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 1171 |
+
if self.gradient_checkpointing and self.training:
|
| 1172 |
+
layer_outputs = self._gradient_checkpointing_func(
|
| 1173 |
+
encoder_layer.__call__,
|
| 1174 |
+
hidden_states,
|
| 1175 |
+
attention_mask,
|
| 1176 |
+
output_attentions,
|
| 1177 |
+
)
|
| 1178 |
+
else:
|
| 1179 |
+
layer_outputs = encoder_layer(
|
| 1180 |
+
hidden_states,
|
| 1181 |
+
attention_mask,
|
| 1182 |
+
output_attentions=output_attentions,
|
| 1183 |
+
)
|
| 1184 |
+
|
| 1185 |
+
hidden_states = layer_outputs[0]
|
| 1186 |
+
|
| 1187 |
+
if output_attentions:
|
| 1188 |
+
all_attentions = all_attentions + (layer_outputs[1],)
|
| 1189 |
+
|
| 1190 |
+
if output_hidden_states:
|
| 1191 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 1192 |
+
|
| 1193 |
+
if not return_dict:
|
| 1194 |
+
return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
|
| 1195 |
+
return BaseModelOutput(
|
| 1196 |
+
last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
|
| 1197 |
+
)
|
| 1198 |
+
|
| 1199 |
+
|
| 1200 |
+
class SiglipTextTransformer(nn.Module):
|
| 1201 |
+
def __init__(self, config: SiglipTextConfig):
|
| 1202 |
+
super().__init__()
|
| 1203 |
+
self.config = config
|
| 1204 |
+
embed_dim = config.hidden_size
|
| 1205 |
+
self.embeddings = SiglipTextEmbeddings(config)
|
| 1206 |
+
self.encoder = SiglipEncoder(config)
|
| 1207 |
+
self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
| 1208 |
+
|
| 1209 |
+
self.head = nn.Linear(embed_dim, embed_dim)
|
| 1210 |
+
|
| 1211 |
+
@add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
|
| 1212 |
+
@replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipTextConfig)
|
| 1213 |
+
def forward(
|
| 1214 |
+
self,
|
| 1215 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 1216 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1217 |
+
position_ids: Optional[torch.Tensor] = None,
|
| 1218 |
+
output_attentions: Optional[bool] = None,
|
| 1219 |
+
output_hidden_states: Optional[bool] = None,
|
| 1220 |
+
return_dict: Optional[bool] = None,
|
| 1221 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 1222 |
+
r"""
|
| 1223 |
+
Returns:
|
| 1224 |
+
"""
|
| 1225 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1226 |
+
output_hidden_states = (
|
| 1227 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1228 |
+
)
|
| 1229 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1230 |
+
|
| 1231 |
+
if input_ids is None:
|
| 1232 |
+
raise ValueError("You have to specify input_ids")
|
| 1233 |
+
|
| 1234 |
+
input_shape = input_ids.size()
|
| 1235 |
+
input_ids = input_ids.view(-1, input_shape[-1])
|
| 1236 |
+
|
| 1237 |
+
hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
|
| 1238 |
+
|
| 1239 |
+
# note: SigLIP's text model does not use a causal mask, unlike the original CLIP model.
|
| 1240 |
+
# expand attention_mask
|
| 1241 |
+
if attention_mask is not None:
|
| 1242 |
+
# [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
|
| 1243 |
+
attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)
|
| 1244 |
+
|
| 1245 |
+
encoder_outputs = self.encoder(
|
| 1246 |
+
inputs_embeds=hidden_states,
|
| 1247 |
+
attention_mask=attention_mask,
|
| 1248 |
+
output_attentions=output_attentions,
|
| 1249 |
+
output_hidden_states=output_hidden_states,
|
| 1250 |
+
return_dict=return_dict,
|
| 1251 |
+
)
|
| 1252 |
+
|
| 1253 |
+
last_hidden_state = encoder_outputs[0]
|
| 1254 |
+
last_hidden_state = self.final_layer_norm(last_hidden_state)
|
| 1255 |
+
|
| 1256 |
+
# Assuming "sticky" EOS tokenization, last token is always EOS.
|
| 1257 |
+
pooled_output = last_hidden_state[:, -1, :]
|
| 1258 |
+
pooled_output = self.head(pooled_output)
|
| 1259 |
+
|
| 1260 |
+
if not return_dict:
|
| 1261 |
+
return (last_hidden_state, pooled_output) + encoder_outputs[1:]
|
| 1262 |
+
|
| 1263 |
+
return BaseModelOutputWithPooling(
|
| 1264 |
+
last_hidden_state=last_hidden_state,
|
| 1265 |
+
pooler_output=pooled_output,
|
| 1266 |
+
hidden_states=encoder_outputs.hidden_states,
|
| 1267 |
+
attentions=encoder_outputs.attentions,
|
| 1268 |
+
)
|
| 1269 |
+
|
| 1270 |
+
|
| 1271 |
+
@add_start_docstrings(
|
| 1272 |
+
"""The text model from SigLIP without any head or projection on top.""",
|
| 1273 |
+
SIGLIP_START_DOCSTRING,
|
| 1274 |
+
)
|
| 1275 |
+
class SiglipTextModel(SiglipPreTrainedModel):
|
| 1276 |
+
config_class = SiglipTextConfig
|
| 1277 |
+
|
| 1278 |
+
_no_split_modules = ["SiglipTextEmbeddings", "SiglipEncoderLayer"]
|
| 1279 |
+
|
| 1280 |
+
def __init__(self, config: SiglipTextConfig):
|
| 1281 |
+
super().__init__(config)
|
| 1282 |
+
self.text_model = SiglipTextTransformer(config)
|
| 1283 |
+
# Initialize weights and apply final processing
|
| 1284 |
+
self.post_init()
|
| 1285 |
+
|
| 1286 |
+
def get_input_embeddings(self) -> nn.Module:
|
| 1287 |
+
return self.text_model.embeddings.token_embedding
|
| 1288 |
+
|
| 1289 |
+
def set_input_embeddings(self, value):
|
| 1290 |
+
self.text_model.embeddings.token_embedding = value
|
| 1291 |
+
|
| 1292 |
+
@add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
|
| 1293 |
+
@replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipTextConfig)
|
| 1294 |
+
def forward(
|
| 1295 |
+
self,
|
| 1296 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 1297 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1298 |
+
position_ids: Optional[torch.Tensor] = None,
|
| 1299 |
+
output_attentions: Optional[bool] = None,
|
| 1300 |
+
output_hidden_states: Optional[bool] = None,
|
| 1301 |
+
return_dict: Optional[bool] = None,
|
| 1302 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 1303 |
+
r"""
|
| 1304 |
+
Returns:
|
| 1305 |
+
Examples:
|
| 1306 |
+
```python
|
| 1307 |
+
>>> from transformers import AutoTokenizer, SiglipTextModel
|
| 1308 |
+
>>> model = SiglipTextModel.from_pretrained("google/siglip-base-patch16-224")
|
| 1309 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
|
| 1310 |
+
>>> # important: make sure to set padding="max_length" as that's how the model was trained
|
| 1311 |
+
>>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt")
|
| 1312 |
+
>>> outputs = model(**inputs)
|
| 1313 |
+
>>> last_hidden_state = outputs.last_hidden_state
|
| 1314 |
+
>>> pooled_output = outputs.pooler_output # pooled (EOS token) states
|
| 1315 |
+
```"""
|
| 1316 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1317 |
+
|
| 1318 |
+
return self.text_model(
|
| 1319 |
+
input_ids=input_ids,
|
| 1320 |
+
attention_mask=attention_mask,
|
| 1321 |
+
position_ids=position_ids,
|
| 1322 |
+
output_attentions=output_attentions,
|
| 1323 |
+
output_hidden_states=output_hidden_states,
|
| 1324 |
+
return_dict=return_dict,
|
| 1325 |
+
)
|
| 1326 |
+
|
| 1327 |
+
|
| 1328 |
+
class SiglipVisionTransformer(nn.Module):
|
| 1329 |
+
def __init__(self, config: SiglipVisionConfig):
|
| 1330 |
+
super().__init__()
|
| 1331 |
+
self.config = config
|
| 1332 |
+
embed_dim = config.hidden_size
|
| 1333 |
+
|
| 1334 |
+
self.embeddings = SiglipVisionEmbeddings(config)
|
| 1335 |
+
self.encoder = SiglipEncoder(config)
|
| 1336 |
+
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
| 1337 |
+
self.head = SiglipMultiheadAttentionPoolingHead(config)
|
| 1338 |
+
|
| 1339 |
+
@add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
|
| 1340 |
+
@replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig)
|
| 1341 |
+
def forward(
|
| 1342 |
+
self,
|
| 1343 |
+
pixel_values,
|
| 1344 |
+
patch_attention_mask: Optional[torch.BoolTensor] = None,
|
| 1345 |
+
output_attentions: Optional[bool] = None,
|
| 1346 |
+
output_hidden_states: Optional[bool] = None,
|
| 1347 |
+
return_dict: Optional[bool] = None,
|
| 1348 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 1349 |
+
r"""
|
| 1350 |
+
Returns:
|
| 1351 |
+
"""
|
| 1352 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1353 |
+
output_hidden_states = (
|
| 1354 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1355 |
+
)
|
| 1356 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1357 |
+
|
| 1358 |
+
batch_size = pixel_values.size(0)
|
| 1359 |
+
if patch_attention_mask is None:
|
| 1360 |
+
patch_attention_mask = torch.ones(
|
| 1361 |
+
size=(
|
| 1362 |
+
batch_size,
|
| 1363 |
+
pixel_values.size(2) // self.config.patch_size,
|
| 1364 |
+
pixel_values.size(3) // self.config.patch_size,
|
| 1365 |
+
),
|
| 1366 |
+
dtype=torch.bool,
|
| 1367 |
+
device=pixel_values.device,
|
| 1368 |
+
)
|
| 1369 |
+
|
| 1370 |
+
hidden_states = self.embeddings(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask)
|
| 1371 |
+
|
| 1372 |
+
patch_attention_mask = patch_attention_mask.view(batch_size, -1)
|
| 1373 |
+
# The call to `_upad_input` in `_flash_attention_forward` is expensive
|
| 1374 |
+
# So when the `patch_attention_mask` is full of 1s (i.e. attending to the whole sequence),
|
| 1375 |
+
# avoiding passing the attention_mask, which is equivalent to attending to the full sequence
|
| 1376 |
+
if not torch.any(~patch_attention_mask):
|
| 1377 |
+
attention_mask=None
|
| 1378 |
+
else:
|
| 1379 |
+
attention_mask = (
|
| 1380 |
+
_prepare_4d_attention_mask(patch_attention_mask, hidden_states.dtype)
|
| 1381 |
+
if not self.config._flash_attn_2_enabled
|
| 1382 |
+
else patch_attention_mask
|
| 1383 |
+
)
|
| 1384 |
+
|
| 1385 |
+
encoder_outputs = self.encoder(
|
| 1386 |
+
inputs_embeds=hidden_states,
|
| 1387 |
+
attention_mask=attention_mask,
|
| 1388 |
+
output_attentions=output_attentions,
|
| 1389 |
+
output_hidden_states=output_hidden_states,
|
| 1390 |
+
return_dict=return_dict,
|
| 1391 |
+
)
|
| 1392 |
+
|
| 1393 |
+
last_hidden_state = encoder_outputs[0]
|
| 1394 |
+
last_hidden_state = self.post_layernorm(last_hidden_state)
|
| 1395 |
+
|
| 1396 |
+
pooled_output = self.head(
|
| 1397 |
+
hidden_state=last_hidden_state,
|
| 1398 |
+
attention_mask=patch_attention_mask,
|
| 1399 |
+
)
|
| 1400 |
+
|
| 1401 |
+
if not return_dict:
|
| 1402 |
+
return (last_hidden_state, pooled_output) + encoder_outputs[1:]
|
| 1403 |
+
|
| 1404 |
+
return BaseModelOutputWithPooling(
|
| 1405 |
+
last_hidden_state=last_hidden_state,
|
| 1406 |
+
pooler_output=pooled_output,
|
| 1407 |
+
hidden_states=encoder_outputs.hidden_states,
|
| 1408 |
+
attentions=encoder_outputs.attentions,
|
| 1409 |
+
)
|
| 1410 |
+
|
| 1411 |
+
|
| 1412 |
+
class SiglipMultiheadAttentionPoolingHead(nn.Module):
|
| 1413 |
+
"""Multihead Attention Pooling."""
|
| 1414 |
+
|
| 1415 |
+
def __init__(self, config: SiglipVisionConfig):
|
| 1416 |
+
super().__init__()
|
| 1417 |
+
|
| 1418 |
+
self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
|
| 1419 |
+
self.attention = torch.nn.MultiheadAttention(config.hidden_size, config.num_attention_heads, batch_first=True)
|
| 1420 |
+
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
| 1421 |
+
self.mlp = SiglipMLP(config)
|
| 1422 |
+
|
| 1423 |
+
def forward(self, hidden_state, attention_mask):
|
| 1424 |
+
batch_size = hidden_state.shape[0]
|
| 1425 |
+
probe = self.probe.repeat(batch_size, 1, 1)
|
| 1426 |
+
|
| 1427 |
+
hidden_state = self.attention(
|
| 1428 |
+
query=probe, key=hidden_state, value=hidden_state, key_padding_mask=~attention_mask
|
| 1429 |
+
)[0]
|
| 1430 |
+
|
| 1431 |
+
residual = hidden_state
|
| 1432 |
+
hidden_state = self.layernorm(hidden_state)
|
| 1433 |
+
hidden_state = residual + self.mlp(hidden_state)
|
| 1434 |
+
|
| 1435 |
+
return hidden_state[:, 0]
|
| 1436 |
+
|
| 1437 |
+
|
| 1438 |
+
@add_start_docstrings(
|
| 1439 |
+
"""The vision model from SigLIP without any head or projection on top.""",
|
| 1440 |
+
SIGLIP_START_DOCSTRING,
|
| 1441 |
+
)
|
| 1442 |
+
class SiglipVisionModel(SiglipPreTrainedModel):
|
| 1443 |
+
config_class = SiglipVisionConfig
|
| 1444 |
+
main_input_name = "pixel_values"
|
| 1445 |
+
|
| 1446 |
+
def __init__(self, config: SiglipVisionConfig):
|
| 1447 |
+
super().__init__(config)
|
| 1448 |
+
|
| 1449 |
+
self.vision_model = SiglipVisionTransformer(config)
|
| 1450 |
+
|
| 1451 |
+
# Initialize weights and apply final processing
|
| 1452 |
+
self.post_init()
|
| 1453 |
+
|
| 1454 |
+
def get_input_embeddings(self) -> nn.Module:
|
| 1455 |
+
return self.vision_model.embeddings.patch_embedding
|
| 1456 |
+
|
| 1457 |
+
@add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
|
| 1458 |
+
@replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig)
|
| 1459 |
+
def forward(
|
| 1460 |
+
self,
|
| 1461 |
+
pixel_values,
|
| 1462 |
+
patch_attention_mask: Optional[torch.BoolTensor] = None,
|
| 1463 |
+
output_attentions: Optional[bool] = None,
|
| 1464 |
+
output_hidden_states: Optional[bool] = None,
|
| 1465 |
+
return_dict: Optional[bool] = None,
|
| 1466 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 1467 |
+
r"""
|
| 1468 |
+
Returns:
|
| 1469 |
+
Examples:
|
| 1470 |
+
```python
|
| 1471 |
+
>>> from PIL import Image
|
| 1472 |
+
>>> import requests
|
| 1473 |
+
>>> from transformers import AutoProcessor, SiglipVisionModel
|
| 1474 |
+
>>> model = SiglipVisionModel.from_pretrained("google/siglip-base-patch16-224")
|
| 1475 |
+
>>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
|
| 1476 |
+
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
| 1477 |
+
>>> image = Image.open(requests.get(url, stream=True).raw)
|
| 1478 |
+
>>> inputs = processor(images=image, return_tensors="pt")
|
| 1479 |
+
>>> outputs = model(**inputs)
|
| 1480 |
+
>>> last_hidden_state = outputs.last_hidden_state
|
| 1481 |
+
>>> pooled_output = outputs.pooler_output # pooled features
|
| 1482 |
+
```"""
|
| 1483 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1484 |
+
|
| 1485 |
+
return self.vision_model(
|
| 1486 |
+
pixel_values=pixel_values,
|
| 1487 |
+
patch_attention_mask=patch_attention_mask,
|
| 1488 |
+
output_attentions=output_attentions,
|
| 1489 |
+
output_hidden_states=output_hidden_states,
|
| 1490 |
+
return_dict=return_dict,
|
| 1491 |
+
)
|
| 1492 |
+
|
| 1493 |
+
|
| 1494 |
+
@add_start_docstrings(SIGLIP_START_DOCSTRING)
|
| 1495 |
+
class SiglipModel(SiglipPreTrainedModel):
|
| 1496 |
+
config_class = SiglipConfig
|
| 1497 |
+
|
| 1498 |
+
def __init__(self, config: SiglipConfig):
|
| 1499 |
+
super().__init__(config)
|
| 1500 |
+
|
| 1501 |
+
if not isinstance(config.text_config, SiglipTextConfig):
|
| 1502 |
+
raise ValueError(
|
| 1503 |
+
"config.text_config is expected to be of type SiglipTextConfig but is of type"
|
| 1504 |
+
f" {type(config.text_config)}."
|
| 1505 |
+
)
|
| 1506 |
+
|
| 1507 |
+
if not isinstance(config.vision_config, SiglipVisionConfig):
|
| 1508 |
+
raise ValueError(
|
| 1509 |
+
"config.vision_config is expected to be of type SiglipVisionConfig but is of type"
|
| 1510 |
+
f" {type(config.vision_config)}."
|
| 1511 |
+
)
|
| 1512 |
+
|
| 1513 |
+
text_config = config.text_config
|
| 1514 |
+
vision_config = config.vision_config
|
| 1515 |
+
|
| 1516 |
+
self.text_model = SiglipTextTransformer(text_config)
|
| 1517 |
+
self.vision_model = SiglipVisionTransformer(vision_config)
|
| 1518 |
+
|
| 1519 |
+
self.logit_scale = nn.Parameter(torch.randn(1))
|
| 1520 |
+
self.logit_bias = nn.Parameter(torch.randn(1))
|
| 1521 |
+
|
| 1522 |
+
# Initialize weights and apply final processing
|
| 1523 |
+
self.post_init()
|
| 1524 |
+
|
| 1525 |
+
@add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
|
| 1526 |
+
def get_text_features(
|
| 1527 |
+
self,
|
| 1528 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 1529 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1530 |
+
position_ids: Optional[torch.Tensor] = None,
|
| 1531 |
+
output_attentions: Optional[bool] = None,
|
| 1532 |
+
output_hidden_states: Optional[bool] = None,
|
| 1533 |
+
return_dict: Optional[bool] = None,
|
| 1534 |
+
) -> torch.FloatTensor:
|
| 1535 |
+
r"""
|
| 1536 |
+
Returns:
|
| 1537 |
+
text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
|
| 1538 |
+
applying the projection layer to the pooled output of [`SiglipTextModel`].
|
| 1539 |
+
Examples:
|
| 1540 |
+
```python
|
| 1541 |
+
>>> from transformers import AutoTokenizer, AutoModel
|
| 1542 |
+
>>> import torch
|
| 1543 |
+
>>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
|
| 1544 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
|
| 1545 |
+
>>> # important: make sure to set padding="max_length" as that's how the model was trained
|
| 1546 |
+
>>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt")
|
| 1547 |
+
>>> with torch.no_grad():
|
| 1548 |
+
... text_features = model.get_text_features(**inputs)
|
| 1549 |
+
```"""
|
| 1550 |
+
# Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
|
| 1551 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1552 |
+
output_hidden_states = (
|
| 1553 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1554 |
+
)
|
| 1555 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1556 |
+
|
| 1557 |
+
text_outputs = self.text_model(
|
| 1558 |
+
input_ids=input_ids,
|
| 1559 |
+
attention_mask=attention_mask,
|
| 1560 |
+
position_ids=position_ids,
|
| 1561 |
+
output_attentions=output_attentions,
|
| 1562 |
+
output_hidden_states=output_hidden_states,
|
| 1563 |
+
return_dict=return_dict,
|
| 1564 |
+
)
|
| 1565 |
+
|
| 1566 |
+
pooled_output = text_outputs[1]
|
| 1567 |
+
|
| 1568 |
+
return pooled_output
|
| 1569 |
+
|
| 1570 |
+
@add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
|
| 1571 |
+
def get_image_features(
|
| 1572 |
+
self,
|
| 1573 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 1574 |
+
output_attentions: Optional[bool] = None,
|
| 1575 |
+
output_hidden_states: Optional[bool] = None,
|
| 1576 |
+
return_dict: Optional[bool] = None,
|
| 1577 |
+
) -> torch.FloatTensor:
|
| 1578 |
+
r"""
|
| 1579 |
+
Returns:
|
| 1580 |
+
image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
|
| 1581 |
+
applying the projection layer to the pooled output of [`SiglipVisionModel`].
|
| 1582 |
+
Examples:
|
| 1583 |
+
```python
|
| 1584 |
+
>>> from PIL import Image
|
| 1585 |
+
>>> import requests
|
| 1586 |
+
>>> from transformers import AutoProcessor, AutoModel
|
| 1587 |
+
>>> import torch
|
| 1588 |
+
>>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
|
| 1589 |
+
>>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
|
| 1590 |
+
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
| 1591 |
+
>>> image = Image.open(requests.get(url, stream=True).raw)
|
| 1592 |
+
>>> inputs = processor(images=image, return_tensors="pt")
|
| 1593 |
+
>>> with torch.no_grad():
|
| 1594 |
+
... image_features = model.get_image_features(**inputs)
|
| 1595 |
+
```"""
|
| 1596 |
+
# Use SiglipModel's config for some fields (if specified) instead of those of vision & text components.
|
| 1597 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1598 |
+
output_hidden_states = (
|
| 1599 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1600 |
+
)
|
| 1601 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1602 |
+
|
| 1603 |
+
vision_outputs = self.vision_model(
|
| 1604 |
+
pixel_values=pixel_values,
|
| 1605 |
+
output_attentions=output_attentions,
|
| 1606 |
+
output_hidden_states=output_hidden_states,
|
| 1607 |
+
return_dict=return_dict,
|
| 1608 |
+
)
|
| 1609 |
+
|
| 1610 |
+
pooled_output = vision_outputs[1]
|
| 1611 |
+
|
| 1612 |
+
return pooled_output
|
| 1613 |
+
|
| 1614 |
+
@add_start_docstrings_to_model_forward(SIGLIP_INPUTS_DOCSTRING)
|
| 1615 |
+
@replace_return_docstrings(output_type=SiglipOutput, config_class=SiglipConfig)
|
| 1616 |
+
def forward(
|
| 1617 |
+
self,
|
| 1618 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 1619 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 1620 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1621 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1622 |
+
return_loss: Optional[bool] = None,
|
| 1623 |
+
output_attentions: Optional[bool] = None,
|
| 1624 |
+
output_hidden_states: Optional[bool] = None,
|
| 1625 |
+
return_dict: Optional[bool] = None,
|
| 1626 |
+
) -> Union[Tuple, SiglipOutput]:
|
| 1627 |
+
r"""
|
| 1628 |
+
Returns:
|
| 1629 |
+
Examples:
|
| 1630 |
+
```python
|
| 1631 |
+
>>> from PIL import Image
|
| 1632 |
+
>>> import requests
|
| 1633 |
+
>>> from transformers import AutoProcessor, AutoModel
|
| 1634 |
+
>>> import torch
|
| 1635 |
+
>>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
|
| 1636 |
+
>>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
|
| 1637 |
+
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
| 1638 |
+
>>> image = Image.open(requests.get(url, stream=True).raw)
|
| 1639 |
+
>>> texts = ["a photo of 2 cats", "a photo of 2 dogs"]
|
| 1640 |
+
>>> # important: we pass `padding=max_length` since the model was trained with this
|
| 1641 |
+
>>> inputs = processor(text=texts, images=image, padding="max_length", return_tensors="pt")
|
| 1642 |
+
>>> with torch.no_grad():
|
| 1643 |
+
... outputs = model(**inputs)
|
| 1644 |
+
>>> logits_per_image = outputs.logits_per_image
|
| 1645 |
+
>>> probs = torch.sigmoid(logits_per_image) # these are the probabilities
|
| 1646 |
+
>>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'")
|
| 1647 |
+
31.9% that image 0 is 'a photo of 2 cats'
|
| 1648 |
+
```"""
|
| 1649 |
+
# Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
|
| 1650 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1651 |
+
output_hidden_states = (
|
| 1652 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1653 |
+
)
|
| 1654 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1655 |
+
|
| 1656 |
+
vision_outputs = self.vision_model(
|
| 1657 |
+
pixel_values=pixel_values,
|
| 1658 |
+
output_attentions=output_attentions,
|
| 1659 |
+
output_hidden_states=output_hidden_states,
|
| 1660 |
+
return_dict=return_dict,
|
| 1661 |
+
)
|
| 1662 |
+
|
| 1663 |
+
text_outputs = self.text_model(
|
| 1664 |
+
input_ids=input_ids,
|
| 1665 |
+
attention_mask=attention_mask,
|
| 1666 |
+
position_ids=position_ids,
|
| 1667 |
+
output_attentions=output_attentions,
|
| 1668 |
+
output_hidden_states=output_hidden_states,
|
| 1669 |
+
return_dict=return_dict,
|
| 1670 |
+
)
|
| 1671 |
+
|
| 1672 |
+
image_embeds = vision_outputs[1]
|
| 1673 |
+
text_embeds = text_outputs[1]
|
| 1674 |
+
|
| 1675 |
+
# normalized features
|
| 1676 |
+
image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True)
|
| 1677 |
+
text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
|
| 1678 |
+
|
| 1679 |
+
# cosine similarity as logits
|
| 1680 |
+
logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * self.logit_scale.exp() + self.logit_bias
|
| 1681 |
+
logits_per_image = logits_per_text.t()
|
| 1682 |
+
|
| 1683 |
+
loss = None
|
| 1684 |
+
if return_loss:
|
| 1685 |
+
raise NotImplementedError("SigLIP loss to be implemented")
|
| 1686 |
+
|
| 1687 |
+
if not return_dict:
|
| 1688 |
+
output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
|
| 1689 |
+
return ((loss,) + output) if loss is not None else output
|
| 1690 |
+
|
| 1691 |
+
return SiglipOutput(
|
| 1692 |
+
loss=loss,
|
| 1693 |
+
logits_per_image=logits_per_image,
|
| 1694 |
+
logits_per_text=logits_per_text,
|
| 1695 |
+
text_embeds=text_embeds,
|
| 1696 |
+
image_embeds=image_embeds,
|
| 1697 |
+
text_model_output=text_outputs,
|
| 1698 |
+
vision_model_output=vision_outputs,
|
| 1699 |
+
)
|
| 1700 |
+
|
| 1701 |
+
|
| 1702 |
+
def get_siglip_vision_model(_flash_attn_2_enabled=True, **kwargs):
|
| 1703 |
+
siglip_vision_config = {
|
| 1704 |
+
"hidden_size": 1152,
|
| 1705 |
+
"image_size": 448,
|
| 1706 |
+
"intermediate_size": 4304,
|
| 1707 |
+
"model_type": "siglip_vision_model",
|
| 1708 |
+
"num_attention_heads": 16,
|
| 1709 |
+
"num_hidden_layers": 27,
|
| 1710 |
+
"patch_size": 14,
|
| 1711 |
+
}
|
| 1712 |
+
|
| 1713 |
+
model_config = SiglipVisionConfig(**siglip_vision_config, _flash_attn_2_enabled=_flash_attn_2_enabled, **kwargs)
|
| 1714 |
+
|
| 1715 |
+
vision_model = SiglipVisionModel(model_config).vision_model
|
| 1716 |
+
|
| 1717 |
+
return vision_model
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|