+ NUM_GPUS=4 + DISTRIBUTED_ARGS=' --nnodes=1 --nproc_per_node 4 --rdzv_backend c10d --rdzv_endpoint localhost:0 ' + MODEL_ID=qwen3vl + MODEL_LOCAL_PATH=/data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ + EMBED_DIM=2048 + DATA_PATH=/data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json + MAX_LINES=10000 + IMAGE_TYPE=both + INFONCE_WEIGHT=1.0 + COT_WEIGHT=1.0 + TEMPERATURE=0.02 + NEG_GATHER_SIZE=4 + MAX_COT_LENGTH=192 + DOC_APPEND_EOS=False + USE_WANDB=False + WANDB_MODE=online + TRAIN_VISION_ENCODER=False + USE_VISION_LORA=False + TRAIN_LLM=True + USE_LORA=True + Q_LORA=False + LORA_R=128 + LORA_ALPHA=256 + TRAIN_LM_HEAD=True + DS_STAGE=zero2 + PER_DEVICE_BATCH_SIZE=4 + GRAD_ACCUM=1 + NUM_EPOCHS=5 + SAVE_STEPS=200 + LR=1e-5 + MODEL_MAX_LEN=768 + OUTPUT_DIR=/data/oss_bucket_0/liqi/cot_embedding/local_train/dapei_lora128_cot_embedding_debug + RUN_ID=qwen3vl_cot_embedding_debug + RESUME=True + echo 'begin training!' begin training! + torchrun --nnodes=1 --nproc_per_node 4 --rdzv_backend c10d --rdzv_endpoint localhost:0 train_cot/train_tuwen_qwen3vl_cot.py --model_id qwen3vl --data_path /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json --max_lines 10000 --output_dir /data/oss_bucket_0/liqi/cot_embedding/local_train/dapei_lora128_cot_embedding_debug/qwen3vl_cot_embedding_debug --report_to tensorboard --run_name qwen3vl_cot_embedding_debug --deepspeed ./ds_configs/zero2.json --bf16 True --num_train_epochs 5 --per_device_train_batch_size 4 --per_device_eval_batch_size 12 --gradient_accumulation_steps 1 --eval_strategy no --save_strategy steps --save_total_limit 200 --learning_rate 1e-5 --weight_decay 0. --warmup_ratio 0.03 --lr_scheduler_type constant_with_warmup --logging_steps 10 --tf32 True --model_max_length 768 --gradient_checkpointing False --dataloader_num_workers 0 --train_vision_encoder False --use_vision_lora False --use_lora True --q_lora False --lora_r 128 --lora_alpha 256 --model_local_path /data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ --use_flash_attn True --save_steps 200 --stage classify --embed_dim 2048 --resume_from_checkpoint True --image_type both --train_llm True --train_lm_head True --infonce_weight 1.0 --cot_weight 1.0 --temperature 0.02 --neg_gather_size 4 --max_cot_length 192 --doc_append_eos False --use_wandb False --wandb_mode online Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ACCL_TUNING_LEVEL=2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO cudaDriverVersion 13010 warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL git version unknown unknown warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO cudaDriverVersion 13010 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ACCL_TUNING_LEVEL=2 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL git version unknown unknown notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO cudaDriverVersion 13010 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ACCL_TUNING_LEVEL=2 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL git version unknown unknown notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO cudaDriverVersion 13010 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ACCL_TUNING_LEVEL=2 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL git version unknown unknown notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL_NET_PLUGIN set by environment to none notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL_NET_PLUGIN set by environment to none notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/IB : No device found. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Failed to initialize NET plugin IB notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/IB : No device found. notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Failed to initialize NET plugin IB notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL_NET_PLUGIN set by environment to none notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/IB : No device found. notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Failed to initialize NET plugin IB notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL_NET_PLUGIN set by environment to none notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/IB : No device found. notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Failed to initialize NET plugin IB notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO [Rank 0] ncclCommInitRankConfig comm 0x11ea92f0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 89000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO [Rank 2] ncclCommInitRankConfig comm 0x11eca310 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId b2000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO [Rank 3] ncclCommInitRankConfig comm 0x11eacf80 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO [Rank 1] ncclCommInitRankConfig comm 0x11ecb740 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 8e000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Bootstrap timings total 0.012297 (create 0.000029, send 0.000117, recv 0.011052, ring 0.000807, delay 0.000007) notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Bootstrap timings total 0.011394 (create 0.000021, send 0.000115, recv 0.000141, ring 0.010813, delay 0.000003) notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO gTaskid = 67e26e210000679c. notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO gTaskid = 67e26e210000679c. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Bootstrap timings total 0.011877 (create 0.000021, send 0.000115, recv 0.000546, ring 0.000758, delay 0.000004) notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Bootstrap timings total 0.001388 (create 0.000036, send 0.000132, renotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO gTaskid = 67e26e210000679c. cv 0.000173, ring 0.000725, delay 0.000004) notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO gTaskid = 67e26e210000679c. notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. ource temporarily unavailable. notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO comm 0x11eacf80 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO comm 0x11ea92f0 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNNVL 0 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO comm 0x11eca310 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0 NVL 0 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO comm 0x11ecb740 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/16 : 0 1 3 2 NVL 0 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Trees [0] -1/-1/-1->2->3 [1] -1/-1/-1->2->3 [2] -1/-1/-1->2->3 [3] -1/-1/-1->2->3 [4] -1/-1/-1->2->3 [5] -1/-1/-1->2->3 [6] -1/-1/-1->2->3 [7] -1/-1/-1->2->3 [8] -1/-1/-1->2->3 [9] -1/-1/-1->2->3 [10] -1/-1/-1->2->3 [11] -1/-1/-1->2->3 [12] -1/-1/-1->2->3 [13] -1/-1/-1->2->3 [14] -1/-1/-1->2->3 [15] -1/-1/-1->2->3 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Trees [0] 2/-1/-1->3->1 [1] 2/-1/-1->3->1 [2] 2/-1/-1->3->1 [3] 2/-1/-1->3->1 [4] 2/-1/-1->3->1 [5] 2/-1/-1->3->1 [6] 2/-1/-1->3->1 [7] 2/-1/-1->3->1 [8] 2/-1/-1->3->1 [9] 2/-1/-1->3->1 [10] 2/-1/-1->3->1 [11] 2/-1/-1->3->1 [12] 2/-1/-1->3->1 [13] 2/-1/-1->3->1 [14] 2/-1/-1->3->1 [15] 2/-1/-1->3->1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Trees [0] 3/-1/-1->1->0 [1] 3/-1/-1->1->0 [2] 3/-1/-1->1->0 [3] 3/-1/-1->1->0 [4] 3/-1/-1->1->0 [5] 3/-1/-1->1->0 [6] 3/-1/-1->1->0 [7] 3/-1/-1->1->0 [8] 3/-1/-1->1->0 [9] 3/-1/-1->1->0 [10] 3/-1/-1->1->0 [11] 3/-1/-1->1->0 [12] 3/-1/-1->1->0 [13] 3/-1/-1->1->0 [14] 3/-1/-1->1->0 [15] 3/-1/-1->1->0 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259326:259825 [0] NCCL INFO [Proxy Service] Device 2 CPU core 62 notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Proxy Service] Device 0 CPU core 126 notebook-622b7fa0e0f7-worker-0:259327:259829 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 110 notebook-622b7fa0e0f7-worker-0:259327:259824 [0] NCCL INFO [Proxy Service] Device 3 CPU core 62 notebook-622b7fa0e0f7-worker-0:259326:259827 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 109 notebook-622b7fa0e0f7-worker-0:259324:259828 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 45 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259325:259830 [0] NCCL INFO [Proxy Service] Device 1 CPU core 45 notebook-622b7fa0e0f7-worker-0:259325:259831 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 46 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO CC Off, workFifoBytes 1048576 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommInitRankConfig comm 0x11ea92f0 rank 0 nranks 4 cudaDev 0 nvmnotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommInitRankConfig comm 0x11eca310 rank 2 nranks 4 cudaDev 2 nvmlDev 0 busId 89000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE lDev 2 busId b2000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 4 total 5.43 (knotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 4 total 5.42 (kernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) ernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommInitRankConfig comm 0x11eacf80 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 4 total 5.42 (knotebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommInitRankConfig comm 0x11ecb740 rank 1 nranks 4 cudaDev 1 nvmernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) lDev 1 busId 8e000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 4 total 5.42 (kernels 0.24, alloc 0.03, bootstrap 0.00, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) Loading Qwen3-VL-CoT-Embedding from /data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ ... Loading weights: 0%| | 0/625 [00:00 组内 4 卡共享，候选池 32，每个 query 31 个负样本 total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Comm config Blocking set to 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommSplit comm 0x2a1307c0 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId b2000 parent 0x11eca310 childCount 1 color 2003953581 key 2 commHash 8923940766810589732 - Init START notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommSplit comm 0x2de13b40 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 parent 0x11eacf80 childCount 1 color 2003953581 key 3 commHash 8923940766810589732 - Init START notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommSplit comm 0x19c91200 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 89000 parent 0x11ea92f0 childCount 1 color 2003953581 key 0 commHash 8923940766810589732 - Init START notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommSplit comm 0x2c7f1100 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 8e000 parent 0x11ecb740 childCount 1 color 2003953581 key 1 commHash 8923940766810589732 - Init START notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO comm 0x2c7f1100 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNNVL 0 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO comm 0x19c91200 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNnotebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO comm 0x2de13b40 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNNVL 0 NVL 0 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO comm 0x2a1307c0 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/16 : 0 1 3 2 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Trees [0] 3/-1/-1->1->0 [1] 3/-1/-1->1->0 [2] 3/-1/-1->1->0 [3] 3/-1/-1->1->0 [4] 3/-1/-1->1->0 [5] 3/-1/-1->1->0 [6] 3/-1/-1->1->0 [7] 3/-1/-1->1->0 [8] 3/-1/-1->1->0 [9] 3/-1/-1->1->0 [10] 3/-1/-1->1->0 [11] 3/-1/-1->1->0 [12] 3/-1/-1->1->0 [13] 3/-1/-1->1->0 [14] 3/-1/-1->1->0 [15] 3/-1/-1->1->0 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Trees [0] 2/-1/-1->3->1 [1] 2/-1/-1->3->1 [2] 2/-1/-1->3->1 [3] 2/-1/-1->3->1 [4] 2/-1/-1->3->1 [5] 2/-1/-1->3->1 [6] 2/-1/-1->3->1 [7] 2/-1/-1->3->1 [8] 2/-1/-1->3->1 [9] 2/-1/-1->3->1 [10] 2/-1/-1->3->1 [11] 2/-1/-1->3->1 [12] 2/-1/-1->3->1 [13] 2/-1/-1->3->1 [14] 2/-1/-1->3->1 [15] 2/-1/-1->3->1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Trees [0] -1/-1/-1->2->3 [1] -1/-1/-1->2->3 [2] -1/-1/-1->2->3 [3] -1/-1/-1->2->3 [4] -1/-1/-1->2->3 [5] -1/-1/-1->2->3 [6] -1/-1/-1->2->3 [7] -1/-1/-1->2->3 [8] -1/-1/-1->2->3 [9] -1/-1/-1->2->3 [10] -1/-1/-1->2->3 [11] -1/-1/-1->2->3 [12] -1/-1/-1->2->3 [13] -1/-1/-1->2->3 [14] -1/-1/-1->2->3 [15] -1/-1/-1->2->3 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO P2P Chunksize set to 524288 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 notebook-622b7fa0e0f7-worker-0:259325:261483 [0] NCCL INFO [Proxy Service] Device 1 CPU core 53 notebook-622b7fa0e0f7-worker-0:259326:261484 [0] NCCL INFO [Proxy Service] Device 2 CPU core 124 notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Proxy Service] Device 0 CPU core 125 notebook-622b7fa0e0f7-worker-0:259327:261485 [0] NCCL INFO [Proxy Service] Device 3 CPU core 60 notebook-622b7fa0e0f7-worker-0:259325:261487 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 62 notebook-622b7fa0e0f7-worker-0:259324:261490 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 45 notebook-622b7fa0e0f7-worker-0:259326:261489 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 44 notebook-622b7fa0e0f7-worker-0:259327:261488 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 108 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO CC Off, workFifoBytes 1048576 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommSplit comm 0x2de13b40 rank 3 nranks 4 cudaDev 3 nvmlDev 3 bunotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommSplit comm 0x19c91200 rank 0 nranks 4 cudaDev 0 nvmlDev 0 bunotebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommSplit comm 0x2c7f1100 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId b8000 parent 0x11eacf80 childCount 1 color 2003953581 key 3 commHash 8923940766810589732 - Init COMPLETE sId 89000 parent 0x11ea92f0 childCount 1 color 2003953581 key 0 commHash 8923940766810589732 - Init COMPLETE sId 8e000 parent 0x11ecb740 childCount 1 color 2003953581 key 1 commHash 8923940766810589732 - Init COMPLETE notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 4 total 0.34 (kernels 0.notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 4 total 0.23 (kernels 0.notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 4 total 0.21 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.13) 00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.03) notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommSplit comm 0x2a1307c0 rank 2 nranks 4 cudaDev 2 nvmlDev 2 bu00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.01) sId b2000 parent 0x11eca310 childCount 1 color 2003953581 key 2 commHash 8923940766810589732 - Init COMPLETE notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 4 total 0.47 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.27) notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 00/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 04/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 07/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 08/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 09/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 12/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 15/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 00/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 04/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 07/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 08/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 09/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 12/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 15/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 14/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 0%| | 0/3125 [00:00 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 00/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 04/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 07/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 08/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 09/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 12/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 15/0 : 2[2] -> 0[0] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Symmetric VA size=80GB notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 00/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 04/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 07/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 08/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 09/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 12/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 15/0 : 1[1] -> 3[3] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 14/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM/read notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 38%|███▊ | 1201/3125 [00:06<00:10, 186.18it/s] 39%|███▊ | 1207/3125 [00:26<00:10, 186.18it/s] 39%|███▊ | 1208/3125 [00:26<00:55, 34.60it/s] 39%|███▊ | 1209/3125 [00:29<01:04, 29.77it/s] {'loss': '1.312', 'grad_norm': '7.882', 'learning_rate': '1e-05', 'train/loss_infonce': '0.5898', 'train/loss_cot': '0.9783', 'epoch': '1.936'} 39%|███▊ | 1210/3125 [00:32<01:04, 29.77it/s] 39%|███▉ | 1215/3125 [00:46<01:04, 29.77it/s] 39%|███▉ | 1216/3125 [00:48<02:31, 12.56it/s] 39%|███▉ | 1217/3125 [00:51<02:47, 11.40it/s] {'loss': '1.349', 'grad_norm': '6.241', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04053', 'train/loss_cot': '1.036', 'epoch': '1.952'} 39%|███▉ | 1220/3125 [01:00<02:47, 11.40it/s] 39%|███▉ | 1222/3125 [01:05<04:47, 6.62it/s] 39%|███▉ | 1223/3125 [01:08<05:21, 5.91it/s] 39%|███▉ | 1226/3125 [01:16<07:43, 4.10it/s] 39%|███▉ | 1228/3125 [01:22<09:53, 3.20it/s] 39%|███▉ | 1230/3125 [01:27<12:29, 2.53it/s] {'loss': '1.36', 'grad_norm': '6.169', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0791', 'train/loss_cot': '0.9644', 'epoch': '1.968'} 39%|███▉ | 1230/3125 [01:28<12:29, 2.53it/s] 39%|███▉ | 1231/3125 [01:31<14:41, 2.15it/s] 39%|███▉ | 1232/3125 [01:33<16:59, 1.86it/s] 39%|███▉ | 1233/3125 [01:36<20:11, 1.56it/s] 39%|███▉ | 1234/3125 [01:39<25:01, 1.26it/s] 40%|███▉ | 1235/3125 [01:42<29:38, 1.06it/s] 40%|███▉ | 1236/3125 [01:45<36:18, 1.15s/it] 40%|███▉ | 1237/3125 [01:47<42:16, 1.34s/it] 40%|███▉ | 1238/3125 [01:50<49:23, 1.57s/it] 40%|███▉ | 1239/3125 [01:53<54:50, 1.74s/it] 40%|███▉ | 1240/3125 [01:55<59:20, 1.89s/it] {'loss': '1.344', 'grad_norm': '6.17', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2715', 'train/loss_cot': '1.272', 'epoch': '1.984'} 40%|███▉ | 1240/3125 [01:55<59:20, 1.89s/it] 40%|███▉ | 1241/3125 [01:58<1:05:48, 2.10s/it] 40%|███▉ | 1242/3125 [02:00<1:08:47, 2.19s/it] 40%|███▉ | 1243/3125 [02:03<1:14:11, 2.37s/it] 40%|███▉ | 1244/3125 [02:06<1:18:48, 2.51s/it] 40%|███▉ | 1245/3125 [02:09<1:18:58, 2.52s/it] 40%|███▉ | 1246/3125 [02:11<1:21:06, 2.59s/it] 40%|███▉ | 1247/3125 [02:14<1:24:50, 2.71s/it] 40%|███▉ | 1248/3125 [02:17<1:24:56, 2.72s/it] 40%|███▉ | 1249/3125 [02:20<1:24:20, 2.70s/it] 40%|████ | 1250/3125 [02:21<1:07:12, 2.15s/it] {'loss': '1.413', 'grad_norm': '7.31', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1162', 'train/loss_cot': '1.073', 'epoch': '2'} 40%|████ | 1250/3125 [02:21<1:07:12, 2.15s/it] 40%|████ | 1251/3125 [02:25<1:25:21, 2.73s/it] 40%|████ | 1252/3125 [02:28<1:28:08, 2.82s/it] 40%|████ | 1253/3125 [02:31<1:31:57, 2.95s/it] 40%|████ | 1254/3125 [02:34<1:30:51, 2.91s/it] 40%|████ | 1255/3125 [02:36<1:27:42, 2.81s/it] 40%|████ | 1256/3125 [02:39<1:26:55, 2.79s/it] 40%|████ | 1257/3125 [02:42<1:28:34, 2.85s/it] 40%|████ | 1258/3125 [02:45<1:27:38, 2.82s/it] 40%|████ | 1259/3125 [02:48<1:27:17, 2.81s/it] 40%|████ | 1260/3125 [02:50<1:25:44, 2.76s/it] {'loss': '1.193', 'grad_norm': '6.144', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1953', 'train/loss_cot': '0.9299', 'epoch': '2.016'} 40%|████ | 1260/3125 [02:51<1:25:44, 2.76s/it] 40%|████ | 1261/3125 [02:53<1:25:13, 2.74s/it] 40%|████ | 1262/3125 [02:56<1:27:25, 2.82s/it] 40%|████ | 1263/3125 [02:59<1:27:53, 2.83s/it] 40%|████ | 1264/3125 [03:02<1:29:47, 2.89s/it] 40%|████ | 1265/3125 [03:04<1:25:04, 2.74s/it] 41%|████ | 1266/3125 [03:07<1:22:43, 2.67s/it] 41%|████ | 1267/3125 [03:10<1:24:04, 2.71s/it] 41%|████ | 1268/3125 [03:13<1:29:47, 2.90s/it] 41%|████ | 1269/3125 [03:16<1:27:26, 2.83s/it] 41%|████ | 1270/3125 [03:18<1:23:25, 2.70s/it] {'loss': '1.15', 'grad_norm': '7.134', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2617', 'train/loss_cot': '1.007', 'epoch': '2.032'} 41%|████ | 1270/3125 [03:18<1:23:25, 2.70s/it] 41%|████ | 1271/3125 [03:21<1:23:20, 2.70s/it] 41%|████ | 1272/3125 [03:24<1:27:30, 2.83s/it] 41%|████ | 1273/3125 [03:27<1:34:06, 3.05s/it] 41%|████ | 1274/3125 [03:30<1:31:46, 2.98s/it] 41%|████ | 1275/3125 [03:34<1:35:25, 3.09s/it] 41%|████ | 1276/3125 [03:37<1:38:26, 3.19s/it] 41%|████ | 1277/3125 [03:40<1:34:51, 3.08s/it] 41%|████ | 1278/3125 [03:43<1:32:22, 3.00s/it] 41%|████ | 1279/3125 [03:45<1:28:36, 2.88s/it] 41%|████ | 1280/3125 [03:48<1:28:35, 2.88s/it] {'loss': '1.236', 'grad_norm': '5.722', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05029', 'train/loss_cot': '1.407', 'epoch': '2.048'} 41%|████ | 1280/3125 [03:49<1:28:35, 2.88s/it] 41%|████ | 1281/3125 [03:51<1:25:59, 2.80s/it] 41%|████ | 1282/3125 [03:54<1:26:27, 2.81s/it] 41%|████ | 1283/3125 [03:56<1:26:01, 2.80s/it] 41%|████ | 1284/3125 [03:59<1:24:56, 2.77s/it] 41%|████ | 1285/3125 [04:02<1:22:38, 2.69s/it] 41%|████ | 1286/3125 [04:04<1:21:26, 2.66s/it] 41%|████ | 1287/3125 [04:07<1:19:24, 2.59s/it] 41%|████ | 1288/3125 [04:09<1:20:52, 2.64s/it] 41%|████ | 1289/3125 [04:13<1:26:06, 2.81s/it] 41%|████▏ | 1290/3125 [04:15<1:26:16, 2.82s/it] {'loss': '1.178', 'grad_norm': '5.351', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01147', 'train/loss_cot': '0.8401', 'epoch': '2.064'} 41%|████▏ | 1290/3125 [04:16<1:26:16, 2.82s/it] 41%|████▏ | 1291/3125 [04:18<1:24:35, 2.77s/it] 41%|████▏ | 1292/3125 [04:21<1:25:49, 2.81s/it] 41%|████▏ | 1293/3125 [04:24<1:24:44, 2.78s/it] 41%|████▏ | 1294/3125 [04:26<1:25:01, 2.79s/it] 41%|████▏ | 1295/3125 [04:29<1:23:37, 2.74s/it] 41%|████▏ | 1296/3125 [04:32<1:21:37, 2.68s/it] 42%|████▏ | 1297/3125 [04:35<1:23:27, 2.74s/it] 42%|████▏ | 1298/3125 [04:37<1:22:19, 2.70s/it] 42%|████▏ | 1299/3125 [04:40<1:22:29, 2.71s/it] 42%|████▏ | 1300/3125 [04:43<1:24:06, 2.77s/it] {'loss': '1.221', 'grad_norm': '8.117', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3984', 'train/loss_cot': '1.056', 'epoch': '2.08'} 42%|████▏ | 1300/3125 [04:43<1:24:06, 2.77s/it] 42%|████▏ | 1301/3125 [04:46<1:23:59, 2.76s/it] 42%|████▏ | 1302/3125 [04:48<1:19:58, 2.63s/it] 42%|████▏ | 1303/3125 [04:51<1:21:05, 2.67s/it] 42%|████▏ | 1304/3125 [04:54<1:26:00, 2.83s/it] 42%|████▏ | 1305/3125 [04:57<1:26:38, 2.86s/it] 42%|████▏ | 1306/3125 [04:59<1:25:01, 2.80s/it] 42%|████▏ | 1307/3125 [05:03<1:28:53, 2.93s/it] 42%|████▏ | 1308/3125 [05:05<1:26:58, 2.87s/it] 42%|████▏ | 1309/3125 [05:08<1:23:22, 2.75s/it] 42%|████▏ | 1310/3125 [05:11<1:23:46, 2.77s/it] {'loss': '1.218', 'grad_norm': '7.923', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02466', 'train/loss_cot': '0.7941', 'epoch': '2.096'} 42%|████▏ | 1310/3125 [05:11<1:23:46, 2.77s/it] 42%|████▏ | 1311/3125 [05:14<1:24:57, 2.81s/it] 42%|████▏ | 1312/3125 [05:16<1:24:05, 2.78s/it] 42%|████▏ | 1313/3125 [05:19<1:23:23, 2.76s/it] 42%|████▏ | 1314/3125 [05:21<1:20:53, 2.68s/it] 42%|████▏ | 1315/3125 [05:24<1:20:20, 2.66s/it] 42%|████▏ | 1316/3125 [05:27<1:21:35, 2.71s/it] 42%|████▏ | 1317/3125 [05:29<1:19:17, 2.63s/it] 42%|████▏ | 1318/3125 [05:32<1:21:07, 2.69s/it] 42%|████▏ | 1319/3125 [05:35<1:20:21, 2.67s/it] 42%|████▏ | 1320/3125 [05:37<1:19:03, 2.63s/it] {'loss': '1.19', 'grad_norm': '6.648', 'learning_rate': '1e-05', 'train/loss_infonce': '0.126', 'train/loss_cot': '1.008', 'epoch': '2.112'} 42%|████▏ | 1320/3125 [05:38<1:19:03, 2.63s/it] 42%|████▏ | 1321/3125 [05:40<1:18:51, 2.62s/it] 42%|████▏ | 1322/3125 [05:43<1:21:58, 2.73s/it] 42%|████▏ | 1323/3125 [05:46<1:20:52, 2.69s/it] 42%|████▏ | 1324/3125 [05:48<1:20:18, 2.68s/it] 42%|████▏ | 1325/3125 [05:51<1:20:44, 2.69s/it] 42%|████▏ | 1326/3125 [05:53<1:19:31, 2.65s/it] 42%|████▏ | 1327/3125 [05:56<1:17:47, 2.60s/it] 42%|████▏ | 1328/3125 [06:00<1:31:11, 3.04s/it] 43%|████▎ | 1329/3125 [06:03<1:27:36, 2.93s/it] 43%|████▎ | 1330/3125 [06:05<1:26:04, 2.88s/it] {'loss': '1.187', 'grad_norm': '6.496', 'learning_rate': '1e-05', 'train/loss_infonce': '0.124', 'train/loss_cot': '0.8749', 'epoch': '2.128'} 43%|████▎ | 1330/3125 [06:06<1:26:04, 2.88s/it] 43%|████▎ | 1331/3125 [06:08<1:23:37, 2.80s/it] 43%|████▎ | 1332/3125 [06:11<1:21:30, 2.73s/it] 43%|████▎ | 1333/3125 [06:13<1:21:22, 2.72s/it] 43%|████▎ | 1334/3125 [06:16<1:19:37, 2.67s/it] 43%|████▎ | 1335/3125 [06:18<1:17:33, 2.60s/it] 43%|████▎ | 1336/3125 [06:21<1:21:16, 2.73s/it] 43%|████▎ | 1337/3125 [06:24<1:19:43, 2.68s/it] 43%|████▎ | 1338/3125 [06:27<1:22:32, 2.77s/it] 43%|████▎ | 1339/3125 [06:30<1:21:17, 2.73s/it] 43%|████▎ | 1340/3125 [06:33<1:27:15, 2.93s/it] {'loss': '1.208', 'grad_norm': '9.786', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02893', 'train/loss_cot': '1.2', 'epoch': '2.144'} 43%|████▎ | 1340/3125 [06:33<1:27:15, 2.93s/it] 43%|████▎ | 1341/3125 [06:36<1:24:46, 2.85s/it] 43%|████▎ | 1342/3125 [06:38<1:22:34, 2.78s/it] 43%|████▎ | 1343/3125 [06:41<1:26:34, 2.91s/it] 43%|████▎ | 1344/3125 [06:44<1:23:08, 2.80s/it] 43%|████▎ | 1345/3125 [06:47<1:22:43, 2.79s/it] 43%|████▎ | 1346/3125 [06:49<1:22:19, 2.78s/it] 43%|████▎ | 1347/3125 [06:52<1:19:22, 2.68s/it] 43%|████▎ | 1348/3125 [06:54<1:17:53, 2.63s/it] 43%|████▎ | 1349/3125 [06:57<1:17:45, 2.63s/it] 43%|████▎ | 1350/3125 [07:00<1:18:40, 2.66s/it] {'loss': '1.181', 'grad_norm': '5.429', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001717', 'train/loss_cot': '1.005', 'epoch': '2.16'} 43%|████▎ | 1350/3125 [07:00<1:18:40, 2.66s/it] 43%|████▎ | 1351/3125 [07:03<1:21:32, 2.76s/it] 43%|████▎ | 1352/3125 [07:06<1:22:33, 2.79s/it] 43%|████▎ | 1353/3125 [07:08<1:21:12, 2.75s/it] 43%|████▎ | 1354/3125 [07:11<1:20:40, 2.73s/it] 43%|████▎ | 1355/3125 [07:14<1:22:41, 2.80s/it] 43%|████▎ | 1356/3125 [07:17<1:21:36, 2.77s/it] 43%|████▎ | 1357/3125 [07:20<1:25:44, 2.91s/it] 43%|████▎ | 1358/3125 [07:23<1:22:56, 2.82s/it] 43%|████▎ | 1359/3125 [07:25<1:21:36, 2.77s/it] 44%|████▎ | 1360/3125 [07:28<1:18:31, 2.67s/it] {'loss': '1.205', 'grad_norm': '5.808', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01843', 'train/loss_cot': '1.143', 'epoch': '2.176'} 44%|████▎ | 1360/3125 [07:28<1:18:31, 2.67s/it] 44%|████▎ | 1361/3125 [07:30<1:18:03, 2.65s/it] 44%|████▎ | 1362/3125 [07:33<1:18:39, 2.68s/it] 44%|████▎ | 1363/3125 [07:36<1:21:21, 2.77s/it] 44%|████▎ | 1364/3125 [07:38<1:18:43, 2.68s/it] 44%|████▎ | 1365/3125 [07:42<1:25:44, 2.92s/it] 44%|████▎ | 1366/3125 [07:45<1:25:39, 2.92s/it] 44%|████▎ | 1367/3125 [07:48<1:28:44, 3.03s/it] 44%|████▍ | 1368/3125 [07:51<1:29:00, 3.04s/it] 44%|████▍ | 1369/3125 [07:54<1:26:44, 2.96s/it] 44%|████▍ | 1370/3125 [07:57<1:24:39, 2.89s/it] {'loss': '1.166', 'grad_norm': '6.158', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006927', 'train/loss_cot': '0.8185', 'epoch': '2.192'} 44%|████▍ | 1370/3125 [07:57<1:24:39, 2.89s/it] 44%|████▍ | 1371/3125 [07:59<1:23:15, 2.85s/it] 44%|████▍ | 1372/3125 [08:02<1:21:09, 2.78s/it] 44%|████▍ | 1373/3125 [08:05<1:20:46, 2.77s/it] 44%|████▍ | 1374/3125 [08:07<1:19:21, 2.72s/it] 44%|████▍ | 1375/3125 [08:11<1:27:41, 3.01s/it] 44%|████▍ | 1376/3125 [08:14<1:24:43, 2.91s/it] 44%|████▍ | 1377/3125 [08:16<1:22:20, 2.83s/it] 44%|████▍ | 1378/3125 [08:20<1:24:53, 2.92s/it] 44%|████▍ | 1379/3125 [08:22<1:21:03, 2.79s/it] 44%|████▍ | 1380/3125 [08:25<1:20:48, 2.78s/it] {'loss': '1.16', 'grad_norm': '5.463', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01624', 'train/loss_cot': '1.312', 'epoch': '2.208'} 44%|████▍ | 1380/3125 [08:25<1:20:48, 2.78s/it] 44%|████▍ | 1381/3125 [08:28<1:22:10, 2.83s/it] 44%|████▍ | 1382/3125 [08:30<1:20:38, 2.78s/it] 44%|████▍ | 1383/3125 [08:33<1:18:37, 2.71s/it] 44%|████▍ | 1384/3125 [08:36<1:19:46, 2.75s/it] 44%|████▍ | 1385/3125 [08:39<1:23:09, 2.87s/it] 44%|████▍ | 1386/3125 [08:42<1:22:23, 2.84s/it] 44%|████▍ | 1387/3125 [08:44<1:19:32, 2.75s/it] 44%|████▍ | 1388/3125 [08:47<1:20:17, 2.77s/it] 44%|████▍ | 1389/3125 [08:50<1:18:35, 2.72s/it] 44%|████▍ | 1390/3125 [08:52<1:19:39, 2.75s/it] {'loss': '1.194', 'grad_norm': '6.727', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2178', 'train/loss_cot': '1.002', 'epoch': '2.224'} 44%|████▍ | 1390/3125 [08:53<1:19:39, 2.75s/it] 45%|████▍ | 1391/3125 [08:55<1:19:46, 2.76s/it] 45%|████▍ | 1392/3125 [08:58<1:18:30, 2.72s/it] 45%|████▍ | 1393/3125 [09:01<1:18:28, 2.72s/it] 45%|████▍ | 1394/3125 [09:03<1:19:08, 2.74s/it] 45%|████▍ | 1395/3125 [09:07<1:25:07, 2.95s/it] 45%|████▍ | 1396/3125 [09:09<1:21:24, 2.82s/it] 45%|████▍ | 1397/3125 [09:12<1:23:17, 2.89s/it] 45%|████▍ | 1398/3125 [09:15<1:21:58, 2.85s/it] 45%|████▍ | 1399/3125 [09:19<1:31:16, 3.17s/it] 45%|████▍ | 1400/3125 [09:22<1:27:11, 3.03s/it] {'loss': '1.23', 'grad_norm': '5.633', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0009041', 'train/loss_cot': '1.322', 'epoch': '2.24'} 45%|████▍ | 1400/3125 [09:22<1:27:11, 3.03s/it]mtime may not be reliable on this filesystem, falling back to numerical ordering 45%|████▍ | 1401/3125 [09:41<3:42:47, 7.75s/it] 45%|████▍ | 1402/3125 [09:43<2:59:45, 6.26s/it] 45%|████▍ | 1403/3125 [09:46<2:28:12, 5.16s/it] 45%|████▍ | 1404/3125 [09:48<2:05:50, 4.39s/it] 45%|████▍ | 1405/3125 [09:52<1:54:36, 4.00s/it] 45%|████▍ | 1406/3125 [09:54<1:43:09, 3.60s/it] 45%|████▌ | 1407/3125 [09:57<1:35:47, 3.35s/it] 45%|████▌ | 1408/3125 [10:00<1:30:31, 3.16s/it] 45%|████▌ | 1409/3125 [10:03<1:31:23, 3.20s/it] 45%|████▌ | 1410/3125 [10:06<1:28:05, 3.08s/it] {'loss': '1.144', 'grad_norm': '9.349', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00386', 'train/loss_cot': '1.238', 'epoch': '2.256'} 45%|████▌ | 1410/3125 [10:06<1:28:05, 3.08s/it] 45%|████▌ | 1411/3125 [10:10<1:33:16, 3.27s/it] 45%|████▌ | 1412/3125 [10:13<1:30:45, 3.18s/it] 45%|████▌ | 1413/3125 [10:15<1:26:23, 3.03s/it] 45%|████▌ | 1414/3125 [10:18<1:23:09, 2.92s/it] 45%|████▌ | 1415/3125 [10:20<1:20:01, 2.81s/it] 45%|████▌ | 1416/3125 [10:23<1:20:47, 2.84s/it] 45%|████▌ | 1417/3125 [10:27<1:24:40, 2.97s/it] 45%|████▌ | 1418/3125 [10:29<1:21:27, 2.86s/it] 45%|████▌ | 1419/3125 [10:32<1:21:59, 2.88s/it] 45%|████▌ | 1420/3125 [10:35<1:18:29, 2.76s/it] {'loss': '1.214', 'grad_norm': '6.526', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0835', 'train/loss_cot': '1.115', 'epoch': '2.272'} 45%|████▌ | 1420/3125 [10:35<1:18:29, 2.76s/it] 45%|████▌ | 1421/3125 [10:38<1:22:23, 2.90s/it] 46%|████▌ | 1422/3125 [10:40<1:19:15, 2.79s/it] 46%|████▌ | 1423/3125 [10:43<1:16:47, 2.71s/it] 46%|████▌ | 1424/3125 [10:46<1:16:18, 2.69s/it] 46%|████▌ | 1425/3125 [10:48<1:17:12, 2.73s/it] 46%|████▌ | 1426/3125 [10:51<1:16:09, 2.69s/it] 46%|████▌ | 1427/3125 [10:54<1:18:29, 2.77s/it] 46%|████▌ | 1428/3125 [10:57<1:21:01, 2.86s/it] 46%|████▌ | 1429/3125 [10:59<1:17:00, 2.72s/it] 46%|████▌ | 1430/3125 [11:02<1:16:00, 2.69s/it] {'loss': '1.255', 'grad_norm': '5.664', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1143', 'train/loss_cot': '0.888', 'epoch': '2.288'} 46%|████▌ | 1430/3125 [11:02<1:16:00, 2.69s/it] 46%|████▌ | 1431/3125 [11:05<1:15:52, 2.69s/it] 46%|████▌ | 1432/3125 [11:07<1:16:14, 2.70s/it] 46%|████▌ | 1433/3125 [11:10<1:15:58, 2.69s/it] 46%|████▌ | 1434/3125 [11:13<1:14:36, 2.65s/it] 46%|████▌ | 1435/3125 [11:16<1:16:37, 2.72s/it] 46%|████▌ | 1436/3125 [11:19<1:19:20, 2.82s/it] 46%|████▌ | 1437/3125 [11:21<1:18:58, 2.81s/it] 46%|████▌ | 1438/3125 [11:25<1:25:03, 3.03s/it] 46%|████▌ | 1439/3125 [11:28<1:22:17, 2.93s/it] 46%|████▌ | 1440/3125 [11:31<1:22:25, 2.94s/it] {'loss': '1.291', 'grad_norm': '10.11', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03369', 'train/loss_cot': '1.043', 'epoch': '2.304'} 46%|████▌ | 1440/3125 [11:31<1:22:25, 2.94s/it] 46%|████▌ | 1441/3125 [11:33<1:18:52, 2.81s/it] 46%|████▌ | 1442/3125 [11:36<1:16:26, 2.73s/it] 46%|████▌ | 1443/3125 [11:38<1:17:00, 2.75s/it] 46%|████▌ | 1444/3125 [11:41<1:14:39, 2.66s/it] 46%|████▌ | 1445/3125 [11:44<1:15:21, 2.69s/it] 46%|████▋ | 1446/3125 [11:46<1:15:37, 2.70s/it] 46%|████▋ | 1447/3125 [11:49<1:15:42, 2.71s/it] 46%|████▋ | 1448/3125 [11:52<1:15:32, 2.70s/it] 46%|████▋ | 1449/3125 [11:54<1:13:56, 2.65s/it] 46%|████▋ | 1450/3125 [11:57<1:13:44, 2.64s/it] {'loss': '1.131', 'grad_norm': '5.072', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04712', 'train/loss_cot': '0.7999', 'epoch': '2.32'} 46%|████▋ | 1450/3125 [11:57<1:13:44, 2.64s/it] 46%|████▋ | 1451/3125 [12:00<1:17:07, 2.76s/it] 46%|████▋ | 1452/3125 [12:02<1:15:08, 2.70s/it] 46%|████▋ | 1453/3125 [12:05<1:17:39, 2.79s/it] 47%|████▋ | 1454/3125 [12:08<1:15:02, 2.69s/it] 47%|████▋ | 1455/3125 [12:11<1:17:42, 2.79s/it] 47%|████▋ | 1456/3125 [12:13<1:14:37, 2.68s/it] 47%|████▋ | 1457/3125 [12:16<1:13:22, 2.64s/it] 47%|████▋ | 1458/3125 [12:19<1:13:05, 2.63s/it] 47%|████▋ | 1459/3125 [12:21<1:13:43, 2.66s/it] 47%|████▋ | 1460/3125 [12:24<1:12:21, 2.61s/it] {'loss': '1.206', 'grad_norm': '7.059', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01117', 'train/loss_cot': '1.28', 'epoch': '2.336'} 47%|████▋ | 1460/3125 [12:24<1:12:21, 2.61s/it] 47%|████▋ | 1461/3125 [12:27<1:13:46, 2.66s/it] 47%|████▋ | 1462/3125 [12:30<1:18:05, 2.82s/it] 47%|████▋ | 1463/3125 [12:33<1:18:57, 2.85s/it] 47%|████▋ | 1464/3125 [12:36<1:19:55, 2.89s/it] 47%|████▋ | 1465/3125 [12:38<1:16:40, 2.77s/it] 47%|████▋ | 1466/3125 [12:41<1:15:28, 2.73s/it] 47%|████▋ | 1467/3125 [12:44<1:16:55, 2.78s/it] 47%|████▋ | 1468/3125 [12:47<1:17:53, 2.82s/it] 47%|████▋ | 1469/3125 [12:50<1:18:50, 2.86s/it] 47%|████▋ | 1470/3125 [12:52<1:15:16, 2.73s/it] {'loss': '1.159', 'grad_norm': '6.841', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08301', 'train/loss_cot': '0.9938', 'epoch': '2.352'} 47%|████▋ | 1470/3125 [12:52<1:15:16, 2.73s/it] 47%|████▋ | 1471/3125 [12:55<1:15:39, 2.74s/it] 47%|████▋ | 1472/3125 [12:57<1:14:59, 2.72s/it] 47%|████▋ | 1473/3125 [13:00<1:13:38, 2.67s/it] 47%|████▋ | 1474/3125 [13:03<1:16:31, 2.78s/it] 47%|████▋ | 1475/3125 [13:06<1:17:31, 2.82s/it] 47%|████▋ | 1476/3125 [13:09<1:16:16, 2.78s/it] 47%|████▋ | 1477/3125 [13:11<1:13:24, 2.67s/it] 47%|████▋ | 1478/3125 [13:14<1:13:42, 2.69s/it] 47%|████▋ | 1479/3125 [13:16<1:12:17, 2.64s/it] 47%|████▋ | 1480/3125 [13:19<1:16:49, 2.80s/it] {'loss': '1.272', 'grad_norm': '6.188', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04712', 'train/loss_cot': '1.353', 'epoch': '2.368'} 47%|████▋ | 1480/3125 [13:20<1:16:49, 2.80s/it] 47%|████▋ | 1481/3125 [13:22<1:16:04, 2.78s/it] 47%|████▋ | 1482/3125 [13:25<1:15:58, 2.77s/it] 47%|████▋ | 1483/3125 [13:28<1:15:06, 2.74s/it] 47%|████▋ | 1484/3125 [13:30<1:11:29, 2.61s/it] 48%|████▊ | 1485/3125 [13:33<1:13:28, 2.69s/it] 48%|████▊ | 1486/3125 [13:36<1:15:20, 2.76s/it] 48%|████▊ | 1487/3125 [13:39<1:15:59, 2.78s/it] 48%|████▊ | 1488/3125 [13:41<1:15:44, 2.78s/it] 48%|████▊ | 1489/3125 [13:44<1:15:16, 2.76s/it] 48%|████▊ | 1490/3125 [13:47<1:14:49, 2.75s/it] {'loss': '1.281', 'grad_norm': '5.982', 'learning_rate': '1e-05', 'train/loss_infonce': '0.332', 'train/loss_cot': '1.231', 'epoch': '2.384'} 48%|████▊ | 1490/3125 [13:47<1:14:49, 2.75s/it] 48%|████▊ | 1491/3125 [13:50<1:17:30, 2.85s/it] 48%|████▊ | 1492/3125 [13:52<1:14:38, 2.74s/it] 48%|████▊ | 1493/3125 [13:56<1:24:19, 3.10s/it] 48%|████▊ | 1494/3125 [13:59<1:23:33, 3.07s/it] 48%|████▊ | 1495/3125 [14:02<1:23:40, 3.08s/it] 48%|████▊ | 1496/3125 [14:05<1:20:38, 2.97s/it] 48%|████▊ | 1497/3125 [14:08<1:19:19, 2.92s/it] 48%|████▊ | 1498/3125 [14:10<1:16:41, 2.83s/it] 48%|████▊ | 1499/3125 [14:13<1:17:32, 2.86s/it] 48%|████▊ | 1500/3125 [14:16<1:18:22, 2.89s/it] {'loss': '1.213', 'grad_norm': '10.42', 'learning_rate': '1e-05', 'train/loss_infonce': '0.06592', 'train/loss_cot': '0.9286', 'epoch': '2.4'} 48%|████▊ | 1500/3125 [14:17<1:18:22, 2.89s/it] 48%|████▊ | 1501/3125 [14:19<1:17:07, 2.85s/it] 48%|████▊ | 1502/3125 [14:22<1:15:23, 2.79s/it] 48%|████▊ | 1503/3125 [14:26<1:23:37, 3.09s/it] 48%|████▊ | 1504/3125 [14:28<1:21:10, 3.00s/it] 48%|████▊ | 1505/3125 [14:31<1:17:37, 2.87s/it] 48%|████▊ | 1506/3125 [14:34<1:16:15, 2.83s/it] 48%|████▊ | 1507/3125 [14:36<1:14:49, 2.77s/it] 48%|████▊ | 1508/3125 [14:39<1:12:16, 2.68s/it] 48%|████▊ | 1509/3125 [14:42<1:19:16, 2.94s/it] 48%|████▊ | 1510/3125 [14:45<1:16:14, 2.83s/it] {'loss': '1.179', 'grad_norm': '8.327', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02014', 'train/loss_cot': '1.373', 'epoch': '2.416'} 48%|████▊ | 1510/3125 [14:45<1:16:14, 2.83s/it] 48%|████▊ | 1511/3125 [14:48<1:15:20, 2.80s/it] 48%|████▊ | 1512/3125 [14:51<1:16:20, 2.84s/it] 48%|████▊ | 1513/3125 [14:53<1:16:41, 2.85s/it] 48%|████▊ | 1514/3125 [14:57<1:18:13, 2.91s/it] 48%|████▊ | 1515/3125 [14:59<1:15:29, 2.81s/it] 49%|████▊ | 1516/3125 [15:02<1:15:27, 2.81s/it] 49%|████▊ | 1517/3125 [15:05<1:15:28, 2.82s/it] 49%|████▊ | 1518/3125 [15:07<1:12:55, 2.72s/it] 49%|████▊ | 1519/3125 [15:10<1:12:31, 2.71s/it] 49%|████▊ | 1520/3125 [15:13<1:14:08, 2.77s/it] {'loss': '1.252', 'grad_norm': '11.81', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1074', 'train/loss_cot': '1.505', 'epoch': '2.432'} 49%|████▊ | 1520/3125 [15:13<1:14:08, 2.77s/it] 49%|████▊ | 1521/3125 [15:16<1:17:10, 2.89s/it] 49%|████▊ | 1522/3125 [15:19<1:21:40, 3.06s/it] 49%|████▊ | 1523/3125 [15:23<1:23:59, 3.15s/it] 49%|████▉ | 1524/3125 [15:26<1:23:28, 3.13s/it] 49%|████▉ | 1525/3125 [15:30<1:28:19, 3.31s/it] 49%|████▉ | 1526/3125 [15:32<1:21:51, 3.07s/it] 49%|████▉ | 1527/3125 [15:35<1:22:11, 3.09s/it] 49%|████▉ | 1528/3125 [15:38<1:20:09, 3.01s/it] 49%|████▉ | 1529/3125 [15:41<1:16:52, 2.89s/it] 49%|████▉ | 1530/3125 [15:44<1:17:56, 2.93s/it] {'loss': '1.194', 'grad_norm': '5.693', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01495', 'train/loss_cot': '1.145', 'epoch': '2.448'} 49%|████▉ | 1530/3125 [15:44<1:17:56, 2.93s/it] 49%|████▉ | 1531/3125 [15:47<1:18:20, 2.95s/it] 49%|████▉ | 1532/3125 [15:49<1:16:51, 2.89s/it] 49%|████▉ | 1533/3125 [15:52<1:14:03, 2.79s/it] 49%|████▉ | 1534/3125 [15:55<1:16:07, 2.87s/it] 49%|████▉ | 1535/3125 [15:58<1:17:24, 2.92s/it] 49%|████▉ | 1536/3125 [16:01<1:16:46, 2.90s/it] 49%|████▉ | 1537/3125 [16:04<1:14:14, 2.81s/it] 49%|████▉ | 1538/3125 [16:06<1:13:39, 2.78s/it] 49%|████▉ | 1539/3125 [16:09<1:15:58, 2.87s/it] 49%|████▉ | 1540/3125 [16:12<1:16:04, 2.88s/it] {'loss': '1.187', 'grad_norm': '5.551', 'learning_rate': '1e-05', 'train/loss_infonce': '0.008179', 'train/loss_cot': '0.8784', 'epoch': '2.464'} 49%|████▉ | 1540/3125 [16:13<1:16:04, 2.88s/it] 49%|████▉ | 1541/3125 [16:16<1:22:17, 3.12s/it] 49%|████▉ | 1542/3125 [16:20<1:27:03, 3.30s/it] 49%|████▉ | 1543/3125 [16:22<1:21:09, 3.08s/it] 49%|████▉ | 1544/3125 [16:25<1:18:17, 2.97s/it] 49%|████▉ | 1545/3125 [16:28<1:17:09, 2.93s/it] 49%|████▉ | 1546/3125 [16:31<1:18:41, 2.99s/it] 50%|████▉ | 1547/3125 [16:34<1:16:56, 2.93s/it] 50%|████▉ | 1548/3125 [16:36<1:15:15, 2.86s/it] 50%|████▉ | 1549/3125 [16:39<1:13:30, 2.80s/it] 50%|████▉ | 1550/3125 [16:42<1:14:25, 2.84s/it] {'loss': '1.28', 'grad_norm': '7.464', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1455', 'train/loss_cot': '1.161', 'epoch': '2.48'} 50%|████▉ | 1550/3125 [16:42<1:14:25, 2.84s/it] 50%|████▉ | 1551/3125 [16:45<1:14:20, 2.83s/it] 50%|████▉ | 1552/3125 [16:47<1:13:01, 2.79s/it] 50%|████▉ | 1553/3125 [16:50<1:12:35, 2.77s/it] 50%|████▉ | 1554/3125 [16:53<1:13:21, 2.80s/it] 50%|████▉ | 1555/3125 [16:56<1:12:06, 2.76s/it] 50%|████▉ | 1556/3125 [16:58<1:10:32, 2.70s/it] 50%|████▉ | 1557/3125 [17:01<1:13:03, 2.80s/it] 50%|████▉ | 1558/3125 [17:04<1:13:37, 2.82s/it] 50%|████▉ | 1559/3125 [17:07<1:11:52, 2.75s/it] 50%|████▉ | 1560/3125 [17:09<1:10:58, 2.72s/it] {'loss': '1.186', 'grad_norm': '8.916', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2246', 'train/loss_cot': '1.121', 'epoch': '2.496'} 50%|████▉ | 1560/3125 [17:10<1:10:58, 2.72s/it] 50%|████▉ | 1561/3125 [17:13<1:14:42, 2.87s/it] 50%|████▉ | 1562/3125 [17:15<1:11:04, 2.73s/it] 50%|█████ | 1563/3125 [17:17<1:08:39, 2.64s/it] 50%|█████ | 1564/3125 [17:21<1:12:43, 2.80s/it] 50%|█████ | 1565/3125 [17:24<1:13:31, 2.83s/it] 50%|█████ | 1566/3125 [17:27<1:17:06, 2.97s/it] 50%|█████ | 1567/3125 [17:30<1:16:03, 2.93s/it] 50%|█████ | 1568/3125 [17:32<1:13:45, 2.84s/it] 50%|█████ | 1569/3125 [17:35<1:11:11, 2.75s/it] 50%|█████ | 1570/3125 [17:38<1:13:57, 2.85s/it] {'loss': '1.141', 'grad_norm': '6.972', 'learning_rate': '1e-05', 'train/loss_infonce': '0.008423', 'train/loss_cot': '1.36', 'epoch': '2.512'} 50%|█████ | 1570/3125 [17:38<1:13:57, 2.85s/it] 50%|█████ | 1571/3125 [17:41<1:12:55, 2.82s/it] 50%|█████ | 1572/3125 [17:43<1:11:55, 2.78s/it] 50%|█████ | 1573/3125 [17:46<1:13:24, 2.84s/it] 50%|█████ | 1574/3125 [17:50<1:17:04, 2.98s/it] 50%|█████ | 1575/3125 [17:52<1:15:35, 2.93s/it] 50%|█████ | 1576/3125 [17:55<1:15:04, 2.91s/it] 50%|█████ | 1577/3125 [17:58<1:14:58, 2.91s/it] 50%|█████ | 1578/3125 [18:01<1:13:39, 2.86s/it] 51%|█████ | 1579/3125 [18:04<1:12:26, 2.81s/it] 51%|█████ | 1580/3125 [18:08<1:22:17, 3.20s/it] {'loss': '1.191', 'grad_norm': '10.7', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005615', 'train/loss_cot': '1.451', 'epoch': '2.528'} 51%|█████ | 1580/3125 [18:08<1:22:17, 3.20s/it] 51%|█████ | 1581/3125 [18:11<1:19:55, 3.11s/it] 51%|█████ | 1582/3125 [18:14<1:18:35, 3.06s/it] 51%|█████ | 1583/3125 [18:16<1:16:59, 3.00s/it] 51%|█████ | 1584/3125 [18:19<1:16:25, 2.98s/it] 51%|█████ | 1585/3125 [18:22<1:15:50, 2.95s/it] 51%|█████ | 1586/3125 [18:25<1:14:10, 2.89s/it] 51%|█████ | 1587/3125 [18:27<1:09:56, 2.73s/it] 51%|█████ | 1588/3125 [18:30<1:09:55, 2.73s/it] 51%|█████ | 1589/3125 [18:34<1:21:12, 3.17s/it] 51%|█████ | 1590/3125 [18:38<1:21:26, 3.18s/it] {'loss': '1.311', 'grad_norm': '6.456', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01831', 'train/loss_cot': '1.249', 'epoch': '2.544'} 51%|█████ | 1590/3125 [18:38<1:21:26, 3.18s/it] 51%|█████ | 1591/3125 [18:42<1:31:45, 3.59s/it] 51%|█████ | 1592/3125 [18:45<1:26:03, 3.37s/it] 51%|█████ | 1593/3125 [18:49<1:28:27, 3.46s/it] 51%|█████ | 1594/3125 [18:51<1:22:52, 3.25s/it] 51%|█████ | 1595/3125 [18:55<1:24:05, 3.30s/it] 51%|█████ | 1596/3125 [18:57<1:18:43, 3.09s/it] 51%|█████ | 1597/3125 [19:00<1:16:15, 2.99s/it] 51%|█████ | 1598/3125 [19:03<1:14:57, 2.95s/it] 51%|█████ | 1599/3125 [19:07<1:20:12, 3.15s/it] 51%|█████ | 1600/3125 [19:09<1:16:19, 3.00s/it] {'loss': '1.176', 'grad_norm': '5.817', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1562', 'train/loss_cot': '1.058', 'epoch': '2.56'} 51%|█████ | 1600/3125 [19:10<1:16:19, 3.00s/it] 51%|█████ | 1601/3125 [19:28<3:18:02, 7.80s/it] 51%|█████▏ | 1602/3125 [19:31<2:40:09, 6.31s/it] 51%|█████▏ | 1603/3125 [19:34<2:14:24, 5.30s/it] 51%|█████▏ | 1604/3125 [19:37<1:55:09, 4.54s/it] 51%|█████▏ | 1605/3125 [19:40<1:43:27, 4.08s/it] 51%|█████▏ | 1606/3125 [19:43<1:35:48, 3.78s/it] 51%|█████▏ | 1607/3125 [19:46<1:27:13, 3.45s/it] 51%|█████▏ | 1608/3125 [19:48<1:23:03, 3.29s/it] 51%|█████▏ | 1609/3125 [19:51<1:18:20, 3.10s/it] 52%|█████▏ | 1610/3125 [19:54<1:15:18, 2.98s/it] {'loss': '1.197', 'grad_norm': '5.939', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02271', 'train/loss_cot': '0.9068', 'epoch': '2.576'} 52%|█████▏ | 1610/3125 [19:54<1:15:18, 2.98s/it] 52%|█████▏ | 1611/3125 [19:56<1:12:05, 2.86s/it] 52%|█████▏ | 1612/3125 [19:59<1:10:40, 2.80s/it] 52%|█████▏ | 1613/3125 [20:02<1:08:35, 2.72s/it] 52%|█████▏ | 1614/3125 [20:04<1:08:53, 2.74s/it] 52%|█████▏ | 1615/3125 [20:07<1:07:49, 2.69s/it] 52%|█████▏ | 1616/3125 [20:10<1:09:17, 2.75s/it] 52%|█████▏ | 1617/3125 [20:13<1:09:11, 2.75s/it] 52%|█████▏ | 1618/3125 [20:16<1:11:52, 2.86s/it] 52%|█████▏ | 1619/3125 [20:18<1:09:42, 2.78s/it] 52%|█████▏ | 1620/3125 [20:21<1:09:54, 2.79s/it] {'loss': '1.222', 'grad_norm': '9.275', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003189', 'train/loss_cot': '1.259', 'epoch': '2.592'} 52%|█████▏ | 1620/3125 [20:22<1:09:54, 2.79s/it] 52%|█████▏ | 1621/3125 [20:24<1:10:02, 2.79s/it] 52%|█████▏ | 1622/3125 [20:26<1:08:05, 2.72s/it] 52%|█████▏ | 1623/3125 [20:29<1:09:27, 2.77s/it] 52%|█████▏ | 1624/3125 [20:33<1:12:08, 2.88s/it] 52%|█████▏ | 1625/3125 [20:35<1:09:41, 2.79s/it] 52%|█████▏ | 1626/3125 [20:38<1:10:07, 2.81s/it] 52%|█████▏ | 1627/3125 [20:41<1:09:51, 2.80s/it] 52%|█████▏ | 1628/3125 [20:43<1:07:35, 2.71s/it] 52%|█████▏ | 1629/3125 [20:46<1:06:01, 2.65s/it] 52%|█████▏ | 1630/3125 [20:49<1:10:04, 2.81s/it] {'loss': '1.241', 'grad_norm': '6.941', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2383', 'train/loss_cot': '0.9823', 'epoch': '2.608'} 52%|█████▏ | 1630/3125 [20:49<1:10:04, 2.81s/it] 52%|█████▏ | 1631/3125 [20:52<1:10:33, 2.83s/it] 52%|█████▏ | 1632/3125 [20:54<1:07:45, 2.72s/it] 52%|█████▏ | 1633/3125 [20:57<1:10:30, 2.84s/it] 52%|█████▏ | 1634/3125 [21:01<1:13:00, 2.94s/it] 52%|█████▏ | 1635/3125 [21:03<1:11:51, 2.89s/it] 52%|█████▏ | 1636/3125 [21:06<1:11:40, 2.89s/it] 52%|█████▏ | 1637/3125 [21:09<1:12:14, 2.91s/it] 52%|█████▏ | 1638/3125 [21:12<1:09:35, 2.81s/it] 52%|█████▏ | 1639/3125 [21:15<1:11:09, 2.87s/it] 52%|█████▏ | 1640/3125 [21:18<1:10:50, 2.86s/it] {'loss': '1.248', 'grad_norm': '6.295', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04272', 'train/loss_cot': '1.242', 'epoch': '2.624'} 52%|█████▏ | 1640/3125 [21:18<1:10:50, 2.86s/it] 53%|█████▎ | 1641/3125 [21:20<1:09:26, 2.81s/it] 53%|█████▎ | 1642/3125 [21:24<1:12:57, 2.95s/it] 53%|█████▎ | 1643/3125 [21:27<1:14:03, 3.00s/it] 53%|█████▎ | 1644/3125 [21:29<1:11:10, 2.88s/it] 53%|█████▎ | 1645/3125 [21:32<1:09:52, 2.83s/it] 53%|█████▎ | 1646/3125 [21:35<1:09:48, 2.83s/it] 53%|█████▎ | 1647/3125 [21:38<1:10:31, 2.86s/it] 53%|█████▎ | 1648/3125 [21:41<1:10:30, 2.86s/it] 53%|█████▎ | 1649/3125 [21:44<1:12:34, 2.95s/it] 53%|█████▎ | 1650/3125 [21:47<1:12:31, 2.95s/it] {'loss': '1.255', 'grad_norm': '5.889', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2324', 'train/loss_cot': '1.162', 'epoch': '2.64'} 53%|█████▎ | 1650/3125 [21:47<1:12:31, 2.95s/it] 53%|█████▎ | 1651/3125 [21:50<1:12:06, 2.94s/it] 53%|█████▎ | 1652/3125 [21:53<1:13:08, 2.98s/it] 53%|█████▎ | 1653/3125 [21:56<1:12:10, 2.94s/it] 53%|█████▎ | 1654/3125 [21:59<1:12:09, 2.94s/it] 53%|█████▎ | 1655/3125 [22:01<1:11:59, 2.94s/it] 53%|█████▎ | 1656/3125 [22:04<1:12:27, 2.96s/it] 53%|█████▎ | 1657/3125 [22:07<1:10:09, 2.87s/it] 53%|█████▎ | 1658/3125 [22:10<1:08:11, 2.79s/it] 53%|█████▎ | 1659/3125 [22:12<1:07:00, 2.74s/it] 53%|█████▎ | 1660/3125 [22:15<1:06:36, 2.73s/it] {'loss': '1.205', 'grad_norm': '5.61', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00647', 'train/loss_cot': '0.9052', 'epoch': '2.656'} 53%|█████▎ | 1660/3125 [22:16<1:06:36, 2.73s/it] 53%|█████▎ | 1661/3125 [22:19<1:12:27, 2.97s/it] 53%|█████▎ | 1662/3125 [22:21<1:09:49, 2.86s/it] 53%|█████▎ | 1663/3125 [22:24<1:05:52, 2.70s/it] 53%|█████▎ | 1664/3125 [22:26<1:05:33, 2.69s/it] 53%|█████▎ | 1665/3125 [22:29<1:05:11, 2.68s/it] 53%|█████▎ | 1666/3125 [22:32<1:06:09, 2.72s/it] 53%|█████▎ | 1667/3125 [22:35<1:09:50, 2.87s/it] 53%|█████▎ | 1668/3125 [22:38<1:10:40, 2.91s/it] 53%|█████▎ | 1669/3125 [22:41<1:09:39, 2.87s/it] 53%|█████▎ | 1670/3125 [22:44<1:10:05, 2.89s/it] {'loss': '1.206', 'grad_norm': '5.497', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007111', 'train/loss_cot': '1.189', 'epoch': '2.672'} 53%|█████▎ | 1670/3125 [22:44<1:10:05, 2.89s/it] 53%|█████▎ | 1671/3125 [22:47<1:13:42, 3.04s/it] 54%|█████▎ | 1672/3125 [22:50<1:16:23, 3.15s/it] 54%|█████▎ | 1673/3125 [22:53<1:15:01, 3.10s/it] 54%|█████▎ | 1674/3125 [22:56<1:11:39, 2.96s/it] 54%|█████▎ | 1675/3125 [22:59<1:11:08, 2.94s/it] 54%|█████▎ | 1676/3125 [23:02<1:08:52, 2.85s/it] 54%|█████▎ | 1677/3125 [23:04<1:06:55, 2.77s/it] 54%|█████▎ | 1678/3125 [23:07<1:07:03, 2.78s/it] 54%|█████▎ | 1679/3125 [23:10<1:05:38, 2.72s/it] 54%|█████▍ | 1680/3125 [23:12<1:06:02, 2.74s/it] {'loss': '1.257', 'grad_norm': '5.173', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '1.105', 'epoch': '2.688'} 54%|█████▍ | 1680/3125 [23:13<1:06:02, 2.74s/it] 54%|█████▍ | 1681/3125 [23:15<1:06:41, 2.77s/it] 54%|█████▍ | 1682/3125 [23:18<1:06:38, 2.77s/it] 54%|█████▍ | 1683/3125 [23:21<1:05:56, 2.74s/it] 54%|█████▍ | 1684/3125 [23:23<1:04:40, 2.69s/it] 54%|█████▍ | 1685/3125 [23:27<1:09:15, 2.89s/it] 54%|█████▍ | 1686/3125 [23:29<1:08:00, 2.84s/it] 54%|█████▍ | 1687/3125 [23:32<1:06:43, 2.78s/it] 54%|█████▍ | 1688/3125 [23:34<1:04:21, 2.69s/it] 54%|█████▍ | 1689/3125 [23:37<1:04:22, 2.69s/it] 54%|█████▍ | 1690/3125 [23:39<1:02:23, 2.61s/it] {'loss': '1.212', 'grad_norm': '6.714', 'learning_rate': '1e-05', 'train/loss_infonce': '0.125', 'train/loss_cot': '0.773', 'epoch': '2.704'} 54%|█████▍ | 1690/3125 [23:40<1:02:23, 2.61s/it] 54%|█████▍ | 1691/3125 [23:43<1:06:35, 2.79s/it] 54%|█████▍ | 1692/3125 [23:46<1:09:47, 2.92s/it] 54%|█████▍ | 1693/3125 [23:48<1:06:13, 2.77s/it] 54%|█████▍ | 1694/3125 [23:51<1:04:52, 2.72s/it] 54%|█████▍ | 1695/3125 [23:54<1:06:01, 2.77s/it] 54%|█████▍ | 1696/3125 [23:57<1:06:58, 2.81s/it] 54%|█████▍ | 1697/3125 [23:59<1:04:36, 2.71s/it] 54%|█████▍ | 1698/3125 [24:02<1:06:10, 2.78s/it] 54%|█████▍ | 1699/3125 [24:05<1:04:06, 2.70s/it] 54%|█████▍ | 1700/3125 [24:07<1:03:40, 2.68s/it] {'loss': '1.21', 'grad_norm': '6.064', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1357', 'train/loss_cot': '0.9565', 'epoch': '2.72'} 54%|█████▍ | 1700/3125 [24:08<1:03:40, 2.68s/it] 54%|█████▍ | 1701/3125 [24:10<1:05:04, 2.74s/it] 54%|█████▍ | 1702/3125 [24:13<1:06:46, 2.82s/it] 54%|█████▍ | 1703/3125 [24:16<1:05:47, 2.78s/it] 55%|█████▍ | 1704/3125 [24:19<1:05:57, 2.79s/it] 55%|█████▍ | 1705/3125 [24:21<1:05:20, 2.76s/it] 55%|█████▍ | 1706/3125 [24:24<1:06:03, 2.79s/it] 55%|█████▍ | 1707/3125 [24:27<1:08:56, 2.92s/it] 55%|█████▍ | 1708/3125 [24:30<1:07:04, 2.84s/it] 55%|█████▍ | 1709/3125 [24:33<1:06:09, 2.80s/it] 55%|█████▍ | 1710/3125 [24:36<1:05:21, 2.77s/it] {'loss': '1.157', 'grad_norm': '5.87', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1504', 'train/loss_cot': '1.076', 'epoch': '2.736'} 55%|█████▍ | 1710/3125 [24:36<1:05:21, 2.77s/it] 55%|█████▍ | 1711/3125 [24:38<1:04:56, 2.76s/it] 55%|█████▍ | 1712/3125 [24:41<1:05:58, 2.80s/it] 55%|█████▍ | 1713/3125 [24:44<1:04:06, 2.72s/it] 55%|█████▍ | 1714/3125 [24:47<1:05:38, 2.79s/it] 55%|█████▍ | 1715/3125 [24:50<1:06:59, 2.85s/it] 55%|█████▍ | 1716/3125 [24:52<1:05:06, 2.77s/it] 55%|█████▍ | 1717/3125 [24:55<1:04:43, 2.76s/it] 55%|█████▍ | 1718/3125 [24:58<1:04:51, 2.77s/it] 55%|█████▌ | 1719/3125 [25:01<1:04:55, 2.77s/it] 55%|█████▌ | 1720/3125 [25:03<1:04:02, 2.73s/it] {'loss': '1.189', 'grad_norm': '5.975', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01471', 'train/loss_cot': '1.276', 'epoch': '2.752'} 55%|█████▌ | 1720/3125 [25:04<1:04:02, 2.73s/it] 55%|█████▌ | 1721/3125 [25:06<1:05:57, 2.82s/it] 55%|█████▌ | 1722/3125 [25:09<1:05:35, 2.81s/it] 55%|█████▌ | 1723/3125 [25:11<1:02:58, 2.70s/it] 55%|█████▌ | 1724/3125 [25:14<1:02:04, 2.66s/it] 55%|█████▌ | 1725/3125 [25:16<1:00:44, 2.60s/it] 55%|█████▌ | 1726/3125 [25:19<1:00:13, 2.58s/it] 55%|█████▌ | 1727/3125 [25:22<59:55, 2.57s/it] 55%|█████▌ | 1728/3125 [25:24<1:02:24, 2.68s/it] 55%|█████▌ | 1729/3125 [25:27<1:03:17, 2.72s/it] 55%|█████▌ | 1730/3125 [25:30<1:03:28, 2.73s/it] {'loss': '1.254', 'grad_norm': '6.991', 'learning_rate': '1e-05', 'train/loss_infonce': '0.104', 'train/loss_cot': '1.089', 'epoch': '2.768'} 55%|█████▌ | 1730/3125 [25:30<1:03:28, 2.73s/it] 55%|█████▌ | 1731/3125 [25:33<1:02:37, 2.70s/it] 55%|█████▌ | 1732/3125 [25:35<1:03:39, 2.74s/it] 55%|█████▌ | 1733/3125 [25:38<1:04:51, 2.80s/it] 55%|█████▌ | 1734/3125 [25:41<1:04:44, 2.79s/it] 56%|█████▌ | 1735/3125 [25:44<1:03:38, 2.75s/it] 56%|█████▌ | 1736/3125 [25:47<1:08:00, 2.94s/it] 56%|█████▌ | 1737/3125 [25:50<1:06:06, 2.86s/it] 56%|█████▌ | 1738/3125 [25:53<1:06:10, 2.86s/it] 56%|█████▌ | 1739/3125 [25:55<1:04:12, 2.78s/it] 56%|█████▌ | 1740/3125 [25:59<1:08:02, 2.95s/it] {'loss': '1.273', 'grad_norm': '6.88', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05713', 'train/loss_cot': '1.284', 'epoch': '2.784'} 56%|█████▌ | 1740/3125 [25:59<1:08:02, 2.95s/it] 56%|█████▌ | 1741/3125 [26:02<1:08:00, 2.95s/it] 56%|█████▌ | 1742/3125 [26:05<1:07:23, 2.92s/it] 56%|█████▌ | 1743/3125 [26:07<1:04:43, 2.81s/it] 56%|█████▌ | 1744/3125 [26:10<1:04:35, 2.81s/it] 56%|█████▌ | 1745/3125 [26:13<1:09:50, 3.04s/it] 56%|█████▌ | 1746/3125 [26:16<1:05:46, 2.86s/it] 56%|█████▌ | 1747/3125 [26:19<1:08:09, 2.97s/it] 56%|█████▌ | 1748/3125 [26:22<1:07:01, 2.92s/it] 56%|█████▌ | 1749/3125 [26:25<1:07:05, 2.93s/it] 56%|█████▌ | 1750/3125 [26:28<1:05:33, 2.86s/it] {'loss': '1.215', 'grad_norm': '7.62', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03076', 'train/loss_cot': '0.9005', 'epoch': '2.8'} 56%|█████▌ | 1750/3125 [26:28<1:05:33, 2.86s/it] 56%|█████▌ | 1751/3125 [26:31<1:11:13, 3.11s/it] 56%|█████▌ | 1752/3125 [26:34<1:07:02, 2.93s/it] 56%|█████▌ | 1753/3125 [26:36<1:04:52, 2.84s/it] 56%|█████▌ | 1754/3125 [26:39<1:01:40, 2.70s/it] 56%|█████▌ | 1755/3125 [26:41<1:00:48, 2.66s/it] 56%|█████▌ | 1756/3125 [26:44<1:01:39, 2.70s/it] 56%|█████▌ | 1757/3125 [26:47<1:02:14, 2.73s/it] 56%|█████▋ | 1758/3125 [26:50<1:02:11, 2.73s/it] 56%|█████▋ | 1759/3125 [26:53<1:05:01, 2.86s/it] 56%|█████▋ | 1760/3125 [26:56<1:04:00, 2.81s/it] {'loss': '1.264', 'grad_norm': '6.701', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3262', 'train/loss_cot': '1.005', 'epoch': '2.816'} 56%|█████▋ | 1760/3125 [26:56<1:04:00, 2.81s/it] 56%|█████▋ | 1761/3125 [26:59<1:05:08, 2.87s/it] 56%|█████▋ | 1762/3125 [27:02<1:06:43, 2.94s/it] 56%|█████▋ | 1763/3125 [27:04<1:05:57, 2.91s/it] 56%|█████▋ | 1764/3125 [27:07<1:04:34, 2.85s/it] 56%|█████▋ | 1765/3125 [27:10<1:02:36, 2.76s/it] 57%|█████▋ | 1766/3125 [27:13<1:02:47, 2.77s/it] 57%|█████▋ | 1767/3125 [27:15<1:00:43, 2.68s/it] 57%|█████▋ | 1768/3125 [27:18<1:01:45, 2.73s/it] 57%|█████▋ | 1769/3125 [27:20<59:07, 2.62s/it] 57%|█████▋ | 1770/3125 [27:23<59:32, 2.64s/it] {'loss': '1.192', 'grad_norm': '6.547', 'learning_rate': '1e-05', 'train/loss_infonce': '0.09131', 'train/loss_cot': '0.9959', 'epoch': '2.832'} 57%|█████▋ | 1770/3125 [27:23<59:32, 2.64s/it] 57%|█████▋ | 1771/3125 [27:26<1:02:29, 2.77s/it] 57%|█████▋ | 1772/3125 [27:29<1:01:59, 2.75s/it] 57%|█████▋ | 1773/3125 [27:33<1:10:38, 3.13s/it] 57%|█████▋ | 1774/3125 [27:36<1:14:29, 3.31s/it] 57%|█████▋ | 1775/3125 [27:39<1:11:28, 3.18s/it] 57%|█████▋ | 1776/3125 [27:42<1:07:02, 2.98s/it] 57%|█████▋ | 1777/3125 [27:45<1:07:10, 2.99s/it] 57%|█████▋ | 1778/3125 [27:48<1:06:33, 2.96s/it] 57%|█████▋ | 1779/3125 [27:51<1:05:47, 2.93s/it] 57%|█████▋ | 1780/3125 [27:54<1:07:25, 3.01s/it] {'loss': '1.209', 'grad_norm': '9.752', 'learning_rate': '1e-05', 'train/loss_infonce': '0.5352', 'train/loss_cot': '1.202', 'epoch': '2.848'} 57%|█████▋ | 1780/3125 [27:54<1:07:25, 3.01s/it] 57%|█████▋ | 1781/3125 [27:57<1:06:18, 2.96s/it] 57%|█████▋ | 1782/3125 [27:59<1:03:29, 2.84s/it] 57%|█████▋ | 1783/3125 [28:02<1:02:13, 2.78s/it] 57%|█████▋ | 1784/3125 [28:05<1:03:42, 2.85s/it] 57%|█████▋ | 1785/3125 [28:08<1:03:54, 2.86s/it] 57%|█████▋ | 1786/3125 [28:10<1:01:53, 2.77s/it] 57%|█████▋ | 1787/3125 [28:13<1:02:42, 2.81s/it] 57%|█████▋ | 1788/3125 [28:16<1:01:07, 2.74s/it] 57%|█████▋ | 1789/3125 [28:19<1:01:32, 2.76s/it] 57%|█████▋ | 1790/3125 [28:22<1:02:50, 2.82s/it] {'loss': '1.186', 'grad_norm': '6.39', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001274', 'train/loss_cot': '0.8117', 'epoch': '2.864'} 57%|█████▋ | 1790/3125 [28:22<1:02:50, 2.82s/it] 57%|█████▋ | 1791/3125 [28:24<1:02:35, 2.82s/it] 57%|█████▋ | 1792/3125 [28:27<1:00:58, 2.74s/it] 57%|█████▋ | 1793/3125 [28:30<1:01:42, 2.78s/it] 57%|█████▋ | 1794/3125 [28:32<1:00:58, 2.75s/it] 57%|█████▋ | 1795/3125 [28:35<59:20, 2.68s/it] 57%|█████▋ | 1796/3125 [28:38<1:01:44, 2.79s/it] 58%|█████▊ | 1797/3125 [28:41<1:01:33, 2.78s/it] 58%|█████▊ | 1798/3125 [28:43<59:29, 2.69s/it] 58%|█████▊ | 1799/3125 [28:46<59:33, 2.69s/it] 58%|█████▊ | 1800/3125 [28:49<58:47, 2.66s/it] {'loss': '1.272', 'grad_norm': '8.465', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2041', 'train/loss_cot': '0.7639', 'epoch': '2.88'} 58%|█████▊ | 1800/3125 [28:49<58:47, 2.66s/it] 58%|█████▊ | 1801/3125 [29:08<2:48:27, 7.63s/it] 58%|█████▊ | 1802/3125 [29:10<2:14:15, 6.09s/it] 58%|█████▊ | 1803/3125 [29:13<1:53:11, 5.14s/it] 58%|█████▊ | 1804/3125 [29:16<1:37:41, 4.44s/it] 58%|█████▊ | 1805/3125 [29:19<1:26:04, 3.91s/it] 58%|█████▊ | 1806/3125 [29:22<1:21:39, 3.71s/it] 58%|█████▊ | 1807/3125 [29:25<1:15:50, 3.45s/it] 58%|█████▊ | 1808/3125 [29:28<1:13:02, 3.33s/it] 58%|█████▊ | 1809/3125 [29:30<1:06:57, 3.05s/it] 58%|█████▊ | 1810/3125 [29:34<1:08:50, 3.14s/it] {'loss': '1.289', 'grad_norm': '6.283', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1562', 'train/loss_cot': '0.9734', 'epoch': '2.896'} 58%|█████▊ | 1810/3125 [29:34<1:08:50, 3.14s/it] 58%|█████▊ | 1811/3125 [29:36<1:05:36, 3.00s/it] 58%|█████▊ | 1812/3125 [29:39<1:03:52, 2.92s/it] 58%|█████▊ | 1813/3125 [29:42<1:02:01, 2.84s/it] 58%|█████▊ | 1814/3125 [29:44<1:01:11, 2.80s/it] 58%|█████▊ | 1815/3125 [29:47<58:54, 2.70s/it] 58%|█████▊ | 1816/3125 [29:49<56:49, 2.60s/it] 58%|█████▊ | 1817/3125 [29:52<58:37, 2.69s/it] 58%|█████▊ | 1818/3125 [29:55<58:04, 2.67s/it] 58%|█████▊ | 1819/3125 [29:57<55:48, 2.56s/it] 58%|█████▊ | 1820/3125 [30:00<57:39, 2.65s/it] {'loss': '1.195', 'grad_norm': '7.439', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2539', 'train/loss_cot': '1.369', 'epoch': '2.912'} 58%|█████▊ | 1820/3125 [30:00<57:39, 2.65s/it] 58%|█████▊ | 1821/3125 [30:03<59:23, 2.73s/it] 58%|█████▊ | 1822/3125 [30:05<58:35, 2.70s/it] 58%|█████▊ | 1823/3125 [30:09<1:04:27, 2.97s/it] 58%|█████▊ | 1824/3125 [30:11<1:01:33, 2.84s/it] 58%|█████▊ | 1825/3125 [30:14<1:00:49, 2.81s/it] 58%|█████▊ | 1826/3125 [30:17<1:01:20, 2.83s/it] 58%|█████▊ | 1827/3125 [30:20<59:35, 2.75s/it] 58%|█████▊ | 1828/3125 [30:23<1:00:26, 2.80s/it] 59%|█████▊ | 1829/3125 [30:25<1:01:04, 2.83s/it] 59%|█████▊ | 1830/3125 [30:29<1:04:49, 3.00s/it] {'loss': '1.197', 'grad_norm': '6.513', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01508', 'train/loss_cot': '1.151', 'epoch': '2.928'} 59%|█████▊ | 1830/3125 [30:29<1:04:49, 3.00s/it] 59%|█████▊ | 1831/3125 [30:32<1:03:07, 2.93s/it] 59%|█████▊ | 1832/3125 [30:34<1:00:19, 2.80s/it] 59%|█████▊ | 1833/3125 [30:36<57:13, 2.66s/it] 59%|█████▊ | 1834/3125 [30:39<59:03, 2.74s/it] 59%|█████▊ | 1835/3125 [30:42<58:33, 2.72s/it] 59%|█████▉ | 1836/3125 [30:45<57:45, 2.69s/it] 59%|█████▉ | 1837/3125 [30:48<1:00:59, 2.84s/it] 59%|█████▉ | 1838/3125 [30:51<1:00:09, 2.80s/it] 59%|█████▉ | 1839/3125 [30:54<1:01:13, 2.86s/it] 59%|█████▉ | 1840/3125 [30:56<59:50, 2.79s/it] {'loss': '1.181', 'grad_norm': '6.85', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005676', 'train/loss_cot': '1.24', 'epoch': '2.944'} 59%|█████▉ | 1840/3125 [30:57<59:50, 2.79s/it] 59%|█████▉ | 1841/3125 [30:59<1:00:46, 2.84s/it] 59%|█████▉ | 1842/3125 [31:02<59:17, 2.77s/it] 59%|█████▉ | 1843/3125 [31:05<1:00:01, 2.81s/it] 59%|█████▉ | 1844/3125 [31:07<57:45, 2.71s/it] 59%|█████▉ | 1845/3125 [31:10<57:09, 2.68s/it] 59%|█████▉ | 1846/3125 [31:13<1:00:03, 2.82s/it] 59%|█████▉ | 1847/3125 [31:15<57:58, 2.72s/it] 59%|█████▉ | 1848/3125 [31:18<57:12, 2.69s/it] 59%|█████▉ | 1849/3125 [31:21<57:10, 2.69s/it] 59%|█████▉ | 1850/3125 [31:23<56:00, 2.64s/it] {'loss': '1.209', 'grad_norm': '6.609', 'learning_rate': '1e-05', 'train/loss_infonce': '0.06641', 'train/loss_cot': '1.055', 'epoch': '2.96'} 59%|█████▉ | 1850/3125 [31:24<56:00, 2.64s/it] 59%|█████▉ | 1851/3125 [31:26<55:27, 2.61s/it] 59%|█████▉ | 1852/3125 [31:28<54:27, 2.57s/it] 59%|█████▉ | 1853/3125 [31:31<55:38, 2.62s/it] 59%|█████▉ | 1854/3125 [31:34<55:58, 2.64s/it] 59%|█████▉ | 1855/3125 [31:36<54:33, 2.58s/it] 59%|█████▉ | 1856/3125 [31:39<55:36, 2.63s/it] 59%|█████▉ | 1857/3125 [31:42<57:20, 2.71s/it] 59%|█████▉ | 1858/3125 [31:45<58:03, 2.75s/it] 59%|█████▉ | 1859/3125 [31:47<56:00, 2.65s/it] 60%|█████▉ | 1860/3125 [31:50<56:09, 2.66s/it] {'loss': '1.239', 'grad_norm': '8.196', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02258', 'train/loss_cot': '1.21', 'epoch': '2.976'} 60%|█████▉ | 1860/3125 [31:50<56:09, 2.66s/it] 60%|█████▉ | 1861/3125 [31:52<56:01, 2.66s/it] 60%|█████▉ | 1862/3125 [31:55<54:59, 2.61s/it] 60%|█████▉ | 1863/3125 [31:59<1:01:26, 2.92s/it] 60%|█████▉ | 1864/3125 [32:01<59:42, 2.84s/it] 60%|█████▉ | 1865/3125 [32:04<1:01:23, 2.92s/it] 60%|█████▉ | 1866/3125 [32:07<1:00:50, 2.90s/it] 60%|█████▉ | 1867/3125 [32:10<58:26, 2.79s/it] 60%|█████▉ | 1868/3125 [32:12<57:52, 2.76s/it] 60%|█████▉ | 1869/3125 [32:15<58:41, 2.80s/it] 60%|█████▉ | 1870/3125 [32:18<57:13, 2.74s/it] {'loss': '1.093', 'grad_norm': '5.23', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01208', 'train/loss_cot': '0.9051', 'epoch': '2.992'} 60%|█████▉ | 1870/3125 [32:18<57:13, 2.74s/it] 60%|█████▉ | 1871/3125 [32:21<59:24, 2.84s/it] 60%|█████▉ | 1872/3125 [32:24<58:10, 2.79s/it] 60%|█████▉ | 1873/3125 [32:26<56:32, 2.71s/it] 60%|█████▉ | 1874/3125 [32:29<56:23, 2.70s/it] 60%|██████ | 1875/3125 [32:30<44:50, 2.15s/it] 60%|██████ | 1876/3125 [32:34<56:10, 2.70s/it] 60%|██████ | 1877/3125 [32:36<54:26, 2.62s/it] 60%|██████ | 1878/3125 [32:39<53:25, 2.57s/it] 60%|██████ | 1879/3125 [32:41<52:21, 2.52s/it] 60%|██████ | 1880/3125 [32:44<54:30, 2.63s/it] {'loss': '1.14', 'grad_norm': '6.176', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01141', 'train/loss_cot': '0.977', 'epoch': '3.008'} 60%|██████ | 1880/3125 [32:44<54:30, 2.63s/it] 60%|██████ | 1881/3125 [32:46<54:13, 2.62s/it] 60%|██████ | 1882/3125 [32:49<51:03, 2.46s/it] 60%|██████ | 1883/3125 [32:51<50:43, 2.45s/it] 60%|██████ | 1884/3125 [32:53<49:31, 2.39s/it] 60%|██████ | 1885/3125 [32:56<50:27, 2.44s/it] 60%|██████ | 1886/3125 [32:58<48:49, 2.36s/it] 60%|██████ | 1887/3125 [33:00<48:41, 2.36s/it] 60%|██████ | 1888/3125 [33:03<49:59, 2.43s/it] 60%|██████ | 1889/3125 [33:06<51:52, 2.52s/it] 60%|██████ | 1890/3125 [33:08<50:21, 2.45s/it] {'loss': '1.045', 'grad_norm': '7.792', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03394', 'train/loss_cot': '1.093', 'epoch': '3.024'} 60%|██████ | 1890/3125 [33:08<50:21, 2.45s/it] 61%|██████ | 1891/3125 [33:10<50:06, 2.44s/it] 61%|██████ | 1892/3125 [33:13<49:30, 2.41s/it] 61%|██████ | 1893/3125 [33:15<48:14, 2.35s/it] 61%|██████ | 1894/3125 [33:17<46:52, 2.29s/it] 61%|██████ | 1895/3125 [33:21<55:06, 2.69s/it] 61%|██████ | 1896/3125 [33:23<52:47, 2.58s/it] 61%|██████ | 1897/3125 [33:25<52:38, 2.57s/it] 61%|██████ | 1898/3125 [33:28<52:46, 2.58s/it] 61%|██████ | 1899/3125 [33:30<50:26, 2.47s/it] 61%|██████ | 1900/3125 [33:33<54:18, 2.66s/it] {'loss': '1.035', 'grad_norm': '7.938', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01343', 'train/loss_cot': '1.245', 'epoch': '3.04'} 61%|██████ | 1900/3125 [33:34<54:18, 2.66s/it] 61%|██████ | 1901/3125 [33:36<55:34, 2.72s/it] 61%|██████ | 1902/3125 [33:39<53:37, 2.63s/it] 61%|██████ | 1903/3125 [33:41<51:04, 2.51s/it] 61%|██████ | 1904/3125 [33:44<51:48, 2.55s/it] 61%|██████ | 1905/3125 [33:46<50:37, 2.49s/it] 61%|██████ | 1906/3125 [33:48<48:51, 2.41s/it] 61%|██████ | 1907/3125 [33:51<51:46, 2.55s/it] 61%|██████ | 1908/3125 [33:53<50:22, 2.48s/it] 61%|██████ | 1909/3125 [33:56<48:59, 2.42s/it] 61%|██████ | 1910/3125 [33:58<47:32, 2.35s/it] {'loss': '1.018', 'grad_norm': '6.123', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05884', 'train/loss_cot': '1.153', 'epoch': '3.056'} 61%|██████ | 1910/3125 [33:58<47:32, 2.35s/it] 61%|██████ | 1911/3125 [34:01<55:02, 2.72s/it] 61%|██████ | 1912/3125 [34:04<52:17, 2.59s/it] 61%|██████ | 1913/3125 [34:06<51:22, 2.54s/it] 61%|██████ | 1914/3125 [34:09<52:51, 2.62s/it] 61%|██████▏ | 1915/3125 [34:11<51:11, 2.54s/it] 61%|██████▏ | 1916/3125 [34:14<51:21, 2.55s/it] 61%|██████▏ | 1917/3125 [34:16<51:19, 2.55s/it] 61%|██████▏ | 1918/3125 [34:19<49:45, 2.47s/it] 61%|██████▏ | 1919/3125 [34:21<48:02, 2.39s/it] 61%|██████▏ | 1920/3125 [34:23<47:37, 2.37s/it] {'loss': '1.051', 'grad_norm': '7.122', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1865', 'train/loss_cot': '0.6729', 'epoch': '3.072'} 61%|██████▏ | 1920/3125 [34:24<47:37, 2.37s/it] 61%|██████▏ | 1921/3125 [34:26<47:59, 2.39s/it] 62%|██████▏ | 1922/3125 [34:28<47:57, 2.39s/it] 62%|██████▏ | 1923/3125 [34:31<53:20, 2.66s/it] 62%|██████▏ | 1924/3125 [34:33<50:19, 2.51s/it] 62%|██████▏ | 1925/3125 [34:36<51:50, 2.59s/it] 62%|██████▏ | 1926/3125 [34:39<50:56, 2.55s/it] 62%|██████▏ | 1927/3125 [34:41<49:03, 2.46s/it] 62%|██████▏ | 1928/3125 [34:44<49:43, 2.49s/it] 62%|██████▏ | 1929/3125 [34:46<51:55, 2.60s/it] 62%|██████▏ | 1930/3125 [34:49<50:09, 2.52s/it] {'loss': '1.018', 'grad_norm': '6.818', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001015', 'train/loss_cot': '0.8492', 'epoch': '3.088'} 62%|██████▏ | 1930/3125 [34:49<50:09, 2.52s/it] 62%|██████▏ | 1931/3125 [34:51<49:50, 2.50s/it] 62%|██████▏ | 1932/3125 [34:54<49:06, 2.47s/it] 62%|██████▏ | 1933/3125 [34:56<47:20, 2.38s/it] 62%|██████▏ | 1934/3125 [34:58<45:58, 2.32s/it] 62%|██████▏ | 1935/3125 [35:01<50:49, 2.56s/it] 62%|██████▏ | 1936/3125 [35:04<52:51, 2.67s/it] 62%|██████▏ | 1937/3125 [35:06<49:57, 2.52s/it] 62%|██████▏ | 1938/3125 [35:09<50:27, 2.55s/it] 62%|██████▏ | 1939/3125 [35:11<48:55, 2.48s/it] 62%|██████▏ | 1940/3125 [35:13<47:53, 2.42s/it] {'loss': '1.051', 'grad_norm': '6.156', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004791', 'train/loss_cot': '0.9085', 'epoch': '3.104'} 62%|██████▏ | 1940/3125 [35:14<47:53, 2.42s/it] 62%|██████▏ | 1941/3125 [35:16<47:54, 2.43s/it] 62%|██████▏ | 1942/3125 [35:19<51:57, 2.64s/it] 62%|██████▏ | 1943/3125 [35:21<50:29, 2.56s/it] 62%|██████▏ | 1944/3125 [35:24<50:23, 2.56s/it] 62%|██████▏ | 1945/3125 [35:26<49:05, 2.50s/it] 62%|██████▏ | 1946/3125 [35:29<49:54, 2.54s/it] 62%|██████▏ | 1947/3125 [35:31<47:38, 2.43s/it] 62%|██████▏ | 1948/3125 [35:33<46:51, 2.39s/it] 62%|██████▏ | 1949/3125 [35:36<45:48, 2.34s/it] 62%|██████▏ | 1950/3125 [35:38<46:20, 2.37s/it] {'loss': '1.08', 'grad_norm': '4.952', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01953', 'train/loss_cot': '0.8', 'epoch': '3.12'} 62%|██████▏ | 1950/3125 [35:38<46:20, 2.37s/it] 62%|██████▏ | 1951/3125 [35:40<45:59, 2.35s/it] 62%|██████▏ | 1952/3125 [35:43<45:21, 2.32s/it] 62%|██████▏ | 1953/3125 [35:45<45:14, 2.32s/it] 63%|██████▎ | 1954/3125 [35:48<48:02, 2.46s/it] 63%|██████▎ | 1955/3125 [35:50<45:57, 2.36s/it] 63%|██████▎ | 1956/3125 [35:52<45:13, 2.32s/it] 63%|██████▎ | 1957/3125 [35:55<48:29, 2.49s/it] 63%|██████▎ | 1958/3125 [35:57<47:45, 2.46s/it] 63%|██████▎ | 1959/3125 [35:59<46:06, 2.37s/it] 63%|██████▎ | 1960/3125 [36:02<45:59, 2.37s/it] {'loss': '1.056', 'grad_norm': '5.82', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02222', 'train/loss_cot': '0.7172', 'epoch': '3.136'} 63%|██████▎ | 1960/3125 [36:02<45:59, 2.37s/it] 63%|██████▎ | 1961/3125 [36:04<44:54, 2.31s/it] 63%|██████▎ | 1962/3125 [36:06<44:36, 2.30s/it] 63%|██████▎ | 1963/3125 [36:08<43:17, 2.24s/it] 63%|██████▎ | 1964/3125 [36:11<43:58, 2.27s/it] 63%|██████▎ | 1965/3125 [36:13<44:30, 2.30s/it] 63%|██████▎ | 1966/3125 [36:16<45:34, 2.36s/it] 63%|██████▎ | 1967/3125 [36:18<45:32, 2.36s/it] 63%|██████▎ | 1968/3125 [36:20<44:54, 2.33s/it] 63%|██████▎ | 1969/3125 [36:22<44:18, 2.30s/it] 63%|██████▎ | 1970/3125 [36:25<44:55, 2.33s/it] {'loss': '1.057', 'grad_norm': '6.039', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01819', 'train/loss_cot': '1.062', 'epoch': '3.152'} 63%|██████▎ | 1970/3125 [36:25<44:55, 2.33s/it] 63%|██████▎ | 1971/3125 [36:27<43:48, 2.28s/it] 63%|██████▎ | 1972/3125 [36:29<43:52, 2.28s/it] 63%|██████▎ | 1973/3125 [36:32<44:52, 2.34s/it] 63%|██████▎ | 1974/3125 [36:35<48:24, 2.52s/it] 63%|██████▎ | 1975/3125 [36:37<46:13, 2.41s/it] 63%|██████▎ | 1976/3125 [36:39<44:49, 2.34s/it] 63%|██████▎ | 1977/3125 [36:41<44:33, 2.33s/it] 63%|██████▎ | 1978/3125 [36:44<44:42, 2.34s/it] 63%|██████▎ | 1979/3125 [36:47<49:39, 2.60s/it] 63%|██████▎ | 1980/3125 [36:49<48:35, 2.55s/it] {'loss': '1.075', 'grad_norm': '6.714', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3125', 'train/loss_cot': '1.149', 'epoch': '3.168'} 63%|██████▎ | 1980/3125 [36:50<48:35, 2.55s/it] 63%|██████▎ | 1981/3125 [36:52<46:44, 2.45s/it] 63%|██████▎ | 1982/3125 [36:54<45:47, 2.40s/it] 63%|██████▎ | 1983/3125 [36:56<45:45, 2.40s/it] 63%|██████▎ | 1984/3125 [36:58<44:37, 2.35s/it] 64%|██████▎ | 1985/3125 [37:01<43:50, 2.31s/it] 64%|██████▎ | 1986/3125 [37:03<43:55, 2.31s/it] 64%|██████▎ | 1987/3125 [37:05<43:47, 2.31s/it] 64%|██████▎ | 1988/3125 [37:08<47:20, 2.50s/it] 64%|██████▎ | 1989/3125 [37:11<46:24, 2.45s/it] 64%|██████▎ | 1990/3125 [37:13<47:40, 2.52s/it] {'loss': '1.034', 'grad_norm': '5.547', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04663', 'train/loss_cot': '1.034', 'epoch': '3.184'} 64%|██████▎ | 1990/3125 [37:14<47:40, 2.52s/it] 64%|██████▎ | 1991/3125 [37:16<46:30, 2.46s/it] 64%|██████▎ | 1992/3125 [37:18<45:21, 2.40s/it] 64%|██████▍ | 1993/3125 [37:20<44:16, 2.35s/it] 64%|██████▍ | 1994/3125 [37:22<43:27, 2.31s/it] 64%|██████▍ | 1995/3125 [37:24<42:42, 2.27s/it] 64%|██████▍ | 1996/3125 [37:27<43:09, 2.29s/it] 64%|██████▍ | 1997/3125 [37:30<46:07, 2.45s/it] 64%|██████▍ | 1998/3125 [37:32<44:08, 2.35s/it] 64%|██████▍ | 1999/3125 [37:34<44:24, 2.37s/it] 64%|██████▍ | 2000/3125 [37:36<44:00, 2.35s/it] {'loss': '0.9929', 'grad_norm': '6.685', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '0.8827', 'epoch': '3.2'} 64%|██████▍ | 2000/3125 [37:37<44:00, 2.35s/it] 64%|██████▍ | 2001/3125 [37:55<2:13:12, 7.11s/it] 64%|██████▍ | 2002/3125 [37:57<1:46:22, 5.68s/it] 64%|██████▍ | 2003/3125 [37:59<1:28:16, 4.72s/it] 64%|██████▍ | 2004/3125 [38:02<1:14:41, 4.00s/it] 64%|██████▍ | 2005/3125 [38:04<1:04:15, 3.44s/it] 64%|██████▍ | 2006/3125 [38:06<58:29, 3.14s/it] 64%|██████▍ | 2007/3125 [38:09<54:42, 2.94s/it] 64%|██████▍ | 2008/3125 [38:11<52:18, 2.81s/it] 64%|██████▍ | 2009/3125 [38:14<51:09, 2.75s/it] 64%|██████▍ | 2010/3125 [38:16<49:08, 2.64s/it] {'loss': '1.085', 'grad_norm': '5.852', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04272', 'train/loss_cot': '0.9627', 'epoch': '3.216'} 64%|██████▍ | 2010/3125 [38:17<49:08, 2.64s/it] 64%|██████▍ | 2011/3125 [38:20<52:07, 2.81s/it] 64%|██████▍ | 2012/3125 [38:22<50:54, 2.74s/it] 64%|██████▍ | 2013/3125 [38:25<50:08, 2.71s/it] 64%|██████▍ | 2014/3125 [38:27<49:02, 2.65s/it] 64%|██████▍ | 2015/3125 [38:30<50:07, 2.71s/it] 65%|██████▍ | 2016/3125 [38:33<51:07, 2.77s/it] 65%|██████▍ | 2017/3125 [38:35<47:48, 2.59s/it] 65%|██████▍ | 2018/3125 [38:38<49:02, 2.66s/it] 65%|██████▍ | 2019/3125 [38:41<48:50, 2.65s/it] 65%|██████▍ | 2020/3125 [38:44<50:40, 2.75s/it] {'loss': '1.097', 'grad_norm': '5.433', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007141', 'train/loss_cot': '1.117', 'epoch': '3.232'} 65%|██████▍ | 2020/3125 [38:44<50:40, 2.75s/it] 65%|██████▍ | 2021/3125 [38:46<49:21, 2.68s/it] 65%|██████▍ | 2022/3125 [38:49<47:44, 2.60s/it] 65%|██████▍ | 2023/3125 [38:51<46:07, 2.51s/it] 65%|██████▍ | 2024/3125 [38:54<49:09, 2.68s/it] 65%|██████▍ | 2025/3125 [38:56<47:27, 2.59s/it] 65%|██████▍ | 2026/3125 [38:59<46:26, 2.54s/it] 65%|██████▍ | 2027/3125 [39:01<47:42, 2.61s/it] 65%|██████▍ | 2028/3125 [39:04<47:06, 2.58s/it] 65%|██████▍ | 2029/3125 [39:06<46:00, 2.52s/it] 65%|██████▍ | 2030/3125 [39:09<44:07, 2.42s/it] {'loss': '1.075', 'grad_norm': '5.627', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0498', 'train/loss_cot': '0.9348', 'epoch': '3.248'} 65%|██████▍ | 2030/3125 [39:09<44:07, 2.42s/it] 65%|██████▍ | 2031/3125 [39:11<44:37, 2.45s/it] 65%|██████▌ | 2032/3125 [39:14<44:52, 2.46s/it] 65%|██████▌ | 2033/3125 [39:16<44:33, 2.45s/it] 65%|██████▌ | 2034/3125 [39:18<44:12, 2.43s/it] 65%|██████▌ | 2035/3125 [39:21<46:48, 2.58s/it] 65%|██████▌ | 2036/3125 [39:24<46:18, 2.55s/it] 65%|██████▌ | 2037/3125 [39:26<44:47, 2.47s/it] 65%|██████▌ | 2038/3125 [39:29<44:48, 2.47s/it] 65%|██████▌ | 2039/3125 [39:31<43:43, 2.42s/it] 65%|██████▌ | 2040/3125 [39:34<45:54, 2.54s/it] {'loss': '1.049', 'grad_norm': '5.633', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00769', 'train/loss_cot': '0.9103', 'epoch': '3.264'} 65%|██████▌ | 2040/3125 [39:34<45:54, 2.54s/it] 65%|██████▌ | 2041/3125 [39:36<45:52, 2.54s/it] 65%|██████▌ | 2042/3125 [39:39<46:16, 2.56s/it] 65%|██████▌ | 2043/3125 [39:41<45:14, 2.51s/it] 65%|██████▌ | 2044/3125 [39:44<45:36, 2.53s/it] 65%|██████▌ | 2045/3125 [39:46<45:12, 2.51s/it] 65%|██████▌ | 2046/3125 [39:49<46:47, 2.60s/it] 66%|██████▌ | 2047/3125 [39:51<45:08, 2.51s/it] 66%|██████▌ | 2048/3125 [39:54<46:08, 2.57s/it] 66%|██████▌ | 2049/3125 [39:57<46:46, 2.61s/it] 66%|██████▌ | 2050/3125 [39:59<44:15, 2.47s/it] {'loss': '1.063', 'grad_norm': '6.172', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005249', 'train/loss_cot': '0.8447', 'epoch': '3.28'} 66%|██████▌ | 2050/3125 [39:59<44:15, 2.47s/it] 66%|██████▌ | 2051/3125 [40:01<43:38, 2.44s/it] 66%|██████▌ | 2052/3125 [40:05<48:07, 2.69s/it] 66%|██████▌ | 2053/3125 [40:07<47:26, 2.66s/it] 66%|██████▌ | 2054/3125 [40:09<44:54, 2.52s/it] 66%|██████▌ | 2055/3125 [40:12<44:31, 2.50s/it] 66%|██████▌ | 2056/3125 [40:14<44:50, 2.52s/it] 66%|██████▌ | 2057/3125 [40:17<44:12, 2.48s/it] 66%|██████▌ | 2058/3125 [40:19<45:12, 2.54s/it] 66%|██████▌ | 2059/3125 [40:22<45:23, 2.55s/it] 66%|██████▌ | 2060/3125 [40:24<44:53, 2.53s/it] {'loss': '1.018', 'grad_norm': '5.294', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006866', 'train/loss_cot': '0.8124', 'epoch': '3.296'} 66%|██████▌ | 2060/3125 [40:25<44:53, 2.53s/it] 66%|██████▌ | 2061/3125 [40:27<44:55, 2.53s/it] 66%|██████▌ | 2062/3125 [40:29<44:26, 2.51s/it] 66%|██████▌ | 2063/3125 [40:32<42:32, 2.40s/it] 66%|██████▌ | 2064/3125 [40:34<42:35, 2.41s/it] 66%|██████▌ | 2065/3125 [40:36<42:14, 2.39s/it] 66%|██████▌ | 2066/3125 [40:39<42:02, 2.38s/it] 66%|██████▌ | 2067/3125 [40:41<42:27, 2.41s/it] 66%|██████▌ | 2068/3125 [40:44<44:10, 2.51s/it] 66%|██████▌ | 2069/3125 [40:46<43:23, 2.47s/it] 66%|██████▌ | 2070/3125 [40:49<42:53, 2.44s/it] {'loss': '1.031', 'grad_norm': '5.682', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07715', 'train/loss_cot': '0.9371', 'epoch': '3.312'} 66%|██████▌ | 2070/3125 [40:49<42:53, 2.44s/it] 66%|██████▋ | 2071/3125 [40:52<44:59, 2.56s/it] 66%|██████▋ | 2072/3125 [40:54<44:05, 2.51s/it] 66%|██████▋ | 2073/3125 [40:56<42:12, 2.41s/it] 66%|██████▋ | 2074/3125 [40:59<42:08, 2.41s/it] 66%|██████▋ | 2075/3125 [41:01<43:18, 2.48s/it] 66%|██████▋ | 2076/3125 [41:03<42:25, 2.43s/it] 66%|██████▋ | 2077/3125 [41:07<46:10, 2.64s/it] 66%|██████▋ | 2078/3125 [41:09<44:36, 2.56s/it] 67%|██████▋ | 2079/3125 [41:11<44:15, 2.54s/it] 67%|██████▋ | 2080/3125 [41:14<41:49, 2.40s/it] {'loss': '1.076', 'grad_norm': '5.887', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009888', 'train/loss_cot': '1.209', 'epoch': '3.328'} 67%|██████▋ | 2080/3125 [41:14<41:49, 2.40s/it] 67%|██████▋ | 2081/3125 [41:16<44:16, 2.54s/it] 67%|██████▋ | 2082/3125 [41:19<43:42, 2.51s/it] 67%|██████▋ | 2083/3125 [41:21<43:24, 2.50s/it] 67%|██████▋ | 2084/3125 [41:23<41:17, 2.38s/it] 67%|██████▋ | 2085/3125 [41:26<40:56, 2.36s/it] 67%|██████▋ | 2086/3125 [41:28<40:58, 2.37s/it] 67%|██████▋ | 2087/3125 [41:30<40:37, 2.35s/it] 67%|██████▋ | 2088/3125 [41:33<41:58, 2.43s/it] 67%|██████▋ | 2089/3125 [41:35<41:43, 2.42s/it] 67%|██████▋ | 2090/3125 [41:38<41:26, 2.40s/it] {'loss': '1.04', 'grad_norm': '6.274', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01031', 'train/loss_cot': '1.146', 'epoch': '3.344'} 67%|██████▋ | 2090/3125 [41:38<41:26, 2.40s/it] 67%|██████▋ | 2091/3125 [41:40<41:36, 2.41s/it] 67%|██████▋ | 2092/3125 [41:43<41:25, 2.41s/it] 67%|██████▋ | 2093/3125 [41:45<42:06, 2.45s/it] 67%|██████▋ | 2094/3125 [41:48<41:53, 2.44s/it] 67%|██████▋ | 2095/3125 [41:50<42:12, 2.46s/it] 67%|██████▋ | 2096/3125 [41:53<44:11, 2.58s/it] 67%|██████▋ | 2097/3125 [41:56<47:17, 2.76s/it] 67%|██████▋ | 2098/3125 [41:59<45:51, 2.68s/it] 67%|██████▋ | 2099/3125 [42:01<45:28, 2.66s/it] 67%|██████▋ | 2100/3125 [42:04<43:43, 2.56s/it] {'loss': '1.012', 'grad_norm': '18.56', 'learning_rate': '1e-05', 'train/loss_infonce': '0.7812', 'train/loss_cot': '0.8486', 'epoch': '3.36'} 67%|██████▋ | 2100/3125 [42:04<43:43, 2.56s/it] 67%|██████▋ | 2101/3125 [42:06<42:11, 2.47s/it] 67%|██████▋ | 2102/3125 [42:08<41:25, 2.43s/it] 67%|██████▋ | 2103/3125 [42:11<41:06, 2.41s/it] 67%|██████▋ | 2104/3125 [42:13<43:06, 2.53s/it] 67%|██████▋ | 2105/3125 [42:16<41:21, 2.43s/it] 67%|██████▋ | 2106/3125 [42:18<39:42, 2.34s/it] 67%|██████▋ | 2107/3125 [42:21<44:15, 2.61s/it] 67%|██████▋ | 2108/3125 [42:23<43:41, 2.58s/it] 67%|██████▋ | 2109/3125 [42:26<43:27, 2.57s/it] 68%|██████▊ | 2110/3125 [42:29<43:19, 2.56s/it] {'loss': '1.133', 'grad_norm': '5.116', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04346', 'train/loss_cot': '1.103', 'epoch': '3.376'} 68%|██████▊ | 2110/3125 [42:29<43:19, 2.56s/it] 68%|██████▊ | 2111/3125 [42:31<42:22, 2.51s/it] 68%|██████▊ | 2112/3125 [42:33<41:31, 2.46s/it] 68%|██████▊ | 2113/3125 [42:36<42:16, 2.51s/it] 68%|██████▊ | 2114/3125 [42:38<40:32, 2.41s/it] 68%|██████▊ | 2115/3125 [42:40<39:55, 2.37s/it] 68%|██████▊ | 2116/3125 [42:43<39:37, 2.36s/it] 68%|██████▊ | 2117/3125 [42:45<40:24, 2.41s/it] 68%|██████▊ | 2118/3125 [42:48<43:58, 2.62s/it] 68%|██████▊ | 2119/3125 [42:52<47:06, 2.81s/it] 68%|██████▊ | 2120/3125 [42:54<45:00, 2.69s/it] {'loss': '1.034', 'grad_norm': '6.444', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006439', 'train/loss_cot': '1.103', 'epoch': '3.392'} 68%|██████▊ | 2120/3125 [42:54<45:00, 2.69s/it] 68%|██████▊ | 2121/3125 [42:57<44:17, 2.65s/it] 68%|██████▊ | 2122/3125 [42:59<42:44, 2.56s/it] 68%|██████▊ | 2123/3125 [43:01<41:04, 2.46s/it] 68%|██████▊ | 2124/3125 [43:03<40:22, 2.42s/it] 68%|██████▊ | 2125/3125 [43:06<41:10, 2.47s/it] 68%|██████▊ | 2126/3125 [43:09<41:18, 2.48s/it] 68%|██████▊ | 2127/3125 [43:12<44:42, 2.69s/it] 68%|██████▊ | 2128/3125 [43:15<47:01, 2.83s/it] 68%|██████▊ | 2129/3125 [43:17<43:17, 2.61s/it] 68%|██████▊ | 2130/3125 [43:20<44:08, 2.66s/it] {'loss': '1.054', 'grad_norm': '6.24', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003464', 'train/loss_cot': '1.269', 'epoch': '3.408'} 68%|██████▊ | 2130/3125 [43:20<44:08, 2.66s/it] 68%|██████▊ | 2131/3125 [43:23<44:59, 2.72s/it] 68%|██████▊ | 2132/3125 [43:25<43:03, 2.60s/it] 68%|██████▊ | 2133/3125 [43:27<42:50, 2.59s/it] 68%|██████▊ | 2134/3125 [43:30<41:21, 2.50s/it] 68%|██████▊ | 2135/3125 [43:33<43:39, 2.65s/it] 68%|██████▊ | 2136/3125 [43:35<41:32, 2.52s/it] 68%|██████▊ | 2137/3125 [43:37<40:09, 2.44s/it] 68%|██████▊ | 2138/3125 [43:40<40:28, 2.46s/it] 68%|██████▊ | 2139/3125 [43:42<39:43, 2.42s/it] 68%|██████▊ | 2140/3125 [43:44<39:37, 2.41s/it] {'loss': '1.068', 'grad_norm': '6.028', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005157', 'train/loss_cot': '0.8323', 'epoch': '3.424'} 68%|██████▊ | 2140/3125 [43:45<39:37, 2.41s/it] 69%|██████▊ | 2141/3125 [43:47<40:03, 2.44s/it] 69%|██████▊ | 2142/3125 [43:49<40:05, 2.45s/it] 69%|██████▊ | 2143/3125 [43:52<40:00, 2.44s/it] 69%|██████▊ | 2144/3125 [43:55<40:58, 2.51s/it] 69%|██████▊ | 2145/3125 [43:57<41:01, 2.51s/it] 69%|██████▊ | 2146/3125 [43:59<40:30, 2.48s/it] 69%|██████▊ | 2147/3125 [44:02<41:53, 2.57s/it] 69%|██████▊ | 2148/3125 [44:06<50:02, 3.07s/it] 69%|██████▉ | 2149/3125 [44:09<49:18, 3.03s/it] 69%|██████▉ | 2150/3125 [44:12<46:20, 2.85s/it] {'loss': '1.06', 'grad_norm': '13.43', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01733', 'train/loss_cot': '1.293', 'epoch': '3.44'} 69%|██████▉ | 2150/3125 [44:12<46:20, 2.85s/it] 69%|██████▉ | 2151/3125 [44:14<44:30, 2.74s/it] 69%|██████▉ | 2152/3125 [44:17<42:17, 2.61s/it] 69%|██████▉ | 2153/3125 [44:19<42:08, 2.60s/it] 69%|██████▉ | 2154/3125 [44:22<42:10, 2.61s/it] 69%|██████▉ | 2155/3125 [44:24<41:00, 2.54s/it] 69%|██████▉ | 2156/3125 [44:26<39:36, 2.45s/it] 69%|██████▉ | 2157/3125 [44:29<39:22, 2.44s/it] 69%|██████▉ | 2158/3125 [44:32<41:15, 2.56s/it] 69%|██████▉ | 2159/3125 [44:34<39:45, 2.47s/it] 69%|██████▉ | 2160/3125 [44:36<38:23, 2.39s/it] {'loss': '0.9993', 'grad_norm': '5.85', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08105', 'train/loss_cot': '0.9752', 'epoch': '3.456'} 69%|██████▉ | 2160/3125 [44:36<38:23, 2.39s/it] 69%|██████▉ | 2161/3125 [44:39<39:18, 2.45s/it] 69%|██████▉ | 2162/3125 [44:41<38:30, 2.40s/it] 69%|██████▉ | 2163/3125 [44:43<38:37, 2.41s/it] 69%|██████▉ | 2164/3125 [44:46<37:51, 2.36s/it] 69%|██████▉ | 2165/3125 [44:48<38:27, 2.40s/it] 69%|██████▉ | 2166/3125 [44:50<37:24, 2.34s/it] 69%|██████▉ | 2167/3125 [44:53<37:32, 2.35s/it] 69%|██████▉ | 2168/3125 [44:55<37:30, 2.35s/it] 69%|██████▉ | 2169/3125 [44:58<37:45, 2.37s/it] 69%|██████▉ | 2170/3125 [45:00<39:01, 2.45s/it] {'loss': '1.011', 'grad_norm': '5.577', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01459', 'train/loss_cot': '1.048', 'epoch': '3.472'} 69%|██████▉ | 2170/3125 [45:00<39:01, 2.45s/it] 69%|██████▉ | 2171/3125 [45:03<39:42, 2.50s/it] 70%|██████▉ | 2172/3125 [45:05<39:42, 2.50s/it] 70%|██████▉ | 2173/3125 [45:08<38:40, 2.44s/it] 70%|██████▉ | 2174/3125 [45:10<39:38, 2.50s/it] 70%|██████▉ | 2175/3125 [45:12<38:25, 2.43s/it] 70%|██████▉ | 2176/3125 [45:15<38:05, 2.41s/it] 70%|██████▉ | 2177/3125 [45:17<38:22, 2.43s/it] 70%|██████▉ | 2178/3125 [45:21<43:29, 2.76s/it] 70%|██████▉ | 2179/3125 [45:23<40:47, 2.59s/it] 70%|██████▉ | 2180/3125 [45:25<39:54, 2.53s/it] {'loss': '1.047', 'grad_norm': '6.482', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1187', 'train/loss_cot': '0.7171', 'epoch': '3.488'} 70%|██████▉ | 2180/3125 [45:26<39:54, 2.53s/it] 70%|██████▉ | 2181/3125 [45:28<39:44, 2.53s/it] 70%|██████▉ | 2182/3125 [45:31<41:02, 2.61s/it] 70%|██████▉ | 2183/3125 [45:33<39:52, 2.54s/it] 70%|██████▉ | 2184/3125 [45:35<38:45, 2.47s/it] 70%|██████▉ | 2185/3125 [45:38<38:43, 2.47s/it] 70%|██████▉ | 2186/3125 [45:40<38:57, 2.49s/it] 70%|██████▉ | 2187/3125 [45:43<38:18, 2.45s/it] 70%|███████ | 2188/3125 [45:46<40:03, 2.57s/it] 70%|███████ | 2189/3125 [45:48<40:09, 2.57s/it] 70%|███████ | 2190/3125 [45:51<38:59, 2.50s/it] {'loss': '1.111', 'grad_norm': '9.047', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01123', 'train/loss_cot': '1.043', 'epoch': '3.504'} 70%|███████ | 2190/3125 [45:51<38:59, 2.50s/it] 70%|███████ | 2191/3125 [45:53<39:52, 2.56s/it] 70%|███████ | 2192/3125 [45:56<38:58, 2.51s/it] 70%|███████ | 2193/3125 [45:58<38:51, 2.50s/it] 70%|███████ | 2194/3125 [46:00<37:38, 2.43s/it] 70%|███████ | 2195/3125 [46:03<36:47, 2.37s/it] 70%|███████ | 2196/3125 [46:05<36:54, 2.38s/it] 70%|███████ | 2197/3125 [46:08<37:33, 2.43s/it] 70%|███████ | 2198/3125 [46:10<36:23, 2.36s/it] 70%|███████ | 2199/3125 [46:12<35:06, 2.28s/it] 70%|███████ | 2200/3125 [46:15<37:32, 2.43s/it] {'loss': '1.09', 'grad_norm': '5.561', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03076', 'train/loss_cot': '1.155', 'epoch': '3.52'} 70%|███████ | 2200/3125 [46:15<37:32, 2.43s/it] 70%|███████ | 2201/3125 [46:33<1:51:18, 7.23s/it] 70%|███████ | 2202/3125 [46:36<1:30:51, 5.91s/it] 70%|███████ | 2203/3125 [46:38<1:14:02, 4.82s/it] 71%|███████ | 2204/3125 [46:41<1:02:49, 4.09s/it] 71%|███████ | 2205/3125 [46:43<54:33, 3.56s/it] 71%|███████ | 2206/3125 [46:46<50:31, 3.30s/it] 71%|███████ | 2207/3125 [46:48<45:51, 3.00s/it] 71%|███████ | 2208/3125 [46:51<45:51, 3.00s/it] 71%|███████ | 2209/3125 [46:53<42:00, 2.75s/it] 71%|███████ | 2210/3125 [46:56<40:59, 2.69s/it] {'loss': '1.011', 'grad_norm': '5.26', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01465', 'train/loss_cot': '0.8401', 'epoch': '3.536'} 71%|███████ | 2210/3125 [46:56<40:59, 2.69s/it] 71%|███████ | 2211/3125 [46:58<40:08, 2.63s/it] 71%|███████ | 2212/3125 [47:01<40:02, 2.63s/it] 71%|███████ | 2213/3125 [47:03<38:57, 2.56s/it] 71%|███████ | 2214/3125 [47:05<37:26, 2.47s/it] 71%|███████ | 2215/3125 [47:08<38:46, 2.56s/it] 71%|███████ | 2216/3125 [47:11<39:07, 2.58s/it] 71%|███████ | 2217/3125 [47:13<37:28, 2.48s/it] 71%|███████ | 2218/3125 [47:15<35:55, 2.38s/it] 71%|███████ | 2219/3125 [47:17<35:02, 2.32s/it] 71%|███████ | 2220/3125 [47:20<35:18, 2.34s/it] {'loss': '1.05', 'grad_norm': '6.493', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03394', 'train/loss_cot': '1.138', 'epoch': '3.552'} 71%|███████ | 2220/3125 [47:20<35:18, 2.34s/it] 71%|███████ | 2221/3125 [47:22<36:47, 2.44s/it] 71%|███████ | 2222/3125 [47:25<36:04, 2.40s/it] 71%|███████ | 2223/3125 [47:27<35:38, 2.37s/it] 71%|███████ | 2224/3125 [47:30<36:44, 2.45s/it] 71%|███████ | 2225/3125 [47:32<37:09, 2.48s/it] 71%|███████ | 2226/3125 [47:35<36:22, 2.43s/it] 71%|███████▏ | 2227/3125 [47:37<35:46, 2.39s/it] 71%|███████▏ | 2228/3125 [47:39<35:43, 2.39s/it] 71%|███████▏ | 2229/3125 [47:42<35:23, 2.37s/it] 71%|███████▏ | 2230/3125 [47:44<36:01, 2.42s/it] {'loss': '1.033', 'grad_norm': '6.419', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02576', 'train/loss_cot': '0.9247', 'epoch': '3.568'} 71%|███████▏ | 2230/3125 [47:44<36:01, 2.42s/it] 71%|███████▏ | 2231/3125 [47:47<36:45, 2.47s/it] 71%|███████▏ | 2232/3125 [47:49<35:26, 2.38s/it] 71%|███████▏ | 2233/3125 [47:51<34:51, 2.34s/it] 71%|███████▏ | 2234/3125 [47:54<36:10, 2.44s/it] 72%|███████▏ | 2235/3125 [47:56<34:23, 2.32s/it] 72%|███████▏ | 2236/3125 [47:58<35:22, 2.39s/it] 72%|███████▏ | 2237/3125 [48:01<36:01, 2.43s/it] 72%|███████▏ | 2238/3125 [48:03<35:49, 2.42s/it] 72%|███████▏ | 2239/3125 [48:06<35:22, 2.40s/it] 72%|███████▏ | 2240/3125 [48:08<36:40, 2.49s/it] {'loss': '1.045', 'grad_norm': '8.428', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007141', 'train/loss_cot': '1.278', 'epoch': '3.584'} 72%|███████▏ | 2240/3125 [48:09<36:40, 2.49s/it] 72%|███████▏ | 2241/3125 [48:11<35:51, 2.43s/it] 72%|███████▏ | 2242/3125 [48:13<35:41, 2.43s/it] 72%|███████▏ | 2243/3125 [48:15<34:38, 2.36s/it] 72%|███████▏ | 2244/3125 [48:17<34:16, 2.33s/it] 72%|███████▏ | 2245/3125 [48:20<34:33, 2.36s/it] 72%|███████▏ | 2246/3125 [48:22<35:19, 2.41s/it] 72%|███████▏ | 2247/3125 [48:25<34:45, 2.38s/it] 72%|███████▏ | 2248/3125 [48:27<34:40, 2.37s/it] 72%|███████▏ | 2249/3125 [48:30<35:27, 2.43s/it] 72%|███████▏ | 2250/3125 [48:32<34:48, 2.39s/it] {'loss': '1.046', 'grad_norm': '5.948', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006409', 'train/loss_cot': '0.889', 'epoch': '3.6'} 72%|███████▏ | 2250/3125 [48:32<34:48, 2.39s/it] 72%|███████▏ | 2251/3125 [48:34<34:26, 2.36s/it] 72%|███████▏ | 2252/3125 [48:36<33:43, 2.32s/it] 72%|███████▏ | 2253/3125 [48:39<34:54, 2.40s/it] 72%|███████▏ | 2254/3125 [48:41<34:49, 2.40s/it] 72%|███████▏ | 2255/3125 [48:44<36:50, 2.54s/it] 72%|███████▏ | 2256/3125 [48:47<36:34, 2.53s/it] 72%|███████▏ | 2257/3125 [48:50<37:55, 2.62s/it] 72%|███████▏ | 2258/3125 [48:52<37:05, 2.57s/it] 72%|███████▏ | 2259/3125 [48:54<36:18, 2.52s/it] 72%|███████▏ | 2260/3125 [48:57<35:23, 2.45s/it] {'loss': '1.039', 'grad_norm': '5.786', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00119', 'train/loss_cot': '1.05', 'epoch': '3.616'} 72%|███████▏ | 2260/3125 [48:57<35:23, 2.45s/it] 72%|███████▏ | 2261/3125 [48:59<36:13, 2.52s/it] 72%|███████▏ | 2262/3125 [49:02<36:46, 2.56s/it] 72%|███████▏ | 2263/3125 [49:04<35:22, 2.46s/it] 72%|███████▏ | 2264/3125 [49:07<35:19, 2.46s/it] 72%|███████▏ | 2265/3125 [49:09<34:44, 2.42s/it] 73%|███████▎ | 2266/3125 [49:12<35:14, 2.46s/it] 73%|███████▎ | 2267/3125 [49:14<35:50, 2.51s/it] 73%|███████▎ | 2268/3125 [49:17<35:53, 2.51s/it] 73%|███████▎ | 2269/3125 [49:19<35:26, 2.48s/it] 73%|███████▎ | 2270/3125 [49:22<35:21, 2.48s/it] {'loss': '1.128', 'grad_norm': '7.582', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04004', 'train/loss_cot': '1.083', 'epoch': '3.632'} 73%|███████▎ | 2270/3125 [49:22<35:21, 2.48s/it] 73%|███████▎ | 2271/3125 [49:24<34:24, 2.42s/it] 73%|███████▎ | 2272/3125 [49:26<34:10, 2.40s/it] 73%|███████▎ | 2273/3125 [49:29<34:43, 2.45s/it] 73%|███████▎ | 2274/3125 [49:31<33:24, 2.36s/it] 73%|███████▎ | 2275/3125 [49:33<33:42, 2.38s/it] 73%|███████▎ | 2276/3125 [49:36<35:35, 2.52s/it] 73%|███████▎ | 2277/3125 [49:39<35:18, 2.50s/it] 73%|███████▎ | 2278/3125 [49:41<34:31, 2.45s/it] 73%|███████▎ | 2279/3125 [49:44<34:25, 2.44s/it] 73%|███████▎ | 2280/3125 [49:46<33:55, 2.41s/it] {'loss': '0.9878', 'grad_norm': '5.992', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05518', 'train/loss_cot': '0.7204', 'epoch': '3.648'} 73%|███████▎ | 2280/3125 [49:46<33:55, 2.41s/it] 73%|███████▎ | 2281/3125 [49:48<34:07, 2.43s/it] 73%|███████▎ | 2282/3125 [49:51<36:50, 2.62s/it] 73%|███████▎ | 2283/3125 [49:54<35:58, 2.56s/it] 73%|███████▎ | 2284/3125 [49:56<34:26, 2.46s/it] 73%|███████▎ | 2285/3125 [49:59<34:28, 2.46s/it] 73%|███████▎ | 2286/3125 [50:01<33:07, 2.37s/it] 73%|███████▎ | 2287/3125 [50:03<32:57, 2.36s/it] 73%|███████▎ | 2288/3125 [50:05<32:46, 2.35s/it] 73%|███████▎ | 2289/3125 [50:08<35:04, 2.52s/it] 73%|███████▎ | 2290/3125 [50:11<34:22, 2.47s/it] {'loss': '1.084', 'grad_norm': '5.384', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01117', 'train/loss_cot': '0.8849', 'epoch': '3.664'} 73%|███████▎ | 2290/3125 [50:11<34:22, 2.47s/it] 73%|███████▎ | 2291/3125 [50:13<35:12, 2.53s/it] 73%|███████▎ | 2292/3125 [50:16<33:58, 2.45s/it] 73%|███████▎ | 2293/3125 [50:18<32:51, 2.37s/it] 73%|███████▎ | 2294/3125 [50:20<32:57, 2.38s/it] 73%|███████▎ | 2295/3125 [50:23<33:29, 2.42s/it] 73%|███████▎ | 2296/3125 [50:25<33:24, 2.42s/it] 74%|███████▎ | 2297/3125 [50:27<32:57, 2.39s/it] 74%|███████▎ | 2298/3125 [50:30<33:53, 2.46s/it] 74%|███████▎ | 2299/3125 [50:32<33:33, 2.44s/it] 74%|███████▎ | 2300/3125 [50:36<37:46, 2.75s/it] {'loss': '1.062', 'grad_norm': '6.839', 'learning_rate': '1e-05', 'train/loss_infonce': '0.002777', 'train/loss_cot': '1.33', 'epoch': '3.68'} 74%|███████▎ | 2300/3125 [50:36<37:46, 2.75s/it] 74%|███████▎ | 2301/3125 [50:39<38:43, 2.82s/it] 74%|███████▎ | 2302/3125 [50:41<36:40, 2.67s/it] 74%|███████▎ | 2303/3125 [50:45<43:10, 3.15s/it] 74%|███████▎ | 2304/3125 [50:48<42:00, 3.07s/it] 74%|███████▍ | 2305/3125 [50:51<39:18, 2.88s/it] 74%|███████▍ | 2306/3125 [50:53<37:47, 2.77s/it] 74%|███████▍ | 2307/3125 [50:56<39:16, 2.88s/it] 74%|███████▍ | 2308/3125 [51:00<40:12, 2.95s/it] 74%|███████▍ | 2309/3125 [51:02<37:23, 2.75s/it] 74%|███████▍ | 2310/3125 [51:04<35:46, 2.63s/it] {'loss': '1.041', 'grad_norm': '9.008', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003586', 'train/loss_cot': '1.135', 'epoch': '3.696'} 74%|███████▍ | 2310/3125 [51:05<35:46, 2.63s/it] 74%|███████▍ | 2311/3125 [51:06<34:25, 2.54s/it] 74%|███████▍ | 2312/3125 [51:09<34:14, 2.53s/it] 74%|███████▍ | 2313/3125 [51:11<33:28, 2.47s/it] 74%|███████▍ | 2314/3125 [51:14<33:13, 2.46s/it] 74%|███████▍ | 2315/3125 [51:17<36:11, 2.68s/it] 74%|███████▍ | 2316/3125 [51:19<34:41, 2.57s/it] 74%|███████▍ | 2317/3125 [51:21<32:47, 2.44s/it] 74%|███████▍ | 2318/3125 [51:24<32:39, 2.43s/it] 74%|███████▍ | 2319/3125 [51:26<33:33, 2.50s/it] 74%|███████▍ | 2320/3125 [51:29<34:58, 2.61s/it] {'loss': '1.041', 'grad_norm': '5.18', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005951', 'train/loss_cot': '1.068', 'epoch': '3.712'} 74%|███████▍ | 2320/3125 [51:30<34:58, 2.61s/it] 74%|███████▍ | 2321/3125 [51:32<34:32, 2.58s/it] 74%|███████▍ | 2322/3125 [51:35<35:11, 2.63s/it] 74%|███████▍ | 2323/3125 [51:38<37:28, 2.80s/it] 74%|███████▍ | 2324/3125 [51:40<35:58, 2.69s/it] 74%|███████▍ | 2325/3125 [51:43<35:15, 2.64s/it] 74%|███████▍ | 2326/3125 [51:45<34:14, 2.57s/it] 74%|███████▍ | 2327/3125 [51:48<34:19, 2.58s/it] 74%|███████▍ | 2328/3125 [51:50<33:39, 2.53s/it] 75%|███████▍ | 2329/3125 [51:53<36:00, 2.71s/it] 75%|███████▍ | 2330/3125 [51:56<35:34, 2.69s/it] {'loss': '1.099', 'grad_norm': '6.674', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02356', 'train/loss_cot': '1.222', 'epoch': '3.728'} 75%|███████▍ | 2330/3125 [51:56<35:34, 2.69s/it] 75%|███████▍ | 2331/3125 [51:59<35:19, 2.67s/it] 75%|███████▍ | 2332/3125 [52:01<34:44, 2.63s/it] 75%|███████▍ | 2333/3125 [52:03<33:35, 2.55s/it] 75%|███████▍ | 2334/3125 [52:06<32:39, 2.48s/it] 75%|███████▍ | 2335/3125 [52:08<31:56, 2.43s/it] 75%|███████▍ | 2336/3125 [52:10<31:26, 2.39s/it] 75%|███████▍ | 2337/3125 [52:13<32:27, 2.47s/it] 75%|███████▍ | 2338/3125 [52:15<31:47, 2.42s/it] 75%|███████▍ | 2339/3125 [52:18<31:49, 2.43s/it] 75%|███████▍ | 2340/3125 [52:20<31:22, 2.40s/it] {'loss': '1.068', 'grad_norm': '6.114', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05322', 'train/loss_cot': '1.074', 'epoch': '3.744'} 75%|███████▍ | 2340/3125 [52:20<31:22, 2.40s/it] 75%|███████▍ | 2341/3125 [52:23<32:53, 2.52s/it] 75%|███████▍ | 2342/3125 [52:26<33:14, 2.55s/it] 75%|███████▍ | 2343/3125 [52:28<34:21, 2.64s/it] 75%|███████▌ | 2344/3125 [52:31<33:51, 2.60s/it] 75%|███████▌ | 2345/3125 [52:34<34:25, 2.65s/it] 75%|███████▌ | 2346/3125 [52:36<32:42, 2.52s/it] 75%|███████▌ | 2347/3125 [52:38<32:17, 2.49s/it] 75%|███████▌ | 2348/3125 [52:41<34:29, 2.66s/it] 75%|███████▌ | 2349/3125 [52:44<34:45, 2.69s/it] 75%|███████▌ | 2350/3125 [52:47<35:07, 2.72s/it] {'loss': '1.073', 'grad_norm': '6.079', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0002089', 'train/loss_cot': '1.133', 'epoch': '3.76'} 75%|███████▌ | 2350/3125 [52:47<35:07, 2.72s/it] 75%|███████▌ | 2351/3125 [52:50<34:46, 2.70s/it] 75%|███████▌ | 2352/3125 [52:52<33:22, 2.59s/it] 75%|███████▌ | 2353/3125 [52:54<33:14, 2.58s/it] 75%|███████▌ | 2354/3125 [52:57<31:39, 2.46s/it] 75%|███████▌ | 2355/3125 [52:59<31:00, 2.42s/it] 75%|███████▌ | 2356/3125 [53:01<30:42, 2.40s/it] 75%|███████▌ | 2357/3125 [53:04<30:12, 2.36s/it] 75%|███████▌ | 2358/3125 [53:06<30:18, 2.37s/it] 75%|███████▌ | 2359/3125 [53:08<30:29, 2.39s/it] 76%|███████▌ | 2360/3125 [53:11<30:02, 2.36s/it] {'loss': '1.146', 'grad_norm': '5.674', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02771', 'train/loss_cot': '1.146', 'epoch': '3.776'} 76%|███████▌ | 2360/3125 [53:11<30:02, 2.36s/it] 76%|███████▌ | 2361/3125 [53:14<32:31, 2.55s/it] 76%|███████▌ | 2362/3125 [53:16<31:52, 2.51s/it] 76%|███████▌ | 2363/3125 [53:19<31:56, 2.52s/it] 76%|███████▌ | 2364/3125 [53:21<31:49, 2.51s/it] 76%|███████▌ | 2365/3125 [53:24<32:45, 2.59s/it] 76%|███████▌ | 2366/3125 [53:27<34:23, 2.72s/it] 76%|███████▌ | 2367/3125 [53:29<32:59, 2.61s/it] 76%|███████▌ | 2368/3125 [53:32<33:21, 2.64s/it] 76%|███████▌ | 2369/3125 [53:34<32:34, 2.59s/it] 76%|███████▌ | 2370/3125 [53:37<33:14, 2.64s/it] {'loss': '1.063', 'grad_norm': '7.171', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02161', 'train/loss_cot': '1.061', 'epoch': '3.792'} 76%|███████▌ | 2370/3125 [53:38<33:14, 2.64s/it] 76%|███████▌ | 2371/3125 [53:40<33:35, 2.67s/it] 76%|███████▌ | 2372/3125 [53:43<33:03, 2.63s/it] 76%|███████▌ | 2373/3125 [53:45<31:28, 2.51s/it] 76%|███████▌ | 2374/3125 [53:47<32:14, 2.58s/it] 76%|███████▌ | 2375/3125 [53:50<31:08, 2.49s/it] 76%|███████▌ | 2376/3125 [53:52<30:43, 2.46s/it] 76%|███████▌ | 2377/3125 [53:54<30:04, 2.41s/it] 76%|███████▌ | 2378/3125 [53:57<29:26, 2.36s/it] 76%|███████▌ | 2379/3125 [53:59<28:34, 2.30s/it] 76%|███████▌ | 2380/3125 [54:01<29:01, 2.34s/it] {'loss': '1.093', 'grad_norm': '5.157', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01544', 'train/loss_cot': '1.136', 'epoch': '3.808'} 76%|███████▌ | 2380/3125 [54:02<29:01, 2.34s/it] 76%|███████▌ | 2381/3125 [54:04<29:17, 2.36s/it] 76%|███████▌ | 2382/3125 [54:06<29:46, 2.40s/it] 76%|███████▋ | 2383/3125 [54:09<30:34, 2.47s/it] 76%|███████▋ | 2384/3125 [54:11<29:25, 2.38s/it] 76%|███████▋ | 2385/3125 [54:14<29:48, 2.42s/it] 76%|███████▋ | 2386/3125 [54:16<29:59, 2.44s/it] 76%|███████▋ | 2387/3125 [54:19<30:21, 2.47s/it] 76%|███████▋ | 2388/3125 [54:21<29:29, 2.40s/it] 76%|███████▋ | 2389/3125 [54:23<29:03, 2.37s/it] 76%|███████▋ | 2390/3125 [54:26<29:51, 2.44s/it] {'loss': '1.05', 'grad_norm': '5.776', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01544', 'train/loss_cot': '1.549', 'epoch': '3.824'} 76%|███████▋ | 2390/3125 [54:26<29:51, 2.44s/it] 77%|███████▋ | 2391/3125 [54:28<30:49, 2.52s/it] 77%|███████▋ | 2392/3125 [54:31<30:13, 2.47s/it] 77%|███████▋ | 2393/3125 [54:33<30:37, 2.51s/it] 77%|███████▋ | 2394/3125 [54:36<29:59, 2.46s/it] 77%|███████▋ | 2395/3125 [54:38<29:00, 2.38s/it] 77%|███████▋ | 2396/3125 [54:40<29:24, 2.42s/it] 77%|███████▋ | 2397/3125 [54:43<28:52, 2.38s/it] 77%|███████▋ | 2398/3125 [54:45<29:18, 2.42s/it] 77%|███████▋ | 2399/3125 [54:48<31:42, 2.62s/it] 77%|███████▋ | 2400/3125 [54:51<31:00, 2.57s/it] {'loss': '1.068', 'grad_norm': '5.884', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004852', 'train/loss_cot': '1.096', 'epoch': '3.84'} 77%|███████▋ | 2400/3125 [54:51<31:00, 2.57s/it] 77%|███████▋ | 2401/3125 [55:09<1:27:41, 7.27s/it] 77%|███████▋ | 2402/3125 [55:11<1:10:23, 5.84s/it] 77%|███████▋ | 2403/3125 [55:14<58:44, 4.88s/it] 77%|███████▋ | 2404/3125 [55:16<49:26, 4.11s/it] 77%|███████▋ | 2405/3125 [55:19<42:17, 3.52s/it] 77%|███████▋ | 2406/3125 [55:21<37:54, 3.16s/it] 77%|███████▋ | 2407/3125 [55:23<35:05, 2.93s/it] 77%|███████▋ | 2408/3125 [55:26<33:23, 2.79s/it] 77%|███████▋ | 2409/3125 [55:28<31:28, 2.64s/it] 77%|███████▋ | 2410/3125 [55:31<30:57, 2.60s/it] {'loss': '1.06', 'grad_norm': '8.105', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05298', 'train/loss_cot': '1.12', 'epoch': '3.856'} 77%|███████▋ | 2410/3125 [55:31<30:57, 2.60s/it] 77%|███████▋ | 2411/3125 [55:34<32:21, 2.72s/it] 77%|███████▋ | 2412/3125 [55:36<30:36, 2.58s/it] 77%|███████▋ | 2413/3125 [55:38<29:26, 2.48s/it] 77%|███████▋ | 2414/3125 [55:41<30:39, 2.59s/it] 77%|███████▋ | 2415/3125 [55:44<31:09, 2.63s/it] 77%|███████▋ | 2416/3125 [55:46<29:38, 2.51s/it] 77%|███████▋ | 2417/3125 [55:48<28:27, 2.41s/it] 77%|███████▋ | 2418/3125 [55:51<29:47, 2.53s/it] 77%|███████▋ | 2419/3125 [55:53<29:02, 2.47s/it] 77%|███████▋ | 2420/3125 [55:56<30:03, 2.56s/it] {'loss': '1.044', 'grad_norm': '5.819', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03149', 'train/loss_cot': '0.7594', 'epoch': '3.872'} 77%|███████▋ | 2420/3125 [55:56<30:03, 2.56s/it] 77%|███████▋ | 2421/3125 [55:58<29:24, 2.51s/it] 78%|███████▊ | 2422/3125 [56:01<29:44, 2.54s/it] 78%|███████▊ | 2423/3125 [56:03<28:31, 2.44s/it] 78%|███████▊ | 2424/3125 [56:06<29:18, 2.51s/it] 78%|███████▊ | 2425/3125 [56:08<28:00, 2.40s/it] 78%|███████▊ | 2426/3125 [56:10<27:39, 2.37s/it] 78%|███████▊ | 2427/3125 [56:13<27:15, 2.34s/it] 78%|███████▊ | 2428/3125 [56:15<28:01, 2.41s/it] 78%|███████▊ | 2429/3125 [56:18<29:56, 2.58s/it] 78%|███████▊ | 2430/3125 [56:20<28:49, 2.49s/it] {'loss': '1.052', 'grad_norm': '6.574', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1299', 'train/loss_cot': '1.008', 'epoch': '3.888'} 78%|███████▊ | 2430/3125 [56:21<28:49, 2.49s/it] 78%|███████▊ | 2431/3125 [56:23<28:58, 2.50s/it] 78%|███████▊ | 2432/3125 [56:25<28:25, 2.46s/it] 78%|███████▊ | 2433/3125 [56:28<27:47, 2.41s/it] 78%|███████▊ | 2434/3125 [56:30<27:57, 2.43s/it] 78%|███████▊ | 2435/3125 [56:33<28:34, 2.48s/it] 78%|███████▊ | 2436/3125 [56:35<28:24, 2.47s/it] 78%|███████▊ | 2437/3125 [56:38<28:17, 2.47s/it] 78%|███████▊ | 2438/3125 [56:40<29:02, 2.54s/it] 78%|███████▊ | 2439/3125 [56:43<28:18, 2.48s/it] 78%|███████▊ | 2440/3125 [56:45<28:23, 2.49s/it] {'loss': '1.071', 'grad_norm': '7.519', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1436', 'train/loss_cot': '1.373', 'epoch': '3.904'} 78%|███████▊ | 2440/3125 [56:45<28:23, 2.49s/it] 78%|███████▊ | 2441/3125 [56:47<27:53, 2.45s/it] 78%|███████▊ | 2442/3125 [56:50<26:56, 2.37s/it] 78%|███████▊ | 2443/3125 [56:52<28:20, 2.49s/it] 78%|███████▊ | 2444/3125 [56:55<29:43, 2.62s/it] 78%|███████▊ | 2445/3125 [56:58<28:38, 2.53s/it] 78%|███████▊ | 2446/3125 [57:00<28:02, 2.48s/it] 78%|███████▊ | 2447/3125 [57:03<28:18, 2.50s/it] 78%|███████▊ | 2448/3125 [57:06<30:24, 2.69s/it] 78%|███████▊ | 2449/3125 [57:08<28:35, 2.54s/it] 78%|███████▊ | 2450/3125 [57:10<28:28, 2.53s/it] {'loss': '1.012', 'grad_norm': '5.586', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009583', 'train/loss_cot': '0.9804', 'epoch': '3.92'} 78%|███████▊ | 2450/3125 [57:11<28:28, 2.53s/it] 78%|███████▊ | 2451/3125 [57:13<28:07, 2.50s/it] 78%|███████▊ | 2452/3125 [57:15<27:12, 2.43s/it] 78%|███████▊ | 2453/3125 [57:18<28:04, 2.51s/it] 79%|███████▊ | 2454/3125 [57:20<27:43, 2.48s/it] 79%|███████▊ | 2455/3125 [57:23<28:16, 2.53s/it] 79%|███████▊ | 2456/3125 [57:26<29:39, 2.66s/it] 79%|███████▊ | 2457/3125 [57:28<29:24, 2.64s/it] 79%|███████▊ | 2458/3125 [57:31<28:25, 2.56s/it] 79%|███████▊ | 2459/3125 [57:33<28:07, 2.53s/it] 79%|███████▊ | 2460/3125 [57:36<28:48, 2.60s/it] {'loss': '1.112', 'grad_norm': '5.726', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07129', 'train/loss_cot': '0.8806', 'epoch': '3.936'} 79%|███████▊ | 2460/3125 [57:36<28:48, 2.60s/it] 79%|███████▉ | 2461/3125 [57:39<29:35, 2.67s/it] 79%|███████▉ | 2462/3125 [57:42<31:10, 2.82s/it] 79%|███████▉ | 2463/3125 [57:44<29:49, 2.70s/it] 79%|███████▉ | 2464/3125 [57:47<29:28, 2.68s/it] 79%|███████▉ | 2465/3125 [57:49<28:21, 2.58s/it] 79%|███████▉ | 2466/3125 [57:52<27:22, 2.49s/it] 79%|███████▉ | 2467/3125 [57:54<27:11, 2.48s/it] 79%|███████▉ | 2468/3125 [57:56<26:34, 2.43s/it] 79%|███████▉ | 2469/3125 [57:59<26:26, 2.42s/it] 79%|███████▉ | 2470/3125 [58:01<26:09, 2.40s/it] {'loss': '1.003', 'grad_norm': '5.478', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01917', 'train/loss_cot': '1.118', 'epoch': '3.952'} 79%|███████▉ | 2470/3125 [58:01<26:09, 2.40s/it] 79%|███████▉ | 2471/3125 [58:04<27:16, 2.50s/it] 79%|███████▉ | 2472/3125 [58:06<27:16, 2.51s/it] 79%|███████▉ | 2473/3125 [58:09<26:19, 2.42s/it] 79%|███████▉ | 2474/3125 [58:11<26:02, 2.40s/it] 79%|███████▉ | 2475/3125 [58:14<29:08, 2.69s/it] 79%|███████▉ | 2476/3125 [58:17<28:35, 2.64s/it] 79%|███████▉ | 2477/3125 [58:19<27:56, 2.59s/it] 79%|███████▉ | 2478/3125 [58:22<27:03, 2.51s/it] 79%|███████▉ | 2479/3125 [58:25<29:49, 2.77s/it] 79%|███████▉ | 2480/3125 [58:28<28:54, 2.69s/it] {'loss': '1.026', 'grad_norm': '6.181', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006287', 'train/loss_cot': '1.017', 'epoch': '3.968'} 79%|███████▉ | 2480/3125 [58:28<28:54, 2.69s/it] 79%|███████▉ | 2481/3125 [58:30<28:32, 2.66s/it] 79%|███████▉ | 2482/3125 [58:32<27:27, 2.56s/it] 79%|███████▉ | 2483/3125 [58:35<27:38, 2.58s/it] 79%|███████▉ | 2484/3125 [58:38<26:59, 2.53s/it] 80%|███████▉ | 2485/3125 [58:40<26:29, 2.48s/it] 80%|███████▉ | 2486/3125 [58:42<26:20, 2.47s/it] 80%|███████▉ | 2487/3125 [58:45<27:18, 2.57s/it] 80%|███████▉ | 2488/3125 [58:48<28:20, 2.67s/it] 80%|███████▉ | 2489/3125 [58:52<32:23, 3.06s/it] 80%|███████▉ | 2490/3125 [58:55<30:55, 2.92s/it] {'loss': '1.086', 'grad_norm': '7.063', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0009918', 'train/loss_cot': '0.9763', 'epoch': '3.984'} 80%|███████▉ | 2490/3125 [58:55<30:55, 2.92s/it] 80%|███████▉ | 2491/3125 [58:57<29:23, 2.78s/it] 80%|███████▉ | 2492/3125 [58:59<27:31, 2.61s/it] 80%|███████▉ | 2493/3125 [59:02<26:44, 2.54s/it] 80%|███████▉ | 2494/3125 [59:05<27:56, 2.66s/it] 80%|███████▉ | 2495/3125 [59:07<27:48, 2.65s/it] 80%|███████▉ | 2496/3125 [59:10<28:10, 2.69s/it] 80%|███████▉ | 2497/3125 [59:13<29:30, 2.82s/it] 80%|███████▉ | 2498/3125 [59:15<27:45, 2.66s/it] 80%|███████▉ | 2499/3125 [59:18<27:04, 2.59s/it] 80%|████████ | 2500/3125 [59:19<21:36, 2.08s/it] {'loss': '1.039', 'grad_norm': '5.045', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0144', 'train/loss_cot': '0.7417', 'epoch': '4'} 80%|████████ | 2500/3125 [59:19<21:36, 2.08s/it] 80%|████████ | 2501/3125 [59:23<27:53, 2.68s/it] 80%|████████ | 2502/3125 [59:25<26:59, 2.60s/it] 80%|████████ | 2503/3125 [59:27<25:13, 2.43s/it] 80%|████████ | 2504/3125 [59:30<25:48, 2.49s/it] 80%|████████ | 2505/3125 [59:32<25:28, 2.47s/it] 80%|████████ | 2506/3125 [59:35<26:32, 2.57s/it] 80%|████████ | 2507/3125 [59:38<26:14, 2.55s/it] 80%|████████ | 2508/3125 [59:40<25:07, 2.44s/it] 80%|████████ | 2509/3125 [59:42<24:12, 2.36s/it] 80%|████████ | 2510/3125 [59:44<23:51, 2.33s/it] {'loss': '0.9051', 'grad_norm': '5.313', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009888', 'train/loss_cot': '0.9877', 'epoch': '4.016'} 80%|████████ | 2510/3125 [59:45<23:51, 2.33s/it] 80%|████████ | 2511/3125 [59:46<23:29, 2.30s/it] 80%|████████ | 2512/3125 [59:49<23:15, 2.28s/it] 80%|████████ | 2513/3125 [59:51<23:22, 2.29s/it] 80%|████████ | 2514/3125 [59:53<23:03, 2.26s/it] 80%|████████ | 2515/3125 [59:56<23:15, 2.29s/it] 81%|████████ | 2516/3125 [59:58<23:30, 2.32s/it] 81%|████████ | 2517/3125 [1:00:00<23:31, 2.32s/it] 81%|████████ | 2518/3125 [1:00:02<22:47, 2.25s/it] 81%|████████ | 2519/3125 [1:00:05<22:44, 2.25s/it] 81%|████████ | 2520/3125 [1:00:07<22:58, 2.28s/it] {'loss': '0.926', 'grad_norm': '8.215', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01526', 'train/loss_cot': '0.8317', 'epoch': '4.032'} 81%|████████ | 2520/3125 [1:00:07<22:58, 2.28s/it] 81%|████████ | 2521/3125 [1:00:09<22:53, 2.27s/it] 81%|████████ | 2522/3125 [1:00:12<23:15, 2.31s/it] 81%|████████ | 2523/3125 [1:00:14<23:28, 2.34s/it] 81%|████████ | 2524/3125 [1:00:16<22:56, 2.29s/it] 81%|████████ | 2525/3125 [1:00:19<23:07, 2.31s/it] 81%|████████ | 2526/3125 [1:00:21<23:11, 2.32s/it] 81%|████████ | 2527/3125 [1:00:23<22:56, 2.30s/it] 81%|████████ | 2528/3125 [1:00:26<23:09, 2.33s/it] 81%|████████ | 2529/3125 [1:00:28<23:40, 2.38s/it] 81%|████████ | 2530/3125 [1:00:31<23:55, 2.41s/it] {'loss': '0.9035', 'grad_norm': '6.4', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001251', 'train/loss_cot': '0.8418', 'epoch': '4.048'} 81%|████████ | 2530/3125 [1:00:31<23:55, 2.41s/it] 81%|████████ | 2531/3125 [1:00:33<24:31, 2.48s/it] 81%|████████ | 2532/3125 [1:00:35<23:48, 2.41s/it] 81%|████████ | 2533/3125 [1:00:38<23:13, 2.35s/it] 81%|████████ | 2534/3125 [1:00:40<23:40, 2.40s/it] 81%|████████ | 2535/3125 [1:00:42<23:23, 2.38s/it] 81%|████████ | 2536/3125 [1:00:45<24:46, 2.52s/it] 81%|████████ | 2537/3125 [1:00:48<24:11, 2.47s/it] 81%|████████ | 2538/3125 [1:00:50<23:31, 2.41s/it] 81%|████████ | 2539/3125 [1:00:52<23:13, 2.38s/it] 81%|████████▏ | 2540/3125 [1:00:57<28:44, 2.95s/it] {'loss': '0.915', 'grad_norm': '5.605', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02515', 'train/loss_cot': '0.7679', 'epoch': '4.064'} 81%|████████▏ | 2540/3125 [1:00:57<28:44, 2.95s/it] 81%|████████▏ | 2541/3125 [1:00:59<26:50, 2.76s/it] 81%|████████▏ | 2542/3125 [1:01:01<25:38, 2.64s/it] 81%|████████▏ | 2543/3125 [1:01:04<25:21, 2.61s/it] 81%|████████▏ | 2544/3125 [1:01:06<23:36, 2.44s/it] 81%|████████▏ | 2545/3125 [1:01:08<22:37, 2.34s/it] 81%|████████▏ | 2546/3125 [1:01:10<22:25, 2.32s/it] 82%|████████▏ | 2547/3125 [1:01:12<22:01, 2.29s/it] 82%|████████▏ | 2548/3125 [1:01:15<21:44, 2.26s/it] 82%|████████▏ | 2549/3125 [1:01:17<21:38, 2.25s/it] 82%|████████▏ | 2550/3125 [1:01:19<21:27, 2.24s/it] {'loss': '0.918', 'grad_norm': '5.706', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05029', 'train/loss_cot': '0.6901', 'epoch': '4.08'} 82%|████████▏ | 2550/3125 [1:01:19<21:27, 2.24s/it] 82%|████████▏ | 2551/3125 [1:01:22<22:37, 2.37s/it] 82%|████████▏ | 2552/3125 [1:01:24<22:10, 2.32s/it] 82%|████████▏ | 2553/3125 [1:01:26<22:23, 2.35s/it] 82%|████████▏ | 2554/3125 [1:01:28<21:49, 2.29s/it] 82%|████████▏ | 2555/3125 [1:01:31<22:00, 2.32s/it] 82%|████████▏ | 2556/3125 [1:01:33<21:42, 2.29s/it] 82%|████████▏ | 2557/3125 [1:01:35<21:42, 2.29s/it] 82%|████████▏ | 2558/3125 [1:01:38<22:19, 2.36s/it] 82%|████████▏ | 2559/3125 [1:01:40<22:31, 2.39s/it] 82%|████████▏ | 2560/3125 [1:01:43<23:10, 2.46s/it] {'loss': '0.8848', 'grad_norm': '5.381', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007385', 'train/loss_cot': '0.8134', 'epoch': '4.096'} 82%|████████▏ | 2560/3125 [1:01:43<23:10, 2.46s/it] 82%|████████▏ | 2561/3125 [1:01:45<23:06, 2.46s/it] 82%|████████▏ | 2562/3125 [1:01:48<22:32, 2.40s/it] 82%|████████▏ | 2563/3125 [1:01:50<22:29, 2.40s/it] 82%|████████▏ | 2564/3125 [1:01:53<24:31, 2.62s/it] 82%|████████▏ | 2565/3125 [1:01:56<24:18, 2.61s/it] 82%|████████▏ | 2566/3125 [1:01:58<23:16, 2.50s/it] 82%|████████▏ | 2567/3125 [1:02:00<22:35, 2.43s/it] 82%|████████▏ | 2568/3125 [1:02:03<21:51, 2.35s/it] 82%|████████▏ | 2569/3125 [1:02:05<22:28, 2.42s/it] 82%|████████▏ | 2570/3125 [1:02:07<21:39, 2.34s/it] {'loss': '0.924', 'grad_norm': '5.49', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02563', 'train/loss_cot': '0.735', 'epoch': '4.112'} 82%|████████▏ | 2570/3125 [1:02:07<21:39, 2.34s/it] 82%|████████▏ | 2571/3125 [1:02:09<21:12, 2.30s/it] 82%|████████▏ | 2572/3125 [1:02:12<20:46, 2.25s/it] 82%|████████▏ | 2573/3125 [1:02:14<21:55, 2.38s/it] 82%|████████▏ | 2574/3125 [1:02:17<21:49, 2.38s/it] 82%|████████▏ | 2575/3125 [1:02:19<21:42, 2.37s/it] 82%|████████▏ | 2576/3125 [1:02:21<20:58, 2.29s/it] 82%|████████▏ | 2577/3125 [1:02:24<21:35, 2.36s/it] 82%|████████▏ | 2578/3125 [1:02:26<21:31, 2.36s/it] 83%|████████▎ | 2579/3125 [1:02:28<21:42, 2.38s/it] 83%|████████▎ | 2580/3125 [1:02:31<20:57, 2.31s/it] {'loss': '0.9002', 'grad_norm': '5.771', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02661', 'train/loss_cot': '1.161', 'epoch': '4.128'} 83%|████████▎ | 2580/3125 [1:02:31<20:57, 2.31s/it] 83%|████████▎ | 2581/3125 [1:02:33<21:10, 2.34s/it] 83%|████████▎ | 2582/3125 [1:02:35<21:07, 2.33s/it] 83%|████████▎ | 2583/3125 [1:02:38<20:46, 2.30s/it] 83%|████████▎ | 2584/3125 [1:02:41<22:41, 2.52s/it] 83%|████████▎ | 2585/3125 [1:02:43<21:43, 2.41s/it] 83%|████████▎ | 2586/3125 [1:02:45<21:28, 2.39s/it] 83%|████████▎ | 2587/3125 [1:02:47<20:49, 2.32s/it] 83%|████████▎ | 2588/3125 [1:02:49<20:10, 2.25s/it] 83%|████████▎ | 2589/3125 [1:02:51<19:55, 2.23s/it] 83%|████████▎ | 2590/3125 [1:02:54<19:34, 2.20s/it] {'loss': '0.9345', 'grad_norm': '6.001', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02026', 'train/loss_cot': '0.8109', 'epoch': '4.144'} 83%|████████▎ | 2590/3125 [1:02:54<19:34, 2.20s/it] 83%|████████▎ | 2591/3125 [1:02:56<19:28, 2.19s/it] 83%|████████▎ | 2592/3125 [1:02:58<19:20, 2.18s/it] 83%|████████▎ | 2593/3125 [1:03:00<19:30, 2.20s/it] 83%|████████▎ | 2594/3125 [1:03:03<19:59, 2.26s/it] 83%|████████▎ | 2595/3125 [1:03:05<20:00, 2.27s/it] 83%|████████▎ | 2596/3125 [1:03:08<21:02, 2.39s/it] 83%|████████▎ | 2597/3125 [1:03:10<20:30, 2.33s/it] 83%|████████▎ | 2598/3125 [1:03:12<20:24, 2.32s/it] 83%|████████▎ | 2599/3125 [1:03:14<19:59, 2.28s/it] 83%|████████▎ | 2600/3125 [1:03:17<20:26, 2.34s/it] {'loss': '0.9016', 'grad_norm': '5.41', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007324', 'train/loss_cot': '1.068', 'epoch': '4.16'} 83%|████████▎ | 2600/3125 [1:03:17<20:26, 2.34s/it] 83%|████████▎ | 2601/3125 [1:03:42<1:20:56, 9.27s/it] 83%|████████▎ | 2602/3125 [1:03:44<1:02:24, 7.16s/it] 83%|████████▎ | 2603/3125 [1:03:47<50:46, 5.84s/it] 83%|████████▎ | 2604/3125 [1:03:49<41:22, 4.76s/it] 83%|████████▎ | 2605/3125 [1:03:52<35:38, 4.11s/it] 83%|████████▎ | 2606/3125 [1:03:54<30:05, 3.48s/it] 83%|████████▎ | 2607/3125 [1:03:56<27:11, 3.15s/it] 83%|████████▎ | 2608/3125 [1:03:59<25:12, 2.93s/it] 83%|████████▎ | 2609/3125 [1:04:01<24:27, 2.84s/it] 84%|████████▎ | 2610/3125 [1:04:04<23:30, 2.74s/it] {'loss': '0.9221', 'grad_norm': '6.122', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003906', 'train/loss_cot': '0.9342', 'epoch': '4.176'} 84%|████████▎ | 2610/3125 [1:04:04<23:30, 2.74s/it] 84%|████████▎ | 2611/3125 [1:04:07<23:18, 2.72s/it] 84%|████████▎ | 2612/3125 [1:04:09<22:23, 2.62s/it] 84%|████████▎ | 2613/3125 [1:04:11<21:33, 2.53s/it] 84%|████████▎ | 2614/3125 [1:04:13<20:36, 2.42s/it] 84%|████████▎ | 2615/3125 [1:04:16<20:07, 2.37s/it] 84%|████████▎ | 2616/3125 [1:04:18<20:10, 2.38s/it] 84%|████████▎ | 2617/3125 [1:04:20<19:28, 2.30s/it] 84%|████████▍ | 2618/3125 [1:04:22<18:42, 2.21s/it] 84%|████████▍ | 2619/3125 [1:04:25<19:46, 2.35s/it] 84%|████████▍ | 2620/3125 [1:04:27<19:35, 2.33s/it] {'loss': '0.9426', 'grad_norm': '5.338', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007439', 'train/loss_cot': '0.8629', 'epoch': '4.192'} 84%|████████▍ | 2620/3125 [1:04:27<19:35, 2.33s/it] 84%|████████▍ | 2621/3125 [1:04:30<19:40, 2.34s/it] 84%|████████▍ | 2622/3125 [1:04:32<19:49, 2.36s/it] 84%|████████▍ | 2623/3125 [1:04:34<20:07, 2.41s/it] 84%|████████▍ | 2624/3125 [1:04:37<19:48, 2.37s/it] 84%|████████▍ | 2625/3125 [1:04:39<19:37, 2.35s/it] 84%|████████▍ | 2626/3125 [1:04:41<18:53, 2.27s/it] 84%|████████▍ | 2627/3125 [1:04:44<19:14, 2.32s/it] 84%|████████▍ | 2628/3125 [1:04:46<20:00, 2.41s/it] 84%|████████▍ | 2629/3125 [1:04:49<21:41, 2.62s/it] 84%|████████▍ | 2630/3125 [1:04:51<20:14, 2.45s/it] {'loss': '0.9196', 'grad_norm': '5.756', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04688', 'train/loss_cot': '1.033', 'epoch': '4.208'} 84%|████████▍ | 2630/3125 [1:04:52<20:14, 2.45s/it] 84%|████████▍ | 2631/3125 [1:04:54<19:39, 2.39s/it] 84%|████████▍ | 2632/3125 [1:04:56<19:42, 2.40s/it] 84%|████████▍ | 2633/3125 [1:04:58<19:15, 2.35s/it] 84%|████████▍ | 2634/3125 [1:05:00<18:46, 2.29s/it] 84%|████████▍ | 2635/3125 [1:05:03<20:08, 2.47s/it] 84%|████████▍ | 2636/3125 [1:05:05<19:12, 2.36s/it] 84%|████████▍ | 2637/3125 [1:05:08<19:05, 2.35s/it] 84%|████████▍ | 2638/3125 [1:05:10<18:29, 2.28s/it] 84%|████████▍ | 2639/3125 [1:05:12<17:45, 2.19s/it] 84%|████████▍ | 2640/3125 [1:05:14<17:17, 2.14s/it] {'loss': '0.936', 'grad_norm': '5.737', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005432', 'train/loss_cot': '0.8158', 'epoch': '4.224'} 84%|████████▍ | 2640/3125 [1:05:14<17:17, 2.14s/it] 85%|████████▍ | 2641/3125 [1:05:16<17:33, 2.18s/it] 85%|████████▍ | 2642/3125 [1:05:18<18:00, 2.24s/it] 85%|████████▍ | 2643/3125 [1:05:21<17:52, 2.23s/it] 85%|████████▍ | 2644/3125 [1:05:23<17:46, 2.22s/it] 85%|████████▍ | 2645/3125 [1:05:25<17:38, 2.20s/it] 85%|████████▍ | 2646/3125 [1:05:27<17:39, 2.21s/it] 85%|████████▍ | 2647/3125 [1:05:30<17:53, 2.25s/it] 85%|████████▍ | 2648/3125 [1:05:32<18:03, 2.27s/it] 85%|████████▍ | 2649/3125 [1:05:34<18:26, 2.32s/it] 85%|████████▍ | 2650/3125 [1:05:37<18:56, 2.39s/it] {'loss': '0.878', 'grad_norm': '8.752', 'learning_rate': '1e-05', 'train/loss_infonce': '0.126', 'train/loss_cot': '0.7221', 'epoch': '4.24'} 85%|████████▍ | 2650/3125 [1:05:37<18:56, 2.39s/it] 85%|████████▍ | 2651/3125 [1:05:40<19:23, 2.45s/it] 85%|████████▍ | 2652/3125 [1:05:42<19:37, 2.49s/it] 85%|████████▍ | 2653/3125 [1:05:45<19:28, 2.48s/it] 85%|████████▍ | 2654/3125 [1:05:47<18:34, 2.37s/it] 85%|████████▍ | 2655/3125 [1:05:49<19:02, 2.43s/it] 85%|████████▍ | 2656/3125 [1:05:51<18:10, 2.33s/it] 85%|████████▌ | 2657/3125 [1:05:54<19:09, 2.46s/it] 85%|████████▌ | 2658/3125 [1:05:56<18:29, 2.38s/it] 85%|████████▌ | 2659/3125 [1:05:58<18:08, 2.34s/it] 85%|████████▌ | 2660/3125 [1:06:01<17:55, 2.31s/it] {'loss': '0.918', 'grad_norm': '6.071', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009521', 'train/loss_cot': '1.002', 'epoch': '4.256'} 85%|████████▌ | 2660/3125 [1:06:01<17:55, 2.31s/it] 85%|████████▌ | 2661/3125 [1:06:03<18:02, 2.33s/it] 85%|████████▌ | 2662/3125 [1:06:05<17:58, 2.33s/it] 85%|████████▌ | 2663/3125 [1:06:08<19:10, 2.49s/it] 85%|████████▌ | 2664/3125 [1:06:11<19:02, 2.48s/it] 85%|████████▌ | 2665/3125 [1:06:13<18:24, 2.40s/it] 85%|████████▌ | 2666/3125 [1:06:16<18:50, 2.46s/it] 85%|████████▌ | 2667/3125 [1:06:18<19:02, 2.49s/it] 85%|████████▌ | 2668/3125 [1:06:20<18:29, 2.43s/it] 85%|████████▌ | 2669/3125 [1:06:23<17:39, 2.32s/it] 85%|████████▌ | 2670/3125 [1:06:25<17:51, 2.35s/it] {'loss': '0.9399', 'grad_norm': '6.179', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003006', 'train/loss_cot': '0.7153', 'epoch': '4.272'} 85%|████████▌ | 2670/3125 [1:06:25<17:51, 2.35s/it] 85%|████████▌ | 2671/3125 [1:06:28<18:57, 2.51s/it] 86%|████████▌ | 2672/3125 [1:06:30<18:05, 2.40s/it] 86%|████████▌ | 2673/3125 [1:06:33<19:08, 2.54s/it] 86%|████████▌ | 2674/3125 [1:06:35<19:08, 2.55s/it] 86%|████████▌ | 2675/3125 [1:06:38<18:47, 2.50s/it] 86%|████████▌ | 2676/3125 [1:06:40<17:56, 2.40s/it] 86%|████████▌ | 2677/3125 [1:06:43<18:24, 2.46s/it] 86%|████████▌ | 2678/3125 [1:06:45<18:50, 2.53s/it] 86%|████████▌ | 2679/3125 [1:06:47<18:02, 2.43s/it] 86%|████████▌ | 2680/3125 [1:06:50<17:18, 2.33s/it] {'loss': '0.9654', 'grad_norm': '10.99', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00148', 'train/loss_cot': '1.084', 'epoch': '4.288'} 86%|████████▌ | 2680/3125 [1:06:50<17:18, 2.33s/it] 86%|████████▌ | 2681/3125 [1:06:52<17:21, 2.35s/it] 86%|████████▌ | 2682/3125 [1:06:54<17:46, 2.41s/it] 86%|████████▌ | 2683/3125 [1:06:57<17:42, 2.40s/it] 86%|████████▌ | 2684/3125 [1:06:59<18:07, 2.47s/it] 86%|████████▌ | 2685/3125 [1:07:02<18:15, 2.49s/it] 86%|████████▌ | 2686/3125 [1:07:05<18:35, 2.54s/it] 86%|████████▌ | 2687/3125 [1:07:07<17:52, 2.45s/it] 86%|████████▌ | 2688/3125 [1:07:09<17:34, 2.41s/it] 86%|████████▌ | 2689/3125 [1:07:11<16:59, 2.34s/it] 86%|████████▌ | 2690/3125 [1:07:14<16:53, 2.33s/it] {'loss': '0.9463', 'grad_norm': '5.868', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006409', 'train/loss_cot': '0.8968', 'epoch': '4.304'} 86%|████████▌ | 2690/3125 [1:07:14<16:53, 2.33s/it] 86%|████████▌ | 2691/3125 [1:07:16<17:07, 2.37s/it] 86%|████████▌ | 2692/3125 [1:07:18<16:33, 2.29s/it] 86%|████████▌ | 2693/3125 [1:07:20<16:13, 2.25s/it] 86%|████████▌ | 2694/3125 [1:07:23<17:07, 2.38s/it] 86%|████████▌ | 2695/3125 [1:07:26<17:02, 2.38s/it] 86%|████████▋ | 2696/3125 [1:07:28<17:07, 2.40s/it] 86%|████████▋ | 2697/3125 [1:07:30<17:08, 2.40s/it] 86%|████████▋ | 2698/3125 [1:07:34<19:01, 2.67s/it] 86%|████████▋ | 2699/3125 [1:07:36<19:12, 2.70s/it] 86%|████████▋ | 2700/3125 [1:07:39<19:13, 2.72s/it] {'loss': '0.9627', 'grad_norm': '6.543', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05688', 'train/loss_cot': '0.9982', 'epoch': '4.32'} 86%|████████▋ | 2700/3125 [1:07:39<19:13, 2.72s/it] 86%|████████▋ | 2701/3125 [1:07:41<18:15, 2.58s/it] 86%|████████▋ | 2702/3125 [1:07:44<18:28, 2.62s/it] 86%|████████▋ | 2703/3125 [1:07:48<20:02, 2.85s/it] 87%|████████▋ | 2704/3125 [1:07:50<18:51, 2.69s/it] 87%|████████▋ | 2705/3125 [1:07:52<18:35, 2.66s/it] 87%|████████▋ | 2706/3125 [1:07:55<17:39, 2.53s/it] 87%|████████▋ | 2707/3125 [1:07:57<16:57, 2.43s/it] 87%|████████▋ | 2708/3125 [1:07:59<16:18, 2.35s/it] 87%|████████▋ | 2709/3125 [1:08:01<16:16, 2.35s/it] 87%|████████▋ | 2710/3125 [1:08:04<16:53, 2.44s/it] {'loss': '0.9318', 'grad_norm': '5.506', 'learning_rate': '1e-05', 'train/loss_infonce': '1.335e-05', 'train/loss_cot': '0.9976', 'epoch': '4.336'} 87%|████████▋ | 2710/3125 [1:08:04<16:53, 2.44s/it] 87%|████████▋ | 2711/3125 [1:08:06<16:41, 2.42s/it] 87%|████████▋ | 2712/3125 [1:08:09<16:25, 2.39s/it] 87%|████████▋ | 2713/3125 [1:08:11<16:05, 2.34s/it] 87%|████████▋ | 2714/3125 [1:08:13<15:35, 2.28s/it] 87%|████████▋ | 2715/3125 [1:08:16<15:59, 2.34s/it] 87%|████████▋ | 2716/3125 [1:08:18<15:34, 2.29s/it] 87%|████████▋ | 2717/3125 [1:08:20<15:14, 2.24s/it] 87%|████████▋ | 2718/3125 [1:08:22<15:46, 2.33s/it] 87%|████████▋ | 2719/3125 [1:08:25<15:56, 2.36s/it] 87%|████████▋ | 2720/3125 [1:08:27<15:40, 2.32s/it] {'loss': '0.9862', 'grad_norm': '5.527', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007401', 'train/loss_cot': '1.246', 'epoch': '4.352'} 87%|████████▋ | 2720/3125 [1:08:27<15:40, 2.32s/it] 87%|████████▋ | 2721/3125 [1:08:29<15:25, 2.29s/it] 87%|████████▋ | 2722/3125 [1:08:32<16:02, 2.39s/it] 87%|████████▋ | 2723/3125 [1:08:34<16:15, 2.43s/it] 87%|████████▋ | 2724/3125 [1:08:37<16:00, 2.40s/it] 87%|████████▋ | 2725/3125 [1:08:39<15:46, 2.37s/it] 87%|████████▋ | 2726/3125 [1:08:41<15:34, 2.34s/it] 87%|████████▋ | 2727/3125 [1:08:44<16:03, 2.42s/it] 87%|████████▋ | 2728/3125 [1:08:46<15:54, 2.41s/it] 87%|████████▋ | 2729/3125 [1:08:49<15:40, 2.38s/it] 87%|████████▋ | 2730/3125 [1:08:51<15:23, 2.34s/it] {'loss': '0.9755', 'grad_norm': '5.466', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0004082', 'train/loss_cot': '0.845', 'epoch': '4.368'} 87%|████████▋ | 2730/3125 [1:08:51<15:23, 2.34s/it] 87%|████████▋ | 2731/3125 [1:08:53<15:20, 2.34s/it] 87%|████████▋ | 2732/3125 [1:08:56<15:36, 2.38s/it] 87%|████████▋ | 2733/3125 [1:08:58<15:17, 2.34s/it] 87%|████████▋ | 2734/3125 [1:09:01<16:46, 2.57s/it] 88%|████████▊ | 2735/3125 [1:09:03<16:12, 2.49s/it] 88%|████████▊ | 2736/3125 [1:09:06<15:31, 2.39s/it] 88%|████████▊ | 2737/3125 [1:09:08<14:57, 2.31s/it] 88%|████████▊ | 2738/3125 [1:09:10<14:46, 2.29s/it] 88%|████████▊ | 2739/3125 [1:09:12<14:56, 2.32s/it] 88%|████████▊ | 2740/3125 [1:09:15<15:58, 2.49s/it] {'loss': '0.9483', 'grad_norm': '6.557', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001228', 'train/loss_cot': '0.6776', 'epoch': '4.384'} 88%|████████▊ | 2740/3125 [1:09:15<15:58, 2.49s/it] 88%|████████▊ | 2741/3125 [1:09:17<15:28, 2.42s/it] 88%|████████▊ | 2742/3125 [1:09:19<14:46, 2.32s/it] 88%|████████▊ | 2743/3125 [1:09:22<14:33, 2.29s/it] 88%|████████▊ | 2744/3125 [1:09:25<15:34, 2.45s/it] 88%|████████▊ | 2745/3125 [1:09:27<15:27, 2.44s/it] 88%|████████▊ | 2746/3125 [1:09:30<15:53, 2.52s/it] 88%|████████▊ | 2747/3125 [1:09:32<15:39, 2.49s/it] 88%|████████▊ | 2748/3125 [1:09:34<15:22, 2.45s/it] 88%|████████▊ | 2749/3125 [1:09:37<15:12, 2.43s/it] 88%|████████▊ | 2750/3125 [1:09:39<14:58, 2.40s/it] {'loss': '0.9502', 'grad_norm': '5.16', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007439', 'train/loss_cot': '0.7696', 'epoch': '4.4'} 88%|████████▊ | 2750/3125 [1:09:39<14:58, 2.40s/it] 88%|████████▊ | 2751/3125 [1:09:41<14:50, 2.38s/it] 88%|████████▊ | 2752/3125 [1:09:43<14:09, 2.28s/it] 88%|████████▊ | 2753/3125 [1:09:46<14:55, 2.41s/it] 88%|████████▊ | 2754/3125 [1:09:48<14:25, 2.33s/it] 88%|████████▊ | 2755/3125 [1:09:51<14:01, 2.28s/it] 88%|████████▊ | 2756/3125 [1:09:53<13:43, 2.23s/it] 88%|████████▊ | 2757/3125 [1:09:55<13:40, 2.23s/it] 88%|████████▊ | 2758/3125 [1:09:58<14:59, 2.45s/it] 88%|████████▊ | 2759/3125 [1:10:01<16:37, 2.73s/it] 88%|████████▊ | 2760/3125 [1:10:04<15:52, 2.61s/it] {'loss': '0.9154', 'grad_norm': '6.214', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0004864', 'train/loss_cot': '0.7799', 'epoch': '4.416'} 88%|████████▊ | 2760/3125 [1:10:04<15:52, 2.61s/it] 88%|████████▊ | 2761/3125 [1:10:06<15:50, 2.61s/it] 88%|████████▊ | 2762/3125 [1:10:08<14:55, 2.47s/it] 88%|████████▊ | 2763/3125 [1:10:11<14:54, 2.47s/it] 88%|████████▊ | 2764/3125 [1:10:14<15:24, 2.56s/it] 88%|████████▊ | 2765/3125 [1:10:16<14:51, 2.48s/it] 89%|████████▊ | 2766/3125 [1:10:18<14:46, 2.47s/it] 89%|████████▊ | 2767/3125 [1:10:20<14:10, 2.38s/it] 89%|████████▊ | 2768/3125 [1:10:23<13:49, 2.32s/it] 89%|████████▊ | 2769/3125 [1:10:25<13:59, 2.36s/it] 89%|████████▊ | 2770/3125 [1:10:28<14:32, 2.46s/it] {'loss': '0.9563', 'grad_norm': '5.857', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01288', 'train/loss_cot': '0.8022', 'epoch': '4.432'} 89%|████████▊ | 2770/3125 [1:10:28<14:32, 2.46s/it] 89%|████████▊ | 2771/3125 [1:10:30<13:57, 2.37s/it] 89%|████████▊ | 2772/3125 [1:10:32<13:39, 2.32s/it] 89%|████████▊ | 2773/3125 [1:10:34<13:09, 2.24s/it] 89%|████████▉ | 2774/3125 [1:10:37<13:18, 2.28s/it] 89%|████████▉ | 2775/3125 [1:10:39<14:10, 2.43s/it] 89%|████████▉ | 2776/3125 [1:10:41<13:36, 2.34s/it] 89%|████████▉ | 2777/3125 [1:10:44<14:18, 2.47s/it] 89%|████████▉ | 2778/3125 [1:10:47<14:01, 2.42s/it] 89%|████████▉ | 2779/3125 [1:10:49<13:47, 2.39s/it] 89%|████████▉ | 2780/3125 [1:10:52<14:22, 2.50s/it] {'loss': '0.9696', 'grad_norm': '5.936', 'learning_rate': '1e-05', 'train/loss_infonce': '0.09912', 'train/loss_cot': '0.9383', 'epoch': '4.448'} 89%|████████▉ | 2780/3125 [1:10:52<14:22, 2.50s/it] 89%|████████▉ | 2781/3125 [1:10:54<14:43, 2.57s/it] 89%|████████▉ | 2782/3125 [1:10:57<14:13, 2.49s/it] 89%|████████▉ | 2783/3125 [1:10:59<14:21, 2.52s/it] 89%|████████▉ | 2784/3125 [1:11:01<13:53, 2.44s/it] 89%|████████▉ | 2785/3125 [1:11:04<14:16, 2.52s/it] 89%|████████▉ | 2786/3125 [1:11:06<13:52, 2.46s/it] 89%|████████▉ | 2787/3125 [1:11:09<13:21, 2.37s/it] 89%|████████▉ | 2788/3125 [1:11:12<14:31, 2.59s/it] 89%|████████▉ | 2789/3125 [1:11:14<14:03, 2.51s/it] 89%|████████▉ | 2790/3125 [1:11:16<13:33, 2.43s/it] {'loss': '0.9493', 'grad_norm': '5.304', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0188', 'train/loss_cot': '1.175', 'epoch': '4.464'} 89%|████████▉ | 2790/3125 [1:11:17<13:33, 2.43s/it] 89%|████████▉ | 2791/3125 [1:11:19<13:24, 2.41s/it] 89%|████████▉ | 2792/3125 [1:11:21<13:14, 2.38s/it] 89%|████████▉ | 2793/3125 [1:11:24<13:55, 2.52s/it] 89%|████████▉ | 2794/3125 [1:11:26<13:39, 2.47s/it] 89%|████████▉ | 2795/3125 [1:11:28<13:04, 2.38s/it] 89%|████████▉ | 2796/3125 [1:11:31<13:01, 2.38s/it] 90%|████████▉ | 2797/3125 [1:11:33<13:17, 2.43s/it] 90%|████████▉ | 2798/3125 [1:11:36<12:57, 2.38s/it] 90%|████████▉ | 2799/3125 [1:11:38<13:09, 2.42s/it] 90%|████████▉ | 2800/3125 [1:11:40<12:51, 2.37s/it] {'loss': '0.9415', 'grad_norm': '5.876', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02368', 'train/loss_cot': '1.129', 'epoch': '4.48'} 90%|████████▉ | 2800/3125 [1:11:41<12:51, 2.37s/it] 90%|████████▉ | 2801/3125 [1:12:07<52:13, 9.67s/it] 90%|████████▉ | 2802/3125 [1:12:09<40:10, 7.46s/it] 90%|████████▉ | 2803/3125 [1:12:12<31:30, 5.87s/it] 90%|████████▉ | 2804/3125 [1:12:14<25:45, 4.81s/it] 90%|████████▉ | 2805/3125 [1:12:16<21:53, 4.10s/it] 90%|████████▉ | 2806/3125 [1:12:19<19:23, 3.65s/it] 90%|████████▉ | 2807/3125 [1:12:21<17:22, 3.28s/it] 90%|████████▉ | 2808/3125 [1:12:24<16:51, 3.19s/it] 90%|████████▉ | 2809/3125 [1:12:27<16:15, 3.09s/it] 90%|████████▉ | 2810/3125 [1:12:29<14:41, 2.80s/it] {'loss': '0.9474', 'grad_norm': '5.484', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009216', 'train/loss_cot': '1.045', 'epoch': '4.496'} 90%|████████▉ | 2810/3125 [1:12:30<14:41, 2.80s/it] 90%|████████▉ | 2811/3125 [1:12:32<13:55, 2.66s/it] 90%|████████▉ | 2812/3125 [1:12:34<13:38, 2.62s/it] 90%|█████████ | 2813/3125 [1:12:37<13:36, 2.62s/it] 90%|█████████ | 2814/3125 [1:12:39<13:13, 2.55s/it] 90%|█████████ | 2815/3125 [1:12:42<12:57, 2.51s/it] 90%|█████████ | 2816/3125 [1:12:44<12:08, 2.36s/it] 90%|█████████ | 2817/3125 [1:12:46<12:02, 2.35s/it] 90%|█████████ | 2818/3125 [1:12:48<11:54, 2.33s/it] 90%|█████████ | 2819/3125 [1:12:50<11:52, 2.33s/it] 90%|█████████ | 2820/3125 [1:12:53<11:42, 2.30s/it] {'loss': '0.9563', 'grad_norm': '5.086', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007996', 'train/loss_cot': '0.8669', 'epoch': '4.512'} 90%|█████████ | 2820/3125 [1:12:53<11:42, 2.30s/it] 90%|█████████ | 2821/3125 [1:12:55<11:55, 2.35s/it] 90%|█████████ | 2822/3125 [1:12:57<11:38, 2.30s/it] 90%|█████████ | 2823/3125 [1:13:00<12:11, 2.42s/it] 90%|█████████ | 2824/3125 [1:13:02<11:50, 2.36s/it] 90%|█████████ | 2825/3125 [1:13:04<11:16, 2.26s/it] 90%|█████████ | 2826/3125 [1:13:07<11:24, 2.29s/it] 90%|█████████ | 2827/3125 [1:13:09<11:39, 2.35s/it] 90%|█████████ | 2828/3125 [1:13:12<11:36, 2.34s/it] 91%|█████████ | 2829/3125 [1:13:14<11:23, 2.31s/it] 91%|█████████ | 2830/3125 [1:13:16<11:52, 2.42s/it] {'loss': '0.9548', 'grad_norm': '5.99', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004578', 'train/loss_cot': '0.936', 'epoch': '4.528'} 91%|█████████ | 2830/3125 [1:13:17<11:52, 2.42s/it] 91%|█████████ | 2831/3125 [1:13:19<11:41, 2.39s/it] 91%|█████████ | 2832/3125 [1:13:21<12:08, 2.49s/it] 91%|█████████ | 2833/3125 [1:13:24<12:09, 2.50s/it] 91%|█████████ | 2834/3125 [1:13:26<11:55, 2.46s/it] 91%|█████████ | 2835/3125 [1:13:29<12:22, 2.56s/it] 91%|█████████ | 2836/3125 [1:13:32<12:12, 2.54s/it] 91%|█████████ | 2837/3125 [1:13:34<11:43, 2.44s/it] 91%|█████████ | 2838/3125 [1:13:36<11:34, 2.42s/it] 91%|█████████ | 2839/3125 [1:13:39<11:58, 2.51s/it] 91%|█████████ | 2840/3125 [1:13:41<11:37, 2.45s/it] {'loss': '1.005', 'grad_norm': '6.94', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00885', 'train/loss_cot': '0.9352', 'epoch': '4.544'} 91%|█████████ | 2840/3125 [1:13:42<11:37, 2.45s/it] 91%|█████████ | 2841/3125 [1:13:44<11:33, 2.44s/it] 91%|█████████ | 2842/3125 [1:13:46<11:53, 2.52s/it] 91%|█████████ | 2843/3125 [1:13:49<11:40, 2.48s/it] 91%|█████████ | 2844/3125 [1:13:52<12:09, 2.60s/it] 91%|█████████ | 2845/3125 [1:13:54<11:58, 2.56s/it] 91%|█████████ | 2846/3125 [1:13:56<11:30, 2.48s/it] 91%|█████████ | 2847/3125 [1:13:58<10:57, 2.36s/it] 91%|█████████ | 2848/3125 [1:14:01<10:32, 2.28s/it] 91%|█████████ | 2849/3125 [1:14:03<10:58, 2.39s/it] 91%|█████████ | 2850/3125 [1:14:06<10:50, 2.37s/it] {'loss': '0.9445', 'grad_norm': '6.037', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04126', 'train/loss_cot': '0.8093', 'epoch': '4.56'} 91%|█████████ | 2850/3125 [1:14:06<10:50, 2.37s/it] 91%|█████████ | 2851/3125 [1:14:08<10:53, 2.38s/it] 91%|█████████▏| 2852/3125 [1:14:10<10:32, 2.32s/it] 91%|█████████▏| 2853/3125 [1:14:12<10:21, 2.29s/it] 91%|█████████▏| 2854/3125 [1:14:15<10:21, 2.29s/it] 91%|█████████▏| 2855/3125 [1:14:17<10:21, 2.30s/it] 91%|█████████▏| 2856/3125 [1:14:20<10:59, 2.45s/it] 91%|█████████▏| 2857/3125 [1:14:22<10:39, 2.39s/it] 91%|█████████▏| 2858/3125 [1:14:24<10:16, 2.31s/it] 91%|█████████▏| 2859/3125 [1:14:26<10:07, 2.29s/it] 92%|█████████▏| 2860/3125 [1:14:29<09:58, 2.26s/it] {'loss': '0.9287', 'grad_norm': '5.349', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001854', 'train/loss_cot': '0.6859', 'epoch': '4.576'} 92%|█████████▏| 2860/3125 [1:14:29<09:58, 2.26s/it] 92%|█████████▏| 2861/3125 [1:14:31<10:05, 2.29s/it] 92%|█████████▏| 2862/3125 [1:14:33<10:15, 2.34s/it] 92%|█████████▏| 2863/3125 [1:14:36<10:49, 2.48s/it] 92%|█████████▏| 2864/3125 [1:14:38<10:25, 2.40s/it] 92%|█████████▏| 2865/3125 [1:14:41<10:41, 2.47s/it] 92%|█████████▏| 2866/3125 [1:14:43<10:04, 2.33s/it] 92%|█████████▏| 2867/3125 [1:14:46<10:38, 2.48s/it] 92%|█████████▏| 2868/3125 [1:14:48<10:21, 2.42s/it] 92%|█████████▏| 2869/3125 [1:14:51<10:18, 2.42s/it] 92%|█████████▏| 2870/3125 [1:14:53<10:08, 2.39s/it] {'loss': '0.9676', 'grad_norm': '7.139', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007385', 'train/loss_cot': '1.016', 'epoch': '4.592'} 92%|█████████▏| 2870/3125 [1:14:53<10:08, 2.39s/it] 92%|█████████▏| 2871/3125 [1:14:56<10:35, 2.50s/it] 92%|█████████▏| 2872/3125 [1:14:58<10:27, 2.48s/it] 92%|█████████▏| 2873/3125 [1:15:00<09:57, 2.37s/it] 92%|█████████▏| 2874/3125 [1:15:02<09:34, 2.29s/it] 92%|█████████▏| 2875/3125 [1:15:05<09:40, 2.32s/it] 92%|█████████▏| 2876/3125 [1:15:07<09:40, 2.33s/it] 92%|█████████▏| 2877/3125 [1:15:10<09:58, 2.41s/it] 92%|█████████▏| 2878/3125 [1:15:12<09:39, 2.35s/it] 92%|█████████▏| 2879/3125 [1:15:14<09:54, 2.42s/it] 92%|█████████▏| 2880/3125 [1:15:17<09:44, 2.39s/it] {'loss': '0.9598', 'grad_norm': '6.255', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005249', 'train/loss_cot': '0.9711', 'epoch': '4.608'} 92%|█████████▏| 2880/3125 [1:15:17<09:44, 2.39s/it] 92%|█████████▏| 2881/3125 [1:15:19<09:44, 2.39s/it] 92%|█████████▏| 2882/3125 [1:15:21<09:24, 2.32s/it] 92%|█████████▏| 2883/3125 [1:15:24<09:27, 2.34s/it] 92%|█████████▏| 2884/3125 [1:15:26<09:13, 2.29s/it] 92%|█████████▏| 2885/3125 [1:15:28<09:02, 2.26s/it] 92%|█████████▏| 2886/3125 [1:15:30<09:09, 2.30s/it] 92%|█████████▏| 2887/3125 [1:15:33<08:53, 2.24s/it] 92%|█████████▏| 2888/3125 [1:15:35<08:44, 2.21s/it] 92%|█████████▏| 2889/3125 [1:15:37<08:42, 2.22s/it] 92%|█████████▏| 2890/3125 [1:15:39<08:33, 2.18s/it] {'loss': '0.9255', 'grad_norm': '5.785', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003677', 'train/loss_cot': '1.039', 'epoch': '4.624'} 92%|█████████▏| 2890/3125 [1:15:39<08:33, 2.18s/it] 93%|█████████▎| 2891/3125 [1:15:41<08:51, 2.27s/it] 93%|█████████▎| 2892/3125 [1:15:45<10:31, 2.71s/it] 93%|█████████▎| 2893/3125 [1:15:48<10:12, 2.64s/it] 93%|█████████▎| 2894/3125 [1:15:50<10:18, 2.68s/it] 93%|█████████▎| 2895/3125 [1:15:52<09:31, 2.49s/it] 93%|█████████▎| 2896/3125 [1:15:55<09:11, 2.41s/it] 93%|█████████▎| 2897/3125 [1:15:57<08:57, 2.36s/it] 93%|█████████▎| 2898/3125 [1:16:00<09:12, 2.43s/it] 93%|█████████▎| 2899/3125 [1:16:02<08:50, 2.35s/it] 93%|█████████▎| 2900/3125 [1:16:04<08:38, 2.30s/it] {'loss': '0.9203', 'grad_norm': '6.336', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07861', 'train/loss_cot': '0.7803', 'epoch': '4.64'} 93%|█████████▎| 2900/3125 [1:16:04<08:38, 2.30s/it] 93%|█████████▎| 2901/3125 [1:16:06<08:42, 2.33s/it] 93%|█████████▎| 2902/3125 [1:16:09<09:25, 2.54s/it] 93%|█████████▎| 2903/3125 [1:16:12<09:09, 2.47s/it] 93%|█████████▎| 2904/3125 [1:16:14<09:22, 2.55s/it] 93%|█████████▎| 2905/3125 [1:16:17<09:20, 2.55s/it] 93%|█████████▎| 2906/3125 [1:16:19<08:52, 2.43s/it] 93%|█████████▎| 2907/3125 [1:16:22<08:59, 2.47s/it] 93%|█████████▎| 2908/3125 [1:16:24<09:16, 2.56s/it] 93%|█████████▎| 2909/3125 [1:16:27<09:10, 2.55s/it] 93%|█████████▎| 2910/3125 [1:16:30<09:20, 2.61s/it] {'loss': '0.9599', 'grad_norm': '8.555', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1719', 'train/loss_cot': '0.9003', 'epoch': '4.656'} 93%|█████████▎| 2910/3125 [1:16:30<09:20, 2.61s/it] 93%|█████████▎| 2911/3125 [1:16:32<09:09, 2.57s/it] 93%|█████████▎| 2912/3125 [1:16:34<08:42, 2.45s/it] 93%|█████████▎| 2913/3125 [1:16:38<09:29, 2.69s/it] 93%|█████████▎| 2914/3125 [1:16:40<08:52, 2.52s/it] 93%|█████████▎| 2915/3125 [1:16:42<08:54, 2.55s/it] 93%|█████████▎| 2916/3125 [1:16:45<08:38, 2.48s/it] 93%|█████████▎| 2917/3125 [1:16:47<08:17, 2.39s/it] 93%|█████████▎| 2918/3125 [1:16:49<07:56, 2.30s/it] 93%|█████████▎| 2919/3125 [1:16:51<07:56, 2.31s/it] 93%|█████████▎| 2920/3125 [1:16:53<07:47, 2.28s/it] {'loss': '0.9655', 'grad_norm': '6.331', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05298', 'train/loss_cot': '0.9183', 'epoch': '4.672'} 93%|█████████▎| 2920/3125 [1:16:54<07:47, 2.28s/it] 93%|█████████▎| 2921/3125 [1:16:56<07:40, 2.26s/it] 94%|█████████▎| 2922/3125 [1:16:58<07:37, 2.25s/it] 94%|█████████▎| 2923/3125 [1:17:00<07:31, 2.24s/it] 94%|█████████▎| 2924/3125 [1:17:03<07:47, 2.32s/it] 94%|█████████▎| 2925/3125 [1:17:05<07:40, 2.30s/it] 94%|█████████▎| 2926/3125 [1:17:07<07:56, 2.39s/it] 94%|█████████▎| 2927/3125 [1:17:10<07:50, 2.38s/it] 94%|█████████▎| 2928/3125 [1:17:12<07:46, 2.37s/it] 94%|█████████▎| 2929/3125 [1:17:14<07:34, 2.32s/it] 94%|█████████▍| 2930/3125 [1:17:16<07:16, 2.24s/it] {'loss': '0.9392', 'grad_norm': '5.799', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00161', 'train/loss_cot': '0.9555', 'epoch': '4.688'} 94%|█████████▍| 2930/3125 [1:17:17<07:16, 2.24s/it] 94%|█████████▍| 2931/3125 [1:17:19<07:28, 2.31s/it] 94%|█████████▍| 2932/3125 [1:17:22<07:58, 2.48s/it] 94%|█████████▍| 2933/3125 [1:17:24<07:46, 2.43s/it] 94%|█████████▍| 2934/3125 [1:17:26<07:34, 2.38s/it] 94%|█████████▍| 2935/3125 [1:17:29<07:46, 2.46s/it] 94%|█████████▍| 2936/3125 [1:17:31<07:32, 2.39s/it] 94%|█████████▍| 2937/3125 [1:17:34<07:25, 2.37s/it] 94%|█████████▍| 2938/3125 [1:17:36<07:12, 2.31s/it] 94%|█████████▍| 2939/3125 [1:17:38<07:19, 2.36s/it] 94%|█████████▍| 2940/3125 [1:17:40<07:08, 2.32s/it] {'loss': '0.969', 'grad_norm': '5.304', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03784', 'train/loss_cot': '0.9394', 'epoch': '4.704'} 94%|█████████▍| 2940/3125 [1:17:41<07:08, 2.32s/it] 94%|█████████▍| 2941/3125 [1:17:43<07:18, 2.38s/it] 94%|█████████▍| 2942/3125 [1:17:45<07:14, 2.37s/it] 94%|█████████▍| 2943/3125 [1:17:47<06:55, 2.28s/it] 94%|█████████▍| 2944/3125 [1:17:50<06:55, 2.29s/it] 94%|█████████▍| 2945/3125 [1:17:52<06:44, 2.25s/it] 94%|█████████▍| 2946/3125 [1:17:55<07:12, 2.41s/it] 94%|█████████▍| 2947/3125 [1:17:57<07:23, 2.49s/it] 94%|█████████▍| 2948/3125 [1:18:00<07:06, 2.41s/it] 94%|█████████▍| 2949/3125 [1:18:02<06:59, 2.38s/it] 94%|█████████▍| 2950/3125 [1:18:04<06:45, 2.32s/it] {'loss': '0.9626', 'grad_norm': '5.614', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02991', 'train/loss_cot': '0.9825', 'epoch': '4.72'} 94%|█████████▍| 2950/3125 [1:18:04<06:45, 2.32s/it] 94%|█████████▍| 2951/3125 [1:18:06<06:40, 2.30s/it] 94%|█████████▍| 2952/3125 [1:18:09<06:42, 2.33s/it] 94%|█████████▍| 2953/3125 [1:18:11<06:40, 2.33s/it] 95%|█████████▍| 2954/3125 [1:18:13<06:34, 2.31s/it] 95%|█████████▍| 2955/3125 [1:18:17<07:28, 2.64s/it] 95%|█████████▍| 2956/3125 [1:18:19<07:04, 2.51s/it] 95%|█████████▍| 2957/3125 [1:18:21<06:37, 2.37s/it] 95%|█████████▍| 2958/3125 [1:18:23<06:33, 2.35s/it] 95%|█████████▍| 2959/3125 [1:18:26<06:34, 2.38s/it] 95%|█████████▍| 2960/3125 [1:18:28<06:28, 2.36s/it] {'loss': '0.9712', 'grad_norm': '6.522', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02722', 'train/loss_cot': '1.158', 'epoch': '4.736'} 95%|█████████▍| 2960/3125 [1:18:28<06:28, 2.36s/it] 95%|█████████▍| 2961/3125 [1:18:30<06:23, 2.34s/it] 95%|█████████▍| 2962/3125 [1:18:33<06:21, 2.34s/it] 95%|█████████▍| 2963/3125 [1:18:35<06:21, 2.36s/it] 95%|█████████▍| 2964/3125 [1:18:37<06:06, 2.28s/it] 95%|█████████▍| 2965/3125 [1:18:39<06:02, 2.27s/it] 95%|█████████▍| 2966/3125 [1:18:42<06:03, 2.29s/it] 95%|█████████▍| 2967/3125 [1:18:44<05:53, 2.24s/it] 95%|█████████▍| 2968/3125 [1:18:46<05:56, 2.27s/it] 95%|█████████▌| 2969/3125 [1:18:49<06:07, 2.35s/it] 95%|█████████▌| 2970/3125 [1:18:51<06:14, 2.42s/it] {'loss': '0.9951', 'grad_norm': '6.371', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03223', 'train/loss_cot': '1.097', 'epoch': '4.752'} 95%|█████████▌| 2970/3125 [1:18:52<06:14, 2.42s/it] 95%|█████████▌| 2971/3125 [1:18:54<06:30, 2.53s/it] 95%|█████████▌| 2972/3125 [1:18:56<06:04, 2.38s/it] 95%|█████████▌| 2973/3125 [1:18:58<05:54, 2.33s/it] 95%|█████████▌| 2974/3125 [1:19:00<05:45, 2.29s/it] 95%|█████████▌| 2975/3125 [1:19:05<07:16, 2.91s/it] 95%|█████████▌| 2976/3125 [1:19:07<06:53, 2.77s/it] 95%|█████████▌| 2977/3125 [1:19:10<06:41, 2.71s/it] 95%|█████████▌| 2978/3125 [1:19:12<06:19, 2.58s/it] 95%|█████████▌| 2979/3125 [1:19:15<06:34, 2.70s/it] 95%|█████████▌| 2980/3125 [1:19:18<06:17, 2.60s/it] {'loss': '0.9387', 'grad_norm': '6.206', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02136', 'train/loss_cot': '0.9229', 'epoch': '4.768'} 95%|█████████▌| 2980/3125 [1:19:18<06:17, 2.60s/it] 95%|█████████▌| 2981/3125 [1:19:20<06:01, 2.51s/it] 95%|█████████▌| 2982/3125 [1:19:22<06:03, 2.54s/it] 95%|█████████▌| 2983/3125 [1:19:24<05:38, 2.38s/it] 95%|█████████▌| 2984/3125 [1:19:27<05:28, 2.33s/it] 96%|█████████▌| 2985/3125 [1:19:29<05:20, 2.29s/it] 96%|█████████▌| 2986/3125 [1:19:32<05:41, 2.46s/it] 96%|█████████▌| 2987/3125 [1:19:34<05:38, 2.45s/it] 96%|█████████▌| 2988/3125 [1:19:37<05:46, 2.53s/it] 96%|█████████▌| 2989/3125 [1:19:39<05:32, 2.44s/it] 96%|█████████▌| 2990/3125 [1:19:41<05:26, 2.42s/it] {'loss': '0.9917', 'grad_norm': '5.709', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04004', 'train/loss_cot': '0.9159', 'epoch': '4.784'} 96%|█████████▌| 2990/3125 [1:19:42<05:26, 2.42s/it] 96%|█████████▌| 2991/3125 [1:19:44<05:38, 2.53s/it] 96%|█████████▌| 2992/3125 [1:19:47<05:35, 2.52s/it] 96%|█████████▌| 2993/3125 [1:19:49<05:20, 2.43s/it] 96%|█████████▌| 2994/3125 [1:19:51<05:16, 2.41s/it] 96%|█████████▌| 2995/3125 [1:19:54<05:09, 2.38s/it] 96%|█████████▌| 2996/3125 [1:19:56<05:00, 2.33s/it] 96%|█████████▌| 2997/3125 [1:19:58<05:02, 2.36s/it] 96%|█████████▌| 2998/3125 [1:20:01<05:01, 2.37s/it] 96%|█████████▌| 2999/3125 [1:20:04<05:16, 2.52s/it] 96%|█████████▌| 3000/3125 [1:20:06<05:08, 2.47s/it] {'loss': '0.9784', 'grad_norm': '6.128', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007294', 'train/loss_cot': '0.816', 'epoch': '4.8'} 96%|█████████▌| 3000/3125 [1:20:06<05:08, 2.47s/it] 96%|█████████▌| 3001/3125 [1:20:24<15:01, 7.27s/it] 96%|█████████▌| 3002/3125 [1:20:27<11:54, 5.81s/it] 96%|█████████▌| 3003/3125 [1:20:29<09:36, 4.73s/it] 96%|█████████▌| 3004/3125 [1:20:31<08:07, 4.03s/it] 96%|█████████▌| 3005/3125 [1:20:34<06:57, 3.48s/it] 96%|█████████▌| 3006/3125 [1:20:36<05:59, 3.02s/it] 96%|█████████▌| 3007/3125 [1:20:38<05:34, 2.84s/it] 96%|█████████▋| 3008/3125 [1:20:41<05:34, 2.86s/it] 96%|█████████▋| 3009/3125 [1:20:43<05:17, 2.74s/it] 96%|█████████▋| 3010/3125 [1:20:46<05:08, 2.68s/it] {'loss': '0.9654', 'grad_norm': '5.905', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006042', 'train/loss_cot': '1.039', 'epoch': '4.816'} 96%|█████████▋| 3010/3125 [1:20:46<05:08, 2.68s/it] 96%|█████████▋| 3011/3125 [1:20:49<05:05, 2.68s/it] 96%|█████████▋| 3012/3125 [1:20:51<04:50, 2.57s/it] 96%|█████████▋| 3013/3125 [1:20:53<04:44, 2.54s/it] 96%|█████████▋| 3014/3125 [1:20:56<04:50, 2.61s/it] 96%|█████████▋| 3015/3125 [1:20:58<04:34, 2.50s/it] 97%|█████████▋| 3016/3125 [1:21:01<04:25, 2.44s/it] 97%|█████████▋| 3017/3125 [1:21:03<04:23, 2.44s/it] 97%|█████████▋| 3018/3125 [1:21:05<04:08, 2.32s/it] 97%|█████████▋| 3019/3125 [1:21:08<04:19, 2.45s/it] 97%|█████████▋| 3020/3125 [1:21:10<04:12, 2.40s/it] {'loss': '1.006', 'grad_norm': '5.958', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006287', 'train/loss_cot': '0.9902', 'epoch': '4.832'} 97%|█████████▋| 3020/3125 [1:21:10<04:12, 2.40s/it] 97%|█████████▋| 3021/3125 [1:21:13<04:34, 2.64s/it] 97%|█████████▋| 3022/3125 [1:21:15<04:13, 2.46s/it] 97%|█████████▋| 3023/3125 [1:21:18<04:04, 2.40s/it] 97%|█████████▋| 3024/3125 [1:21:20<04:04, 2.42s/it] 97%|█████████▋| 3025/3125 [1:21:23<04:12, 2.52s/it] 97%|█████████▋| 3026/3125 [1:21:25<03:58, 2.41s/it] 97%|█████████▋| 3027/3125 [1:21:27<03:49, 2.34s/it] 97%|█████████▋| 3028/3125 [1:21:30<03:53, 2.41s/it] 97%|█████████▋| 3029/3125 [1:21:32<03:43, 2.33s/it] 97%|█████████▋| 3030/3125 [1:21:34<03:43, 2.35s/it] {'loss': '0.9478', 'grad_norm': '5.495', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00325', 'train/loss_cot': '0.9661', 'epoch': '4.848'} 97%|█████████▋| 3030/3125 [1:21:35<03:43, 2.35s/it] 97%|█████████▋| 3031/3125 [1:21:37<03:40, 2.34s/it] 97%|█████████▋| 3032/3125 [1:21:39<03:42, 2.39s/it] 97%|█████████▋| 3033/3125 [1:21:42<03:42, 2.42s/it] 97%|█████████▋| 3034/3125 [1:21:44<03:47, 2.50s/it] 97%|█████████▋| 3035/3125 [1:21:47<03:45, 2.50s/it] 97%|█████████▋| 3036/3125 [1:21:50<03:55, 2.65s/it] 97%|█████████▋| 3037/3125 [1:21:52<03:44, 2.55s/it] 97%|█████████▋| 3038/3125 [1:21:54<03:34, 2.47s/it] 97%|█████████▋| 3039/3125 [1:21:57<03:32, 2.47s/it] 97%|█████████▋| 3040/3125 [1:21:59<03:23, 2.40s/it] {'loss': '0.9572', 'grad_norm': '5.489', 'learning_rate': '1e-05', 'train/loss_infonce': '0.002045', 'train/loss_cot': '0.9033', 'epoch': '4.864'} 97%|█████████▋| 3040/3125 [1:21:59<03:23, 2.40s/it] 97%|█████████▋| 3041/3125 [1:22:01<03:20, 2.39s/it] 97%|█████████▋| 3042/3125 [1:22:04<03:17, 2.37s/it] 97%|█████████▋| 3043/3125 [1:22:06<03:08, 2.29s/it] 97%|█████████▋| 3044/3125 [1:22:08<03:05, 2.29s/it] 97%|█████████▋| 3045/3125 [1:22:11<03:05, 2.32s/it] 97%|█████████▋| 3046/3125 [1:22:13<03:01, 2.30s/it] 98%|█████████▊| 3047/3125 [1:22:16<03:10, 2.45s/it] 98%|█████████▊| 3048/3125 [1:22:18<03:08, 2.45s/it] 98%|█████████▊| 3049/3125 [1:22:20<03:05, 2.44s/it] 98%|█████████▊| 3050/3125 [1:22:23<03:02, 2.44s/it] {'loss': '1.014', 'grad_norm': '7.566', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0108', 'train/loss_cot': '1.04', 'epoch': '4.88'} 98%|█████████▊| 3050/3125 [1:22:23<03:02, 2.44s/it] 98%|█████████▊| 3051/3125 [1:22:27<03:28, 2.81s/it] 98%|█████████▊| 3052/3125 [1:22:29<03:15, 2.68s/it] 98%|█████████▊| 3053/3125 [1:22:32<03:25, 2.86s/it] 98%|█████████▊| 3054/3125 [1:22:34<03:07, 2.64s/it] 98%|█████████▊| 3055/3125 [1:22:37<02:56, 2.53s/it] 98%|█████████▊| 3056/3125 [1:22:39<02:43, 2.37s/it] 98%|█████████▊| 3057/3125 [1:22:41<02:40, 2.36s/it] 98%|█████████▊| 3058/3125 [1:22:43<02:31, 2.26s/it] 98%|█████████▊| 3059/3125 [1:22:46<02:40, 2.43s/it] 98%|█████████▊| 3060/3125 [1:22:48<02:35, 2.40s/it] {'loss': '0.9426', 'grad_norm': '5.976', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005646', 'train/loss_cot': '1.075', 'epoch': '4.896'} 98%|█████████▊| 3060/3125 [1:22:49<02:35, 2.40s/it] 98%|█████████▊| 3061/3125 [1:22:51<02:36, 2.45s/it] 98%|█████████▊| 3062/3125 [1:22:53<02:33, 2.43s/it] 98%|█████████▊| 3063/3125 [1:22:55<02:26, 2.37s/it] 98%|█████████▊| 3064/3125 [1:22:58<02:26, 2.40s/it] 98%|█████████▊| 3065/3125 [1:23:00<02:22, 2.37s/it] 98%|█████████▊| 3066/3125 [1:23:02<02:16, 2.31s/it] 98%|█████████▊| 3067/3125 [1:23:05<02:19, 2.40s/it] 98%|█████████▊| 3068/3125 [1:23:07<02:16, 2.39s/it] 98%|█████████▊| 3069/3125 [1:23:10<02:15, 2.42s/it] 98%|█████████▊| 3070/3125 [1:23:12<02:15, 2.46s/it] {'loss': '0.9363', 'grad_norm': '5.658', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02258', 'train/loss_cot': '1.09', 'epoch': '4.912'} 98%|█████████▊| 3070/3125 [1:23:13<02:15, 2.46s/it] 98%|█████████▊| 3071/3125 [1:23:15<02:10, 2.42s/it] 98%|█████████▊| 3072/3125 [1:23:17<02:06, 2.38s/it] 98%|█████████▊| 3073/3125 [1:23:19<01:59, 2.31s/it] 98%|█████████▊| 3074/3125 [1:23:21<01:58, 2.32s/it] 98%|█████████▊| 3075/3125 [1:23:24<01:56, 2.34s/it] 98%|█████████▊| 3076/3125 [1:23:26<01:56, 2.37s/it] 98%|█████████▊| 3077/3125 [1:23:29<01:54, 2.39s/it] 98%|█████████▊| 3078/3125 [1:23:31<01:50, 2.35s/it] 99%|█████████▊| 3079/3125 [1:23:33<01:46, 2.33s/it] 99%|█████████▊| 3080/3125 [1:23:36<01:46, 2.36s/it] {'loss': '1.002', 'grad_norm': '5.522', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '0.7513', 'epoch': '4.928'} 99%|█████████▊| 3080/3125 [1:23:36<01:46, 2.36s/it] 99%|█████████▊| 3081/3125 [1:23:38<01:46, 2.42s/it] 99%|█████████▊| 3082/3125 [1:23:41<01:43, 2.42s/it] 99%|█████████▊| 3083/3125 [1:23:43<01:39, 2.37s/it] 99%|█████████▊| 3084/3125 [1:23:45<01:33, 2.29s/it] 99%|█████████▊| 3085/3125 [1:23:47<01:33, 2.33s/it] 99%|█████████▉| 3086/3125 [1:23:50<01:31, 2.35s/it] 99%|█████████▉| 3087/3125 [1:23:52<01:26, 2.28s/it] 99%|█████████▉| 3088/3125 [1:23:55<01:34, 2.56s/it] 99%|█████████▉| 3089/3125 [1:23:58<01:34, 2.62s/it] 99%|█████████▉| 3090/3125 [1:24:00<01:26, 2.48s/it] {'loss': '0.9326', 'grad_norm': '5.445', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006226', 'train/loss_cot': '1.042', 'epoch': '4.944'} 99%|█████████▉| 3090/3125 [1:24:00<01:26, 2.48s/it] 99%|█████████▉| 3091/3125 [1:24:02<01:23, 2.47s/it] 99%|█████████▉| 3092/3125 [1:24:05<01:20, 2.44s/it] 99%|█████████▉| 3093/3125 [1:24:07<01:17, 2.42s/it] 99%|█████████▉| 3094/3125 [1:24:10<01:17, 2.51s/it] 99%|█████████▉| 3095/3125 [1:24:13<01:16, 2.54s/it] 99%|█████████▉| 3096/3125 [1:24:15<01:12, 2.49s/it] 99%|█████████▉| 3097/3125 [1:24:17<01:06, 2.38s/it] 99%|█████████▉| 3098/3125 [1:24:20<01:06, 2.47s/it] 99%|█████████▉| 3099/3125 [1:24:22<01:04, 2.47s/it] 99%|█████████▉| 3100/3125 [1:24:25<01:00, 2.44s/it] {'loss': '0.9142', 'grad_norm': '5.25', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005615', 'train/loss_cot': '1.028', 'epoch': '4.96'} 99%|█████████▉| 3100/3125 [1:24:25<01:00, 2.44s/it] 99%|█████████▉| 3101/3125 [1:24:27<01:01, 2.55s/it] 99%|█████████▉| 3102/3125 [1:24:30<01:01, 2.66s/it] 99%|█████████▉| 3103/3125 [1:24:33<00:56, 2.59s/it] 99%|█████████▉| 3104/3125 [1:24:35<00:52, 2.51s/it] 99%|█████████▉| 3105/3125 [1:24:38<00:54, 2.74s/it] 99%|█████████▉| 3106/3125 [1:24:41<00:49, 2.60s/it] 99%|█████████▉| 3107/3125 [1:24:43<00:44, 2.48s/it] 99%|█████████▉| 3108/3125 [1:24:45<00:41, 2.46s/it] 99%|█████████▉| 3109/3125 [1:24:47<00:37, 2.36s/it] 100%|█████████▉| 3110/3125 [1:24:49<00:33, 2.24s/it] {'loss': '0.9832', 'grad_norm': '6.69', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04028', 'train/loss_cot': '0.9677', 'epoch': '4.976'} 100%|█████████▉| 3110/3125 [1:24:50<00:33, 2.24s/it] 100%|█████████▉| 3111/3125 [1:24:52<00:32, 2.31s/it] 100%|█████████▉| 3112/3125 [1:24:54<00:29, 2.27s/it] 100%|█████████▉| 3113/3125 [1:24:56<00:27, 2.27s/it] 100%|█████████▉| 3114/3125 [1:24:59<00:25, 2.31s/it] 100%|█████████▉| 3115/3125 [1:25:01<00:22, 2.27s/it] 100%|█████████▉| 3116/3125 [1:25:03<00:20, 2.24s/it] 100%|█████████▉| 3117/3125 [1:25:05<00:17, 2.25s/it] 100%|█████████▉| 3118/3125 [1:25:07<00:15, 2.21s/it] 100%|█████████▉| 3119/3125 [1:25:09<00:12, 2.14s/it] 100%|█████████▉| 3120/3125 [1:25:13<00:12, 2.58s/it] {'loss': '0.9366', 'grad_norm': '5.909', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03296', 'train/loss_cot': '0.9476', 'epoch': '4.992'} 100%|█████████▉| 3120/3125 [1:25:13<00:12, 2.58s/it] 100%|█████████▉| 3121/3125 [1:25:16<00:10, 2.73s/it] 100%|█████████▉| 3122/3125 [1:25:19<00:08, 2.88s/it] 100%|█████████▉| 3123/3125 [1:25:21<00:05, 2.68s/it] 100%|█████████▉| 3124/3125 [1:25:24<00:02, 2.56s/it] 100%|██████████| 3125/3125 [1:25:25<00:00, 2.05s/it] {'train_runtime': '5148', 'train_samples_per_second': '9.712', 'train_steps_per_second': '0.607', 'train_loss': '0.664', 'train/loss_infonce': '0.04126', 'train/loss_cot': '0.7905', 'epoch': '5'} 100%|██████████| 3125/3125 [1:25:48<00:00, 2.05s/it] 100%|██████████| 3125/3125 [1:25:48<00:00, 1.65s/it] notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault notebook-622b7fa0e0f7-worker-0:259327:259824 [3] NCCL INFO [Service thread] Connection closed by localRank 1 notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Service thread] Connection closed by localRank 1 notebook-622b7fa0e0f7-worker-0:259327:261485 [3] NCCL INFO [Service thread] Connection closed by localRank 1 notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Service thread] Connection closed by localRank 1 notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault notebook-622b7fa0e0f7-worker-0:259327:259824 [3] NCCL INFO [Service thread] Connection closed by localRank 2 notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Service thread] Connection closed by localRank 2 notebook-622b7fa0e0f7-worker-0:259327:261485 [3] NCCL INFO [Service thread] Connection closed by localRank 2 notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Service thread] Connection closed by localRank 2 notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault