| + NUM_GPUS=4 |
| + DISTRIBUTED_ARGS=' |
| --nnodes=1 --nproc_per_node 4 --rdzv_backend c10d --rdzv_endpoint localhost:0 ' |
| + MODEL_ID=qwen3vl |
| + MODEL_LOCAL_PATH=/data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ |
| + EMBED_DIM=2048 |
| + DATA_PATH=/data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json |
| + MAX_LINES=10000 |
| + IMAGE_TYPE=both |
| + INFONCE_WEIGHT=1.0 |
| + COT_WEIGHT=1.0 |
| + TEMPERATURE=0.02 |
| + NEG_GATHER_SIZE=4 |
| + MAX_COT_LENGTH=192 |
| + DOC_APPEND_EOS=False |
| + USE_WANDB=False |
| + WANDB_MODE=online |
| + TRAIN_VISION_ENCODER=False |
| + USE_VISION_LORA=False |
| + TRAIN_LLM=True |
| + USE_LORA=True |
| + Q_LORA=False |
| + LORA_R=128 |
| + LORA_ALPHA=256 |
| + TRAIN_LM_HEAD=True |
| + DS_STAGE=zero2 |
| + PER_DEVICE_BATCH_SIZE=4 |
| + GRAD_ACCUM=1 |
| + NUM_EPOCHS=5 |
| + SAVE_STEPS=200 |
| + LR=1e-5 |
| + MODEL_MAX_LEN=768 |
| + OUTPUT_DIR=/data/oss_bucket_0/liqi/cot_embedding/local_train/dapei_lora128_cot_embedding_debug |
| + RUN_ID=qwen3vl_cot_embedding_debug |
| + RESUME=True |
| + echo 'begin training!' |
| begin training! |
| + torchrun --nnodes=1 --nproc_per_node 4 --rdzv_backend c10d --rdzv_endpoint localhost:0 train_cot/train_tuwen_qwen3vl_cot.py --model_id qwen3vl --data_path /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json --max_lines 10000 --output_dir /data/oss_bucket_0/liqi/cot_embedding/local_train/dapei_lora128_cot_embedding_debug/qwen3vl_cot_embedding_debug --report_to tensorboard --run_name qwen3vl_cot_embedding_debug --deepspeed ./ds_configs/zero2.json --bf16 True --num_train_epochs 5 --per_device_train_batch_size 4 --per_device_eval_batch_size 12 --gradient_accumulation_steps 1 --eval_strategy no --save_strategy steps --save_total_limit 200 --learning_rate 1e-5 --weight_decay 0. --warmup_ratio 0.03 --lr_scheduler_type constant_with_warmup --logging_steps 10 --tf32 True --model_max_length 768 --gradient_checkpointing False --dataloader_num_workers 0 --train_vision_encoder False --use_vision_lora False --use_lora True --q_lora False --lora_r 128 --lora_alpha 256 --model_local_path /data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ --use_flash_attn True --save_steps 200 --stage classify --embed_dim 2048 --resume_from_checkpoint True --image_type both --train_llm True --train_lm_head True --infonce_weight 1.0 --cot_weight 1.0 --temperature 0.02 --neg_gather_size 4 --max_cot_length 192 --doc_append_eos False --use_wandb False --wandb_mode online |
| Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. |
| Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. |
| Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. |
| Supported flash-attn versions are >= 2.1.1, <= 2.8.3. Found flash-attn 2.8.4. |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) |
| warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable |
| warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) |
| warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable |
| warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) |
| warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../loaders/__init__.py:15: UserWarning: tbstars_ecom_emb loader unavailable (skipped): cannot import name 'is_torch_sdpa_available' from 'transformers.utils' (/opt/venv/python3.13.12/lib/python3.13/site-packages/transformers/utils/__init__.py) |
| warnings.warn(f"tbstars_ecom_emb loader unavailable (skipped): {e}") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable |
| warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") |
| /mnt/workspace/liqi/multi_vlm2vec_train/train_cot/../utils/supported_models.py:51: UserWarning: Loader not found for model family: tbstars_ecom_emb_3b, it will be unavailable |
| warnings.warn(f"Loader not found for model family: {model_family_id}, it will be unavailable") |
| warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. |
| warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ACCL_TUNING_LEVEL=2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO cudaDriverVersion 13010 |
| warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL git version unknown unknown |
| warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead. |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO cudaDriverVersion 13010 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ACCL_TUNING_LEVEL=2 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL git version unknown unknown |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO cudaDriverVersion 13010 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ACCL_TUNING_LEVEL=2 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL git version unknown unknown |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ENV/Plugin: Could not find: libnccl-env.so |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO cudaDriverVersion 13010 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Bootstrap: Using eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ACCL_TUNING_LEVEL=2 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL version 2.29.2.18-accl-n+cuda13.1, COMMIT_ID , BUILD_TIME 2026-02-25 20:37:33 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL git version unknown unknown |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO C4 stats mode none, reduce 1, send/recv 0. |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NCCL_NET_PLUGIN set by environment to none |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NCCL_NET_PLUGIN set by environment to none |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/IB : No device found. |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Failed to initialize NET plugin IB |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/IB : No device found. |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Failed to initialize NET plugin IB |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NCCL_NET_PLUGIN set by environment to none |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/IB : No device found. |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Failed to initialize NET plugin IB |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NCCL_NET_PLUGIN set by environment to none |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/IB : No device found. |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/IB : Using [RO]; OOB eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Failed to initialize NET plugin IB |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NET/Socket : Using [0]eth0:33.110.226.103<0> |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO [Rank 0] ncclCommInitRankConfig comm 0x11ea92f0 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 89000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO [Rank 2] ncclCommInitRankConfig comm 0x11eca310 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId b2000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO [Rank 3] ncclCommInitRankConfig comm 0x11eacf80 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO RAS client listening socket at 127.0.0.1<28028> |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO [Rank 1] ncclCommInitRankConfig comm 0x11ecb740 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 8e000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init START |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO RAS client listening socket at 127.0.0.1<28028> |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO RAS client listening socket at 127.0.0.1<28028> |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO RAS client listening socket at 127.0.0.1<28028> |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Bootstrap timings total 0.012297 (create 0.000029, send 0.000117, recv 0.011052, ring 0.000807, delay 0.000007) |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Bootstrap timings total 0.011394 (create 0.000021, send 0.000115, recv 0.000141, ring 0.010813, delay 0.000003) |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO gTaskid = 67e26e210000679c. |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO gTaskid = 67e26e210000679c. |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Bootstrap timings total 0.011877 (create 0.000021, send 0.000115, recv 0.000546, ring 0.000758, delay 0.000004) |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Bootstrap timings total 0.001388 (create 0.000036, send 0.000132, renotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO gTaskid = 67e26e210000679c. |
| cv 0.000173, ring 0.000725, delay 0.000004) |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO gTaskid = 67e26e210000679c. |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. |
|
|
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. |
|
|
| ource temporarily unavailable. |
|
|
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO hpn_license_register fail: Error occurred during zmq_msg_recv(): Resource temporarily unavailable. |
|
|
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 14-31,44-47,52-53,59-62,78-95,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO comm 0x11eacf80 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO comm 0x11ea92f0 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNNVL 0 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO comm 0x11eca310 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0 |
| NVL 0 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO comm 0x11ecb740 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNnotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/16 : 0 1 3 2 |
| NVL 0 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Trees [0] -1/-1/-1->2->3 [1] -1/-1/-1->2->3 [2] -1/-1/-1->2->3 [3] -1/-1/-1->2->3 [4] -1/-1/-1->2->3 [5] -1/-1/-1->2->3 [6] -1/-1/-1->2->3 [7] -1/-1/-1->2->3 [8] -1/-1/-1->2->3 [9] -1/-1/-1->2->3 [10] -1/-1/-1->2->3 [11] -1/-1/-1->2->3 [12] -1/-1/-1->2->3 [13] -1/-1/-1->2->3 [14] -1/-1/-1->2->3 [15] -1/-1/-1->2->3 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Trees [0] 2/-1/-1->3->1 [1] 2/-1/-1->3->1 [2] 2/-1/-1->3->1 [3] 2/-1/-1->3->1 [4] 2/-1/-1->3->1 [5] 2/-1/-1->3->1 [6] 2/-1/-1->3->1 [7] 2/-1/-1->3->1 [8] 2/-1/-1->3->1 [9] 2/-1/-1->3->1 [10] 2/-1/-1->3->1 [11] 2/-1/-1->3->1 [12] 2/-1/-1->3->1 [13] 2/-1/-1->3->1 [14] 2/-1/-1->3->1 [15] 2/-1/-1->3->1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Trees [0] 3/-1/-1->1->0 [1] 3/-1/-1->1->0 [2] 3/-1/-1->1->0 [3] 3/-1/-1->1->0 [4] 3/-1/-1->1->0 [5] 3/-1/-1->1->0 [6] 3/-1/-1->1->0 [7] 3/-1/-1->1->0 [8] 3/-1/-1->1->0 [9] 3/-1/-1->1->0 [10] 3/-1/-1->1->0 [11] 3/-1/-1->1->0 [12] 3/-1/-1->1->0 [13] 3/-1/-1->1->0 [14] 3/-1/-1->1->0 [15] 3/-1/-1->1->0 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259825 [0] NCCL INFO [Proxy Service] Device 2 CPU core 62 |
| notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Proxy Service] Device 0 CPU core 126 |
| notebook-622b7fa0e0f7-worker-0:259327:259829 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 110 |
| notebook-622b7fa0e0f7-worker-0:259327:259824 [0] NCCL INFO [Proxy Service] Device 3 CPU core 62 |
| notebook-622b7fa0e0f7-worker-0:259326:259827 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 109 |
| notebook-622b7fa0e0f7-worker-0:259324:259828 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 45 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259325:259830 [0] NCCL INFO [Proxy Service] Device 1 CPU core 45 |
| notebook-622b7fa0e0f7-worker-0:259325:259831 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 46 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO CC Off, workFifoBytes 1048576 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO TUNER/Plugin: Could not find: libnccl-tuner.so |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommInitRankConfig comm 0x11ea92f0 rank 0 nranks 4 cudaDev 0 nvmnotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommInitRankConfig comm 0x11eca310 rank 2 nranks 4 cudaDev 2 nvmlDev 0 busId 89000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE |
| lDev 2 busId b2000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 4 total 5.43 (knotebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 4 total 5.42 (kernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) |
| ernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommInitRankConfig comm 0x11eacf80 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 4 total 5.42 (knotebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommInitRankConfig comm 0x11ecb740 rank 1 nranks 4 cudaDev 1 nvmernels 0.23, alloc 0.03, bootstrap 0.01, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) |
| lDev 1 busId 8e000 commId 0x84a1fba13acef023 commHash 9557196554151850019 - Init COMPLETE |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 4 total 5.42 (kernels 0.24, alloc 0.03, bootstrap 0.00, allgathers 0.00, topo 0.12, graphs 0.00, connections 0.02, rest 5.01) |
| Loading Qwen3-VL-CoT-Embedding from /data/oss_bucket_0/peifeng/vlm2vec/models/qwen/Qwen3-VL-Embedding-2B/ ... |
|
Loading weights: 0%| | 0/625 [00:00<?, ?it/s]
Loading weights: 0%| | 0/625 [00:00<?, ?it/s]
Loading weights: 0%| | 0/625 [00:00<?, ?it/s]
Loading weights: 100%|ββββββββββ| 625/625 [00:00<00:00, 8596.94it/s] |
| The module name (originally ) is not a valid Python identifier. Please rename the original module to avoid import issues. |
|
Loading weights: 100%|ββββββββββ| 625/625 [00:00<00:00, 9038.11it/s] |
|
Loading weights: 100%|ββββββββββ| 625/625 [00:00<00:00, 7969.55it/s] |
| The module name (originally ) is not a valid Python identifier. Please rename the original module to avoid import issues. |
| The module name (originally ) is not a valid Python identifier. Please rename the original module to avoid import issues. |
|
Loading weights: 0%| | 0/625 [00:00<?, ?it/s]
Loading weights: 100%|ββββββββββ| 625/625 [00:00<00:00, 8753.28it/s] |
| The module name (originally ) is not a valid Python identifier. Please rename the original module to avoid import issues. |
| /opt/venv/python3.13.12/lib/python3.13/site-packages/peft/tuners/tuners_utils.py:1377: UserWarning: Model has `tie_word_embeddings=True` and a tied layer is part of the adapter, but `ensure_weight_tying` is not set to True. This can lead to complications, for example when merging the adapter or converting your model to formats other than safetensors. Check the discussion here: https://github.com/huggingface/peft/issues/2777 |
| warnings.warn(msg) |
| Freezing vision encoder... |
| Training lm_head via modules_to_save (decoupled from embed_tokens)... |
| /opt/venv/python3.13.12/lib/python3.13/site-packages/peft/tuners/tuners_utils.py:1377: UserWarning: Model has `tie_word_embeddings=True` and a tied layer is part of the adapter, but `ensure_weight_tying` is not set to True. This can lead to complications, for example when merging the adapter or converting your model to formats other than safetensors. Check the discussion here: https://github.com/huggingface/peft/issues/2777 |
| warnings.warn(msg) |
| /opt/venv/python3.13.12/lib/python3.13/site-packages/peft/tuners/tuners_utils.py:1377: UserWarning: Model has `tie_word_embeddings=True` and a tied layer is part of the adapter, but `ensure_weight_tying` is not set to True. This can lead to complications, for example when merging the adapter or converting your model to formats other than safetensors. Check the discussion here: https://github.com/huggingface/peft/issues/2777 |
| warnings.warn(msg) |
| /opt/venv/python3.13.12/lib/python3.13/site-packages/peft/tuners/tuners_utils.py:1377: UserWarning: Model has `tie_word_embeddings=True` and a tied layer is part of the adapter, but `ensure_weight_tying` is not set to True. This can lead to complications, for example when merging the adapter or converting your model to formats other than safetensors. Check the discussion here: https://github.com/huggingface/peft/issues/2777 |
| warnings.warn(msg) |
| LoRA applied. |
| Trainable parameters: |
| base_model.model.model.language_model.layers.0.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.0.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.0.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.1.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.1.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.2.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.2.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.3.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.3.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.4.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.4.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.5.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.5.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.6.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.6.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.7.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.7.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.8.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.8.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.9.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.9.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.10.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.10.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.11.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.11.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.12.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.12.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.13.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.13.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.14.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.14.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.15.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.15.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.16.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.16.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.17.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.17.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.18.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.18.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.19.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.19.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.20.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.20.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.21.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.21.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.22.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.22.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.23.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.23.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.24.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.24.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.25.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.25.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.26.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.26.mlp.down_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.q_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.q_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.k_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.k_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.v_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.v_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.o_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.self_attn.o_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.mlp.gate_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.mlp.gate_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.mlp.up_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.mlp.up_proj.lora_B.default.weight |
| base_model.model.model.language_model.layers.27.mlp.down_proj.lora_A.default.weight |
| base_model.model.model.language_model.layers.27.mlp.down_proj.lora_B.default.weight |
| base_model.model.lm_head.modules_to_save.default.weight |
| Loading data... |
| total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). |
| total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). |
| No eval during training. |
| [infonce] neg_gather_size=4 -> η»ε
4 ε‘ε
±δΊ«οΌειζ± 32οΌζ―δΈͺ query 31 δΈͺθ΄ζ ·ζ¬ |
| total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). |
| total of 10000 items (local json /data/oss_bucket_0/liqi/dataset/embedding_pf/cot_output_new/dapei_train_cot.json). |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Comm config Blocking set to 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommSplit comm 0x2a1307c0 rank 2 nranks 4 cudaDev 2 nvmlDev 2 busId b2000 parent 0x11eca310 childCount 1 color 2003953581 key 2 commHash 8923940766810589732 - Init START |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Initialized NET plugin Socket |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommSplit comm 0x2de13b40 rank 3 nranks 4 cudaDev 3 nvmlDev 3 busId b8000 parent 0x11eacf80 childCount 1 color 2003953581 key 3 commHash 8923940766810589732 - Init START |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommSplit comm 0x19c91200 rank 0 nranks 4 cudaDev 0 nvmlDev 0 busId 89000 parent 0x11ea92f0 childCount 1 color 2003953581 key 0 commHash 8923940766810589732 - Init START |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Assigned NET plugin Socket to comm |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Using network Socket |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommSplit comm 0x2c7f1100 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId 8e000 parent 0x11ecb740 childCount 1 color 2003953581 key 1 commHash 8923940766810589732 - Init START |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 3 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO NVLS multicast support is not available on dev 3 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 2 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO NVLS multicast support is not available on dev 2 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 0 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO NVLS multicast support is not available on dev 0 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclTopoGetCpuAffinity: Affinity for GPU 1 is 44-47,52-53,59-62,108-111,116-117,123-126. (GPU affinity = 32-63,96-127 ; CPU affinity = 44-47,52-53,59-62,108-111,116-117,123-126). |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO NVLS multicast support is not available on dev 1 (NVLS_NCHANNELS 0) |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO comm 0x2c7f1100 rank 1 nRanks 4 nNodes 1 localRanks 4 localRank 1 MNNVL 0 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO comm 0x19c91200 rank 0 nRanks 4 nNodes 1 localRanks 4 localRank 0 MNnotebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO comm 0x2de13b40 rank 3 nRanks 4 nNodes 1 localRanks 4 localRank 3 MNNVL 0 |
| NVL 0 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO comm 0x2a1307c0 rank 2 nRanks 4 nNodes 1 localRanks 4 localRank 2 MNNVL 0 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/16 : 0 1 3 2 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Trees [0] 3/-1/-1->1->0 [1] 3/-1/-1->1->0 [2] 3/-1/-1->1->0 [3] 3/-1/-1->1->0 [4] 3/-1/-1->1->0 [5] 3/-1/-1->1->0 [6] 3/-1/-1->1->0 [7] 3/-1/-1->1->0 [8] 3/-1/-1->1->0 [9] 3/-1/-1->1->0 [10] 3/-1/-1->1->0 [11] 3/-1/-1->1->0 [12] 3/-1/-1->1->0 [13] 3/-1/-1->1->0 [14] 3/-1/-1->1->0 [15] 3/-1/-1->1->0 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Trees [0] 2/-1/-1->3->1 [1] 2/-1/-1->3->1 [2] 2/-1/-1->3->1 [3] 2/-1/-1->3->1 [4] 2/-1/-1->3->1 [5] 2/-1/-1->3->1 [6] 2/-1/-1->3->1 [7] 2/-1/-1->3->1 [8] 2/-1/-1->3->1 [9] 2/-1/-1->3->1 [10] 2/-1/-1->3->1 [11] 2/-1/-1->3->1 [12] 2/-1/-1->3->1 [13] 2/-1/-1->3->1 [14] 2/-1/-1->3->1 [15] 2/-1/-1->3->1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Trees [0] -1/-1/-1->2->3 [1] -1/-1/-1->2->3 [2] -1/-1/-1->2->3 [3] -1/-1/-1->2->3 [4] -1/-1/-1->2->3 [5] -1/-1/-1->2->3 [6] -1/-1/-1->2->3 [7] -1/-1/-1->2->3 [8] -1/-1/-1->2->3 [9] -1/-1/-1->2->3 [10] -1/-1/-1->2->3 [11] -1/-1/-1->2->3 [12] -1/-1/-1->2->3 [13] -1/-1/-1->2->3 [14] -1/-1/-1->2->3 [15] -1/-1/-1->2->3 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO P2P Chunksize set to 524288 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Check P2P Type isAllDirectP2p 1 directMode 0 isAllCudaP2p 1 |
| notebook-622b7fa0e0f7-worker-0:259325:261483 [0] NCCL INFO [Proxy Service] Device 1 CPU core 53 |
| notebook-622b7fa0e0f7-worker-0:259326:261484 [0] NCCL INFO [Proxy Service] Device 2 CPU core 124 |
| notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Proxy Service] Device 0 CPU core 125 |
| notebook-622b7fa0e0f7-worker-0:259327:261485 [0] NCCL INFO [Proxy Service] Device 3 CPU core 60 |
| notebook-622b7fa0e0f7-worker-0:259325:261487 [0] NCCL INFO [Proxy Service UDS] Device 1 CPU core 62 |
| notebook-622b7fa0e0f7-worker-0:259324:261490 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 45 |
| notebook-622b7fa0e0f7-worker-0:259326:261489 [0] NCCL INFO [Proxy Service UDS] Device 2 CPU core 44 |
| notebook-622b7fa0e0f7-worker-0:259327:261488 [0] NCCL INFO [Proxy Service UDS] Device 3 CPU core 108 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 512 | 512 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO 16 coll channels, 16 collnet channels, 0 nvls channels, 16 p2p channels, 16 p2p channels per peer |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO CC Off, workFifoBytes 1048576 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ncclCommSplit comm 0x2de13b40 rank 3 nranks 4 cudaDev 3 nvmlDev 3 bunotebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ncclCommSplit comm 0x19c91200 rank 0 nranks 4 cudaDev 0 nvmlDev 0 bunotebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ncclCommSplit comm 0x2c7f1100 rank 1 nranks 4 cudaDev 1 nvmlDev 1 busId b8000 parent 0x11eacf80 childCount 1 color 2003953581 key 3 commHash 8923940766810589732 - Init COMPLETE |
| sId 89000 parent 0x11ea92f0 childCount 1 color 2003953581 key 0 commHash 8923940766810589732 - Init COMPLETE |
| sId 8e000 parent 0x11ecb740 childCount 1 color 2003953581 key 1 commHash 8923940766810589732 - Init COMPLETE |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Init timings - ncclCommSplit: rank 0 nranks 4 total 0.34 (kernels 0.notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Init timings - ncclCommSplit: rank 3 nranks 4 total 0.23 (kernels 0.notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Init timings - ncclCommSplit: rank 1 nranks 4 total 0.21 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.13) |
| 00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.03) |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ncclCommSplit comm 0x2a1307c0 rank 2 nranks 4 cudaDev 2 nvmlDev 2 bu00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.01) |
| sId b2000 parent 0x11eca310 childCount 1 color 2003953581 key 2 commHash 8923940766810589732 - Init COMPLETE |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Init timings - ncclCommSplit: rank 2 nranks 4 total 0.47 (kernels 0.00, alloc 0.00, bootstrap 0.00, allgathers 0.00, topo 0.18, graphs 0.00, connections 0.02, rest 0.27) |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 00/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 04/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 07/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 08/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 09/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 12/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 15/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 00/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 04/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 07/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 08/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 09/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 12/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 15/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 14/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
|
0%| | 0/3125 [00:00<?, ?it/s]notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 00/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 01/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 02/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 03/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 04/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 05/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 06/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 07/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 08/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 09/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 10/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 11/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 12/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 13/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 14/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Channel 15/0 : 2[2] -> 0[0] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Symmetric VA size=80GB |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 00/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 01/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 02/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 03/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 04/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 05/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 06/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 07/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 08/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 09/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 10/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 11/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 12/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 13/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 14/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Channel 15/0 : 1[1] -> 3[3] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 01/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 02/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 03/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 05/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 07/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 09/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 10/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 11/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 13/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 14/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Channel 15/0 : 3[3] -> 2[2] via P2P/CUMEM/read |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 |
|
38%|ββββ | 1201/3125 [00:06<00:10, 186.18it/s]
39%|ββββ | 1207/3125 [00:26<00:10, 186.18it/s]
39%|ββββ | 1208/3125 [00:26<00:55, 34.60it/s]
39%|ββββ | 1209/3125 [00:29<01:04, 29.77it/s]
{'loss': '1.312', 'grad_norm': '7.882', 'learning_rate': '1e-05', 'train/loss_infonce': '0.5898', 'train/loss_cot': '0.9783', 'epoch': '1.936'} |
|
39%|ββββ | 1210/3125 [00:32<01:04, 29.77it/s]
39%|ββββ | 1215/3125 [00:46<01:04, 29.77it/s]
39%|ββββ | 1216/3125 [00:48<02:31, 12.56it/s]
39%|ββββ | 1217/3125 [00:51<02:47, 11.40it/s]
{'loss': '1.349', 'grad_norm': '6.241', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04053', 'train/loss_cot': '1.036', 'epoch': '1.952'} |
|
39%|ββββ | 1220/3125 [01:00<02:47, 11.40it/s]
39%|ββββ | 1222/3125 [01:05<04:47, 6.62it/s]
39%|ββββ | 1223/3125 [01:08<05:21, 5.91it/s]
39%|ββββ | 1226/3125 [01:16<07:43, 4.10it/s]
39%|ββββ | 1228/3125 [01:22<09:53, 3.20it/s]
39%|ββββ | 1230/3125 [01:27<12:29, 2.53it/s]
{'loss': '1.36', 'grad_norm': '6.169', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0791', 'train/loss_cot': '0.9644', 'epoch': '1.968'} |
|
39%|ββββ | 1230/3125 [01:28<12:29, 2.53it/s]
39%|ββββ | 1231/3125 [01:31<14:41, 2.15it/s]
39%|ββββ | 1232/3125 [01:33<16:59, 1.86it/s]
39%|ββββ | 1233/3125 [01:36<20:11, 1.56it/s]
39%|ββββ | 1234/3125 [01:39<25:01, 1.26it/s]
40%|ββββ | 1235/3125 [01:42<29:38, 1.06it/s]
40%|ββββ | 1236/3125 [01:45<36:18, 1.15s/it]
40%|ββββ | 1237/3125 [01:47<42:16, 1.34s/it]
40%|ββββ | 1238/3125 [01:50<49:23, 1.57s/it]
40%|ββββ | 1239/3125 [01:53<54:50, 1.74s/it]
40%|ββββ | 1240/3125 [01:55<59:20, 1.89s/it]
{'loss': '1.344', 'grad_norm': '6.17', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2715', 'train/loss_cot': '1.272', 'epoch': '1.984'} |
|
40%|ββββ | 1240/3125 [01:55<59:20, 1.89s/it]
40%|ββββ | 1241/3125 [01:58<1:05:48, 2.10s/it]
40%|ββββ | 1242/3125 [02:00<1:08:47, 2.19s/it]
40%|ββββ | 1243/3125 [02:03<1:14:11, 2.37s/it]
40%|ββββ | 1244/3125 [02:06<1:18:48, 2.51s/it]
40%|ββββ | 1245/3125 [02:09<1:18:58, 2.52s/it]
40%|ββββ | 1246/3125 [02:11<1:21:06, 2.59s/it]
40%|ββββ | 1247/3125 [02:14<1:24:50, 2.71s/it]
40%|ββββ | 1248/3125 [02:17<1:24:56, 2.72s/it]
40%|ββββ | 1249/3125 [02:20<1:24:20, 2.70s/it]
40%|ββββ | 1250/3125 [02:21<1:07:12, 2.15s/it]
{'loss': '1.413', 'grad_norm': '7.31', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1162', 'train/loss_cot': '1.073', 'epoch': '2'} |
|
40%|ββββ | 1250/3125 [02:21<1:07:12, 2.15s/it]
40%|ββββ | 1251/3125 [02:25<1:25:21, 2.73s/it]
40%|ββββ | 1252/3125 [02:28<1:28:08, 2.82s/it]
40%|ββββ | 1253/3125 [02:31<1:31:57, 2.95s/it]
40%|ββββ | 1254/3125 [02:34<1:30:51, 2.91s/it]
40%|ββββ | 1255/3125 [02:36<1:27:42, 2.81s/it]
40%|ββββ | 1256/3125 [02:39<1:26:55, 2.79s/it]
40%|ββββ | 1257/3125 [02:42<1:28:34, 2.85s/it]
40%|ββββ | 1258/3125 [02:45<1:27:38, 2.82s/it]
40%|ββββ | 1259/3125 [02:48<1:27:17, 2.81s/it]
40%|ββββ | 1260/3125 [02:50<1:25:44, 2.76s/it]
{'loss': '1.193', 'grad_norm': '6.144', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1953', 'train/loss_cot': '0.9299', 'epoch': '2.016'} |
|
40%|ββββ | 1260/3125 [02:51<1:25:44, 2.76s/it]
40%|ββββ | 1261/3125 [02:53<1:25:13, 2.74s/it]
40%|ββββ | 1262/3125 [02:56<1:27:25, 2.82s/it]
40%|ββββ | 1263/3125 [02:59<1:27:53, 2.83s/it]
40%|ββββ | 1264/3125 [03:02<1:29:47, 2.89s/it]
40%|ββββ | 1265/3125 [03:04<1:25:04, 2.74s/it]
41%|ββββ | 1266/3125 [03:07<1:22:43, 2.67s/it]
41%|ββββ | 1267/3125 [03:10<1:24:04, 2.71s/it]
41%|ββββ | 1268/3125 [03:13<1:29:47, 2.90s/it]
41%|ββββ | 1269/3125 [03:16<1:27:26, 2.83s/it]
41%|ββββ | 1270/3125 [03:18<1:23:25, 2.70s/it]
{'loss': '1.15', 'grad_norm': '7.134', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2617', 'train/loss_cot': '1.007', 'epoch': '2.032'} |
|
41%|ββββ | 1270/3125 [03:18<1:23:25, 2.70s/it]
41%|ββββ | 1271/3125 [03:21<1:23:20, 2.70s/it]
41%|ββββ | 1272/3125 [03:24<1:27:30, 2.83s/it]
41%|ββββ | 1273/3125 [03:27<1:34:06, 3.05s/it]
41%|ββββ | 1274/3125 [03:30<1:31:46, 2.98s/it]
41%|ββββ | 1275/3125 [03:34<1:35:25, 3.09s/it]
41%|ββββ | 1276/3125 [03:37<1:38:26, 3.19s/it]
41%|ββββ | 1277/3125 [03:40<1:34:51, 3.08s/it]
41%|ββββ | 1278/3125 [03:43<1:32:22, 3.00s/it]
41%|ββββ | 1279/3125 [03:45<1:28:36, 2.88s/it]
41%|ββββ | 1280/3125 [03:48<1:28:35, 2.88s/it]
{'loss': '1.236', 'grad_norm': '5.722', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05029', 'train/loss_cot': '1.407', 'epoch': '2.048'} |
|
41%|ββββ | 1280/3125 [03:49<1:28:35, 2.88s/it]
41%|ββββ | 1281/3125 [03:51<1:25:59, 2.80s/it]
41%|ββββ | 1282/3125 [03:54<1:26:27, 2.81s/it]
41%|ββββ | 1283/3125 [03:56<1:26:01, 2.80s/it]
41%|ββββ | 1284/3125 [03:59<1:24:56, 2.77s/it]
41%|ββββ | 1285/3125 [04:02<1:22:38, 2.69s/it]
41%|ββββ | 1286/3125 [04:04<1:21:26, 2.66s/it]
41%|ββββ | 1287/3125 [04:07<1:19:24, 2.59s/it]
41%|ββββ | 1288/3125 [04:09<1:20:52, 2.64s/it]
41%|ββββ | 1289/3125 [04:13<1:26:06, 2.81s/it]
41%|βββββ | 1290/3125 [04:15<1:26:16, 2.82s/it]
{'loss': '1.178', 'grad_norm': '5.351', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01147', 'train/loss_cot': '0.8401', 'epoch': '2.064'} |
|
41%|βββββ | 1290/3125 [04:16<1:26:16, 2.82s/it]
41%|βββββ | 1291/3125 [04:18<1:24:35, 2.77s/it]
41%|βββββ | 1292/3125 [04:21<1:25:49, 2.81s/it]
41%|βββββ | 1293/3125 [04:24<1:24:44, 2.78s/it]
41%|βββββ | 1294/3125 [04:26<1:25:01, 2.79s/it]
41%|βββββ | 1295/3125 [04:29<1:23:37, 2.74s/it]
41%|βββββ | 1296/3125 [04:32<1:21:37, 2.68s/it]
42%|βββββ | 1297/3125 [04:35<1:23:27, 2.74s/it]
42%|βββββ | 1298/3125 [04:37<1:22:19, 2.70s/it]
42%|βββββ | 1299/3125 [04:40<1:22:29, 2.71s/it]
42%|βββββ | 1300/3125 [04:43<1:24:06, 2.77s/it]
{'loss': '1.221', 'grad_norm': '8.117', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3984', 'train/loss_cot': '1.056', 'epoch': '2.08'} |
|
42%|βββββ | 1300/3125 [04:43<1:24:06, 2.77s/it]
42%|βββββ | 1301/3125 [04:46<1:23:59, 2.76s/it]
42%|βββββ | 1302/3125 [04:48<1:19:58, 2.63s/it]
42%|βββββ | 1303/3125 [04:51<1:21:05, 2.67s/it]
42%|βββββ | 1304/3125 [04:54<1:26:00, 2.83s/it]
42%|βββββ | 1305/3125 [04:57<1:26:38, 2.86s/it]
42%|βββββ | 1306/3125 [04:59<1:25:01, 2.80s/it]
42%|βββββ | 1307/3125 [05:03<1:28:53, 2.93s/it]
42%|βββββ | 1308/3125 [05:05<1:26:58, 2.87s/it]
42%|βββββ | 1309/3125 [05:08<1:23:22, 2.75s/it]
42%|βββββ | 1310/3125 [05:11<1:23:46, 2.77s/it]
{'loss': '1.218', 'grad_norm': '7.923', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02466', 'train/loss_cot': '0.7941', 'epoch': '2.096'} |
|
42%|βββββ | 1310/3125 [05:11<1:23:46, 2.77s/it]
42%|βββββ | 1311/3125 [05:14<1:24:57, 2.81s/it]
42%|βββββ | 1312/3125 [05:16<1:24:05, 2.78s/it]
42%|βββββ | 1313/3125 [05:19<1:23:23, 2.76s/it]
42%|βββββ | 1314/3125 [05:21<1:20:53, 2.68s/it]
42%|βββββ | 1315/3125 [05:24<1:20:20, 2.66s/it]
42%|βββββ | 1316/3125 [05:27<1:21:35, 2.71s/it]
42%|βββββ | 1317/3125 [05:29<1:19:17, 2.63s/it]
42%|βββββ | 1318/3125 [05:32<1:21:07, 2.69s/it]
42%|βββββ | 1319/3125 [05:35<1:20:21, 2.67s/it]
42%|βββββ | 1320/3125 [05:37<1:19:03, 2.63s/it]
{'loss': '1.19', 'grad_norm': '6.648', 'learning_rate': '1e-05', 'train/loss_infonce': '0.126', 'train/loss_cot': '1.008', 'epoch': '2.112'} |
|
42%|βββββ | 1320/3125 [05:38<1:19:03, 2.63s/it]
42%|βββββ | 1321/3125 [05:40<1:18:51, 2.62s/it]
42%|βββββ | 1322/3125 [05:43<1:21:58, 2.73s/it]
42%|βββββ | 1323/3125 [05:46<1:20:52, 2.69s/it]
42%|βββββ | 1324/3125 [05:48<1:20:18, 2.68s/it]
42%|βββββ | 1325/3125 [05:51<1:20:44, 2.69s/it]
42%|βββββ | 1326/3125 [05:53<1:19:31, 2.65s/it]
42%|βββββ | 1327/3125 [05:56<1:17:47, 2.60s/it]
42%|βββββ | 1328/3125 [06:00<1:31:11, 3.04s/it]
43%|βββββ | 1329/3125 [06:03<1:27:36, 2.93s/it]
43%|βββββ | 1330/3125 [06:05<1:26:04, 2.88s/it]
{'loss': '1.187', 'grad_norm': '6.496', 'learning_rate': '1e-05', 'train/loss_infonce': '0.124', 'train/loss_cot': '0.8749', 'epoch': '2.128'} |
|
43%|βββββ | 1330/3125 [06:06<1:26:04, 2.88s/it]
43%|βββββ | 1331/3125 [06:08<1:23:37, 2.80s/it]
43%|βββββ | 1332/3125 [06:11<1:21:30, 2.73s/it]
43%|βββββ | 1333/3125 [06:13<1:21:22, 2.72s/it]
43%|βββββ | 1334/3125 [06:16<1:19:37, 2.67s/it]
43%|βββββ | 1335/3125 [06:18<1:17:33, 2.60s/it]
43%|βββββ | 1336/3125 [06:21<1:21:16, 2.73s/it]
43%|βββββ | 1337/3125 [06:24<1:19:43, 2.68s/it]
43%|βββββ | 1338/3125 [06:27<1:22:32, 2.77s/it]
43%|βββββ | 1339/3125 [06:30<1:21:17, 2.73s/it]
43%|βββββ | 1340/3125 [06:33<1:27:15, 2.93s/it]
{'loss': '1.208', 'grad_norm': '9.786', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02893', 'train/loss_cot': '1.2', 'epoch': '2.144'} |
|
43%|βββββ | 1340/3125 [06:33<1:27:15, 2.93s/it]
43%|βββββ | 1341/3125 [06:36<1:24:46, 2.85s/it]
43%|βββββ | 1342/3125 [06:38<1:22:34, 2.78s/it]
43%|βββββ | 1343/3125 [06:41<1:26:34, 2.91s/it]
43%|βββββ | 1344/3125 [06:44<1:23:08, 2.80s/it]
43%|βββββ | 1345/3125 [06:47<1:22:43, 2.79s/it]
43%|βββββ | 1346/3125 [06:49<1:22:19, 2.78s/it]
43%|βββββ | 1347/3125 [06:52<1:19:22, 2.68s/it]
43%|βββββ | 1348/3125 [06:54<1:17:53, 2.63s/it]
43%|βββββ | 1349/3125 [06:57<1:17:45, 2.63s/it]
43%|βββββ | 1350/3125 [07:00<1:18:40, 2.66s/it]
{'loss': '1.181', 'grad_norm': '5.429', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001717', 'train/loss_cot': '1.005', 'epoch': '2.16'} |
|
43%|βββββ | 1350/3125 [07:00<1:18:40, 2.66s/it]
43%|βββββ | 1351/3125 [07:03<1:21:32, 2.76s/it]
43%|βββββ | 1352/3125 [07:06<1:22:33, 2.79s/it]
43%|βββββ | 1353/3125 [07:08<1:21:12, 2.75s/it]
43%|βββββ | 1354/3125 [07:11<1:20:40, 2.73s/it]
43%|βββββ | 1355/3125 [07:14<1:22:41, 2.80s/it]
43%|βββββ | 1356/3125 [07:17<1:21:36, 2.77s/it]
43%|βββββ | 1357/3125 [07:20<1:25:44, 2.91s/it]
43%|βββββ | 1358/3125 [07:23<1:22:56, 2.82s/it]
43%|βββββ | 1359/3125 [07:25<1:21:36, 2.77s/it]
44%|βββββ | 1360/3125 [07:28<1:18:31, 2.67s/it]
{'loss': '1.205', 'grad_norm': '5.808', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01843', 'train/loss_cot': '1.143', 'epoch': '2.176'} |
|
44%|βββββ | 1360/3125 [07:28<1:18:31, 2.67s/it]
44%|βββββ | 1361/3125 [07:30<1:18:03, 2.65s/it]
44%|βββββ | 1362/3125 [07:33<1:18:39, 2.68s/it]
44%|βββββ | 1363/3125 [07:36<1:21:21, 2.77s/it]
44%|βββββ | 1364/3125 [07:38<1:18:43, 2.68s/it]
44%|βββββ | 1365/3125 [07:42<1:25:44, 2.92s/it]
44%|βββββ | 1366/3125 [07:45<1:25:39, 2.92s/it]
44%|βββββ | 1367/3125 [07:48<1:28:44, 3.03s/it]
44%|βββββ | 1368/3125 [07:51<1:29:00, 3.04s/it]
44%|βββββ | 1369/3125 [07:54<1:26:44, 2.96s/it]
44%|βββββ | 1370/3125 [07:57<1:24:39, 2.89s/it]
{'loss': '1.166', 'grad_norm': '6.158', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006927', 'train/loss_cot': '0.8185', 'epoch': '2.192'} |
|
44%|βββββ | 1370/3125 [07:57<1:24:39, 2.89s/it]
44%|βββββ | 1371/3125 [07:59<1:23:15, 2.85s/it]
44%|βββββ | 1372/3125 [08:02<1:21:09, 2.78s/it]
44%|βββββ | 1373/3125 [08:05<1:20:46, 2.77s/it]
44%|βββββ | 1374/3125 [08:07<1:19:21, 2.72s/it]
44%|βββββ | 1375/3125 [08:11<1:27:41, 3.01s/it]
44%|βββββ | 1376/3125 [08:14<1:24:43, 2.91s/it]
44%|βββββ | 1377/3125 [08:16<1:22:20, 2.83s/it]
44%|βββββ | 1378/3125 [08:20<1:24:53, 2.92s/it]
44%|βββββ | 1379/3125 [08:22<1:21:03, 2.79s/it]
44%|βββββ | 1380/3125 [08:25<1:20:48, 2.78s/it]
{'loss': '1.16', 'grad_norm': '5.463', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01624', 'train/loss_cot': '1.312', 'epoch': '2.208'} |
|
44%|βββββ | 1380/3125 [08:25<1:20:48, 2.78s/it]
44%|βββββ | 1381/3125 [08:28<1:22:10, 2.83s/it]
44%|βββββ | 1382/3125 [08:30<1:20:38, 2.78s/it]
44%|βββββ | 1383/3125 [08:33<1:18:37, 2.71s/it]
44%|βββββ | 1384/3125 [08:36<1:19:46, 2.75s/it]
44%|βββββ | 1385/3125 [08:39<1:23:09, 2.87s/it]
44%|βββββ | 1386/3125 [08:42<1:22:23, 2.84s/it]
44%|βββββ | 1387/3125 [08:44<1:19:32, 2.75s/it]
44%|βββββ | 1388/3125 [08:47<1:20:17, 2.77s/it]
44%|βββββ | 1389/3125 [08:50<1:18:35, 2.72s/it]
44%|βββββ | 1390/3125 [08:52<1:19:39, 2.75s/it]
{'loss': '1.194', 'grad_norm': '6.727', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2178', 'train/loss_cot': '1.002', 'epoch': '2.224'} |
|
44%|βββββ | 1390/3125 [08:53<1:19:39, 2.75s/it]
45%|βββββ | 1391/3125 [08:55<1:19:46, 2.76s/it]
45%|βββββ | 1392/3125 [08:58<1:18:30, 2.72s/it]
45%|βββββ | 1393/3125 [09:01<1:18:28, 2.72s/it]
45%|βββββ | 1394/3125 [09:03<1:19:08, 2.74s/it]
45%|βββββ | 1395/3125 [09:07<1:25:07, 2.95s/it]
45%|βββββ | 1396/3125 [09:09<1:21:24, 2.82s/it]
45%|βββββ | 1397/3125 [09:12<1:23:17, 2.89s/it]
45%|βββββ | 1398/3125 [09:15<1:21:58, 2.85s/it]
45%|βββββ | 1399/3125 [09:19<1:31:16, 3.17s/it]
45%|βββββ | 1400/3125 [09:22<1:27:11, 3.03s/it]
{'loss': '1.23', 'grad_norm': '5.633', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0009041', 'train/loss_cot': '1.322', 'epoch': '2.24'} |
|
45%|βββββ | 1400/3125 [09:22<1:27:11, 3.03s/it]mtime may not be reliable on this filesystem, falling back to numerical ordering |
|
45%|βββββ | 1401/3125 [09:41<3:42:47, 7.75s/it]
45%|βββββ | 1402/3125 [09:43<2:59:45, 6.26s/it]
45%|βββββ | 1403/3125 [09:46<2:28:12, 5.16s/it]
45%|βββββ | 1404/3125 [09:48<2:05:50, 4.39s/it]
45%|βββββ | 1405/3125 [09:52<1:54:36, 4.00s/it]
45%|βββββ | 1406/3125 [09:54<1:43:09, 3.60s/it]
45%|βββββ | 1407/3125 [09:57<1:35:47, 3.35s/it]
45%|βββββ | 1408/3125 [10:00<1:30:31, 3.16s/it]
45%|βββββ | 1409/3125 [10:03<1:31:23, 3.20s/it]
45%|βββββ | 1410/3125 [10:06<1:28:05, 3.08s/it]
{'loss': '1.144', 'grad_norm': '9.349', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00386', 'train/loss_cot': '1.238', 'epoch': '2.256'} |
|
45%|βββββ | 1410/3125 [10:06<1:28:05, 3.08s/it]
45%|βββββ | 1411/3125 [10:10<1:33:16, 3.27s/it]
45%|βββββ | 1412/3125 [10:13<1:30:45, 3.18s/it]
45%|βββββ | 1413/3125 [10:15<1:26:23, 3.03s/it]
45%|βββββ | 1414/3125 [10:18<1:23:09, 2.92s/it]
45%|βββββ | 1415/3125 [10:20<1:20:01, 2.81s/it]
45%|βββββ | 1416/3125 [10:23<1:20:47, 2.84s/it]
45%|βββββ | 1417/3125 [10:27<1:24:40, 2.97s/it]
45%|βββββ | 1418/3125 [10:29<1:21:27, 2.86s/it]
45%|βββββ | 1419/3125 [10:32<1:21:59, 2.88s/it]
45%|βββββ | 1420/3125 [10:35<1:18:29, 2.76s/it]
{'loss': '1.214', 'grad_norm': '6.526', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0835', 'train/loss_cot': '1.115', 'epoch': '2.272'} |
|
45%|βββββ | 1420/3125 [10:35<1:18:29, 2.76s/it]
45%|βββββ | 1421/3125 [10:38<1:22:23, 2.90s/it]
46%|βββββ | 1422/3125 [10:40<1:19:15, 2.79s/it]
46%|βββββ | 1423/3125 [10:43<1:16:47, 2.71s/it]
46%|βββββ | 1424/3125 [10:46<1:16:18, 2.69s/it]
46%|βββββ | 1425/3125 [10:48<1:17:12, 2.73s/it]
46%|βββββ | 1426/3125 [10:51<1:16:09, 2.69s/it]
46%|βββββ | 1427/3125 [10:54<1:18:29, 2.77s/it]
46%|βββββ | 1428/3125 [10:57<1:21:01, 2.86s/it]
46%|βββββ | 1429/3125 [10:59<1:17:00, 2.72s/it]
46%|βββββ | 1430/3125 [11:02<1:16:00, 2.69s/it]
{'loss': '1.255', 'grad_norm': '5.664', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1143', 'train/loss_cot': '0.888', 'epoch': '2.288'} |
|
46%|βββββ | 1430/3125 [11:02<1:16:00, 2.69s/it]
46%|βββββ | 1431/3125 [11:05<1:15:52, 2.69s/it]
46%|βββββ | 1432/3125 [11:07<1:16:14, 2.70s/it]
46%|βββββ | 1433/3125 [11:10<1:15:58, 2.69s/it]
46%|βββββ | 1434/3125 [11:13<1:14:36, 2.65s/it]
46%|βββββ | 1435/3125 [11:16<1:16:37, 2.72s/it]
46%|βββββ | 1436/3125 [11:19<1:19:20, 2.82s/it]
46%|βββββ | 1437/3125 [11:21<1:18:58, 2.81s/it]
46%|βββββ | 1438/3125 [11:25<1:25:03, 3.03s/it]
46%|βββββ | 1439/3125 [11:28<1:22:17, 2.93s/it]
46%|βββββ | 1440/3125 [11:31<1:22:25, 2.94s/it]
{'loss': '1.291', 'grad_norm': '10.11', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03369', 'train/loss_cot': '1.043', 'epoch': '2.304'} |
|
46%|βββββ | 1440/3125 [11:31<1:22:25, 2.94s/it]
46%|βββββ | 1441/3125 [11:33<1:18:52, 2.81s/it]
46%|βββββ | 1442/3125 [11:36<1:16:26, 2.73s/it]
46%|βββββ | 1443/3125 [11:38<1:17:00, 2.75s/it]
46%|βββββ | 1444/3125 [11:41<1:14:39, 2.66s/it]
46%|βββββ | 1445/3125 [11:44<1:15:21, 2.69s/it]
46%|βββββ | 1446/3125 [11:46<1:15:37, 2.70s/it]
46%|βββββ | 1447/3125 [11:49<1:15:42, 2.71s/it]
46%|βββββ | 1448/3125 [11:52<1:15:32, 2.70s/it]
46%|βββββ | 1449/3125 [11:54<1:13:56, 2.65s/it]
46%|βββββ | 1450/3125 [11:57<1:13:44, 2.64s/it]
{'loss': '1.131', 'grad_norm': '5.072', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04712', 'train/loss_cot': '0.7999', 'epoch': '2.32'} |
|
46%|βββββ | 1450/3125 [11:57<1:13:44, 2.64s/it]
46%|βββββ | 1451/3125 [12:00<1:17:07, 2.76s/it]
46%|βββββ | 1452/3125 [12:02<1:15:08, 2.70s/it]
46%|βββββ | 1453/3125 [12:05<1:17:39, 2.79s/it]
47%|βββββ | 1454/3125 [12:08<1:15:02, 2.69s/it]
47%|βββββ | 1455/3125 [12:11<1:17:42, 2.79s/it]
47%|βββββ | 1456/3125 [12:13<1:14:37, 2.68s/it]
47%|βββββ | 1457/3125 [12:16<1:13:22, 2.64s/it]
47%|βββββ | 1458/3125 [12:19<1:13:05, 2.63s/it]
47%|βββββ | 1459/3125 [12:21<1:13:43, 2.66s/it]
47%|βββββ | 1460/3125 [12:24<1:12:21, 2.61s/it]
{'loss': '1.206', 'grad_norm': '7.059', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01117', 'train/loss_cot': '1.28', 'epoch': '2.336'} |
|
47%|βββββ | 1460/3125 [12:24<1:12:21, 2.61s/it]
47%|βββββ | 1461/3125 [12:27<1:13:46, 2.66s/it]
47%|βββββ | 1462/3125 [12:30<1:18:05, 2.82s/it]
47%|βββββ | 1463/3125 [12:33<1:18:57, 2.85s/it]
47%|βββββ | 1464/3125 [12:36<1:19:55, 2.89s/it]
47%|βββββ | 1465/3125 [12:38<1:16:40, 2.77s/it]
47%|βββββ | 1466/3125 [12:41<1:15:28, 2.73s/it]
47%|βββββ | 1467/3125 [12:44<1:16:55, 2.78s/it]
47%|βββββ | 1468/3125 [12:47<1:17:53, 2.82s/it]
47%|βββββ | 1469/3125 [12:50<1:18:50, 2.86s/it]
47%|βββββ | 1470/3125 [12:52<1:15:16, 2.73s/it]
{'loss': '1.159', 'grad_norm': '6.841', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08301', 'train/loss_cot': '0.9938', 'epoch': '2.352'} |
|
47%|βββββ | 1470/3125 [12:52<1:15:16, 2.73s/it]
47%|βββββ | 1471/3125 [12:55<1:15:39, 2.74s/it]
47%|βββββ | 1472/3125 [12:57<1:14:59, 2.72s/it]
47%|βββββ | 1473/3125 [13:00<1:13:38, 2.67s/it]
47%|βββββ | 1474/3125 [13:03<1:16:31, 2.78s/it]
47%|βββββ | 1475/3125 [13:06<1:17:31, 2.82s/it]
47%|βββββ | 1476/3125 [13:09<1:16:16, 2.78s/it]
47%|βββββ | 1477/3125 [13:11<1:13:24, 2.67s/it]
47%|βββββ | 1478/3125 [13:14<1:13:42, 2.69s/it]
47%|βββββ | 1479/3125 [13:16<1:12:17, 2.64s/it]
47%|βββββ | 1480/3125 [13:19<1:16:49, 2.80s/it]
{'loss': '1.272', 'grad_norm': '6.188', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04712', 'train/loss_cot': '1.353', 'epoch': '2.368'} |
|
47%|βββββ | 1480/3125 [13:20<1:16:49, 2.80s/it]
47%|βββββ | 1481/3125 [13:22<1:16:04, 2.78s/it]
47%|βββββ | 1482/3125 [13:25<1:15:58, 2.77s/it]
47%|βββββ | 1483/3125 [13:28<1:15:06, 2.74s/it]
47%|βββββ | 1484/3125 [13:30<1:11:29, 2.61s/it]
48%|βββββ | 1485/3125 [13:33<1:13:28, 2.69s/it]
48%|βββββ | 1486/3125 [13:36<1:15:20, 2.76s/it]
48%|βββββ | 1487/3125 [13:39<1:15:59, 2.78s/it]
48%|βββββ | 1488/3125 [13:41<1:15:44, 2.78s/it]
48%|βββββ | 1489/3125 [13:44<1:15:16, 2.76s/it]
48%|βββββ | 1490/3125 [13:47<1:14:49, 2.75s/it]
{'loss': '1.281', 'grad_norm': '5.982', 'learning_rate': '1e-05', 'train/loss_infonce': '0.332', 'train/loss_cot': '1.231', 'epoch': '2.384'} |
|
48%|βββββ | 1490/3125 [13:47<1:14:49, 2.75s/it]
48%|βββββ | 1491/3125 [13:50<1:17:30, 2.85s/it]
48%|βββββ | 1492/3125 [13:52<1:14:38, 2.74s/it]
48%|βββββ | 1493/3125 [13:56<1:24:19, 3.10s/it]
48%|βββββ | 1494/3125 [13:59<1:23:33, 3.07s/it]
48%|βββββ | 1495/3125 [14:02<1:23:40, 3.08s/it]
48%|βββββ | 1496/3125 [14:05<1:20:38, 2.97s/it]
48%|βββββ | 1497/3125 [14:08<1:19:19, 2.92s/it]
48%|βββββ | 1498/3125 [14:10<1:16:41, 2.83s/it]
48%|βββββ | 1499/3125 [14:13<1:17:32, 2.86s/it]
48%|βββββ | 1500/3125 [14:16<1:18:22, 2.89s/it]
{'loss': '1.213', 'grad_norm': '10.42', 'learning_rate': '1e-05', 'train/loss_infonce': '0.06592', 'train/loss_cot': '0.9286', 'epoch': '2.4'} |
|
48%|βββββ | 1500/3125 [14:17<1:18:22, 2.89s/it]
48%|βββββ | 1501/3125 [14:19<1:17:07, 2.85s/it]
48%|βββββ | 1502/3125 [14:22<1:15:23, 2.79s/it]
48%|βββββ | 1503/3125 [14:26<1:23:37, 3.09s/it]
48%|βββββ | 1504/3125 [14:28<1:21:10, 3.00s/it]
48%|βββββ | 1505/3125 [14:31<1:17:37, 2.87s/it]
48%|βββββ | 1506/3125 [14:34<1:16:15, 2.83s/it]
48%|βββββ | 1507/3125 [14:36<1:14:49, 2.77s/it]
48%|βββββ | 1508/3125 [14:39<1:12:16, 2.68s/it]
48%|βββββ | 1509/3125 [14:42<1:19:16, 2.94s/it]
48%|βββββ | 1510/3125 [14:45<1:16:14, 2.83s/it]
{'loss': '1.179', 'grad_norm': '8.327', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02014', 'train/loss_cot': '1.373', 'epoch': '2.416'} |
|
48%|βββββ | 1510/3125 [14:45<1:16:14, 2.83s/it]
48%|βββββ | 1511/3125 [14:48<1:15:20, 2.80s/it]
48%|βββββ | 1512/3125 [14:51<1:16:20, 2.84s/it]
48%|βββββ | 1513/3125 [14:53<1:16:41, 2.85s/it]
48%|βββββ | 1514/3125 [14:57<1:18:13, 2.91s/it]
48%|βββββ | 1515/3125 [14:59<1:15:29, 2.81s/it]
49%|βββββ | 1516/3125 [15:02<1:15:27, 2.81s/it]
49%|βββββ | 1517/3125 [15:05<1:15:28, 2.82s/it]
49%|βββββ | 1518/3125 [15:07<1:12:55, 2.72s/it]
49%|βββββ | 1519/3125 [15:10<1:12:31, 2.71s/it]
49%|βββββ | 1520/3125 [15:13<1:14:08, 2.77s/it]
{'loss': '1.252', 'grad_norm': '11.81', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1074', 'train/loss_cot': '1.505', 'epoch': '2.432'} |
|
49%|βββββ | 1520/3125 [15:13<1:14:08, 2.77s/it]
49%|βββββ | 1521/3125 [15:16<1:17:10, 2.89s/it]
49%|βββββ | 1522/3125 [15:19<1:21:40, 3.06s/it]
49%|βββββ | 1523/3125 [15:23<1:23:59, 3.15s/it]
49%|βββββ | 1524/3125 [15:26<1:23:28, 3.13s/it]
49%|βββββ | 1525/3125 [15:30<1:28:19, 3.31s/it]
49%|βββββ | 1526/3125 [15:32<1:21:51, 3.07s/it]
49%|βββββ | 1527/3125 [15:35<1:22:11, 3.09s/it]
49%|βββββ | 1528/3125 [15:38<1:20:09, 3.01s/it]
49%|βββββ | 1529/3125 [15:41<1:16:52, 2.89s/it]
49%|βββββ | 1530/3125 [15:44<1:17:56, 2.93s/it]
{'loss': '1.194', 'grad_norm': '5.693', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01495', 'train/loss_cot': '1.145', 'epoch': '2.448'} |
|
49%|βββββ | 1530/3125 [15:44<1:17:56, 2.93s/it]
49%|βββββ | 1531/3125 [15:47<1:18:20, 2.95s/it]
49%|βββββ | 1532/3125 [15:49<1:16:51, 2.89s/it]
49%|βββββ | 1533/3125 [15:52<1:14:03, 2.79s/it]
49%|βββββ | 1534/3125 [15:55<1:16:07, 2.87s/it]
49%|βββββ | 1535/3125 [15:58<1:17:24, 2.92s/it]
49%|βββββ | 1536/3125 [16:01<1:16:46, 2.90s/it]
49%|βββββ | 1537/3125 [16:04<1:14:14, 2.81s/it]
49%|βββββ | 1538/3125 [16:06<1:13:39, 2.78s/it]
49%|βββββ | 1539/3125 [16:09<1:15:58, 2.87s/it]
49%|βββββ | 1540/3125 [16:12<1:16:04, 2.88s/it]
{'loss': '1.187', 'grad_norm': '5.551', 'learning_rate': '1e-05', 'train/loss_infonce': '0.008179', 'train/loss_cot': '0.8784', 'epoch': '2.464'} |
|
49%|βββββ | 1540/3125 [16:13<1:16:04, 2.88s/it]
49%|βββββ | 1541/3125 [16:16<1:22:17, 3.12s/it]
49%|βββββ | 1542/3125 [16:20<1:27:03, 3.30s/it]
49%|βββββ | 1543/3125 [16:22<1:21:09, 3.08s/it]
49%|βββββ | 1544/3125 [16:25<1:18:17, 2.97s/it]
49%|βββββ | 1545/3125 [16:28<1:17:09, 2.93s/it]
49%|βββββ | 1546/3125 [16:31<1:18:41, 2.99s/it]
50%|βββββ | 1547/3125 [16:34<1:16:56, 2.93s/it]
50%|βββββ | 1548/3125 [16:36<1:15:15, 2.86s/it]
50%|βββββ | 1549/3125 [16:39<1:13:30, 2.80s/it]
50%|βββββ | 1550/3125 [16:42<1:14:25, 2.84s/it]
{'loss': '1.28', 'grad_norm': '7.464', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1455', 'train/loss_cot': '1.161', 'epoch': '2.48'} |
|
50%|βββββ | 1550/3125 [16:42<1:14:25, 2.84s/it]
50%|βββββ | 1551/3125 [16:45<1:14:20, 2.83s/it]
50%|βββββ | 1552/3125 [16:47<1:13:01, 2.79s/it]
50%|βββββ | 1553/3125 [16:50<1:12:35, 2.77s/it]
50%|βββββ | 1554/3125 [16:53<1:13:21, 2.80s/it]
50%|βββββ | 1555/3125 [16:56<1:12:06, 2.76s/it]
50%|βββββ | 1556/3125 [16:58<1:10:32, 2.70s/it]
50%|βββββ | 1557/3125 [17:01<1:13:03, 2.80s/it]
50%|βββββ | 1558/3125 [17:04<1:13:37, 2.82s/it]
50%|βββββ | 1559/3125 [17:07<1:11:52, 2.75s/it]
50%|βββββ | 1560/3125 [17:09<1:10:58, 2.72s/it]
{'loss': '1.186', 'grad_norm': '8.916', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2246', 'train/loss_cot': '1.121', 'epoch': '2.496'} |
|
50%|βββββ | 1560/3125 [17:10<1:10:58, 2.72s/it]
50%|βββββ | 1561/3125 [17:13<1:14:42, 2.87s/it]
50%|βββββ | 1562/3125 [17:15<1:11:04, 2.73s/it]
50%|βββββ | 1563/3125 [17:17<1:08:39, 2.64s/it]
50%|βββββ | 1564/3125 [17:21<1:12:43, 2.80s/it]
50%|βββββ | 1565/3125 [17:24<1:13:31, 2.83s/it]
50%|βββββ | 1566/3125 [17:27<1:17:06, 2.97s/it]
50%|βββββ | 1567/3125 [17:30<1:16:03, 2.93s/it]
50%|βββββ | 1568/3125 [17:32<1:13:45, 2.84s/it]
50%|βββββ | 1569/3125 [17:35<1:11:11, 2.75s/it]
50%|βββββ | 1570/3125 [17:38<1:13:57, 2.85s/it]
{'loss': '1.141', 'grad_norm': '6.972', 'learning_rate': '1e-05', 'train/loss_infonce': '0.008423', 'train/loss_cot': '1.36', 'epoch': '2.512'} |
|
50%|βββββ | 1570/3125 [17:38<1:13:57, 2.85s/it]
50%|βββββ | 1571/3125 [17:41<1:12:55, 2.82s/it]
50%|βββββ | 1572/3125 [17:43<1:11:55, 2.78s/it]
50%|βββββ | 1573/3125 [17:46<1:13:24, 2.84s/it]
50%|βββββ | 1574/3125 [17:50<1:17:04, 2.98s/it]
50%|βββββ | 1575/3125 [17:52<1:15:35, 2.93s/it]
50%|βββββ | 1576/3125 [17:55<1:15:04, 2.91s/it]
50%|βββββ | 1577/3125 [17:58<1:14:58, 2.91s/it]
50%|βββββ | 1578/3125 [18:01<1:13:39, 2.86s/it]
51%|βββββ | 1579/3125 [18:04<1:12:26, 2.81s/it]
51%|βββββ | 1580/3125 [18:08<1:22:17, 3.20s/it]
{'loss': '1.191', 'grad_norm': '10.7', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005615', 'train/loss_cot': '1.451', 'epoch': '2.528'} |
|
51%|βββββ | 1580/3125 [18:08<1:22:17, 3.20s/it]
51%|βββββ | 1581/3125 [18:11<1:19:55, 3.11s/it]
51%|βββββ | 1582/3125 [18:14<1:18:35, 3.06s/it]
51%|βββββ | 1583/3125 [18:16<1:16:59, 3.00s/it]
51%|βββββ | 1584/3125 [18:19<1:16:25, 2.98s/it]
51%|βββββ | 1585/3125 [18:22<1:15:50, 2.95s/it]
51%|βββββ | 1586/3125 [18:25<1:14:10, 2.89s/it]
51%|βββββ | 1587/3125 [18:27<1:09:56, 2.73s/it]
51%|βββββ | 1588/3125 [18:30<1:09:55, 2.73s/it]
51%|βββββ | 1589/3125 [18:34<1:21:12, 3.17s/it]
51%|βββββ | 1590/3125 [18:38<1:21:26, 3.18s/it]
{'loss': '1.311', 'grad_norm': '6.456', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01831', 'train/loss_cot': '1.249', 'epoch': '2.544'} |
|
51%|βββββ | 1590/3125 [18:38<1:21:26, 3.18s/it]
51%|βββββ | 1591/3125 [18:42<1:31:45, 3.59s/it]
51%|βββββ | 1592/3125 [18:45<1:26:03, 3.37s/it]
51%|βββββ | 1593/3125 [18:49<1:28:27, 3.46s/it]
51%|βββββ | 1594/3125 [18:51<1:22:52, 3.25s/it]
51%|βββββ | 1595/3125 [18:55<1:24:05, 3.30s/it]
51%|βββββ | 1596/3125 [18:57<1:18:43, 3.09s/it]
51%|βββββ | 1597/3125 [19:00<1:16:15, 2.99s/it]
51%|βββββ | 1598/3125 [19:03<1:14:57, 2.95s/it]
51%|βββββ | 1599/3125 [19:07<1:20:12, 3.15s/it]
51%|βββββ | 1600/3125 [19:09<1:16:19, 3.00s/it]
{'loss': '1.176', 'grad_norm': '5.817', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1562', 'train/loss_cot': '1.058', 'epoch': '2.56'} |
|
51%|βββββ | 1600/3125 [19:10<1:16:19, 3.00s/it]
51%|βββββ | 1601/3125 [19:28<3:18:02, 7.80s/it]
51%|ββββββ | 1602/3125 [19:31<2:40:09, 6.31s/it]
51%|ββββββ | 1603/3125 [19:34<2:14:24, 5.30s/it]
51%|ββββββ | 1604/3125 [19:37<1:55:09, 4.54s/it]
51%|ββββββ | 1605/3125 [19:40<1:43:27, 4.08s/it]
51%|ββββββ | 1606/3125 [19:43<1:35:48, 3.78s/it]
51%|ββββββ | 1607/3125 [19:46<1:27:13, 3.45s/it]
51%|ββββββ | 1608/3125 [19:48<1:23:03, 3.29s/it]
51%|ββββββ | 1609/3125 [19:51<1:18:20, 3.10s/it]
52%|ββββββ | 1610/3125 [19:54<1:15:18, 2.98s/it]
{'loss': '1.197', 'grad_norm': '5.939', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02271', 'train/loss_cot': '0.9068', 'epoch': '2.576'} |
|
52%|ββββββ | 1610/3125 [19:54<1:15:18, 2.98s/it]
52%|ββββββ | 1611/3125 [19:56<1:12:05, 2.86s/it]
52%|ββββββ | 1612/3125 [19:59<1:10:40, 2.80s/it]
52%|ββββββ | 1613/3125 [20:02<1:08:35, 2.72s/it]
52%|ββββββ | 1614/3125 [20:04<1:08:53, 2.74s/it]
52%|ββββββ | 1615/3125 [20:07<1:07:49, 2.69s/it]
52%|ββββββ | 1616/3125 [20:10<1:09:17, 2.75s/it]
52%|ββββββ | 1617/3125 [20:13<1:09:11, 2.75s/it]
52%|ββββββ | 1618/3125 [20:16<1:11:52, 2.86s/it]
52%|ββββββ | 1619/3125 [20:18<1:09:42, 2.78s/it]
52%|ββββββ | 1620/3125 [20:21<1:09:54, 2.79s/it]
{'loss': '1.222', 'grad_norm': '9.275', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003189', 'train/loss_cot': '1.259', 'epoch': '2.592'} |
|
52%|ββββββ | 1620/3125 [20:22<1:09:54, 2.79s/it]
52%|ββββββ | 1621/3125 [20:24<1:10:02, 2.79s/it]
52%|ββββββ | 1622/3125 [20:26<1:08:05, 2.72s/it]
52%|ββββββ | 1623/3125 [20:29<1:09:27, 2.77s/it]
52%|ββββββ | 1624/3125 [20:33<1:12:08, 2.88s/it]
52%|ββββββ | 1625/3125 [20:35<1:09:41, 2.79s/it]
52%|ββββββ | 1626/3125 [20:38<1:10:07, 2.81s/it]
52%|ββββββ | 1627/3125 [20:41<1:09:51, 2.80s/it]
52%|ββββββ | 1628/3125 [20:43<1:07:35, 2.71s/it]
52%|ββββββ | 1629/3125 [20:46<1:06:01, 2.65s/it]
52%|ββββββ | 1630/3125 [20:49<1:10:04, 2.81s/it]
{'loss': '1.241', 'grad_norm': '6.941', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2383', 'train/loss_cot': '0.9823', 'epoch': '2.608'} |
|
52%|ββββββ | 1630/3125 [20:49<1:10:04, 2.81s/it]
52%|ββββββ | 1631/3125 [20:52<1:10:33, 2.83s/it]
52%|ββββββ | 1632/3125 [20:54<1:07:45, 2.72s/it]
52%|ββββββ | 1633/3125 [20:57<1:10:30, 2.84s/it]
52%|ββββββ | 1634/3125 [21:01<1:13:00, 2.94s/it]
52%|ββββββ | 1635/3125 [21:03<1:11:51, 2.89s/it]
52%|ββββββ | 1636/3125 [21:06<1:11:40, 2.89s/it]
52%|ββββββ | 1637/3125 [21:09<1:12:14, 2.91s/it]
52%|ββββββ | 1638/3125 [21:12<1:09:35, 2.81s/it]
52%|ββββββ | 1639/3125 [21:15<1:11:09, 2.87s/it]
52%|ββββββ | 1640/3125 [21:18<1:10:50, 2.86s/it]
{'loss': '1.248', 'grad_norm': '6.295', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04272', 'train/loss_cot': '1.242', 'epoch': '2.624'} |
|
52%|ββββββ | 1640/3125 [21:18<1:10:50, 2.86s/it]
53%|ββββββ | 1641/3125 [21:20<1:09:26, 2.81s/it]
53%|ββββββ | 1642/3125 [21:24<1:12:57, 2.95s/it]
53%|ββββββ | 1643/3125 [21:27<1:14:03, 3.00s/it]
53%|ββββββ | 1644/3125 [21:29<1:11:10, 2.88s/it]
53%|ββββββ | 1645/3125 [21:32<1:09:52, 2.83s/it]
53%|ββββββ | 1646/3125 [21:35<1:09:48, 2.83s/it]
53%|ββββββ | 1647/3125 [21:38<1:10:31, 2.86s/it]
53%|ββββββ | 1648/3125 [21:41<1:10:30, 2.86s/it]
53%|ββββββ | 1649/3125 [21:44<1:12:34, 2.95s/it]
53%|ββββββ | 1650/3125 [21:47<1:12:31, 2.95s/it]
{'loss': '1.255', 'grad_norm': '5.889', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2324', 'train/loss_cot': '1.162', 'epoch': '2.64'} |
|
53%|ββββββ | 1650/3125 [21:47<1:12:31, 2.95s/it]
53%|ββββββ | 1651/3125 [21:50<1:12:06, 2.94s/it]
53%|ββββββ | 1652/3125 [21:53<1:13:08, 2.98s/it]
53%|ββββββ | 1653/3125 [21:56<1:12:10, 2.94s/it]
53%|ββββββ | 1654/3125 [21:59<1:12:09, 2.94s/it]
53%|ββββββ | 1655/3125 [22:01<1:11:59, 2.94s/it]
53%|ββββββ | 1656/3125 [22:04<1:12:27, 2.96s/it]
53%|ββββββ | 1657/3125 [22:07<1:10:09, 2.87s/it]
53%|ββββββ | 1658/3125 [22:10<1:08:11, 2.79s/it]
53%|ββββββ | 1659/3125 [22:12<1:07:00, 2.74s/it]
53%|ββββββ | 1660/3125 [22:15<1:06:36, 2.73s/it]
{'loss': '1.205', 'grad_norm': '5.61', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00647', 'train/loss_cot': '0.9052', 'epoch': '2.656'} |
|
53%|ββββββ | 1660/3125 [22:16<1:06:36, 2.73s/it]
53%|ββββββ | 1661/3125 [22:19<1:12:27, 2.97s/it]
53%|ββββββ | 1662/3125 [22:21<1:09:49, 2.86s/it]
53%|ββββββ | 1663/3125 [22:24<1:05:52, 2.70s/it]
53%|ββββββ | 1664/3125 [22:26<1:05:33, 2.69s/it]
53%|ββββββ | 1665/3125 [22:29<1:05:11, 2.68s/it]
53%|ββββββ | 1666/3125 [22:32<1:06:09, 2.72s/it]
53%|ββββββ | 1667/3125 [22:35<1:09:50, 2.87s/it]
53%|ββββββ | 1668/3125 [22:38<1:10:40, 2.91s/it]
53%|ββββββ | 1669/3125 [22:41<1:09:39, 2.87s/it]
53%|ββββββ | 1670/3125 [22:44<1:10:05, 2.89s/it]
{'loss': '1.206', 'grad_norm': '5.497', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007111', 'train/loss_cot': '1.189', 'epoch': '2.672'} |
|
53%|ββββββ | 1670/3125 [22:44<1:10:05, 2.89s/it]
53%|ββββββ | 1671/3125 [22:47<1:13:42, 3.04s/it]
54%|ββββββ | 1672/3125 [22:50<1:16:23, 3.15s/it]
54%|ββββββ | 1673/3125 [22:53<1:15:01, 3.10s/it]
54%|ββββββ | 1674/3125 [22:56<1:11:39, 2.96s/it]
54%|ββββββ | 1675/3125 [22:59<1:11:08, 2.94s/it]
54%|ββββββ | 1676/3125 [23:02<1:08:52, 2.85s/it]
54%|ββββββ | 1677/3125 [23:04<1:06:55, 2.77s/it]
54%|ββββββ | 1678/3125 [23:07<1:07:03, 2.78s/it]
54%|ββββββ | 1679/3125 [23:10<1:05:38, 2.72s/it]
54%|ββββββ | 1680/3125 [23:12<1:06:02, 2.74s/it]
{'loss': '1.257', 'grad_norm': '5.173', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '1.105', 'epoch': '2.688'} |
|
54%|ββββββ | 1680/3125 [23:13<1:06:02, 2.74s/it]
54%|ββββββ | 1681/3125 [23:15<1:06:41, 2.77s/it]
54%|ββββββ | 1682/3125 [23:18<1:06:38, 2.77s/it]
54%|ββββββ | 1683/3125 [23:21<1:05:56, 2.74s/it]
54%|ββββββ | 1684/3125 [23:23<1:04:40, 2.69s/it]
54%|ββββββ | 1685/3125 [23:27<1:09:15, 2.89s/it]
54%|ββββββ | 1686/3125 [23:29<1:08:00, 2.84s/it]
54%|ββββββ | 1687/3125 [23:32<1:06:43, 2.78s/it]
54%|ββββββ | 1688/3125 [23:34<1:04:21, 2.69s/it]
54%|ββββββ | 1689/3125 [23:37<1:04:22, 2.69s/it]
54%|ββββββ | 1690/3125 [23:39<1:02:23, 2.61s/it]
{'loss': '1.212', 'grad_norm': '6.714', 'learning_rate': '1e-05', 'train/loss_infonce': '0.125', 'train/loss_cot': '0.773', 'epoch': '2.704'} |
|
54%|ββββββ | 1690/3125 [23:40<1:02:23, 2.61s/it]
54%|ββββββ | 1691/3125 [23:43<1:06:35, 2.79s/it]
54%|ββββββ | 1692/3125 [23:46<1:09:47, 2.92s/it]
54%|ββββββ | 1693/3125 [23:48<1:06:13, 2.77s/it]
54%|ββββββ | 1694/3125 [23:51<1:04:52, 2.72s/it]
54%|ββββββ | 1695/3125 [23:54<1:06:01, 2.77s/it]
54%|ββββββ | 1696/3125 [23:57<1:06:58, 2.81s/it]
54%|ββββββ | 1697/3125 [23:59<1:04:36, 2.71s/it]
54%|ββββββ | 1698/3125 [24:02<1:06:10, 2.78s/it]
54%|ββββββ | 1699/3125 [24:05<1:04:06, 2.70s/it]
54%|ββββββ | 1700/3125 [24:07<1:03:40, 2.68s/it]
{'loss': '1.21', 'grad_norm': '6.064', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1357', 'train/loss_cot': '0.9565', 'epoch': '2.72'} |
|
54%|ββββββ | 1700/3125 [24:08<1:03:40, 2.68s/it]
54%|ββββββ | 1701/3125 [24:10<1:05:04, 2.74s/it]
54%|ββββββ | 1702/3125 [24:13<1:06:46, 2.82s/it]
54%|ββββββ | 1703/3125 [24:16<1:05:47, 2.78s/it]
55%|ββββββ | 1704/3125 [24:19<1:05:57, 2.79s/it]
55%|ββββββ | 1705/3125 [24:21<1:05:20, 2.76s/it]
55%|ββββββ | 1706/3125 [24:24<1:06:03, 2.79s/it]
55%|ββββββ | 1707/3125 [24:27<1:08:56, 2.92s/it]
55%|ββββββ | 1708/3125 [24:30<1:07:04, 2.84s/it]
55%|ββββββ | 1709/3125 [24:33<1:06:09, 2.80s/it]
55%|ββββββ | 1710/3125 [24:36<1:05:21, 2.77s/it]
{'loss': '1.157', 'grad_norm': '5.87', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1504', 'train/loss_cot': '1.076', 'epoch': '2.736'} |
|
55%|ββββββ | 1710/3125 [24:36<1:05:21, 2.77s/it]
55%|ββββββ | 1711/3125 [24:38<1:04:56, 2.76s/it]
55%|ββββββ | 1712/3125 [24:41<1:05:58, 2.80s/it]
55%|ββββββ | 1713/3125 [24:44<1:04:06, 2.72s/it]
55%|ββββββ | 1714/3125 [24:47<1:05:38, 2.79s/it]
55%|ββββββ | 1715/3125 [24:50<1:06:59, 2.85s/it]
55%|ββββββ | 1716/3125 [24:52<1:05:06, 2.77s/it]
55%|ββββββ | 1717/3125 [24:55<1:04:43, 2.76s/it]
55%|ββββββ | 1718/3125 [24:58<1:04:51, 2.77s/it]
55%|ββββββ | 1719/3125 [25:01<1:04:55, 2.77s/it]
55%|ββββββ | 1720/3125 [25:03<1:04:02, 2.73s/it]
{'loss': '1.189', 'grad_norm': '5.975', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01471', 'train/loss_cot': '1.276', 'epoch': '2.752'} |
|
55%|ββββββ | 1720/3125 [25:04<1:04:02, 2.73s/it]
55%|ββββββ | 1721/3125 [25:06<1:05:57, 2.82s/it]
55%|ββββββ | 1722/3125 [25:09<1:05:35, 2.81s/it]
55%|ββββββ | 1723/3125 [25:11<1:02:58, 2.70s/it]
55%|ββββββ | 1724/3125 [25:14<1:02:04, 2.66s/it]
55%|ββββββ | 1725/3125 [25:16<1:00:44, 2.60s/it]
55%|ββββββ | 1726/3125 [25:19<1:00:13, 2.58s/it]
55%|ββββββ | 1727/3125 [25:22<59:55, 2.57s/it]
55%|ββββββ | 1728/3125 [25:24<1:02:24, 2.68s/it]
55%|ββββββ | 1729/3125 [25:27<1:03:17, 2.72s/it]
55%|ββββββ | 1730/3125 [25:30<1:03:28, 2.73s/it]
{'loss': '1.254', 'grad_norm': '6.991', 'learning_rate': '1e-05', 'train/loss_infonce': '0.104', 'train/loss_cot': '1.089', 'epoch': '2.768'} |
|
55%|ββββββ | 1730/3125 [25:30<1:03:28, 2.73s/it]
55%|ββββββ | 1731/3125 [25:33<1:02:37, 2.70s/it]
55%|ββββββ | 1732/3125 [25:35<1:03:39, 2.74s/it]
55%|ββββββ | 1733/3125 [25:38<1:04:51, 2.80s/it]
55%|ββββββ | 1734/3125 [25:41<1:04:44, 2.79s/it]
56%|ββββββ | 1735/3125 [25:44<1:03:38, 2.75s/it]
56%|ββββββ | 1736/3125 [25:47<1:08:00, 2.94s/it]
56%|ββββββ | 1737/3125 [25:50<1:06:06, 2.86s/it]
56%|ββββββ | 1738/3125 [25:53<1:06:10, 2.86s/it]
56%|ββββββ | 1739/3125 [25:55<1:04:12, 2.78s/it]
56%|ββββββ | 1740/3125 [25:59<1:08:02, 2.95s/it]
{'loss': '1.273', 'grad_norm': '6.88', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05713', 'train/loss_cot': '1.284', 'epoch': '2.784'} |
|
56%|ββββββ | 1740/3125 [25:59<1:08:02, 2.95s/it]
56%|ββββββ | 1741/3125 [26:02<1:08:00, 2.95s/it]
56%|ββββββ | 1742/3125 [26:05<1:07:23, 2.92s/it]
56%|ββββββ | 1743/3125 [26:07<1:04:43, 2.81s/it]
56%|ββββββ | 1744/3125 [26:10<1:04:35, 2.81s/it]
56%|ββββββ | 1745/3125 [26:13<1:09:50, 3.04s/it]
56%|ββββββ | 1746/3125 [26:16<1:05:46, 2.86s/it]
56%|ββββββ | 1747/3125 [26:19<1:08:09, 2.97s/it]
56%|ββββββ | 1748/3125 [26:22<1:07:01, 2.92s/it]
56%|ββββββ | 1749/3125 [26:25<1:07:05, 2.93s/it]
56%|ββββββ | 1750/3125 [26:28<1:05:33, 2.86s/it]
{'loss': '1.215', 'grad_norm': '7.62', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03076', 'train/loss_cot': '0.9005', 'epoch': '2.8'} |
|
56%|ββββββ | 1750/3125 [26:28<1:05:33, 2.86s/it]
56%|ββββββ | 1751/3125 [26:31<1:11:13, 3.11s/it]
56%|ββββββ | 1752/3125 [26:34<1:07:02, 2.93s/it]
56%|ββββββ | 1753/3125 [26:36<1:04:52, 2.84s/it]
56%|ββββββ | 1754/3125 [26:39<1:01:40, 2.70s/it]
56%|ββββββ | 1755/3125 [26:41<1:00:48, 2.66s/it]
56%|ββββββ | 1756/3125 [26:44<1:01:39, 2.70s/it]
56%|ββββββ | 1757/3125 [26:47<1:02:14, 2.73s/it]
56%|ββββββ | 1758/3125 [26:50<1:02:11, 2.73s/it]
56%|ββββββ | 1759/3125 [26:53<1:05:01, 2.86s/it]
56%|ββββββ | 1760/3125 [26:56<1:04:00, 2.81s/it]
{'loss': '1.264', 'grad_norm': '6.701', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3262', 'train/loss_cot': '1.005', 'epoch': '2.816'} |
|
56%|ββββββ | 1760/3125 [26:56<1:04:00, 2.81s/it]
56%|ββββββ | 1761/3125 [26:59<1:05:08, 2.87s/it]
56%|ββββββ | 1762/3125 [27:02<1:06:43, 2.94s/it]
56%|ββββββ | 1763/3125 [27:04<1:05:57, 2.91s/it]
56%|ββββββ | 1764/3125 [27:07<1:04:34, 2.85s/it]
56%|ββββββ | 1765/3125 [27:10<1:02:36, 2.76s/it]
57%|ββββββ | 1766/3125 [27:13<1:02:47, 2.77s/it]
57%|ββββββ | 1767/3125 [27:15<1:00:43, 2.68s/it]
57%|ββββββ | 1768/3125 [27:18<1:01:45, 2.73s/it]
57%|ββββββ | 1769/3125 [27:20<59:07, 2.62s/it]
57%|ββββββ | 1770/3125 [27:23<59:32, 2.64s/it]
{'loss': '1.192', 'grad_norm': '6.547', 'learning_rate': '1e-05', 'train/loss_infonce': '0.09131', 'train/loss_cot': '0.9959', 'epoch': '2.832'} |
|
57%|ββββββ | 1770/3125 [27:23<59:32, 2.64s/it]
57%|ββββββ | 1771/3125 [27:26<1:02:29, 2.77s/it]
57%|ββββββ | 1772/3125 [27:29<1:01:59, 2.75s/it]
57%|ββββββ | 1773/3125 [27:33<1:10:38, 3.13s/it]
57%|ββββββ | 1774/3125 [27:36<1:14:29, 3.31s/it]
57%|ββββββ | 1775/3125 [27:39<1:11:28, 3.18s/it]
57%|ββββββ | 1776/3125 [27:42<1:07:02, 2.98s/it]
57%|ββββββ | 1777/3125 [27:45<1:07:10, 2.99s/it]
57%|ββββββ | 1778/3125 [27:48<1:06:33, 2.96s/it]
57%|ββββββ | 1779/3125 [27:51<1:05:47, 2.93s/it]
57%|ββββββ | 1780/3125 [27:54<1:07:25, 3.01s/it]
{'loss': '1.209', 'grad_norm': '9.752', 'learning_rate': '1e-05', 'train/loss_infonce': '0.5352', 'train/loss_cot': '1.202', 'epoch': '2.848'} |
|
57%|ββββββ | 1780/3125 [27:54<1:07:25, 3.01s/it]
57%|ββββββ | 1781/3125 [27:57<1:06:18, 2.96s/it]
57%|ββββββ | 1782/3125 [27:59<1:03:29, 2.84s/it]
57%|ββββββ | 1783/3125 [28:02<1:02:13, 2.78s/it]
57%|ββββββ | 1784/3125 [28:05<1:03:42, 2.85s/it]
57%|ββββββ | 1785/3125 [28:08<1:03:54, 2.86s/it]
57%|ββββββ | 1786/3125 [28:10<1:01:53, 2.77s/it]
57%|ββββββ | 1787/3125 [28:13<1:02:42, 2.81s/it]
57%|ββββββ | 1788/3125 [28:16<1:01:07, 2.74s/it]
57%|ββββββ | 1789/3125 [28:19<1:01:32, 2.76s/it]
57%|ββββββ | 1790/3125 [28:22<1:02:50, 2.82s/it]
{'loss': '1.186', 'grad_norm': '6.39', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001274', 'train/loss_cot': '0.8117', 'epoch': '2.864'} |
|
57%|ββββββ | 1790/3125 [28:22<1:02:50, 2.82s/it]
57%|ββββββ | 1791/3125 [28:24<1:02:35, 2.82s/it]
57%|ββββββ | 1792/3125 [28:27<1:00:58, 2.74s/it]
57%|ββββββ | 1793/3125 [28:30<1:01:42, 2.78s/it]
57%|ββββββ | 1794/3125 [28:32<1:00:58, 2.75s/it]
57%|ββββββ | 1795/3125 [28:35<59:20, 2.68s/it]
57%|ββββββ | 1796/3125 [28:38<1:01:44, 2.79s/it]
58%|ββββββ | 1797/3125 [28:41<1:01:33, 2.78s/it]
58%|ββββββ | 1798/3125 [28:43<59:29, 2.69s/it]
58%|ββββββ | 1799/3125 [28:46<59:33, 2.69s/it]
58%|ββββββ | 1800/3125 [28:49<58:47, 2.66s/it]
{'loss': '1.272', 'grad_norm': '8.465', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2041', 'train/loss_cot': '0.7639', 'epoch': '2.88'} |
|
58%|ββββββ | 1800/3125 [28:49<58:47, 2.66s/it]
58%|ββββββ | 1801/3125 [29:08<2:48:27, 7.63s/it]
58%|ββββββ | 1802/3125 [29:10<2:14:15, 6.09s/it]
58%|ββββββ | 1803/3125 [29:13<1:53:11, 5.14s/it]
58%|ββββββ | 1804/3125 [29:16<1:37:41, 4.44s/it]
58%|ββββββ | 1805/3125 [29:19<1:26:04, 3.91s/it]
58%|ββββββ | 1806/3125 [29:22<1:21:39, 3.71s/it]
58%|ββββββ | 1807/3125 [29:25<1:15:50, 3.45s/it]
58%|ββββββ | 1808/3125 [29:28<1:13:02, 3.33s/it]
58%|ββββββ | 1809/3125 [29:30<1:06:57, 3.05s/it]
58%|ββββββ | 1810/3125 [29:34<1:08:50, 3.14s/it]
{'loss': '1.289', 'grad_norm': '6.283', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1562', 'train/loss_cot': '0.9734', 'epoch': '2.896'} |
|
58%|ββββββ | 1810/3125 [29:34<1:08:50, 3.14s/it]
58%|ββββββ | 1811/3125 [29:36<1:05:36, 3.00s/it]
58%|ββββββ | 1812/3125 [29:39<1:03:52, 2.92s/it]
58%|ββββββ | 1813/3125 [29:42<1:02:01, 2.84s/it]
58%|ββββββ | 1814/3125 [29:44<1:01:11, 2.80s/it]
58%|ββββββ | 1815/3125 [29:47<58:54, 2.70s/it]
58%|ββββββ | 1816/3125 [29:49<56:49, 2.60s/it]
58%|ββββββ | 1817/3125 [29:52<58:37, 2.69s/it]
58%|ββββββ | 1818/3125 [29:55<58:04, 2.67s/it]
58%|ββββββ | 1819/3125 [29:57<55:48, 2.56s/it]
58%|ββββββ | 1820/3125 [30:00<57:39, 2.65s/it]
{'loss': '1.195', 'grad_norm': '7.439', 'learning_rate': '1e-05', 'train/loss_infonce': '0.2539', 'train/loss_cot': '1.369', 'epoch': '2.912'} |
|
58%|ββββββ | 1820/3125 [30:00<57:39, 2.65s/it]
58%|ββββββ | 1821/3125 [30:03<59:23, 2.73s/it]
58%|ββββββ | 1822/3125 [30:05<58:35, 2.70s/it]
58%|ββββββ | 1823/3125 [30:09<1:04:27, 2.97s/it]
58%|ββββββ | 1824/3125 [30:11<1:01:33, 2.84s/it]
58%|ββββββ | 1825/3125 [30:14<1:00:49, 2.81s/it]
58%|ββββββ | 1826/3125 [30:17<1:01:20, 2.83s/it]
58%|ββββββ | 1827/3125 [30:20<59:35, 2.75s/it]
58%|ββββββ | 1828/3125 [30:23<1:00:26, 2.80s/it]
59%|ββββββ | 1829/3125 [30:25<1:01:04, 2.83s/it]
59%|ββββββ | 1830/3125 [30:29<1:04:49, 3.00s/it]
{'loss': '1.197', 'grad_norm': '6.513', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01508', 'train/loss_cot': '1.151', 'epoch': '2.928'} |
|
59%|ββββββ | 1830/3125 [30:29<1:04:49, 3.00s/it]
59%|ββββββ | 1831/3125 [30:32<1:03:07, 2.93s/it]
59%|ββββββ | 1832/3125 [30:34<1:00:19, 2.80s/it]
59%|ββββββ | 1833/3125 [30:36<57:13, 2.66s/it]
59%|ββββββ | 1834/3125 [30:39<59:03, 2.74s/it]
59%|ββββββ | 1835/3125 [30:42<58:33, 2.72s/it]
59%|ββββββ | 1836/3125 [30:45<57:45, 2.69s/it]
59%|ββββββ | 1837/3125 [30:48<1:00:59, 2.84s/it]
59%|ββββββ | 1838/3125 [30:51<1:00:09, 2.80s/it]
59%|ββββββ | 1839/3125 [30:54<1:01:13, 2.86s/it]
59%|ββββββ | 1840/3125 [30:56<59:50, 2.79s/it]
{'loss': '1.181', 'grad_norm': '6.85', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005676', 'train/loss_cot': '1.24', 'epoch': '2.944'} |
|
59%|ββββββ | 1840/3125 [30:57<59:50, 2.79s/it]
59%|ββββββ | 1841/3125 [30:59<1:00:46, 2.84s/it]
59%|ββββββ | 1842/3125 [31:02<59:17, 2.77s/it]
59%|ββββββ | 1843/3125 [31:05<1:00:01, 2.81s/it]
59%|ββββββ | 1844/3125 [31:07<57:45, 2.71s/it]
59%|ββββββ | 1845/3125 [31:10<57:09, 2.68s/it]
59%|ββββββ | 1846/3125 [31:13<1:00:03, 2.82s/it]
59%|ββββββ | 1847/3125 [31:15<57:58, 2.72s/it]
59%|ββββββ | 1848/3125 [31:18<57:12, 2.69s/it]
59%|ββββββ | 1849/3125 [31:21<57:10, 2.69s/it]
59%|ββββββ | 1850/3125 [31:23<56:00, 2.64s/it]
{'loss': '1.209', 'grad_norm': '6.609', 'learning_rate': '1e-05', 'train/loss_infonce': '0.06641', 'train/loss_cot': '1.055', 'epoch': '2.96'} |
|
59%|ββββββ | 1850/3125 [31:24<56:00, 2.64s/it]
59%|ββββββ | 1851/3125 [31:26<55:27, 2.61s/it]
59%|ββββββ | 1852/3125 [31:28<54:27, 2.57s/it]
59%|ββββββ | 1853/3125 [31:31<55:38, 2.62s/it]
59%|ββββββ | 1854/3125 [31:34<55:58, 2.64s/it]
59%|ββββββ | 1855/3125 [31:36<54:33, 2.58s/it]
59%|ββββββ | 1856/3125 [31:39<55:36, 2.63s/it]
59%|ββββββ | 1857/3125 [31:42<57:20, 2.71s/it]
59%|ββββββ | 1858/3125 [31:45<58:03, 2.75s/it]
59%|ββββββ | 1859/3125 [31:47<56:00, 2.65s/it]
60%|ββββββ | 1860/3125 [31:50<56:09, 2.66s/it]
{'loss': '1.239', 'grad_norm': '8.196', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02258', 'train/loss_cot': '1.21', 'epoch': '2.976'} |
|
60%|ββββββ | 1860/3125 [31:50<56:09, 2.66s/it]
60%|ββββββ | 1861/3125 [31:52<56:01, 2.66s/it]
60%|ββββββ | 1862/3125 [31:55<54:59, 2.61s/it]
60%|ββββββ | 1863/3125 [31:59<1:01:26, 2.92s/it]
60%|ββββββ | 1864/3125 [32:01<59:42, 2.84s/it]
60%|ββββββ | 1865/3125 [32:04<1:01:23, 2.92s/it]
60%|ββββββ | 1866/3125 [32:07<1:00:50, 2.90s/it]
60%|ββββββ | 1867/3125 [32:10<58:26, 2.79s/it]
60%|ββββββ | 1868/3125 [32:12<57:52, 2.76s/it]
60%|ββββββ | 1869/3125 [32:15<58:41, 2.80s/it]
60%|ββββββ | 1870/3125 [32:18<57:13, 2.74s/it]
{'loss': '1.093', 'grad_norm': '5.23', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01208', 'train/loss_cot': '0.9051', 'epoch': '2.992'} |
|
60%|ββββββ | 1870/3125 [32:18<57:13, 2.74s/it]
60%|ββββββ | 1871/3125 [32:21<59:24, 2.84s/it]
60%|ββββββ | 1872/3125 [32:24<58:10, 2.79s/it]
60%|ββββββ | 1873/3125 [32:26<56:32, 2.71s/it]
60%|ββββββ | 1874/3125 [32:29<56:23, 2.70s/it]
60%|ββββββ | 1875/3125 [32:30<44:50, 2.15s/it]
60%|ββββββ | 1876/3125 [32:34<56:10, 2.70s/it]
60%|ββββββ | 1877/3125 [32:36<54:26, 2.62s/it]
60%|ββββββ | 1878/3125 [32:39<53:25, 2.57s/it]
60%|ββββββ | 1879/3125 [32:41<52:21, 2.52s/it]
60%|ββββββ | 1880/3125 [32:44<54:30, 2.63s/it]
{'loss': '1.14', 'grad_norm': '6.176', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01141', 'train/loss_cot': '0.977', 'epoch': '3.008'} |
|
60%|ββββββ | 1880/3125 [32:44<54:30, 2.63s/it]
60%|ββββββ | 1881/3125 [32:46<54:13, 2.62s/it]
60%|ββββββ | 1882/3125 [32:49<51:03, 2.46s/it]
60%|ββββββ | 1883/3125 [32:51<50:43, 2.45s/it]
60%|ββββββ | 1884/3125 [32:53<49:31, 2.39s/it]
60%|ββββββ | 1885/3125 [32:56<50:27, 2.44s/it]
60%|ββββββ | 1886/3125 [32:58<48:49, 2.36s/it]
60%|ββββββ | 1887/3125 [33:00<48:41, 2.36s/it]
60%|ββββββ | 1888/3125 [33:03<49:59, 2.43s/it]
60%|ββββββ | 1889/3125 [33:06<51:52, 2.52s/it]
60%|ββββββ | 1890/3125 [33:08<50:21, 2.45s/it]
{'loss': '1.045', 'grad_norm': '7.792', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03394', 'train/loss_cot': '1.093', 'epoch': '3.024'} |
|
60%|ββββββ | 1890/3125 [33:08<50:21, 2.45s/it]
61%|ββββββ | 1891/3125 [33:10<50:06, 2.44s/it]
61%|ββββββ | 1892/3125 [33:13<49:30, 2.41s/it]
61%|ββββββ | 1893/3125 [33:15<48:14, 2.35s/it]
61%|ββββββ | 1894/3125 [33:17<46:52, 2.29s/it]
61%|ββββββ | 1895/3125 [33:21<55:06, 2.69s/it]
61%|ββββββ | 1896/3125 [33:23<52:47, 2.58s/it]
61%|ββββββ | 1897/3125 [33:25<52:38, 2.57s/it]
61%|ββββββ | 1898/3125 [33:28<52:46, 2.58s/it]
61%|ββββββ | 1899/3125 [33:30<50:26, 2.47s/it]
61%|ββββββ | 1900/3125 [33:33<54:18, 2.66s/it]
{'loss': '1.035', 'grad_norm': '7.938', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01343', 'train/loss_cot': '1.245', 'epoch': '3.04'} |
|
61%|ββββββ | 1900/3125 [33:34<54:18, 2.66s/it]
61%|ββββββ | 1901/3125 [33:36<55:34, 2.72s/it]
61%|ββββββ | 1902/3125 [33:39<53:37, 2.63s/it]
61%|ββββββ | 1903/3125 [33:41<51:04, 2.51s/it]
61%|ββββββ | 1904/3125 [33:44<51:48, 2.55s/it]
61%|ββββββ | 1905/3125 [33:46<50:37, 2.49s/it]
61%|ββββββ | 1906/3125 [33:48<48:51, 2.41s/it]
61%|ββββββ | 1907/3125 [33:51<51:46, 2.55s/it]
61%|ββββββ | 1908/3125 [33:53<50:22, 2.48s/it]
61%|ββββββ | 1909/3125 [33:56<48:59, 2.42s/it]
61%|ββββββ | 1910/3125 [33:58<47:32, 2.35s/it]
{'loss': '1.018', 'grad_norm': '6.123', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05884', 'train/loss_cot': '1.153', 'epoch': '3.056'} |
|
61%|ββββββ | 1910/3125 [33:58<47:32, 2.35s/it]
61%|ββββββ | 1911/3125 [34:01<55:02, 2.72s/it]
61%|ββββββ | 1912/3125 [34:04<52:17, 2.59s/it]
61%|ββββββ | 1913/3125 [34:06<51:22, 2.54s/it]
61%|ββββββ | 1914/3125 [34:09<52:51, 2.62s/it]
61%|βββββββ | 1915/3125 [34:11<51:11, 2.54s/it]
61%|βββββββ | 1916/3125 [34:14<51:21, 2.55s/it]
61%|βββββββ | 1917/3125 [34:16<51:19, 2.55s/it]
61%|βββββββ | 1918/3125 [34:19<49:45, 2.47s/it]
61%|βββββββ | 1919/3125 [34:21<48:02, 2.39s/it]
61%|βββββββ | 1920/3125 [34:23<47:37, 2.37s/it]
{'loss': '1.051', 'grad_norm': '7.122', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1865', 'train/loss_cot': '0.6729', 'epoch': '3.072'} |
|
61%|βββββββ | 1920/3125 [34:24<47:37, 2.37s/it]
61%|βββββββ | 1921/3125 [34:26<47:59, 2.39s/it]
62%|βββββββ | 1922/3125 [34:28<47:57, 2.39s/it]
62%|βββββββ | 1923/3125 [34:31<53:20, 2.66s/it]
62%|βββββββ | 1924/3125 [34:33<50:19, 2.51s/it]
62%|βββββββ | 1925/3125 [34:36<51:50, 2.59s/it]
62%|βββββββ | 1926/3125 [34:39<50:56, 2.55s/it]
62%|βββββββ | 1927/3125 [34:41<49:03, 2.46s/it]
62%|βββββββ | 1928/3125 [34:44<49:43, 2.49s/it]
62%|βββββββ | 1929/3125 [34:46<51:55, 2.60s/it]
62%|βββββββ | 1930/3125 [34:49<50:09, 2.52s/it]
{'loss': '1.018', 'grad_norm': '6.818', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001015', 'train/loss_cot': '0.8492', 'epoch': '3.088'} |
|
62%|βββββββ | 1930/3125 [34:49<50:09, 2.52s/it]
62%|βββββββ | 1931/3125 [34:51<49:50, 2.50s/it]
62%|βββββββ | 1932/3125 [34:54<49:06, 2.47s/it]
62%|βββββββ | 1933/3125 [34:56<47:20, 2.38s/it]
62%|βββββββ | 1934/3125 [34:58<45:58, 2.32s/it]
62%|βββββββ | 1935/3125 [35:01<50:49, 2.56s/it]
62%|βββββββ | 1936/3125 [35:04<52:51, 2.67s/it]
62%|βββββββ | 1937/3125 [35:06<49:57, 2.52s/it]
62%|βββββββ | 1938/3125 [35:09<50:27, 2.55s/it]
62%|βββββββ | 1939/3125 [35:11<48:55, 2.48s/it]
62%|βββββββ | 1940/3125 [35:13<47:53, 2.42s/it]
{'loss': '1.051', 'grad_norm': '6.156', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004791', 'train/loss_cot': '0.9085', 'epoch': '3.104'} |
|
62%|βββββββ | 1940/3125 [35:14<47:53, 2.42s/it]
62%|βββββββ | 1941/3125 [35:16<47:54, 2.43s/it]
62%|βββββββ | 1942/3125 [35:19<51:57, 2.64s/it]
62%|βββββββ | 1943/3125 [35:21<50:29, 2.56s/it]
62%|βββββββ | 1944/3125 [35:24<50:23, 2.56s/it]
62%|βββββββ | 1945/3125 [35:26<49:05, 2.50s/it]
62%|βββββββ | 1946/3125 [35:29<49:54, 2.54s/it]
62%|βββββββ | 1947/3125 [35:31<47:38, 2.43s/it]
62%|βββββββ | 1948/3125 [35:33<46:51, 2.39s/it]
62%|βββββββ | 1949/3125 [35:36<45:48, 2.34s/it]
62%|βββββββ | 1950/3125 [35:38<46:20, 2.37s/it]
{'loss': '1.08', 'grad_norm': '4.952', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01953', 'train/loss_cot': '0.8', 'epoch': '3.12'} |
|
62%|βββββββ | 1950/3125 [35:38<46:20, 2.37s/it]
62%|βββββββ | 1951/3125 [35:40<45:59, 2.35s/it]
62%|βββββββ | 1952/3125 [35:43<45:21, 2.32s/it]
62%|βββββββ | 1953/3125 [35:45<45:14, 2.32s/it]
63%|βββββββ | 1954/3125 [35:48<48:02, 2.46s/it]
63%|βββββββ | 1955/3125 [35:50<45:57, 2.36s/it]
63%|βββββββ | 1956/3125 [35:52<45:13, 2.32s/it]
63%|βββββββ | 1957/3125 [35:55<48:29, 2.49s/it]
63%|βββββββ | 1958/3125 [35:57<47:45, 2.46s/it]
63%|βββββββ | 1959/3125 [35:59<46:06, 2.37s/it]
63%|βββββββ | 1960/3125 [36:02<45:59, 2.37s/it]
{'loss': '1.056', 'grad_norm': '5.82', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02222', 'train/loss_cot': '0.7172', 'epoch': '3.136'} |
|
63%|βββββββ | 1960/3125 [36:02<45:59, 2.37s/it]
63%|βββββββ | 1961/3125 [36:04<44:54, 2.31s/it]
63%|βββββββ | 1962/3125 [36:06<44:36, 2.30s/it]
63%|βββββββ | 1963/3125 [36:08<43:17, 2.24s/it]
63%|βββββββ | 1964/3125 [36:11<43:58, 2.27s/it]
63%|βββββββ | 1965/3125 [36:13<44:30, 2.30s/it]
63%|βββββββ | 1966/3125 [36:16<45:34, 2.36s/it]
63%|βββββββ | 1967/3125 [36:18<45:32, 2.36s/it]
63%|βββββββ | 1968/3125 [36:20<44:54, 2.33s/it]
63%|βββββββ | 1969/3125 [36:22<44:18, 2.30s/it]
63%|βββββββ | 1970/3125 [36:25<44:55, 2.33s/it]
{'loss': '1.057', 'grad_norm': '6.039', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01819', 'train/loss_cot': '1.062', 'epoch': '3.152'} |
|
63%|βββββββ | 1970/3125 [36:25<44:55, 2.33s/it]
63%|βββββββ | 1971/3125 [36:27<43:48, 2.28s/it]
63%|βββββββ | 1972/3125 [36:29<43:52, 2.28s/it]
63%|βββββββ | 1973/3125 [36:32<44:52, 2.34s/it]
63%|βββββββ | 1974/3125 [36:35<48:24, 2.52s/it]
63%|βββββββ | 1975/3125 [36:37<46:13, 2.41s/it]
63%|βββββββ | 1976/3125 [36:39<44:49, 2.34s/it]
63%|βββββββ | 1977/3125 [36:41<44:33, 2.33s/it]
63%|βββββββ | 1978/3125 [36:44<44:42, 2.34s/it]
63%|βββββββ | 1979/3125 [36:47<49:39, 2.60s/it]
63%|βββββββ | 1980/3125 [36:49<48:35, 2.55s/it]
{'loss': '1.075', 'grad_norm': '6.714', 'learning_rate': '1e-05', 'train/loss_infonce': '0.3125', 'train/loss_cot': '1.149', 'epoch': '3.168'} |
|
63%|βββββββ | 1980/3125 [36:50<48:35, 2.55s/it]
63%|βββββββ | 1981/3125 [36:52<46:44, 2.45s/it]
63%|βββββββ | 1982/3125 [36:54<45:47, 2.40s/it]
63%|βββββββ | 1983/3125 [36:56<45:45, 2.40s/it]
63%|βββββββ | 1984/3125 [36:58<44:37, 2.35s/it]
64%|βββββββ | 1985/3125 [37:01<43:50, 2.31s/it]
64%|βββββββ | 1986/3125 [37:03<43:55, 2.31s/it]
64%|βββββββ | 1987/3125 [37:05<43:47, 2.31s/it]
64%|βββββββ | 1988/3125 [37:08<47:20, 2.50s/it]
64%|βββββββ | 1989/3125 [37:11<46:24, 2.45s/it]
64%|βββββββ | 1990/3125 [37:13<47:40, 2.52s/it]
{'loss': '1.034', 'grad_norm': '5.547', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04663', 'train/loss_cot': '1.034', 'epoch': '3.184'} |
|
64%|βββββββ | 1990/3125 [37:14<47:40, 2.52s/it]
64%|βββββββ | 1991/3125 [37:16<46:30, 2.46s/it]
64%|βββββββ | 1992/3125 [37:18<45:21, 2.40s/it]
64%|βββββββ | 1993/3125 [37:20<44:16, 2.35s/it]
64%|βββββββ | 1994/3125 [37:22<43:27, 2.31s/it]
64%|βββββββ | 1995/3125 [37:24<42:42, 2.27s/it]
64%|βββββββ | 1996/3125 [37:27<43:09, 2.29s/it]
64%|βββββββ | 1997/3125 [37:30<46:07, 2.45s/it]
64%|βββββββ | 1998/3125 [37:32<44:08, 2.35s/it]
64%|βββββββ | 1999/3125 [37:34<44:24, 2.37s/it]
64%|βββββββ | 2000/3125 [37:36<44:00, 2.35s/it]
{'loss': '0.9929', 'grad_norm': '6.685', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '0.8827', 'epoch': '3.2'} |
|
64%|βββββββ | 2000/3125 [37:37<44:00, 2.35s/it]
64%|βββββββ | 2001/3125 [37:55<2:13:12, 7.11s/it]
64%|βββββββ | 2002/3125 [37:57<1:46:22, 5.68s/it]
64%|βββββββ | 2003/3125 [37:59<1:28:16, 4.72s/it]
64%|βββββββ | 2004/3125 [38:02<1:14:41, 4.00s/it]
64%|βββββββ | 2005/3125 [38:04<1:04:15, 3.44s/it]
64%|βββββββ | 2006/3125 [38:06<58:29, 3.14s/it]
64%|βββββββ | 2007/3125 [38:09<54:42, 2.94s/it]
64%|βββββββ | 2008/3125 [38:11<52:18, 2.81s/it]
64%|βββββββ | 2009/3125 [38:14<51:09, 2.75s/it]
64%|βββββββ | 2010/3125 [38:16<49:08, 2.64s/it]
{'loss': '1.085', 'grad_norm': '5.852', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04272', 'train/loss_cot': '0.9627', 'epoch': '3.216'} |
|
64%|βββββββ | 2010/3125 [38:17<49:08, 2.64s/it]
64%|βββββββ | 2011/3125 [38:20<52:07, 2.81s/it]
64%|βββββββ | 2012/3125 [38:22<50:54, 2.74s/it]
64%|βββββββ | 2013/3125 [38:25<50:08, 2.71s/it]
64%|βββββββ | 2014/3125 [38:27<49:02, 2.65s/it]
64%|βββββββ | 2015/3125 [38:30<50:07, 2.71s/it]
65%|βββββββ | 2016/3125 [38:33<51:07, 2.77s/it]
65%|βββββββ | 2017/3125 [38:35<47:48, 2.59s/it]
65%|βββββββ | 2018/3125 [38:38<49:02, 2.66s/it]
65%|βββββββ | 2019/3125 [38:41<48:50, 2.65s/it]
65%|βββββββ | 2020/3125 [38:44<50:40, 2.75s/it]
{'loss': '1.097', 'grad_norm': '5.433', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007141', 'train/loss_cot': '1.117', 'epoch': '3.232'} |
|
65%|βββββββ | 2020/3125 [38:44<50:40, 2.75s/it]
65%|βββββββ | 2021/3125 [38:46<49:21, 2.68s/it]
65%|βββββββ | 2022/3125 [38:49<47:44, 2.60s/it]
65%|βββββββ | 2023/3125 [38:51<46:07, 2.51s/it]
65%|βββββββ | 2024/3125 [38:54<49:09, 2.68s/it]
65%|βββββββ | 2025/3125 [38:56<47:27, 2.59s/it]
65%|βββββββ | 2026/3125 [38:59<46:26, 2.54s/it]
65%|βββββββ | 2027/3125 [39:01<47:42, 2.61s/it]
65%|βββββββ | 2028/3125 [39:04<47:06, 2.58s/it]
65%|βββββββ | 2029/3125 [39:06<46:00, 2.52s/it]
65%|βββββββ | 2030/3125 [39:09<44:07, 2.42s/it]
{'loss': '1.075', 'grad_norm': '5.627', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0498', 'train/loss_cot': '0.9348', 'epoch': '3.248'} |
|
65%|βββββββ | 2030/3125 [39:09<44:07, 2.42s/it]
65%|βββββββ | 2031/3125 [39:11<44:37, 2.45s/it]
65%|βββββββ | 2032/3125 [39:14<44:52, 2.46s/it]
65%|βββββββ | 2033/3125 [39:16<44:33, 2.45s/it]
65%|βββββββ | 2034/3125 [39:18<44:12, 2.43s/it]
65%|βββββββ | 2035/3125 [39:21<46:48, 2.58s/it]
65%|βββββββ | 2036/3125 [39:24<46:18, 2.55s/it]
65%|βββββββ | 2037/3125 [39:26<44:47, 2.47s/it]
65%|βββββββ | 2038/3125 [39:29<44:48, 2.47s/it]
65%|βββββββ | 2039/3125 [39:31<43:43, 2.42s/it]
65%|βββββββ | 2040/3125 [39:34<45:54, 2.54s/it]
{'loss': '1.049', 'grad_norm': '5.633', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00769', 'train/loss_cot': '0.9103', 'epoch': '3.264'} |
|
65%|βββββββ | 2040/3125 [39:34<45:54, 2.54s/it]
65%|βββββββ | 2041/3125 [39:36<45:52, 2.54s/it]
65%|βββββββ | 2042/3125 [39:39<46:16, 2.56s/it]
65%|βββββββ | 2043/3125 [39:41<45:14, 2.51s/it]
65%|βββββββ | 2044/3125 [39:44<45:36, 2.53s/it]
65%|βββββββ | 2045/3125 [39:46<45:12, 2.51s/it]
65%|βββββββ | 2046/3125 [39:49<46:47, 2.60s/it]
66%|βββββββ | 2047/3125 [39:51<45:08, 2.51s/it]
66%|βββββββ | 2048/3125 [39:54<46:08, 2.57s/it]
66%|βββββββ | 2049/3125 [39:57<46:46, 2.61s/it]
66%|βββββββ | 2050/3125 [39:59<44:15, 2.47s/it]
{'loss': '1.063', 'grad_norm': '6.172', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005249', 'train/loss_cot': '0.8447', 'epoch': '3.28'} |
|
66%|βββββββ | 2050/3125 [39:59<44:15, 2.47s/it]
66%|βββββββ | 2051/3125 [40:01<43:38, 2.44s/it]
66%|βββββββ | 2052/3125 [40:05<48:07, 2.69s/it]
66%|βββββββ | 2053/3125 [40:07<47:26, 2.66s/it]
66%|βββββββ | 2054/3125 [40:09<44:54, 2.52s/it]
66%|βββββββ | 2055/3125 [40:12<44:31, 2.50s/it]
66%|βββββββ | 2056/3125 [40:14<44:50, 2.52s/it]
66%|βββββββ | 2057/3125 [40:17<44:12, 2.48s/it]
66%|βββββββ | 2058/3125 [40:19<45:12, 2.54s/it]
66%|βββββββ | 2059/3125 [40:22<45:23, 2.55s/it]
66%|βββββββ | 2060/3125 [40:24<44:53, 2.53s/it]
{'loss': '1.018', 'grad_norm': '5.294', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006866', 'train/loss_cot': '0.8124', 'epoch': '3.296'} |
|
66%|βββββββ | 2060/3125 [40:25<44:53, 2.53s/it]
66%|βββββββ | 2061/3125 [40:27<44:55, 2.53s/it]
66%|βββββββ | 2062/3125 [40:29<44:26, 2.51s/it]
66%|βββββββ | 2063/3125 [40:32<42:32, 2.40s/it]
66%|βββββββ | 2064/3125 [40:34<42:35, 2.41s/it]
66%|βββββββ | 2065/3125 [40:36<42:14, 2.39s/it]
66%|βββββββ | 2066/3125 [40:39<42:02, 2.38s/it]
66%|βββββββ | 2067/3125 [40:41<42:27, 2.41s/it]
66%|βββββββ | 2068/3125 [40:44<44:10, 2.51s/it]
66%|βββββββ | 2069/3125 [40:46<43:23, 2.47s/it]
66%|βββββββ | 2070/3125 [40:49<42:53, 2.44s/it]
{'loss': '1.031', 'grad_norm': '5.682', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07715', 'train/loss_cot': '0.9371', 'epoch': '3.312'} |
|
66%|βββββββ | 2070/3125 [40:49<42:53, 2.44s/it]
66%|βββββββ | 2071/3125 [40:52<44:59, 2.56s/it]
66%|βββββββ | 2072/3125 [40:54<44:05, 2.51s/it]
66%|βββββββ | 2073/3125 [40:56<42:12, 2.41s/it]
66%|βββββββ | 2074/3125 [40:59<42:08, 2.41s/it]
66%|βββββββ | 2075/3125 [41:01<43:18, 2.48s/it]
66%|βββββββ | 2076/3125 [41:03<42:25, 2.43s/it]
66%|βββββββ | 2077/3125 [41:07<46:10, 2.64s/it]
66%|βββββββ | 2078/3125 [41:09<44:36, 2.56s/it]
67%|βββββββ | 2079/3125 [41:11<44:15, 2.54s/it]
67%|βββββββ | 2080/3125 [41:14<41:49, 2.40s/it]
{'loss': '1.076', 'grad_norm': '5.887', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009888', 'train/loss_cot': '1.209', 'epoch': '3.328'} |
|
67%|βββββββ | 2080/3125 [41:14<41:49, 2.40s/it]
67%|βββββββ | 2081/3125 [41:16<44:16, 2.54s/it]
67%|βββββββ | 2082/3125 [41:19<43:42, 2.51s/it]
67%|βββββββ | 2083/3125 [41:21<43:24, 2.50s/it]
67%|βββββββ | 2084/3125 [41:23<41:17, 2.38s/it]
67%|βββββββ | 2085/3125 [41:26<40:56, 2.36s/it]
67%|βββββββ | 2086/3125 [41:28<40:58, 2.37s/it]
67%|βββββββ | 2087/3125 [41:30<40:37, 2.35s/it]
67%|βββββββ | 2088/3125 [41:33<41:58, 2.43s/it]
67%|βββββββ | 2089/3125 [41:35<41:43, 2.42s/it]
67%|βββββββ | 2090/3125 [41:38<41:26, 2.40s/it]
{'loss': '1.04', 'grad_norm': '6.274', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01031', 'train/loss_cot': '1.146', 'epoch': '3.344'} |
|
67%|βββββββ | 2090/3125 [41:38<41:26, 2.40s/it]
67%|βββββββ | 2091/3125 [41:40<41:36, 2.41s/it]
67%|βββββββ | 2092/3125 [41:43<41:25, 2.41s/it]
67%|βββββββ | 2093/3125 [41:45<42:06, 2.45s/it]
67%|βββββββ | 2094/3125 [41:48<41:53, 2.44s/it]
67%|βββββββ | 2095/3125 [41:50<42:12, 2.46s/it]
67%|βββββββ | 2096/3125 [41:53<44:11, 2.58s/it]
67%|βββββββ | 2097/3125 [41:56<47:17, 2.76s/it]
67%|βββββββ | 2098/3125 [41:59<45:51, 2.68s/it]
67%|βββββββ | 2099/3125 [42:01<45:28, 2.66s/it]
67%|βββββββ | 2100/3125 [42:04<43:43, 2.56s/it]
{'loss': '1.012', 'grad_norm': '18.56', 'learning_rate': '1e-05', 'train/loss_infonce': '0.7812', 'train/loss_cot': '0.8486', 'epoch': '3.36'} |
|
67%|βββββββ | 2100/3125 [42:04<43:43, 2.56s/it]
67%|βββββββ | 2101/3125 [42:06<42:11, 2.47s/it]
67%|βββββββ | 2102/3125 [42:08<41:25, 2.43s/it]
67%|βββββββ | 2103/3125 [42:11<41:06, 2.41s/it]
67%|βββββββ | 2104/3125 [42:13<43:06, 2.53s/it]
67%|βββββββ | 2105/3125 [42:16<41:21, 2.43s/it]
67%|βββββββ | 2106/3125 [42:18<39:42, 2.34s/it]
67%|βββββββ | 2107/3125 [42:21<44:15, 2.61s/it]
67%|βββββββ | 2108/3125 [42:23<43:41, 2.58s/it]
67%|βββββββ | 2109/3125 [42:26<43:27, 2.57s/it]
68%|βββββββ | 2110/3125 [42:29<43:19, 2.56s/it]
{'loss': '1.133', 'grad_norm': '5.116', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04346', 'train/loss_cot': '1.103', 'epoch': '3.376'} |
|
68%|βββββββ | 2110/3125 [42:29<43:19, 2.56s/it]
68%|βββββββ | 2111/3125 [42:31<42:22, 2.51s/it]
68%|βββββββ | 2112/3125 [42:33<41:31, 2.46s/it]
68%|βββββββ | 2113/3125 [42:36<42:16, 2.51s/it]
68%|βββββββ | 2114/3125 [42:38<40:32, 2.41s/it]
68%|βββββββ | 2115/3125 [42:40<39:55, 2.37s/it]
68%|βββββββ | 2116/3125 [42:43<39:37, 2.36s/it]
68%|βββββββ | 2117/3125 [42:45<40:24, 2.41s/it]
68%|βββββββ | 2118/3125 [42:48<43:58, 2.62s/it]
68%|βββββββ | 2119/3125 [42:52<47:06, 2.81s/it]
68%|βββββββ | 2120/3125 [42:54<45:00, 2.69s/it]
{'loss': '1.034', 'grad_norm': '6.444', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006439', 'train/loss_cot': '1.103', 'epoch': '3.392'} |
|
68%|βββββββ | 2120/3125 [42:54<45:00, 2.69s/it]
68%|βββββββ | 2121/3125 [42:57<44:17, 2.65s/it]
68%|βββββββ | 2122/3125 [42:59<42:44, 2.56s/it]
68%|βββββββ | 2123/3125 [43:01<41:04, 2.46s/it]
68%|βββββββ | 2124/3125 [43:03<40:22, 2.42s/it]
68%|βββββββ | 2125/3125 [43:06<41:10, 2.47s/it]
68%|βββββββ | 2126/3125 [43:09<41:18, 2.48s/it]
68%|βββββββ | 2127/3125 [43:12<44:42, 2.69s/it]
68%|βββββββ | 2128/3125 [43:15<47:01, 2.83s/it]
68%|βββββββ | 2129/3125 [43:17<43:17, 2.61s/it]
68%|βββββββ | 2130/3125 [43:20<44:08, 2.66s/it]
{'loss': '1.054', 'grad_norm': '6.24', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003464', 'train/loss_cot': '1.269', 'epoch': '3.408'} |
|
68%|βββββββ | 2130/3125 [43:20<44:08, 2.66s/it]
68%|βββββββ | 2131/3125 [43:23<44:59, 2.72s/it]
68%|βββββββ | 2132/3125 [43:25<43:03, 2.60s/it]
68%|βββββββ | 2133/3125 [43:27<42:50, 2.59s/it]
68%|βββββββ | 2134/3125 [43:30<41:21, 2.50s/it]
68%|βββββββ | 2135/3125 [43:33<43:39, 2.65s/it]
68%|βββββββ | 2136/3125 [43:35<41:32, 2.52s/it]
68%|βββββββ | 2137/3125 [43:37<40:09, 2.44s/it]
68%|βββββββ | 2138/3125 [43:40<40:28, 2.46s/it]
68%|βββββββ | 2139/3125 [43:42<39:43, 2.42s/it]
68%|βββββββ | 2140/3125 [43:44<39:37, 2.41s/it]
{'loss': '1.068', 'grad_norm': '6.028', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005157', 'train/loss_cot': '0.8323', 'epoch': '3.424'} |
|
68%|βββββββ | 2140/3125 [43:45<39:37, 2.41s/it]
69%|βββββββ | 2141/3125 [43:47<40:03, 2.44s/it]
69%|βββββββ | 2142/3125 [43:49<40:05, 2.45s/it]
69%|βββββββ | 2143/3125 [43:52<40:00, 2.44s/it]
69%|βββββββ | 2144/3125 [43:55<40:58, 2.51s/it]
69%|βββββββ | 2145/3125 [43:57<41:01, 2.51s/it]
69%|βββββββ | 2146/3125 [43:59<40:30, 2.48s/it]
69%|βββββββ | 2147/3125 [44:02<41:53, 2.57s/it]
69%|βββββββ | 2148/3125 [44:06<50:02, 3.07s/it]
69%|βββββββ | 2149/3125 [44:09<49:18, 3.03s/it]
69%|βββββββ | 2150/3125 [44:12<46:20, 2.85s/it]
{'loss': '1.06', 'grad_norm': '13.43', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01733', 'train/loss_cot': '1.293', 'epoch': '3.44'} |
|
69%|βββββββ | 2150/3125 [44:12<46:20, 2.85s/it]
69%|βββββββ | 2151/3125 [44:14<44:30, 2.74s/it]
69%|βββββββ | 2152/3125 [44:17<42:17, 2.61s/it]
69%|βββββββ | 2153/3125 [44:19<42:08, 2.60s/it]
69%|βββββββ | 2154/3125 [44:22<42:10, 2.61s/it]
69%|βββββββ | 2155/3125 [44:24<41:00, 2.54s/it]
69%|βββββββ | 2156/3125 [44:26<39:36, 2.45s/it]
69%|βββββββ | 2157/3125 [44:29<39:22, 2.44s/it]
69%|βββββββ | 2158/3125 [44:32<41:15, 2.56s/it]
69%|βββββββ | 2159/3125 [44:34<39:45, 2.47s/it]
69%|βββββββ | 2160/3125 [44:36<38:23, 2.39s/it]
{'loss': '0.9993', 'grad_norm': '5.85', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08105', 'train/loss_cot': '0.9752', 'epoch': '3.456'} |
|
69%|βββββββ | 2160/3125 [44:36<38:23, 2.39s/it]
69%|βββββββ | 2161/3125 [44:39<39:18, 2.45s/it]
69%|βββββββ | 2162/3125 [44:41<38:30, 2.40s/it]
69%|βββββββ | 2163/3125 [44:43<38:37, 2.41s/it]
69%|βββββββ | 2164/3125 [44:46<37:51, 2.36s/it]
69%|βββββββ | 2165/3125 [44:48<38:27, 2.40s/it]
69%|βββββββ | 2166/3125 [44:50<37:24, 2.34s/it]
69%|βββββββ | 2167/3125 [44:53<37:32, 2.35s/it]
69%|βββββββ | 2168/3125 [44:55<37:30, 2.35s/it]
69%|βββββββ | 2169/3125 [44:58<37:45, 2.37s/it]
69%|βββββββ | 2170/3125 [45:00<39:01, 2.45s/it]
{'loss': '1.011', 'grad_norm': '5.577', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01459', 'train/loss_cot': '1.048', 'epoch': '3.472'} |
|
69%|βββββββ | 2170/3125 [45:00<39:01, 2.45s/it]
69%|βββββββ | 2171/3125 [45:03<39:42, 2.50s/it]
70%|βββββββ | 2172/3125 [45:05<39:42, 2.50s/it]
70%|βββββββ | 2173/3125 [45:08<38:40, 2.44s/it]
70%|βββββββ | 2174/3125 [45:10<39:38, 2.50s/it]
70%|βββββββ | 2175/3125 [45:12<38:25, 2.43s/it]
70%|βββββββ | 2176/3125 [45:15<38:05, 2.41s/it]
70%|βββββββ | 2177/3125 [45:17<38:22, 2.43s/it]
70%|βββββββ | 2178/3125 [45:21<43:29, 2.76s/it]
70%|βββββββ | 2179/3125 [45:23<40:47, 2.59s/it]
70%|βββββββ | 2180/3125 [45:25<39:54, 2.53s/it]
{'loss': '1.047', 'grad_norm': '6.482', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1187', 'train/loss_cot': '0.7171', 'epoch': '3.488'} |
|
70%|βββββββ | 2180/3125 [45:26<39:54, 2.53s/it]
70%|βββββββ | 2181/3125 [45:28<39:44, 2.53s/it]
70%|βββββββ | 2182/3125 [45:31<41:02, 2.61s/it]
70%|βββββββ | 2183/3125 [45:33<39:52, 2.54s/it]
70%|βββββββ | 2184/3125 [45:35<38:45, 2.47s/it]
70%|βββββββ | 2185/3125 [45:38<38:43, 2.47s/it]
70%|βββββββ | 2186/3125 [45:40<38:57, 2.49s/it]
70%|βββββββ | 2187/3125 [45:43<38:18, 2.45s/it]
70%|βββββββ | 2188/3125 [45:46<40:03, 2.57s/it]
70%|βββββββ | 2189/3125 [45:48<40:09, 2.57s/it]
70%|βββββββ | 2190/3125 [45:51<38:59, 2.50s/it]
{'loss': '1.111', 'grad_norm': '9.047', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01123', 'train/loss_cot': '1.043', 'epoch': '3.504'} |
|
70%|βββββββ | 2190/3125 [45:51<38:59, 2.50s/it]
70%|βββββββ | 2191/3125 [45:53<39:52, 2.56s/it]
70%|βββββββ | 2192/3125 [45:56<38:58, 2.51s/it]
70%|βββββββ | 2193/3125 [45:58<38:51, 2.50s/it]
70%|βββββββ | 2194/3125 [46:00<37:38, 2.43s/it]
70%|βββββββ | 2195/3125 [46:03<36:47, 2.37s/it]
70%|βββββββ | 2196/3125 [46:05<36:54, 2.38s/it]
70%|βββββββ | 2197/3125 [46:08<37:33, 2.43s/it]
70%|βββββββ | 2198/3125 [46:10<36:23, 2.36s/it]
70%|βββββββ | 2199/3125 [46:12<35:06, 2.28s/it]
70%|βββββββ | 2200/3125 [46:15<37:32, 2.43s/it]
{'loss': '1.09', 'grad_norm': '5.561', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03076', 'train/loss_cot': '1.155', 'epoch': '3.52'} |
|
70%|βββββββ | 2200/3125 [46:15<37:32, 2.43s/it]
70%|βββββββ | 2201/3125 [46:33<1:51:18, 7.23s/it]
70%|βββββββ | 2202/3125 [46:36<1:30:51, 5.91s/it]
70%|βββββββ | 2203/3125 [46:38<1:14:02, 4.82s/it]
71%|βββββββ | 2204/3125 [46:41<1:02:49, 4.09s/it]
71%|βββββββ | 2205/3125 [46:43<54:33, 3.56s/it]
71%|βββββββ | 2206/3125 [46:46<50:31, 3.30s/it]
71%|βββββββ | 2207/3125 [46:48<45:51, 3.00s/it]
71%|βββββββ | 2208/3125 [46:51<45:51, 3.00s/it]
71%|βββββββ | 2209/3125 [46:53<42:00, 2.75s/it]
71%|βββββββ | 2210/3125 [46:56<40:59, 2.69s/it]
{'loss': '1.011', 'grad_norm': '5.26', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01465', 'train/loss_cot': '0.8401', 'epoch': '3.536'} |
|
71%|βββββββ | 2210/3125 [46:56<40:59, 2.69s/it]
71%|βββββββ | 2211/3125 [46:58<40:08, 2.63s/it]
71%|βββββββ | 2212/3125 [47:01<40:02, 2.63s/it]
71%|βββββββ | 2213/3125 [47:03<38:57, 2.56s/it]
71%|βββββββ | 2214/3125 [47:05<37:26, 2.47s/it]
71%|βββββββ | 2215/3125 [47:08<38:46, 2.56s/it]
71%|βββββββ | 2216/3125 [47:11<39:07, 2.58s/it]
71%|βββββββ | 2217/3125 [47:13<37:28, 2.48s/it]
71%|βββββββ | 2218/3125 [47:15<35:55, 2.38s/it]
71%|βββββββ | 2219/3125 [47:17<35:02, 2.32s/it]
71%|βββββββ | 2220/3125 [47:20<35:18, 2.34s/it]
{'loss': '1.05', 'grad_norm': '6.493', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03394', 'train/loss_cot': '1.138', 'epoch': '3.552'} |
|
71%|βββββββ | 2220/3125 [47:20<35:18, 2.34s/it]
71%|βββββββ | 2221/3125 [47:22<36:47, 2.44s/it]
71%|βββββββ | 2222/3125 [47:25<36:04, 2.40s/it]
71%|βββββββ | 2223/3125 [47:27<35:38, 2.37s/it]
71%|βββββββ | 2224/3125 [47:30<36:44, 2.45s/it]
71%|βββββββ | 2225/3125 [47:32<37:09, 2.48s/it]
71%|βββββββ | 2226/3125 [47:35<36:22, 2.43s/it]
71%|ββββββββ | 2227/3125 [47:37<35:46, 2.39s/it]
71%|ββββββββ | 2228/3125 [47:39<35:43, 2.39s/it]
71%|ββββββββ | 2229/3125 [47:42<35:23, 2.37s/it]
71%|ββββββββ | 2230/3125 [47:44<36:01, 2.42s/it]
{'loss': '1.033', 'grad_norm': '6.419', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02576', 'train/loss_cot': '0.9247', 'epoch': '3.568'} |
|
71%|ββββββββ | 2230/3125 [47:44<36:01, 2.42s/it]
71%|ββββββββ | 2231/3125 [47:47<36:45, 2.47s/it]
71%|ββββββββ | 2232/3125 [47:49<35:26, 2.38s/it]
71%|ββββββββ | 2233/3125 [47:51<34:51, 2.34s/it]
71%|ββββββββ | 2234/3125 [47:54<36:10, 2.44s/it]
72%|ββββββββ | 2235/3125 [47:56<34:23, 2.32s/it]
72%|ββββββββ | 2236/3125 [47:58<35:22, 2.39s/it]
72%|ββββββββ | 2237/3125 [48:01<36:01, 2.43s/it]
72%|ββββββββ | 2238/3125 [48:03<35:49, 2.42s/it]
72%|ββββββββ | 2239/3125 [48:06<35:22, 2.40s/it]
72%|ββββββββ | 2240/3125 [48:08<36:40, 2.49s/it]
{'loss': '1.045', 'grad_norm': '8.428', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007141', 'train/loss_cot': '1.278', 'epoch': '3.584'} |
|
72%|ββββββββ | 2240/3125 [48:09<36:40, 2.49s/it]
72%|ββββββββ | 2241/3125 [48:11<35:51, 2.43s/it]
72%|ββββββββ | 2242/3125 [48:13<35:41, 2.43s/it]
72%|ββββββββ | 2243/3125 [48:15<34:38, 2.36s/it]
72%|ββββββββ | 2244/3125 [48:17<34:16, 2.33s/it]
72%|ββββββββ | 2245/3125 [48:20<34:33, 2.36s/it]
72%|ββββββββ | 2246/3125 [48:22<35:19, 2.41s/it]
72%|ββββββββ | 2247/3125 [48:25<34:45, 2.38s/it]
72%|ββββββββ | 2248/3125 [48:27<34:40, 2.37s/it]
72%|ββββββββ | 2249/3125 [48:30<35:27, 2.43s/it]
72%|ββββββββ | 2250/3125 [48:32<34:48, 2.39s/it]
{'loss': '1.046', 'grad_norm': '5.948', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006409', 'train/loss_cot': '0.889', 'epoch': '3.6'} |
|
72%|ββββββββ | 2250/3125 [48:32<34:48, 2.39s/it]
72%|ββββββββ | 2251/3125 [48:34<34:26, 2.36s/it]
72%|ββββββββ | 2252/3125 [48:36<33:43, 2.32s/it]
72%|ββββββββ | 2253/3125 [48:39<34:54, 2.40s/it]
72%|ββββββββ | 2254/3125 [48:41<34:49, 2.40s/it]
72%|ββββββββ | 2255/3125 [48:44<36:50, 2.54s/it]
72%|ββββββββ | 2256/3125 [48:47<36:34, 2.53s/it]
72%|ββββββββ | 2257/3125 [48:50<37:55, 2.62s/it]
72%|ββββββββ | 2258/3125 [48:52<37:05, 2.57s/it]
72%|ββββββββ | 2259/3125 [48:54<36:18, 2.52s/it]
72%|ββββββββ | 2260/3125 [48:57<35:23, 2.45s/it]
{'loss': '1.039', 'grad_norm': '5.786', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00119', 'train/loss_cot': '1.05', 'epoch': '3.616'} |
|
72%|ββββββββ | 2260/3125 [48:57<35:23, 2.45s/it]
72%|ββββββββ | 2261/3125 [48:59<36:13, 2.52s/it]
72%|ββββββββ | 2262/3125 [49:02<36:46, 2.56s/it]
72%|ββββββββ | 2263/3125 [49:04<35:22, 2.46s/it]
72%|ββββββββ | 2264/3125 [49:07<35:19, 2.46s/it]
72%|ββββββββ | 2265/3125 [49:09<34:44, 2.42s/it]
73%|ββββββββ | 2266/3125 [49:12<35:14, 2.46s/it]
73%|ββββββββ | 2267/3125 [49:14<35:50, 2.51s/it]
73%|ββββββββ | 2268/3125 [49:17<35:53, 2.51s/it]
73%|ββββββββ | 2269/3125 [49:19<35:26, 2.48s/it]
73%|ββββββββ | 2270/3125 [49:22<35:21, 2.48s/it]
{'loss': '1.128', 'grad_norm': '7.582', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04004', 'train/loss_cot': '1.083', 'epoch': '3.632'} |
|
73%|ββββββββ | 2270/3125 [49:22<35:21, 2.48s/it]
73%|ββββββββ | 2271/3125 [49:24<34:24, 2.42s/it]
73%|ββββββββ | 2272/3125 [49:26<34:10, 2.40s/it]
73%|ββββββββ | 2273/3125 [49:29<34:43, 2.45s/it]
73%|ββββββββ | 2274/3125 [49:31<33:24, 2.36s/it]
73%|ββββββββ | 2275/3125 [49:33<33:42, 2.38s/it]
73%|ββββββββ | 2276/3125 [49:36<35:35, 2.52s/it]
73%|ββββββββ | 2277/3125 [49:39<35:18, 2.50s/it]
73%|ββββββββ | 2278/3125 [49:41<34:31, 2.45s/it]
73%|ββββββββ | 2279/3125 [49:44<34:25, 2.44s/it]
73%|ββββββββ | 2280/3125 [49:46<33:55, 2.41s/it]
{'loss': '0.9878', 'grad_norm': '5.992', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05518', 'train/loss_cot': '0.7204', 'epoch': '3.648'} |
|
73%|ββββββββ | 2280/3125 [49:46<33:55, 2.41s/it]
73%|ββββββββ | 2281/3125 [49:48<34:07, 2.43s/it]
73%|ββββββββ | 2282/3125 [49:51<36:50, 2.62s/it]
73%|ββββββββ | 2283/3125 [49:54<35:58, 2.56s/it]
73%|ββββββββ | 2284/3125 [49:56<34:26, 2.46s/it]
73%|ββββββββ | 2285/3125 [49:59<34:28, 2.46s/it]
73%|ββββββββ | 2286/3125 [50:01<33:07, 2.37s/it]
73%|ββββββββ | 2287/3125 [50:03<32:57, 2.36s/it]
73%|ββββββββ | 2288/3125 [50:05<32:46, 2.35s/it]
73%|ββββββββ | 2289/3125 [50:08<35:04, 2.52s/it]
73%|ββββββββ | 2290/3125 [50:11<34:22, 2.47s/it]
{'loss': '1.084', 'grad_norm': '5.384', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01117', 'train/loss_cot': '0.8849', 'epoch': '3.664'} |
|
73%|ββββββββ | 2290/3125 [50:11<34:22, 2.47s/it]
73%|ββββββββ | 2291/3125 [50:13<35:12, 2.53s/it]
73%|ββββββββ | 2292/3125 [50:16<33:58, 2.45s/it]
73%|ββββββββ | 2293/3125 [50:18<32:51, 2.37s/it]
73%|ββββββββ | 2294/3125 [50:20<32:57, 2.38s/it]
73%|ββββββββ | 2295/3125 [50:23<33:29, 2.42s/it]
73%|ββββββββ | 2296/3125 [50:25<33:24, 2.42s/it]
74%|ββββββββ | 2297/3125 [50:27<32:57, 2.39s/it]
74%|ββββββββ | 2298/3125 [50:30<33:53, 2.46s/it]
74%|ββββββββ | 2299/3125 [50:32<33:33, 2.44s/it]
74%|ββββββββ | 2300/3125 [50:36<37:46, 2.75s/it]
{'loss': '1.062', 'grad_norm': '6.839', 'learning_rate': '1e-05', 'train/loss_infonce': '0.002777', 'train/loss_cot': '1.33', 'epoch': '3.68'} |
|
74%|ββββββββ | 2300/3125 [50:36<37:46, 2.75s/it]
74%|ββββββββ | 2301/3125 [50:39<38:43, 2.82s/it]
74%|ββββββββ | 2302/3125 [50:41<36:40, 2.67s/it]
74%|ββββββββ | 2303/3125 [50:45<43:10, 3.15s/it]
74%|ββββββββ | 2304/3125 [50:48<42:00, 3.07s/it]
74%|ββββββββ | 2305/3125 [50:51<39:18, 2.88s/it]
74%|ββββββββ | 2306/3125 [50:53<37:47, 2.77s/it]
74%|ββββββββ | 2307/3125 [50:56<39:16, 2.88s/it]
74%|ββββββββ | 2308/3125 [51:00<40:12, 2.95s/it]
74%|ββββββββ | 2309/3125 [51:02<37:23, 2.75s/it]
74%|ββββββββ | 2310/3125 [51:04<35:46, 2.63s/it]
{'loss': '1.041', 'grad_norm': '9.008', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003586', 'train/loss_cot': '1.135', 'epoch': '3.696'} |
|
74%|ββββββββ | 2310/3125 [51:05<35:46, 2.63s/it]
74%|ββββββββ | 2311/3125 [51:06<34:25, 2.54s/it]
74%|ββββββββ | 2312/3125 [51:09<34:14, 2.53s/it]
74%|ββββββββ | 2313/3125 [51:11<33:28, 2.47s/it]
74%|ββββββββ | 2314/3125 [51:14<33:13, 2.46s/it]
74%|ββββββββ | 2315/3125 [51:17<36:11, 2.68s/it]
74%|ββββββββ | 2316/3125 [51:19<34:41, 2.57s/it]
74%|ββββββββ | 2317/3125 [51:21<32:47, 2.44s/it]
74%|ββββββββ | 2318/3125 [51:24<32:39, 2.43s/it]
74%|ββββββββ | 2319/3125 [51:26<33:33, 2.50s/it]
74%|ββββββββ | 2320/3125 [51:29<34:58, 2.61s/it]
{'loss': '1.041', 'grad_norm': '5.18', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005951', 'train/loss_cot': '1.068', 'epoch': '3.712'} |
|
74%|ββββββββ | 2320/3125 [51:30<34:58, 2.61s/it]
74%|ββββββββ | 2321/3125 [51:32<34:32, 2.58s/it]
74%|ββββββββ | 2322/3125 [51:35<35:11, 2.63s/it]
74%|ββββββββ | 2323/3125 [51:38<37:28, 2.80s/it]
74%|ββββββββ | 2324/3125 [51:40<35:58, 2.69s/it]
74%|ββββββββ | 2325/3125 [51:43<35:15, 2.64s/it]
74%|ββββββββ | 2326/3125 [51:45<34:14, 2.57s/it]
74%|ββββββββ | 2327/3125 [51:48<34:19, 2.58s/it]
74%|ββββββββ | 2328/3125 [51:50<33:39, 2.53s/it]
75%|ββββββββ | 2329/3125 [51:53<36:00, 2.71s/it]
75%|ββββββββ | 2330/3125 [51:56<35:34, 2.69s/it]
{'loss': '1.099', 'grad_norm': '6.674', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02356', 'train/loss_cot': '1.222', 'epoch': '3.728'} |
|
75%|ββββββββ | 2330/3125 [51:56<35:34, 2.69s/it]
75%|ββββββββ | 2331/3125 [51:59<35:19, 2.67s/it]
75%|ββββββββ | 2332/3125 [52:01<34:44, 2.63s/it]
75%|ββββββββ | 2333/3125 [52:03<33:35, 2.55s/it]
75%|ββββββββ | 2334/3125 [52:06<32:39, 2.48s/it]
75%|ββββββββ | 2335/3125 [52:08<31:56, 2.43s/it]
75%|ββββββββ | 2336/3125 [52:10<31:26, 2.39s/it]
75%|ββββββββ | 2337/3125 [52:13<32:27, 2.47s/it]
75%|ββββββββ | 2338/3125 [52:15<31:47, 2.42s/it]
75%|ββββββββ | 2339/3125 [52:18<31:49, 2.43s/it]
75%|ββββββββ | 2340/3125 [52:20<31:22, 2.40s/it]
{'loss': '1.068', 'grad_norm': '6.114', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05322', 'train/loss_cot': '1.074', 'epoch': '3.744'} |
|
75%|ββββββββ | 2340/3125 [52:20<31:22, 2.40s/it]
75%|ββββββββ | 2341/3125 [52:23<32:53, 2.52s/it]
75%|ββββββββ | 2342/3125 [52:26<33:14, 2.55s/it]
75%|ββββββββ | 2343/3125 [52:28<34:21, 2.64s/it]
75%|ββββββββ | 2344/3125 [52:31<33:51, 2.60s/it]
75%|ββββββββ | 2345/3125 [52:34<34:25, 2.65s/it]
75%|ββββββββ | 2346/3125 [52:36<32:42, 2.52s/it]
75%|ββββββββ | 2347/3125 [52:38<32:17, 2.49s/it]
75%|ββββββββ | 2348/3125 [52:41<34:29, 2.66s/it]
75%|ββββββββ | 2349/3125 [52:44<34:45, 2.69s/it]
75%|ββββββββ | 2350/3125 [52:47<35:07, 2.72s/it]
{'loss': '1.073', 'grad_norm': '6.079', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0002089', 'train/loss_cot': '1.133', 'epoch': '3.76'} |
|
75%|ββββββββ | 2350/3125 [52:47<35:07, 2.72s/it]
75%|ββββββββ | 2351/3125 [52:50<34:46, 2.70s/it]
75%|ββββββββ | 2352/3125 [52:52<33:22, 2.59s/it]
75%|ββββββββ | 2353/3125 [52:54<33:14, 2.58s/it]
75%|ββββββββ | 2354/3125 [52:57<31:39, 2.46s/it]
75%|ββββββββ | 2355/3125 [52:59<31:00, 2.42s/it]
75%|ββββββββ | 2356/3125 [53:01<30:42, 2.40s/it]
75%|ββββββββ | 2357/3125 [53:04<30:12, 2.36s/it]
75%|ββββββββ | 2358/3125 [53:06<30:18, 2.37s/it]
75%|ββββββββ | 2359/3125 [53:08<30:29, 2.39s/it]
76%|ββββββββ | 2360/3125 [53:11<30:02, 2.36s/it]
{'loss': '1.146', 'grad_norm': '5.674', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02771', 'train/loss_cot': '1.146', 'epoch': '3.776'} |
|
76%|ββββββββ | 2360/3125 [53:11<30:02, 2.36s/it]
76%|ββββββββ | 2361/3125 [53:14<32:31, 2.55s/it]
76%|ββββββββ | 2362/3125 [53:16<31:52, 2.51s/it]
76%|ββββββββ | 2363/3125 [53:19<31:56, 2.52s/it]
76%|ββββββββ | 2364/3125 [53:21<31:49, 2.51s/it]
76%|ββββββββ | 2365/3125 [53:24<32:45, 2.59s/it]
76%|ββββββββ | 2366/3125 [53:27<34:23, 2.72s/it]
76%|ββββββββ | 2367/3125 [53:29<32:59, 2.61s/it]
76%|ββββββββ | 2368/3125 [53:32<33:21, 2.64s/it]
76%|ββββββββ | 2369/3125 [53:34<32:34, 2.59s/it]
76%|ββββββββ | 2370/3125 [53:37<33:14, 2.64s/it]
{'loss': '1.063', 'grad_norm': '7.171', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02161', 'train/loss_cot': '1.061', 'epoch': '3.792'} |
|
76%|ββββββββ | 2370/3125 [53:38<33:14, 2.64s/it]
76%|ββββββββ | 2371/3125 [53:40<33:35, 2.67s/it]
76%|ββββββββ | 2372/3125 [53:43<33:03, 2.63s/it]
76%|ββββββββ | 2373/3125 [53:45<31:28, 2.51s/it]
76%|ββββββββ | 2374/3125 [53:47<32:14, 2.58s/it]
76%|ββββββββ | 2375/3125 [53:50<31:08, 2.49s/it]
76%|ββββββββ | 2376/3125 [53:52<30:43, 2.46s/it]
76%|ββββββββ | 2377/3125 [53:54<30:04, 2.41s/it]
76%|ββββββββ | 2378/3125 [53:57<29:26, 2.36s/it]
76%|ββββββββ | 2379/3125 [53:59<28:34, 2.30s/it]
76%|ββββββββ | 2380/3125 [54:01<29:01, 2.34s/it]
{'loss': '1.093', 'grad_norm': '5.157', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01544', 'train/loss_cot': '1.136', 'epoch': '3.808'} |
|
76%|ββββββββ | 2380/3125 [54:02<29:01, 2.34s/it]
76%|ββββββββ | 2381/3125 [54:04<29:17, 2.36s/it]
76%|ββββββββ | 2382/3125 [54:06<29:46, 2.40s/it]
76%|ββββββββ | 2383/3125 [54:09<30:34, 2.47s/it]
76%|ββββββββ | 2384/3125 [54:11<29:25, 2.38s/it]
76%|ββββββββ | 2385/3125 [54:14<29:48, 2.42s/it]
76%|ββββββββ | 2386/3125 [54:16<29:59, 2.44s/it]
76%|ββββββββ | 2387/3125 [54:19<30:21, 2.47s/it]
76%|ββββββββ | 2388/3125 [54:21<29:29, 2.40s/it]
76%|ββββββββ | 2389/3125 [54:23<29:03, 2.37s/it]
76%|ββββββββ | 2390/3125 [54:26<29:51, 2.44s/it]
{'loss': '1.05', 'grad_norm': '5.776', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01544', 'train/loss_cot': '1.549', 'epoch': '3.824'} |
|
76%|ββββββββ | 2390/3125 [54:26<29:51, 2.44s/it]
77%|ββββββββ | 2391/3125 [54:28<30:49, 2.52s/it]
77%|ββββββββ | 2392/3125 [54:31<30:13, 2.47s/it]
77%|ββββββββ | 2393/3125 [54:33<30:37, 2.51s/it]
77%|ββββββββ | 2394/3125 [54:36<29:59, 2.46s/it]
77%|ββββββββ | 2395/3125 [54:38<29:00, 2.38s/it]
77%|ββββββββ | 2396/3125 [54:40<29:24, 2.42s/it]
77%|ββββββββ | 2397/3125 [54:43<28:52, 2.38s/it]
77%|ββββββββ | 2398/3125 [54:45<29:18, 2.42s/it]
77%|ββββββββ | 2399/3125 [54:48<31:42, 2.62s/it]
77%|ββββββββ | 2400/3125 [54:51<31:00, 2.57s/it]
{'loss': '1.068', 'grad_norm': '5.884', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004852', 'train/loss_cot': '1.096', 'epoch': '3.84'} |
|
77%|ββββββββ | 2400/3125 [54:51<31:00, 2.57s/it]
77%|ββββββββ | 2401/3125 [55:09<1:27:41, 7.27s/it]
77%|ββββββββ | 2402/3125 [55:11<1:10:23, 5.84s/it]
77%|ββββββββ | 2403/3125 [55:14<58:44, 4.88s/it]
77%|ββββββββ | 2404/3125 [55:16<49:26, 4.11s/it]
77%|ββββββββ | 2405/3125 [55:19<42:17, 3.52s/it]
77%|ββββββββ | 2406/3125 [55:21<37:54, 3.16s/it]
77%|ββββββββ | 2407/3125 [55:23<35:05, 2.93s/it]
77%|ββββββββ | 2408/3125 [55:26<33:23, 2.79s/it]
77%|ββββββββ | 2409/3125 [55:28<31:28, 2.64s/it]
77%|ββββββββ | 2410/3125 [55:31<30:57, 2.60s/it]
{'loss': '1.06', 'grad_norm': '8.105', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05298', 'train/loss_cot': '1.12', 'epoch': '3.856'} |
|
77%|ββββββββ | 2410/3125 [55:31<30:57, 2.60s/it]
77%|ββββββββ | 2411/3125 [55:34<32:21, 2.72s/it]
77%|ββββββββ | 2412/3125 [55:36<30:36, 2.58s/it]
77%|ββββββββ | 2413/3125 [55:38<29:26, 2.48s/it]
77%|ββββββββ | 2414/3125 [55:41<30:39, 2.59s/it]
77%|ββββββββ | 2415/3125 [55:44<31:09, 2.63s/it]
77%|ββββββββ | 2416/3125 [55:46<29:38, 2.51s/it]
77%|ββββββββ | 2417/3125 [55:48<28:27, 2.41s/it]
77%|ββββββββ | 2418/3125 [55:51<29:47, 2.53s/it]
77%|ββββββββ | 2419/3125 [55:53<29:02, 2.47s/it]
77%|ββββββββ | 2420/3125 [55:56<30:03, 2.56s/it]
{'loss': '1.044', 'grad_norm': '5.819', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03149', 'train/loss_cot': '0.7594', 'epoch': '3.872'} |
|
77%|ββββββββ | 2420/3125 [55:56<30:03, 2.56s/it]
77%|ββββββββ | 2421/3125 [55:58<29:24, 2.51s/it]
78%|ββββββββ | 2422/3125 [56:01<29:44, 2.54s/it]
78%|ββββββββ | 2423/3125 [56:03<28:31, 2.44s/it]
78%|ββββββββ | 2424/3125 [56:06<29:18, 2.51s/it]
78%|ββββββββ | 2425/3125 [56:08<28:00, 2.40s/it]
78%|ββββββββ | 2426/3125 [56:10<27:39, 2.37s/it]
78%|ββββββββ | 2427/3125 [56:13<27:15, 2.34s/it]
78%|ββββββββ | 2428/3125 [56:15<28:01, 2.41s/it]
78%|ββββββββ | 2429/3125 [56:18<29:56, 2.58s/it]
78%|ββββββββ | 2430/3125 [56:20<28:49, 2.49s/it]
{'loss': '1.052', 'grad_norm': '6.574', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1299', 'train/loss_cot': '1.008', 'epoch': '3.888'} |
|
78%|ββββββββ | 2430/3125 [56:21<28:49, 2.49s/it]
78%|ββββββββ | 2431/3125 [56:23<28:58, 2.50s/it]
78%|ββββββββ | 2432/3125 [56:25<28:25, 2.46s/it]
78%|ββββββββ | 2433/3125 [56:28<27:47, 2.41s/it]
78%|ββββββββ | 2434/3125 [56:30<27:57, 2.43s/it]
78%|ββββββββ | 2435/3125 [56:33<28:34, 2.48s/it]
78%|ββββββββ | 2436/3125 [56:35<28:24, 2.47s/it]
78%|ββββββββ | 2437/3125 [56:38<28:17, 2.47s/it]
78%|ββββββββ | 2438/3125 [56:40<29:02, 2.54s/it]
78%|ββββββββ | 2439/3125 [56:43<28:18, 2.48s/it]
78%|ββββββββ | 2440/3125 [56:45<28:23, 2.49s/it]
{'loss': '1.071', 'grad_norm': '7.519', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1436', 'train/loss_cot': '1.373', 'epoch': '3.904'} |
|
78%|ββββββββ | 2440/3125 [56:45<28:23, 2.49s/it]
78%|ββββββββ | 2441/3125 [56:47<27:53, 2.45s/it]
78%|ββββββββ | 2442/3125 [56:50<26:56, 2.37s/it]
78%|ββββββββ | 2443/3125 [56:52<28:20, 2.49s/it]
78%|ββββββββ | 2444/3125 [56:55<29:43, 2.62s/it]
78%|ββββββββ | 2445/3125 [56:58<28:38, 2.53s/it]
78%|ββββββββ | 2446/3125 [57:00<28:02, 2.48s/it]
78%|ββββββββ | 2447/3125 [57:03<28:18, 2.50s/it]
78%|ββββββββ | 2448/3125 [57:06<30:24, 2.69s/it]
78%|ββββββββ | 2449/3125 [57:08<28:35, 2.54s/it]
78%|ββββββββ | 2450/3125 [57:10<28:28, 2.53s/it]
{'loss': '1.012', 'grad_norm': '5.586', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009583', 'train/loss_cot': '0.9804', 'epoch': '3.92'} |
|
78%|ββββββββ | 2450/3125 [57:11<28:28, 2.53s/it]
78%|ββββββββ | 2451/3125 [57:13<28:07, 2.50s/it]
78%|ββββββββ | 2452/3125 [57:15<27:12, 2.43s/it]
78%|ββββββββ | 2453/3125 [57:18<28:04, 2.51s/it]
79%|ββββββββ | 2454/3125 [57:20<27:43, 2.48s/it]
79%|ββββββββ | 2455/3125 [57:23<28:16, 2.53s/it]
79%|ββββββββ | 2456/3125 [57:26<29:39, 2.66s/it]
79%|ββββββββ | 2457/3125 [57:28<29:24, 2.64s/it]
79%|ββββββββ | 2458/3125 [57:31<28:25, 2.56s/it]
79%|ββββββββ | 2459/3125 [57:33<28:07, 2.53s/it]
79%|ββββββββ | 2460/3125 [57:36<28:48, 2.60s/it]
{'loss': '1.112', 'grad_norm': '5.726', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07129', 'train/loss_cot': '0.8806', 'epoch': '3.936'} |
|
79%|ββββββββ | 2460/3125 [57:36<28:48, 2.60s/it]
79%|ββββββββ | 2461/3125 [57:39<29:35, 2.67s/it]
79%|ββββββββ | 2462/3125 [57:42<31:10, 2.82s/it]
79%|ββββββββ | 2463/3125 [57:44<29:49, 2.70s/it]
79%|ββββββββ | 2464/3125 [57:47<29:28, 2.68s/it]
79%|ββββββββ | 2465/3125 [57:49<28:21, 2.58s/it]
79%|ββββββββ | 2466/3125 [57:52<27:22, 2.49s/it]
79%|ββββββββ | 2467/3125 [57:54<27:11, 2.48s/it]
79%|ββββββββ | 2468/3125 [57:56<26:34, 2.43s/it]
79%|ββββββββ | 2469/3125 [57:59<26:26, 2.42s/it]
79%|ββββββββ | 2470/3125 [58:01<26:09, 2.40s/it]
{'loss': '1.003', 'grad_norm': '5.478', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01917', 'train/loss_cot': '1.118', 'epoch': '3.952'} |
|
79%|ββββββββ | 2470/3125 [58:01<26:09, 2.40s/it]
79%|ββββββββ | 2471/3125 [58:04<27:16, 2.50s/it]
79%|ββββββββ | 2472/3125 [58:06<27:16, 2.51s/it]
79%|ββββββββ | 2473/3125 [58:09<26:19, 2.42s/it]
79%|ββββββββ | 2474/3125 [58:11<26:02, 2.40s/it]
79%|ββββββββ | 2475/3125 [58:14<29:08, 2.69s/it]
79%|ββββββββ | 2476/3125 [58:17<28:35, 2.64s/it]
79%|ββββββββ | 2477/3125 [58:19<27:56, 2.59s/it]
79%|ββββββββ | 2478/3125 [58:22<27:03, 2.51s/it]
79%|ββββββββ | 2479/3125 [58:25<29:49, 2.77s/it]
79%|ββββββββ | 2480/3125 [58:28<28:54, 2.69s/it]
{'loss': '1.026', 'grad_norm': '6.181', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006287', 'train/loss_cot': '1.017', 'epoch': '3.968'} |
|
79%|ββββββββ | 2480/3125 [58:28<28:54, 2.69s/it]
79%|ββββββββ | 2481/3125 [58:30<28:32, 2.66s/it]
79%|ββββββββ | 2482/3125 [58:32<27:27, 2.56s/it]
79%|ββββββββ | 2483/3125 [58:35<27:38, 2.58s/it]
79%|ββββββββ | 2484/3125 [58:38<26:59, 2.53s/it]
80%|ββββββββ | 2485/3125 [58:40<26:29, 2.48s/it]
80%|ββββββββ | 2486/3125 [58:42<26:20, 2.47s/it]
80%|ββββββββ | 2487/3125 [58:45<27:18, 2.57s/it]
80%|ββββββββ | 2488/3125 [58:48<28:20, 2.67s/it]
80%|ββββββββ | 2489/3125 [58:52<32:23, 3.06s/it]
80%|ββββββββ | 2490/3125 [58:55<30:55, 2.92s/it]
{'loss': '1.086', 'grad_norm': '7.063', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0009918', 'train/loss_cot': '0.9763', 'epoch': '3.984'} |
|
80%|ββββββββ | 2490/3125 [58:55<30:55, 2.92s/it]
80%|ββββββββ | 2491/3125 [58:57<29:23, 2.78s/it]
80%|ββββββββ | 2492/3125 [58:59<27:31, 2.61s/it]
80%|ββββββββ | 2493/3125 [59:02<26:44, 2.54s/it]
80%|ββββββββ | 2494/3125 [59:05<27:56, 2.66s/it]
80%|ββββββββ | 2495/3125 [59:07<27:48, 2.65s/it]
80%|ββββββββ | 2496/3125 [59:10<28:10, 2.69s/it]
80%|ββββββββ | 2497/3125 [59:13<29:30, 2.82s/it]
80%|ββββββββ | 2498/3125 [59:15<27:45, 2.66s/it]
80%|ββββββββ | 2499/3125 [59:18<27:04, 2.59s/it]
80%|ββββββββ | 2500/3125 [59:19<21:36, 2.08s/it]
{'loss': '1.039', 'grad_norm': '5.045', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0144', 'train/loss_cot': '0.7417', 'epoch': '4'} |
|
80%|ββββββββ | 2500/3125 [59:19<21:36, 2.08s/it]
80%|ββββββββ | 2501/3125 [59:23<27:53, 2.68s/it]
80%|ββββββββ | 2502/3125 [59:25<26:59, 2.60s/it]
80%|ββββββββ | 2503/3125 [59:27<25:13, 2.43s/it]
80%|ββββββββ | 2504/3125 [59:30<25:48, 2.49s/it]
80%|ββββββββ | 2505/3125 [59:32<25:28, 2.47s/it]
80%|ββββββββ | 2506/3125 [59:35<26:32, 2.57s/it]
80%|ββββββββ | 2507/3125 [59:38<26:14, 2.55s/it]
80%|ββββββββ | 2508/3125 [59:40<25:07, 2.44s/it]
80%|ββββββββ | 2509/3125 [59:42<24:12, 2.36s/it]
80%|ββββββββ | 2510/3125 [59:44<23:51, 2.33s/it]
{'loss': '0.9051', 'grad_norm': '5.313', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009888', 'train/loss_cot': '0.9877', 'epoch': '4.016'} |
|
80%|ββββββββ | 2510/3125 [59:45<23:51, 2.33s/it]
80%|ββββββββ | 2511/3125 [59:46<23:29, 2.30s/it]
80%|ββββββββ | 2512/3125 [59:49<23:15, 2.28s/it]
80%|ββββββββ | 2513/3125 [59:51<23:22, 2.29s/it]
80%|ββββββββ | 2514/3125 [59:53<23:03, 2.26s/it]
80%|ββββββββ | 2515/3125 [59:56<23:15, 2.29s/it]
81%|ββββββββ | 2516/3125 [59:58<23:30, 2.32s/it]
81%|ββββββββ | 2517/3125 [1:00:00<23:31, 2.32s/it]
81%|ββββββββ | 2518/3125 [1:00:02<22:47, 2.25s/it]
81%|ββββββββ | 2519/3125 [1:00:05<22:44, 2.25s/it]
81%|ββββββββ | 2520/3125 [1:00:07<22:58, 2.28s/it]
{'loss': '0.926', 'grad_norm': '8.215', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01526', 'train/loss_cot': '0.8317', 'epoch': '4.032'} |
|
81%|ββββββββ | 2520/3125 [1:00:07<22:58, 2.28s/it]
81%|ββββββββ | 2521/3125 [1:00:09<22:53, 2.27s/it]
81%|ββββββββ | 2522/3125 [1:00:12<23:15, 2.31s/it]
81%|ββββββββ | 2523/3125 [1:00:14<23:28, 2.34s/it]
81%|ββββββββ | 2524/3125 [1:00:16<22:56, 2.29s/it]
81%|ββββββββ | 2525/3125 [1:00:19<23:07, 2.31s/it]
81%|ββββββββ | 2526/3125 [1:00:21<23:11, 2.32s/it]
81%|ββββββββ | 2527/3125 [1:00:23<22:56, 2.30s/it]
81%|ββββββββ | 2528/3125 [1:00:26<23:09, 2.33s/it]
81%|ββββββββ | 2529/3125 [1:00:28<23:40, 2.38s/it]
81%|ββββββββ | 2530/3125 [1:00:31<23:55, 2.41s/it]
{'loss': '0.9035', 'grad_norm': '6.4', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001251', 'train/loss_cot': '0.8418', 'epoch': '4.048'} |
|
81%|ββββββββ | 2530/3125 [1:00:31<23:55, 2.41s/it]
81%|ββββββββ | 2531/3125 [1:00:33<24:31, 2.48s/it]
81%|ββββββββ | 2532/3125 [1:00:35<23:48, 2.41s/it]
81%|ββββββββ | 2533/3125 [1:00:38<23:13, 2.35s/it]
81%|ββββββββ | 2534/3125 [1:00:40<23:40, 2.40s/it]
81%|ββββββββ | 2535/3125 [1:00:42<23:23, 2.38s/it]
81%|ββββββββ | 2536/3125 [1:00:45<24:46, 2.52s/it]
81%|ββββββββ | 2537/3125 [1:00:48<24:11, 2.47s/it]
81%|ββββββββ | 2538/3125 [1:00:50<23:31, 2.41s/it]
81%|ββββββββ | 2539/3125 [1:00:52<23:13, 2.38s/it]
81%|βββββββββ | 2540/3125 [1:00:57<28:44, 2.95s/it]
{'loss': '0.915', 'grad_norm': '5.605', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02515', 'train/loss_cot': '0.7679', 'epoch': '4.064'} |
|
81%|βββββββββ | 2540/3125 [1:00:57<28:44, 2.95s/it]
81%|βββββββββ | 2541/3125 [1:00:59<26:50, 2.76s/it]
81%|βββββββββ | 2542/3125 [1:01:01<25:38, 2.64s/it]
81%|βββββββββ | 2543/3125 [1:01:04<25:21, 2.61s/it]
81%|βββββββββ | 2544/3125 [1:01:06<23:36, 2.44s/it]
81%|βββββββββ | 2545/3125 [1:01:08<22:37, 2.34s/it]
81%|βββββββββ | 2546/3125 [1:01:10<22:25, 2.32s/it]
82%|βββββββββ | 2547/3125 [1:01:12<22:01, 2.29s/it]
82%|βββββββββ | 2548/3125 [1:01:15<21:44, 2.26s/it]
82%|βββββββββ | 2549/3125 [1:01:17<21:38, 2.25s/it]
82%|βββββββββ | 2550/3125 [1:01:19<21:27, 2.24s/it]
{'loss': '0.918', 'grad_norm': '5.706', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05029', 'train/loss_cot': '0.6901', 'epoch': '4.08'} |
|
82%|βββββββββ | 2550/3125 [1:01:19<21:27, 2.24s/it]
82%|βββββββββ | 2551/3125 [1:01:22<22:37, 2.37s/it]
82%|βββββββββ | 2552/3125 [1:01:24<22:10, 2.32s/it]
82%|βββββββββ | 2553/3125 [1:01:26<22:23, 2.35s/it]
82%|βββββββββ | 2554/3125 [1:01:28<21:49, 2.29s/it]
82%|βββββββββ | 2555/3125 [1:01:31<22:00, 2.32s/it]
82%|βββββββββ | 2556/3125 [1:01:33<21:42, 2.29s/it]
82%|βββββββββ | 2557/3125 [1:01:35<21:42, 2.29s/it]
82%|βββββββββ | 2558/3125 [1:01:38<22:19, 2.36s/it]
82%|βββββββββ | 2559/3125 [1:01:40<22:31, 2.39s/it]
82%|βββββββββ | 2560/3125 [1:01:43<23:10, 2.46s/it]
{'loss': '0.8848', 'grad_norm': '5.381', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007385', 'train/loss_cot': '0.8134', 'epoch': '4.096'} |
|
82%|βββββββββ | 2560/3125 [1:01:43<23:10, 2.46s/it]
82%|βββββββββ | 2561/3125 [1:01:45<23:06, 2.46s/it]
82%|βββββββββ | 2562/3125 [1:01:48<22:32, 2.40s/it]
82%|βββββββββ | 2563/3125 [1:01:50<22:29, 2.40s/it]
82%|βββββββββ | 2564/3125 [1:01:53<24:31, 2.62s/it]
82%|βββββββββ | 2565/3125 [1:01:56<24:18, 2.61s/it]
82%|βββββββββ | 2566/3125 [1:01:58<23:16, 2.50s/it]
82%|βββββββββ | 2567/3125 [1:02:00<22:35, 2.43s/it]
82%|βββββββββ | 2568/3125 [1:02:03<21:51, 2.35s/it]
82%|βββββββββ | 2569/3125 [1:02:05<22:28, 2.42s/it]
82%|βββββββββ | 2570/3125 [1:02:07<21:39, 2.34s/it]
{'loss': '0.924', 'grad_norm': '5.49', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02563', 'train/loss_cot': '0.735', 'epoch': '4.112'} |
|
82%|βββββββββ | 2570/3125 [1:02:07<21:39, 2.34s/it]
82%|βββββββββ | 2571/3125 [1:02:09<21:12, 2.30s/it]
82%|βββββββββ | 2572/3125 [1:02:12<20:46, 2.25s/it]
82%|βββββββββ | 2573/3125 [1:02:14<21:55, 2.38s/it]
82%|βββββββββ | 2574/3125 [1:02:17<21:49, 2.38s/it]
82%|βββββββββ | 2575/3125 [1:02:19<21:42, 2.37s/it]
82%|βββββββββ | 2576/3125 [1:02:21<20:58, 2.29s/it]
82%|βββββββββ | 2577/3125 [1:02:24<21:35, 2.36s/it]
82%|βββββββββ | 2578/3125 [1:02:26<21:31, 2.36s/it]
83%|βββββββββ | 2579/3125 [1:02:28<21:42, 2.38s/it]
83%|βββββββββ | 2580/3125 [1:02:31<20:57, 2.31s/it]
{'loss': '0.9002', 'grad_norm': '5.771', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02661', 'train/loss_cot': '1.161', 'epoch': '4.128'} |
|
83%|βββββββββ | 2580/3125 [1:02:31<20:57, 2.31s/it]
83%|βββββββββ | 2581/3125 [1:02:33<21:10, 2.34s/it]
83%|βββββββββ | 2582/3125 [1:02:35<21:07, 2.33s/it]
83%|βββββββββ | 2583/3125 [1:02:38<20:46, 2.30s/it]
83%|βββββββββ | 2584/3125 [1:02:41<22:41, 2.52s/it]
83%|βββββββββ | 2585/3125 [1:02:43<21:43, 2.41s/it]
83%|βββββββββ | 2586/3125 [1:02:45<21:28, 2.39s/it]
83%|βββββββββ | 2587/3125 [1:02:47<20:49, 2.32s/it]
83%|βββββββββ | 2588/3125 [1:02:49<20:10, 2.25s/it]
83%|βββββββββ | 2589/3125 [1:02:51<19:55, 2.23s/it]
83%|βββββββββ | 2590/3125 [1:02:54<19:34, 2.20s/it]
{'loss': '0.9345', 'grad_norm': '6.001', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02026', 'train/loss_cot': '0.8109', 'epoch': '4.144'} |
|
83%|βββββββββ | 2590/3125 [1:02:54<19:34, 2.20s/it]
83%|βββββββββ | 2591/3125 [1:02:56<19:28, 2.19s/it]
83%|βββββββββ | 2592/3125 [1:02:58<19:20, 2.18s/it]
83%|βββββββββ | 2593/3125 [1:03:00<19:30, 2.20s/it]
83%|βββββββββ | 2594/3125 [1:03:03<19:59, 2.26s/it]
83%|βββββββββ | 2595/3125 [1:03:05<20:00, 2.27s/it]
83%|βββββββββ | 2596/3125 [1:03:08<21:02, 2.39s/it]
83%|βββββββββ | 2597/3125 [1:03:10<20:30, 2.33s/it]
83%|βββββββββ | 2598/3125 [1:03:12<20:24, 2.32s/it]
83%|βββββββββ | 2599/3125 [1:03:14<19:59, 2.28s/it]
83%|βββββββββ | 2600/3125 [1:03:17<20:26, 2.34s/it]
{'loss': '0.9016', 'grad_norm': '5.41', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007324', 'train/loss_cot': '1.068', 'epoch': '4.16'} |
|
83%|βββββββββ | 2600/3125 [1:03:17<20:26, 2.34s/it]
83%|βββββββββ | 2601/3125 [1:03:42<1:20:56, 9.27s/it]
83%|βββββββββ | 2602/3125 [1:03:44<1:02:24, 7.16s/it]
83%|βββββββββ | 2603/3125 [1:03:47<50:46, 5.84s/it]
83%|βββββββββ | 2604/3125 [1:03:49<41:22, 4.76s/it]
83%|βββββββββ | 2605/3125 [1:03:52<35:38, 4.11s/it]
83%|βββββββββ | 2606/3125 [1:03:54<30:05, 3.48s/it]
83%|βββββββββ | 2607/3125 [1:03:56<27:11, 3.15s/it]
83%|βββββββββ | 2608/3125 [1:03:59<25:12, 2.93s/it]
83%|βββββββββ | 2609/3125 [1:04:01<24:27, 2.84s/it]
84%|βββββββββ | 2610/3125 [1:04:04<23:30, 2.74s/it]
{'loss': '0.9221', 'grad_norm': '6.122', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003906', 'train/loss_cot': '0.9342', 'epoch': '4.176'} |
|
84%|βββββββββ | 2610/3125 [1:04:04<23:30, 2.74s/it]
84%|βββββββββ | 2611/3125 [1:04:07<23:18, 2.72s/it]
84%|βββββββββ | 2612/3125 [1:04:09<22:23, 2.62s/it]
84%|βββββββββ | 2613/3125 [1:04:11<21:33, 2.53s/it]
84%|βββββββββ | 2614/3125 [1:04:13<20:36, 2.42s/it]
84%|βββββββββ | 2615/3125 [1:04:16<20:07, 2.37s/it]
84%|βββββββββ | 2616/3125 [1:04:18<20:10, 2.38s/it]
84%|βββββββββ | 2617/3125 [1:04:20<19:28, 2.30s/it]
84%|βββββββββ | 2618/3125 [1:04:22<18:42, 2.21s/it]
84%|βββββββββ | 2619/3125 [1:04:25<19:46, 2.35s/it]
84%|βββββββββ | 2620/3125 [1:04:27<19:35, 2.33s/it]
{'loss': '0.9426', 'grad_norm': '5.338', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007439', 'train/loss_cot': '0.8629', 'epoch': '4.192'} |
|
84%|βββββββββ | 2620/3125 [1:04:27<19:35, 2.33s/it]
84%|βββββββββ | 2621/3125 [1:04:30<19:40, 2.34s/it]
84%|βββββββββ | 2622/3125 [1:04:32<19:49, 2.36s/it]
84%|βββββββββ | 2623/3125 [1:04:34<20:07, 2.41s/it]
84%|βββββββββ | 2624/3125 [1:04:37<19:48, 2.37s/it]
84%|βββββββββ | 2625/3125 [1:04:39<19:37, 2.35s/it]
84%|βββββββββ | 2626/3125 [1:04:41<18:53, 2.27s/it]
84%|βββββββββ | 2627/3125 [1:04:44<19:14, 2.32s/it]
84%|βββββββββ | 2628/3125 [1:04:46<20:00, 2.41s/it]
84%|βββββββββ | 2629/3125 [1:04:49<21:41, 2.62s/it]
84%|βββββββββ | 2630/3125 [1:04:51<20:14, 2.45s/it]
{'loss': '0.9196', 'grad_norm': '5.756', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04688', 'train/loss_cot': '1.033', 'epoch': '4.208'} |
|
84%|βββββββββ | 2630/3125 [1:04:52<20:14, 2.45s/it]
84%|βββββββββ | 2631/3125 [1:04:54<19:39, 2.39s/it]
84%|βββββββββ | 2632/3125 [1:04:56<19:42, 2.40s/it]
84%|βββββββββ | 2633/3125 [1:04:58<19:15, 2.35s/it]
84%|βββββββββ | 2634/3125 [1:05:00<18:46, 2.29s/it]
84%|βββββββββ | 2635/3125 [1:05:03<20:08, 2.47s/it]
84%|βββββββββ | 2636/3125 [1:05:05<19:12, 2.36s/it]
84%|βββββββββ | 2637/3125 [1:05:08<19:05, 2.35s/it]
84%|βββββββββ | 2638/3125 [1:05:10<18:29, 2.28s/it]
84%|βββββββββ | 2639/3125 [1:05:12<17:45, 2.19s/it]
84%|βββββββββ | 2640/3125 [1:05:14<17:17, 2.14s/it]
{'loss': '0.936', 'grad_norm': '5.737', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005432', 'train/loss_cot': '0.8158', 'epoch': '4.224'} |
|
84%|βββββββββ | 2640/3125 [1:05:14<17:17, 2.14s/it]
85%|βββββββββ | 2641/3125 [1:05:16<17:33, 2.18s/it]
85%|βββββββββ | 2642/3125 [1:05:18<18:00, 2.24s/it]
85%|βββββββββ | 2643/3125 [1:05:21<17:52, 2.23s/it]
85%|βββββββββ | 2644/3125 [1:05:23<17:46, 2.22s/it]
85%|βββββββββ | 2645/3125 [1:05:25<17:38, 2.20s/it]
85%|βββββββββ | 2646/3125 [1:05:27<17:39, 2.21s/it]
85%|βββββββββ | 2647/3125 [1:05:30<17:53, 2.25s/it]
85%|βββββββββ | 2648/3125 [1:05:32<18:03, 2.27s/it]
85%|βββββββββ | 2649/3125 [1:05:34<18:26, 2.32s/it]
85%|βββββββββ | 2650/3125 [1:05:37<18:56, 2.39s/it]
{'loss': '0.878', 'grad_norm': '8.752', 'learning_rate': '1e-05', 'train/loss_infonce': '0.126', 'train/loss_cot': '0.7221', 'epoch': '4.24'} |
|
85%|βββββββββ | 2650/3125 [1:05:37<18:56, 2.39s/it]
85%|βββββββββ | 2651/3125 [1:05:40<19:23, 2.45s/it]
85%|βββββββββ | 2652/3125 [1:05:42<19:37, 2.49s/it]
85%|βββββββββ | 2653/3125 [1:05:45<19:28, 2.48s/it]
85%|βββββββββ | 2654/3125 [1:05:47<18:34, 2.37s/it]
85%|βββββββββ | 2655/3125 [1:05:49<19:02, 2.43s/it]
85%|βββββββββ | 2656/3125 [1:05:51<18:10, 2.33s/it]
85%|βββββββββ | 2657/3125 [1:05:54<19:09, 2.46s/it]
85%|βββββββββ | 2658/3125 [1:05:56<18:29, 2.38s/it]
85%|βββββββββ | 2659/3125 [1:05:58<18:08, 2.34s/it]
85%|βββββββββ | 2660/3125 [1:06:01<17:55, 2.31s/it]
{'loss': '0.918', 'grad_norm': '6.071', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009521', 'train/loss_cot': '1.002', 'epoch': '4.256'} |
|
85%|βββββββββ | 2660/3125 [1:06:01<17:55, 2.31s/it]
85%|βββββββββ | 2661/3125 [1:06:03<18:02, 2.33s/it]
85%|βββββββββ | 2662/3125 [1:06:05<17:58, 2.33s/it]
85%|βββββββββ | 2663/3125 [1:06:08<19:10, 2.49s/it]
85%|βββββββββ | 2664/3125 [1:06:11<19:02, 2.48s/it]
85%|βββββββββ | 2665/3125 [1:06:13<18:24, 2.40s/it]
85%|βββββββββ | 2666/3125 [1:06:16<18:50, 2.46s/it]
85%|βββββββββ | 2667/3125 [1:06:18<19:02, 2.49s/it]
85%|βββββββββ | 2668/3125 [1:06:20<18:29, 2.43s/it]
85%|βββββββββ | 2669/3125 [1:06:23<17:39, 2.32s/it]
85%|βββββββββ | 2670/3125 [1:06:25<17:51, 2.35s/it]
{'loss': '0.9399', 'grad_norm': '6.179', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003006', 'train/loss_cot': '0.7153', 'epoch': '4.272'} |
|
85%|βββββββββ | 2670/3125 [1:06:25<17:51, 2.35s/it]
85%|βββββββββ | 2671/3125 [1:06:28<18:57, 2.51s/it]
86%|βββββββββ | 2672/3125 [1:06:30<18:05, 2.40s/it]
86%|βββββββββ | 2673/3125 [1:06:33<19:08, 2.54s/it]
86%|βββββββββ | 2674/3125 [1:06:35<19:08, 2.55s/it]
86%|βββββββββ | 2675/3125 [1:06:38<18:47, 2.50s/it]
86%|βββββββββ | 2676/3125 [1:06:40<17:56, 2.40s/it]
86%|βββββββββ | 2677/3125 [1:06:43<18:24, 2.46s/it]
86%|βββββββββ | 2678/3125 [1:06:45<18:50, 2.53s/it]
86%|βββββββββ | 2679/3125 [1:06:47<18:02, 2.43s/it]
86%|βββββββββ | 2680/3125 [1:06:50<17:18, 2.33s/it]
{'loss': '0.9654', 'grad_norm': '10.99', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00148', 'train/loss_cot': '1.084', 'epoch': '4.288'} |
|
86%|βββββββββ | 2680/3125 [1:06:50<17:18, 2.33s/it]
86%|βββββββββ | 2681/3125 [1:06:52<17:21, 2.35s/it]
86%|βββββββββ | 2682/3125 [1:06:54<17:46, 2.41s/it]
86%|βββββββββ | 2683/3125 [1:06:57<17:42, 2.40s/it]
86%|βββββββββ | 2684/3125 [1:06:59<18:07, 2.47s/it]
86%|βββββββββ | 2685/3125 [1:07:02<18:15, 2.49s/it]
86%|βββββββββ | 2686/3125 [1:07:05<18:35, 2.54s/it]
86%|βββββββββ | 2687/3125 [1:07:07<17:52, 2.45s/it]
86%|βββββββββ | 2688/3125 [1:07:09<17:34, 2.41s/it]
86%|βββββββββ | 2689/3125 [1:07:11<16:59, 2.34s/it]
86%|βββββββββ | 2690/3125 [1:07:14<16:53, 2.33s/it]
{'loss': '0.9463', 'grad_norm': '5.868', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0006409', 'train/loss_cot': '0.8968', 'epoch': '4.304'} |
|
86%|βββββββββ | 2690/3125 [1:07:14<16:53, 2.33s/it]
86%|βββββββββ | 2691/3125 [1:07:16<17:07, 2.37s/it]
86%|βββββββββ | 2692/3125 [1:07:18<16:33, 2.29s/it]
86%|βββββββββ | 2693/3125 [1:07:20<16:13, 2.25s/it]
86%|βββββββββ | 2694/3125 [1:07:23<17:07, 2.38s/it]
86%|βββββββββ | 2695/3125 [1:07:26<17:02, 2.38s/it]
86%|βββββββββ | 2696/3125 [1:07:28<17:07, 2.40s/it]
86%|βββββββββ | 2697/3125 [1:07:30<17:08, 2.40s/it]
86%|βββββββββ | 2698/3125 [1:07:34<19:01, 2.67s/it]
86%|βββββββββ | 2699/3125 [1:07:36<19:12, 2.70s/it]
86%|βββββββββ | 2700/3125 [1:07:39<19:13, 2.72s/it]
{'loss': '0.9627', 'grad_norm': '6.543', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05688', 'train/loss_cot': '0.9982', 'epoch': '4.32'} |
|
86%|βββββββββ | 2700/3125 [1:07:39<19:13, 2.72s/it]
86%|βββββββββ | 2701/3125 [1:07:41<18:15, 2.58s/it]
86%|βββββββββ | 2702/3125 [1:07:44<18:28, 2.62s/it]
86%|βββββββββ | 2703/3125 [1:07:48<20:02, 2.85s/it]
87%|βββββββββ | 2704/3125 [1:07:50<18:51, 2.69s/it]
87%|βββββββββ | 2705/3125 [1:07:52<18:35, 2.66s/it]
87%|βββββββββ | 2706/3125 [1:07:55<17:39, 2.53s/it]
87%|βββββββββ | 2707/3125 [1:07:57<16:57, 2.43s/it]
87%|βββββββββ | 2708/3125 [1:07:59<16:18, 2.35s/it]
87%|βββββββββ | 2709/3125 [1:08:01<16:16, 2.35s/it]
87%|βββββββββ | 2710/3125 [1:08:04<16:53, 2.44s/it]
{'loss': '0.9318', 'grad_norm': '5.506', 'learning_rate': '1e-05', 'train/loss_infonce': '1.335e-05', 'train/loss_cot': '0.9976', 'epoch': '4.336'} |
|
87%|βββββββββ | 2710/3125 [1:08:04<16:53, 2.44s/it]
87%|βββββββββ | 2711/3125 [1:08:06<16:41, 2.42s/it]
87%|βββββββββ | 2712/3125 [1:08:09<16:25, 2.39s/it]
87%|βββββββββ | 2713/3125 [1:08:11<16:05, 2.34s/it]
87%|βββββββββ | 2714/3125 [1:08:13<15:35, 2.28s/it]
87%|βββββββββ | 2715/3125 [1:08:16<15:59, 2.34s/it]
87%|βββββββββ | 2716/3125 [1:08:18<15:34, 2.29s/it]
87%|βββββββββ | 2717/3125 [1:08:20<15:14, 2.24s/it]
87%|βββββββββ | 2718/3125 [1:08:22<15:46, 2.33s/it]
87%|βββββββββ | 2719/3125 [1:08:25<15:56, 2.36s/it]
87%|βββββββββ | 2720/3125 [1:08:27<15:40, 2.32s/it]
{'loss': '0.9862', 'grad_norm': '5.527', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007401', 'train/loss_cot': '1.246', 'epoch': '4.352'} |
|
87%|βββββββββ | 2720/3125 [1:08:27<15:40, 2.32s/it]
87%|βββββββββ | 2721/3125 [1:08:29<15:25, 2.29s/it]
87%|βββββββββ | 2722/3125 [1:08:32<16:02, 2.39s/it]
87%|βββββββββ | 2723/3125 [1:08:34<16:15, 2.43s/it]
87%|βββββββββ | 2724/3125 [1:08:37<16:00, 2.40s/it]
87%|βββββββββ | 2725/3125 [1:08:39<15:46, 2.37s/it]
87%|βββββββββ | 2726/3125 [1:08:41<15:34, 2.34s/it]
87%|βββββββββ | 2727/3125 [1:08:44<16:03, 2.42s/it]
87%|βββββββββ | 2728/3125 [1:08:46<15:54, 2.41s/it]
87%|βββββββββ | 2729/3125 [1:08:49<15:40, 2.38s/it]
87%|βββββββββ | 2730/3125 [1:08:51<15:23, 2.34s/it]
{'loss': '0.9755', 'grad_norm': '5.466', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0004082', 'train/loss_cot': '0.845', 'epoch': '4.368'} |
|
87%|βββββββββ | 2730/3125 [1:08:51<15:23, 2.34s/it]
87%|βββββββββ | 2731/3125 [1:08:53<15:20, 2.34s/it]
87%|βββββββββ | 2732/3125 [1:08:56<15:36, 2.38s/it]
87%|βββββββββ | 2733/3125 [1:08:58<15:17, 2.34s/it]
87%|βββββββββ | 2734/3125 [1:09:01<16:46, 2.57s/it]
88%|βββββββββ | 2735/3125 [1:09:03<16:12, 2.49s/it]
88%|βββββββββ | 2736/3125 [1:09:06<15:31, 2.39s/it]
88%|βββββββββ | 2737/3125 [1:09:08<14:57, 2.31s/it]
88%|βββββββββ | 2738/3125 [1:09:10<14:46, 2.29s/it]
88%|βββββββββ | 2739/3125 [1:09:12<14:56, 2.32s/it]
88%|βββββββββ | 2740/3125 [1:09:15<15:58, 2.49s/it]
{'loss': '0.9483', 'grad_norm': '6.557', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001228', 'train/loss_cot': '0.6776', 'epoch': '4.384'} |
|
88%|βββββββββ | 2740/3125 [1:09:15<15:58, 2.49s/it]
88%|βββββββββ | 2741/3125 [1:09:17<15:28, 2.42s/it]
88%|βββββββββ | 2742/3125 [1:09:19<14:46, 2.32s/it]
88%|βββββββββ | 2743/3125 [1:09:22<14:33, 2.29s/it]
88%|βββββββββ | 2744/3125 [1:09:25<15:34, 2.45s/it]
88%|βββββββββ | 2745/3125 [1:09:27<15:27, 2.44s/it]
88%|βββββββββ | 2746/3125 [1:09:30<15:53, 2.52s/it]
88%|βββββββββ | 2747/3125 [1:09:32<15:39, 2.49s/it]
88%|βββββββββ | 2748/3125 [1:09:34<15:22, 2.45s/it]
88%|βββββββββ | 2749/3125 [1:09:37<15:12, 2.43s/it]
88%|βββββββββ | 2750/3125 [1:09:39<14:58, 2.40s/it]
{'loss': '0.9502', 'grad_norm': '5.16', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0007439', 'train/loss_cot': '0.7696', 'epoch': '4.4'} |
|
88%|βββββββββ | 2750/3125 [1:09:39<14:58, 2.40s/it]
88%|βββββββββ | 2751/3125 [1:09:41<14:50, 2.38s/it]
88%|βββββββββ | 2752/3125 [1:09:43<14:09, 2.28s/it]
88%|βββββββββ | 2753/3125 [1:09:46<14:55, 2.41s/it]
88%|βββββββββ | 2754/3125 [1:09:48<14:25, 2.33s/it]
88%|βββββββββ | 2755/3125 [1:09:51<14:01, 2.28s/it]
88%|βββββββββ | 2756/3125 [1:09:53<13:43, 2.23s/it]
88%|βββββββββ | 2757/3125 [1:09:55<13:40, 2.23s/it]
88%|βββββββββ | 2758/3125 [1:09:58<14:59, 2.45s/it]
88%|βββββββββ | 2759/3125 [1:10:01<16:37, 2.73s/it]
88%|βββββββββ | 2760/3125 [1:10:04<15:52, 2.61s/it]
{'loss': '0.9154', 'grad_norm': '6.214', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0004864', 'train/loss_cot': '0.7799', 'epoch': '4.416'} |
|
88%|βββββββββ | 2760/3125 [1:10:04<15:52, 2.61s/it]
88%|βββββββββ | 2761/3125 [1:10:06<15:50, 2.61s/it]
88%|βββββββββ | 2762/3125 [1:10:08<14:55, 2.47s/it]
88%|βββββββββ | 2763/3125 [1:10:11<14:54, 2.47s/it]
88%|βββββββββ | 2764/3125 [1:10:14<15:24, 2.56s/it]
88%|βββββββββ | 2765/3125 [1:10:16<14:51, 2.48s/it]
89%|βββββββββ | 2766/3125 [1:10:18<14:46, 2.47s/it]
89%|βββββββββ | 2767/3125 [1:10:20<14:10, 2.38s/it]
89%|βββββββββ | 2768/3125 [1:10:23<13:49, 2.32s/it]
89%|βββββββββ | 2769/3125 [1:10:25<13:59, 2.36s/it]
89%|βββββββββ | 2770/3125 [1:10:28<14:32, 2.46s/it]
{'loss': '0.9563', 'grad_norm': '5.857', 'learning_rate': '1e-05', 'train/loss_infonce': '0.01288', 'train/loss_cot': '0.8022', 'epoch': '4.432'} |
|
89%|βββββββββ | 2770/3125 [1:10:28<14:32, 2.46s/it]
89%|βββββββββ | 2771/3125 [1:10:30<13:57, 2.37s/it]
89%|βββββββββ | 2772/3125 [1:10:32<13:39, 2.32s/it]
89%|βββββββββ | 2773/3125 [1:10:34<13:09, 2.24s/it]
89%|βββββββββ | 2774/3125 [1:10:37<13:18, 2.28s/it]
89%|βββββββββ | 2775/3125 [1:10:39<14:10, 2.43s/it]
89%|βββββββββ | 2776/3125 [1:10:41<13:36, 2.34s/it]
89%|βββββββββ | 2777/3125 [1:10:44<14:18, 2.47s/it]
89%|βββββββββ | 2778/3125 [1:10:47<14:01, 2.42s/it]
89%|βββββββββ | 2779/3125 [1:10:49<13:47, 2.39s/it]
89%|βββββββββ | 2780/3125 [1:10:52<14:22, 2.50s/it]
{'loss': '0.9696', 'grad_norm': '5.936', 'learning_rate': '1e-05', 'train/loss_infonce': '0.09912', 'train/loss_cot': '0.9383', 'epoch': '4.448'} |
|
89%|βββββββββ | 2780/3125 [1:10:52<14:22, 2.50s/it]
89%|βββββββββ | 2781/3125 [1:10:54<14:43, 2.57s/it]
89%|βββββββββ | 2782/3125 [1:10:57<14:13, 2.49s/it]
89%|βββββββββ | 2783/3125 [1:10:59<14:21, 2.52s/it]
89%|βββββββββ | 2784/3125 [1:11:01<13:53, 2.44s/it]
89%|βββββββββ | 2785/3125 [1:11:04<14:16, 2.52s/it]
89%|βββββββββ | 2786/3125 [1:11:06<13:52, 2.46s/it]
89%|βββββββββ | 2787/3125 [1:11:09<13:21, 2.37s/it]
89%|βββββββββ | 2788/3125 [1:11:12<14:31, 2.59s/it]
89%|βββββββββ | 2789/3125 [1:11:14<14:03, 2.51s/it]
89%|βββββββββ | 2790/3125 [1:11:16<13:33, 2.43s/it]
{'loss': '0.9493', 'grad_norm': '5.304', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0188', 'train/loss_cot': '1.175', 'epoch': '4.464'} |
|
89%|βββββββββ | 2790/3125 [1:11:17<13:33, 2.43s/it]
89%|βββββββββ | 2791/3125 [1:11:19<13:24, 2.41s/it]
89%|βββββββββ | 2792/3125 [1:11:21<13:14, 2.38s/it]
89%|βββββββββ | 2793/3125 [1:11:24<13:55, 2.52s/it]
89%|βββββββββ | 2794/3125 [1:11:26<13:39, 2.47s/it]
89%|βββββββββ | 2795/3125 [1:11:28<13:04, 2.38s/it]
89%|βββββββββ | 2796/3125 [1:11:31<13:01, 2.38s/it]
90%|βββββββββ | 2797/3125 [1:11:33<13:17, 2.43s/it]
90%|βββββββββ | 2798/3125 [1:11:36<12:57, 2.38s/it]
90%|βββββββββ | 2799/3125 [1:11:38<13:09, 2.42s/it]
90%|βββββββββ | 2800/3125 [1:11:40<12:51, 2.37s/it]
{'loss': '0.9415', 'grad_norm': '5.876', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02368', 'train/loss_cot': '1.129', 'epoch': '4.48'} |
|
90%|βββββββββ | 2800/3125 [1:11:41<12:51, 2.37s/it]
90%|βββββββββ | 2801/3125 [1:12:07<52:13, 9.67s/it]
90%|βββββββββ | 2802/3125 [1:12:09<40:10, 7.46s/it]
90%|βββββββββ | 2803/3125 [1:12:12<31:30, 5.87s/it]
90%|βββββββββ | 2804/3125 [1:12:14<25:45, 4.81s/it]
90%|βββββββββ | 2805/3125 [1:12:16<21:53, 4.10s/it]
90%|βββββββββ | 2806/3125 [1:12:19<19:23, 3.65s/it]
90%|βββββββββ | 2807/3125 [1:12:21<17:22, 3.28s/it]
90%|βββββββββ | 2808/3125 [1:12:24<16:51, 3.19s/it]
90%|βββββββββ | 2809/3125 [1:12:27<16:15, 3.09s/it]
90%|βββββββββ | 2810/3125 [1:12:29<14:41, 2.80s/it]
{'loss': '0.9474', 'grad_norm': '5.484', 'learning_rate': '1e-05', 'train/loss_infonce': '0.009216', 'train/loss_cot': '1.045', 'epoch': '4.496'} |
|
90%|βββββββββ | 2810/3125 [1:12:30<14:41, 2.80s/it]
90%|βββββββββ | 2811/3125 [1:12:32<13:55, 2.66s/it]
90%|βββββββββ | 2812/3125 [1:12:34<13:38, 2.62s/it]
90%|βββββββββ | 2813/3125 [1:12:37<13:36, 2.62s/it]
90%|βββββββββ | 2814/3125 [1:12:39<13:13, 2.55s/it]
90%|βββββββββ | 2815/3125 [1:12:42<12:57, 2.51s/it]
90%|βββββββββ | 2816/3125 [1:12:44<12:08, 2.36s/it]
90%|βββββββββ | 2817/3125 [1:12:46<12:02, 2.35s/it]
90%|βββββββββ | 2818/3125 [1:12:48<11:54, 2.33s/it]
90%|βββββββββ | 2819/3125 [1:12:50<11:52, 2.33s/it]
90%|βββββββββ | 2820/3125 [1:12:53<11:42, 2.30s/it]
{'loss': '0.9563', 'grad_norm': '5.086', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007996', 'train/loss_cot': '0.8669', 'epoch': '4.512'} |
|
90%|βββββββββ | 2820/3125 [1:12:53<11:42, 2.30s/it]
90%|βββββββββ | 2821/3125 [1:12:55<11:55, 2.35s/it]
90%|βββββββββ | 2822/3125 [1:12:57<11:38, 2.30s/it]
90%|βββββββββ | 2823/3125 [1:13:00<12:11, 2.42s/it]
90%|βββββββββ | 2824/3125 [1:13:02<11:50, 2.36s/it]
90%|βββββββββ | 2825/3125 [1:13:04<11:16, 2.26s/it]
90%|βββββββββ | 2826/3125 [1:13:07<11:24, 2.29s/it]
90%|βββββββββ | 2827/3125 [1:13:09<11:39, 2.35s/it]
90%|βββββββββ | 2828/3125 [1:13:12<11:36, 2.34s/it]
91%|βββββββββ | 2829/3125 [1:13:14<11:23, 2.31s/it]
91%|βββββββββ | 2830/3125 [1:13:16<11:52, 2.42s/it]
{'loss': '0.9548', 'grad_norm': '5.99', 'learning_rate': '1e-05', 'train/loss_infonce': '0.004578', 'train/loss_cot': '0.936', 'epoch': '4.528'} |
|
91%|βββββββββ | 2830/3125 [1:13:17<11:52, 2.42s/it]
91%|βββββββββ | 2831/3125 [1:13:19<11:41, 2.39s/it]
91%|βββββββββ | 2832/3125 [1:13:21<12:08, 2.49s/it]
91%|βββββββββ | 2833/3125 [1:13:24<12:09, 2.50s/it]
91%|βββββββββ | 2834/3125 [1:13:26<11:55, 2.46s/it]
91%|βββββββββ | 2835/3125 [1:13:29<12:22, 2.56s/it]
91%|βββββββββ | 2836/3125 [1:13:32<12:12, 2.54s/it]
91%|βββββββββ | 2837/3125 [1:13:34<11:43, 2.44s/it]
91%|βββββββββ | 2838/3125 [1:13:36<11:34, 2.42s/it]
91%|βββββββββ | 2839/3125 [1:13:39<11:58, 2.51s/it]
91%|βββββββββ | 2840/3125 [1:13:41<11:37, 2.45s/it]
{'loss': '1.005', 'grad_norm': '6.94', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00885', 'train/loss_cot': '0.9352', 'epoch': '4.544'} |
|
91%|βββββββββ | 2840/3125 [1:13:42<11:37, 2.45s/it]
91%|βββββββββ | 2841/3125 [1:13:44<11:33, 2.44s/it]
91%|βββββββββ | 2842/3125 [1:13:46<11:53, 2.52s/it]
91%|βββββββββ | 2843/3125 [1:13:49<11:40, 2.48s/it]
91%|βββββββββ | 2844/3125 [1:13:52<12:09, 2.60s/it]
91%|βββββββββ | 2845/3125 [1:13:54<11:58, 2.56s/it]
91%|βββββββββ | 2846/3125 [1:13:56<11:30, 2.48s/it]
91%|βββββββββ | 2847/3125 [1:13:58<10:57, 2.36s/it]
91%|βββββββββ | 2848/3125 [1:14:01<10:32, 2.28s/it]
91%|βββββββββ | 2849/3125 [1:14:03<10:58, 2.39s/it]
91%|βββββββββ | 2850/3125 [1:14:06<10:50, 2.37s/it]
{'loss': '0.9445', 'grad_norm': '6.037', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04126', 'train/loss_cot': '0.8093', 'epoch': '4.56'} |
|
91%|βββββββββ | 2850/3125 [1:14:06<10:50, 2.37s/it]
91%|βββββββββ | 2851/3125 [1:14:08<10:53, 2.38s/it]
91%|ββββββββββ| 2852/3125 [1:14:10<10:32, 2.32s/it]
91%|ββββββββββ| 2853/3125 [1:14:12<10:21, 2.29s/it]
91%|ββββββββββ| 2854/3125 [1:14:15<10:21, 2.29s/it]
91%|ββββββββββ| 2855/3125 [1:14:17<10:21, 2.30s/it]
91%|ββββββββββ| 2856/3125 [1:14:20<10:59, 2.45s/it]
91%|ββββββββββ| 2857/3125 [1:14:22<10:39, 2.39s/it]
91%|ββββββββββ| 2858/3125 [1:14:24<10:16, 2.31s/it]
91%|ββββββββββ| 2859/3125 [1:14:26<10:07, 2.29s/it]
92%|ββββββββββ| 2860/3125 [1:14:29<09:58, 2.26s/it]
{'loss': '0.9287', 'grad_norm': '5.349', 'learning_rate': '1e-05', 'train/loss_infonce': '0.001854', 'train/loss_cot': '0.6859', 'epoch': '4.576'} |
|
92%|ββββββββββ| 2860/3125 [1:14:29<09:58, 2.26s/it]
92%|ββββββββββ| 2861/3125 [1:14:31<10:05, 2.29s/it]
92%|ββββββββββ| 2862/3125 [1:14:33<10:15, 2.34s/it]
92%|ββββββββββ| 2863/3125 [1:14:36<10:49, 2.48s/it]
92%|ββββββββββ| 2864/3125 [1:14:38<10:25, 2.40s/it]
92%|ββββββββββ| 2865/3125 [1:14:41<10:41, 2.47s/it]
92%|ββββββββββ| 2866/3125 [1:14:43<10:04, 2.33s/it]
92%|ββββββββββ| 2867/3125 [1:14:46<10:38, 2.48s/it]
92%|ββββββββββ| 2868/3125 [1:14:48<10:21, 2.42s/it]
92%|ββββββββββ| 2869/3125 [1:14:51<10:18, 2.42s/it]
92%|ββββββββββ| 2870/3125 [1:14:53<10:08, 2.39s/it]
{'loss': '0.9676', 'grad_norm': '7.139', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007385', 'train/loss_cot': '1.016', 'epoch': '4.592'} |
|
92%|ββββββββββ| 2870/3125 [1:14:53<10:08, 2.39s/it]
92%|ββββββββββ| 2871/3125 [1:14:56<10:35, 2.50s/it]
92%|ββββββββββ| 2872/3125 [1:14:58<10:27, 2.48s/it]
92%|ββββββββββ| 2873/3125 [1:15:00<09:57, 2.37s/it]
92%|ββββββββββ| 2874/3125 [1:15:02<09:34, 2.29s/it]
92%|ββββββββββ| 2875/3125 [1:15:05<09:40, 2.32s/it]
92%|ββββββββββ| 2876/3125 [1:15:07<09:40, 2.33s/it]
92%|ββββββββββ| 2877/3125 [1:15:10<09:58, 2.41s/it]
92%|ββββββββββ| 2878/3125 [1:15:12<09:39, 2.35s/it]
92%|ββββββββββ| 2879/3125 [1:15:14<09:54, 2.42s/it]
92%|ββββββββββ| 2880/3125 [1:15:17<09:44, 2.39s/it]
{'loss': '0.9598', 'grad_norm': '6.255', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005249', 'train/loss_cot': '0.9711', 'epoch': '4.608'} |
|
92%|ββββββββββ| 2880/3125 [1:15:17<09:44, 2.39s/it]
92%|ββββββββββ| 2881/3125 [1:15:19<09:44, 2.39s/it]
92%|ββββββββββ| 2882/3125 [1:15:21<09:24, 2.32s/it]
92%|ββββββββββ| 2883/3125 [1:15:24<09:27, 2.34s/it]
92%|ββββββββββ| 2884/3125 [1:15:26<09:13, 2.29s/it]
92%|ββββββββββ| 2885/3125 [1:15:28<09:02, 2.26s/it]
92%|ββββββββββ| 2886/3125 [1:15:30<09:09, 2.30s/it]
92%|ββββββββββ| 2887/3125 [1:15:33<08:53, 2.24s/it]
92%|ββββββββββ| 2888/3125 [1:15:35<08:44, 2.21s/it]
92%|ββββββββββ| 2889/3125 [1:15:37<08:42, 2.22s/it]
92%|ββββββββββ| 2890/3125 [1:15:39<08:33, 2.18s/it]
{'loss': '0.9255', 'grad_norm': '5.785', 'learning_rate': '1e-05', 'train/loss_infonce': '0.003677', 'train/loss_cot': '1.039', 'epoch': '4.624'} |
|
92%|ββββββββββ| 2890/3125 [1:15:39<08:33, 2.18s/it]
93%|ββββββββββ| 2891/3125 [1:15:41<08:51, 2.27s/it]
93%|ββββββββββ| 2892/3125 [1:15:45<10:31, 2.71s/it]
93%|ββββββββββ| 2893/3125 [1:15:48<10:12, 2.64s/it]
93%|ββββββββββ| 2894/3125 [1:15:50<10:18, 2.68s/it]
93%|ββββββββββ| 2895/3125 [1:15:52<09:31, 2.49s/it]
93%|ββββββββββ| 2896/3125 [1:15:55<09:11, 2.41s/it]
93%|ββββββββββ| 2897/3125 [1:15:57<08:57, 2.36s/it]
93%|ββββββββββ| 2898/3125 [1:16:00<09:12, 2.43s/it]
93%|ββββββββββ| 2899/3125 [1:16:02<08:50, 2.35s/it]
93%|ββββββββββ| 2900/3125 [1:16:04<08:38, 2.30s/it]
{'loss': '0.9203', 'grad_norm': '6.336', 'learning_rate': '1e-05', 'train/loss_infonce': '0.07861', 'train/loss_cot': '0.7803', 'epoch': '4.64'} |
|
93%|ββββββββββ| 2900/3125 [1:16:04<08:38, 2.30s/it]
93%|ββββββββββ| 2901/3125 [1:16:06<08:42, 2.33s/it]
93%|ββββββββββ| 2902/3125 [1:16:09<09:25, 2.54s/it]
93%|ββββββββββ| 2903/3125 [1:16:12<09:09, 2.47s/it]
93%|ββββββββββ| 2904/3125 [1:16:14<09:22, 2.55s/it]
93%|ββββββββββ| 2905/3125 [1:16:17<09:20, 2.55s/it]
93%|ββββββββββ| 2906/3125 [1:16:19<08:52, 2.43s/it]
93%|ββββββββββ| 2907/3125 [1:16:22<08:59, 2.47s/it]
93%|ββββββββββ| 2908/3125 [1:16:24<09:16, 2.56s/it]
93%|ββββββββββ| 2909/3125 [1:16:27<09:10, 2.55s/it]
93%|ββββββββββ| 2910/3125 [1:16:30<09:20, 2.61s/it]
{'loss': '0.9599', 'grad_norm': '8.555', 'learning_rate': '1e-05', 'train/loss_infonce': '0.1719', 'train/loss_cot': '0.9003', 'epoch': '4.656'} |
|
93%|ββββββββββ| 2910/3125 [1:16:30<09:20, 2.61s/it]
93%|ββββββββββ| 2911/3125 [1:16:32<09:09, 2.57s/it]
93%|ββββββββββ| 2912/3125 [1:16:34<08:42, 2.45s/it]
93%|ββββββββββ| 2913/3125 [1:16:38<09:29, 2.69s/it]
93%|ββββββββββ| 2914/3125 [1:16:40<08:52, 2.52s/it]
93%|ββββββββββ| 2915/3125 [1:16:42<08:54, 2.55s/it]
93%|ββββββββββ| 2916/3125 [1:16:45<08:38, 2.48s/it]
93%|ββββββββββ| 2917/3125 [1:16:47<08:17, 2.39s/it]
93%|ββββββββββ| 2918/3125 [1:16:49<07:56, 2.30s/it]
93%|ββββββββββ| 2919/3125 [1:16:51<07:56, 2.31s/it]
93%|ββββββββββ| 2920/3125 [1:16:53<07:47, 2.28s/it]
{'loss': '0.9655', 'grad_norm': '6.331', 'learning_rate': '1e-05', 'train/loss_infonce': '0.05298', 'train/loss_cot': '0.9183', 'epoch': '4.672'} |
|
93%|ββββββββββ| 2920/3125 [1:16:54<07:47, 2.28s/it]
93%|ββββββββββ| 2921/3125 [1:16:56<07:40, 2.26s/it]
94%|ββββββββββ| 2922/3125 [1:16:58<07:37, 2.25s/it]
94%|ββββββββββ| 2923/3125 [1:17:00<07:31, 2.24s/it]
94%|ββββββββββ| 2924/3125 [1:17:03<07:47, 2.32s/it]
94%|ββββββββββ| 2925/3125 [1:17:05<07:40, 2.30s/it]
94%|ββββββββββ| 2926/3125 [1:17:07<07:56, 2.39s/it]
94%|ββββββββββ| 2927/3125 [1:17:10<07:50, 2.38s/it]
94%|ββββββββββ| 2928/3125 [1:17:12<07:46, 2.37s/it]
94%|ββββββββββ| 2929/3125 [1:17:14<07:34, 2.32s/it]
94%|ββββββββββ| 2930/3125 [1:17:16<07:16, 2.24s/it]
{'loss': '0.9392', 'grad_norm': '5.799', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00161', 'train/loss_cot': '0.9555', 'epoch': '4.688'} |
|
94%|ββββββββββ| 2930/3125 [1:17:17<07:16, 2.24s/it]
94%|ββββββββββ| 2931/3125 [1:17:19<07:28, 2.31s/it]
94%|ββββββββββ| 2932/3125 [1:17:22<07:58, 2.48s/it]
94%|ββββββββββ| 2933/3125 [1:17:24<07:46, 2.43s/it]
94%|ββββββββββ| 2934/3125 [1:17:26<07:34, 2.38s/it]
94%|ββββββββββ| 2935/3125 [1:17:29<07:46, 2.46s/it]
94%|ββββββββββ| 2936/3125 [1:17:31<07:32, 2.39s/it]
94%|ββββββββββ| 2937/3125 [1:17:34<07:25, 2.37s/it]
94%|ββββββββββ| 2938/3125 [1:17:36<07:12, 2.31s/it]
94%|ββββββββββ| 2939/3125 [1:17:38<07:19, 2.36s/it]
94%|ββββββββββ| 2940/3125 [1:17:40<07:08, 2.32s/it]
{'loss': '0.969', 'grad_norm': '5.304', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03784', 'train/loss_cot': '0.9394', 'epoch': '4.704'} |
|
94%|ββββββββββ| 2940/3125 [1:17:41<07:08, 2.32s/it]
94%|ββββββββββ| 2941/3125 [1:17:43<07:18, 2.38s/it]
94%|ββββββββββ| 2942/3125 [1:17:45<07:14, 2.37s/it]
94%|ββββββββββ| 2943/3125 [1:17:47<06:55, 2.28s/it]
94%|ββββββββββ| 2944/3125 [1:17:50<06:55, 2.29s/it]
94%|ββββββββββ| 2945/3125 [1:17:52<06:44, 2.25s/it]
94%|ββββββββββ| 2946/3125 [1:17:55<07:12, 2.41s/it]
94%|ββββββββββ| 2947/3125 [1:17:57<07:23, 2.49s/it]
94%|ββββββββββ| 2948/3125 [1:18:00<07:06, 2.41s/it]
94%|ββββββββββ| 2949/3125 [1:18:02<06:59, 2.38s/it]
94%|ββββββββββ| 2950/3125 [1:18:04<06:45, 2.32s/it]
{'loss': '0.9626', 'grad_norm': '5.614', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02991', 'train/loss_cot': '0.9825', 'epoch': '4.72'} |
|
94%|ββββββββββ| 2950/3125 [1:18:04<06:45, 2.32s/it]
94%|ββββββββββ| 2951/3125 [1:18:06<06:40, 2.30s/it]
94%|ββββββββββ| 2952/3125 [1:18:09<06:42, 2.33s/it]
94%|ββββββββββ| 2953/3125 [1:18:11<06:40, 2.33s/it]
95%|ββββββββββ| 2954/3125 [1:18:13<06:34, 2.31s/it]
95%|ββββββββββ| 2955/3125 [1:18:17<07:28, 2.64s/it]
95%|ββββββββββ| 2956/3125 [1:18:19<07:04, 2.51s/it]
95%|ββββββββββ| 2957/3125 [1:18:21<06:37, 2.37s/it]
95%|ββββββββββ| 2958/3125 [1:18:23<06:33, 2.35s/it]
95%|ββββββββββ| 2959/3125 [1:18:26<06:34, 2.38s/it]
95%|ββββββββββ| 2960/3125 [1:18:28<06:28, 2.36s/it]
{'loss': '0.9712', 'grad_norm': '6.522', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02722', 'train/loss_cot': '1.158', 'epoch': '4.736'} |
|
95%|ββββββββββ| 2960/3125 [1:18:28<06:28, 2.36s/it]
95%|ββββββββββ| 2961/3125 [1:18:30<06:23, 2.34s/it]
95%|ββββββββββ| 2962/3125 [1:18:33<06:21, 2.34s/it]
95%|ββββββββββ| 2963/3125 [1:18:35<06:21, 2.36s/it]
95%|ββββββββββ| 2964/3125 [1:18:37<06:06, 2.28s/it]
95%|ββββββββββ| 2965/3125 [1:18:39<06:02, 2.27s/it]
95%|ββββββββββ| 2966/3125 [1:18:42<06:03, 2.29s/it]
95%|ββββββββββ| 2967/3125 [1:18:44<05:53, 2.24s/it]
95%|ββββββββββ| 2968/3125 [1:18:46<05:56, 2.27s/it]
95%|ββββββββββ| 2969/3125 [1:18:49<06:07, 2.35s/it]
95%|ββββββββββ| 2970/3125 [1:18:51<06:14, 2.42s/it]
{'loss': '0.9951', 'grad_norm': '6.371', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03223', 'train/loss_cot': '1.097', 'epoch': '4.752'} |
|
95%|ββββββββββ| 2970/3125 [1:18:52<06:14, 2.42s/it]
95%|ββββββββββ| 2971/3125 [1:18:54<06:30, 2.53s/it]
95%|ββββββββββ| 2972/3125 [1:18:56<06:04, 2.38s/it]
95%|ββββββββββ| 2973/3125 [1:18:58<05:54, 2.33s/it]
95%|ββββββββββ| 2974/3125 [1:19:00<05:45, 2.29s/it]
95%|ββββββββββ| 2975/3125 [1:19:05<07:16, 2.91s/it]
95%|ββββββββββ| 2976/3125 [1:19:07<06:53, 2.77s/it]
95%|ββββββββββ| 2977/3125 [1:19:10<06:41, 2.71s/it]
95%|ββββββββββ| 2978/3125 [1:19:12<06:19, 2.58s/it]
95%|ββββββββββ| 2979/3125 [1:19:15<06:34, 2.70s/it]
95%|ββββββββββ| 2980/3125 [1:19:18<06:17, 2.60s/it]
{'loss': '0.9387', 'grad_norm': '6.206', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02136', 'train/loss_cot': '0.9229', 'epoch': '4.768'} |
|
95%|ββββββββββ| 2980/3125 [1:19:18<06:17, 2.60s/it]
95%|ββββββββββ| 2981/3125 [1:19:20<06:01, 2.51s/it]
95%|ββββββββββ| 2982/3125 [1:19:22<06:03, 2.54s/it]
95%|ββββββββββ| 2983/3125 [1:19:24<05:38, 2.38s/it]
95%|ββββββββββ| 2984/3125 [1:19:27<05:28, 2.33s/it]
96%|ββββββββββ| 2985/3125 [1:19:29<05:20, 2.29s/it]
96%|ββββββββββ| 2986/3125 [1:19:32<05:41, 2.46s/it]
96%|ββββββββββ| 2987/3125 [1:19:34<05:38, 2.45s/it]
96%|ββββββββββ| 2988/3125 [1:19:37<05:46, 2.53s/it]
96%|ββββββββββ| 2989/3125 [1:19:39<05:32, 2.44s/it]
96%|ββββββββββ| 2990/3125 [1:19:41<05:26, 2.42s/it]
{'loss': '0.9917', 'grad_norm': '5.709', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04004', 'train/loss_cot': '0.9159', 'epoch': '4.784'} |
|
96%|ββββββββββ| 2990/3125 [1:19:42<05:26, 2.42s/it]
96%|ββββββββββ| 2991/3125 [1:19:44<05:38, 2.53s/it]
96%|ββββββββββ| 2992/3125 [1:19:47<05:35, 2.52s/it]
96%|ββββββββββ| 2993/3125 [1:19:49<05:20, 2.43s/it]
96%|ββββββββββ| 2994/3125 [1:19:51<05:16, 2.41s/it]
96%|ββββββββββ| 2995/3125 [1:19:54<05:09, 2.38s/it]
96%|ββββββββββ| 2996/3125 [1:19:56<05:00, 2.33s/it]
96%|ββββββββββ| 2997/3125 [1:19:58<05:02, 2.36s/it]
96%|ββββββββββ| 2998/3125 [1:20:01<05:01, 2.37s/it]
96%|ββββββββββ| 2999/3125 [1:20:04<05:16, 2.52s/it]
96%|ββββββββββ| 3000/3125 [1:20:06<05:08, 2.47s/it]
{'loss': '0.9784', 'grad_norm': '6.128', 'learning_rate': '1e-05', 'train/loss_infonce': '0.007294', 'train/loss_cot': '0.816', 'epoch': '4.8'} |
|
96%|ββββββββββ| 3000/3125 [1:20:06<05:08, 2.47s/it]
96%|ββββββββββ| 3001/3125 [1:20:24<15:01, 7.27s/it]
96%|ββββββββββ| 3002/3125 [1:20:27<11:54, 5.81s/it]
96%|ββββββββββ| 3003/3125 [1:20:29<09:36, 4.73s/it]
96%|ββββββββββ| 3004/3125 [1:20:31<08:07, 4.03s/it]
96%|ββββββββββ| 3005/3125 [1:20:34<06:57, 3.48s/it]
96%|ββββββββββ| 3006/3125 [1:20:36<05:59, 3.02s/it]
96%|ββββββββββ| 3007/3125 [1:20:38<05:34, 2.84s/it]
96%|ββββββββββ| 3008/3125 [1:20:41<05:34, 2.86s/it]
96%|ββββββββββ| 3009/3125 [1:20:43<05:17, 2.74s/it]
96%|ββββββββββ| 3010/3125 [1:20:46<05:08, 2.68s/it]
{'loss': '0.9654', 'grad_norm': '5.905', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006042', 'train/loss_cot': '1.039', 'epoch': '4.816'} |
|
96%|ββββββββββ| 3010/3125 [1:20:46<05:08, 2.68s/it]
96%|ββββββββββ| 3011/3125 [1:20:49<05:05, 2.68s/it]
96%|ββββββββββ| 3012/3125 [1:20:51<04:50, 2.57s/it]
96%|ββββββββββ| 3013/3125 [1:20:53<04:44, 2.54s/it]
96%|ββββββββββ| 3014/3125 [1:20:56<04:50, 2.61s/it]
96%|ββββββββββ| 3015/3125 [1:20:58<04:34, 2.50s/it]
97%|ββββββββββ| 3016/3125 [1:21:01<04:25, 2.44s/it]
97%|ββββββββββ| 3017/3125 [1:21:03<04:23, 2.44s/it]
97%|ββββββββββ| 3018/3125 [1:21:05<04:08, 2.32s/it]
97%|ββββββββββ| 3019/3125 [1:21:08<04:19, 2.45s/it]
97%|ββββββββββ| 3020/3125 [1:21:10<04:12, 2.40s/it]
{'loss': '1.006', 'grad_norm': '5.958', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006287', 'train/loss_cot': '0.9902', 'epoch': '4.832'} |
|
97%|ββββββββββ| 3020/3125 [1:21:10<04:12, 2.40s/it]
97%|ββββββββββ| 3021/3125 [1:21:13<04:34, 2.64s/it]
97%|ββββββββββ| 3022/3125 [1:21:15<04:13, 2.46s/it]
97%|ββββββββββ| 3023/3125 [1:21:18<04:04, 2.40s/it]
97%|ββββββββββ| 3024/3125 [1:21:20<04:04, 2.42s/it]
97%|ββββββββββ| 3025/3125 [1:21:23<04:12, 2.52s/it]
97%|ββββββββββ| 3026/3125 [1:21:25<03:58, 2.41s/it]
97%|ββββββββββ| 3027/3125 [1:21:27<03:49, 2.34s/it]
97%|ββββββββββ| 3028/3125 [1:21:30<03:53, 2.41s/it]
97%|ββββββββββ| 3029/3125 [1:21:32<03:43, 2.33s/it]
97%|ββββββββββ| 3030/3125 [1:21:34<03:43, 2.35s/it]
{'loss': '0.9478', 'grad_norm': '5.495', 'learning_rate': '1e-05', 'train/loss_infonce': '0.00325', 'train/loss_cot': '0.9661', 'epoch': '4.848'} |
|
97%|ββββββββββ| 3030/3125 [1:21:35<03:43, 2.35s/it]
97%|ββββββββββ| 3031/3125 [1:21:37<03:40, 2.34s/it]
97%|ββββββββββ| 3032/3125 [1:21:39<03:42, 2.39s/it]
97%|ββββββββββ| 3033/3125 [1:21:42<03:42, 2.42s/it]
97%|ββββββββββ| 3034/3125 [1:21:44<03:47, 2.50s/it]
97%|ββββββββββ| 3035/3125 [1:21:47<03:45, 2.50s/it]
97%|ββββββββββ| 3036/3125 [1:21:50<03:55, 2.65s/it]
97%|ββββββββββ| 3037/3125 [1:21:52<03:44, 2.55s/it]
97%|ββββββββββ| 3038/3125 [1:21:54<03:34, 2.47s/it]
97%|ββββββββββ| 3039/3125 [1:21:57<03:32, 2.47s/it]
97%|ββββββββββ| 3040/3125 [1:21:59<03:23, 2.40s/it]
{'loss': '0.9572', 'grad_norm': '5.489', 'learning_rate': '1e-05', 'train/loss_infonce': '0.002045', 'train/loss_cot': '0.9033', 'epoch': '4.864'} |
|
97%|ββββββββββ| 3040/3125 [1:21:59<03:23, 2.40s/it]
97%|ββββββββββ| 3041/3125 [1:22:01<03:20, 2.39s/it]
97%|ββββββββββ| 3042/3125 [1:22:04<03:17, 2.37s/it]
97%|ββββββββββ| 3043/3125 [1:22:06<03:08, 2.29s/it]
97%|ββββββββββ| 3044/3125 [1:22:08<03:05, 2.29s/it]
97%|ββββββββββ| 3045/3125 [1:22:11<03:05, 2.32s/it]
97%|ββββββββββ| 3046/3125 [1:22:13<03:01, 2.30s/it]
98%|ββββββββββ| 3047/3125 [1:22:16<03:10, 2.45s/it]
98%|ββββββββββ| 3048/3125 [1:22:18<03:08, 2.45s/it]
98%|ββββββββββ| 3049/3125 [1:22:20<03:05, 2.44s/it]
98%|ββββββββββ| 3050/3125 [1:22:23<03:02, 2.44s/it]
{'loss': '1.014', 'grad_norm': '7.566', 'learning_rate': '1e-05', 'train/loss_infonce': '0.0108', 'train/loss_cot': '1.04', 'epoch': '4.88'} |
|
98%|ββββββββββ| 3050/3125 [1:22:23<03:02, 2.44s/it]
98%|ββββββββββ| 3051/3125 [1:22:27<03:28, 2.81s/it]
98%|ββββββββββ| 3052/3125 [1:22:29<03:15, 2.68s/it]
98%|ββββββββββ| 3053/3125 [1:22:32<03:25, 2.86s/it]
98%|ββββββββββ| 3054/3125 [1:22:34<03:07, 2.64s/it]
98%|ββββββββββ| 3055/3125 [1:22:37<02:56, 2.53s/it]
98%|ββββββββββ| 3056/3125 [1:22:39<02:43, 2.37s/it]
98%|ββββββββββ| 3057/3125 [1:22:41<02:40, 2.36s/it]
98%|ββββββββββ| 3058/3125 [1:22:43<02:31, 2.26s/it]
98%|ββββββββββ| 3059/3125 [1:22:46<02:40, 2.43s/it]
98%|ββββββββββ| 3060/3125 [1:22:48<02:35, 2.40s/it]
{'loss': '0.9426', 'grad_norm': '5.976', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005646', 'train/loss_cot': '1.075', 'epoch': '4.896'} |
|
98%|ββββββββββ| 3060/3125 [1:22:49<02:35, 2.40s/it]
98%|ββββββββββ| 3061/3125 [1:22:51<02:36, 2.45s/it]
98%|ββββββββββ| 3062/3125 [1:22:53<02:33, 2.43s/it]
98%|ββββββββββ| 3063/3125 [1:22:55<02:26, 2.37s/it]
98%|ββββββββββ| 3064/3125 [1:22:58<02:26, 2.40s/it]
98%|ββββββββββ| 3065/3125 [1:23:00<02:22, 2.37s/it]
98%|ββββββββββ| 3066/3125 [1:23:02<02:16, 2.31s/it]
98%|ββββββββββ| 3067/3125 [1:23:05<02:19, 2.40s/it]
98%|ββββββββββ| 3068/3125 [1:23:07<02:16, 2.39s/it]
98%|ββββββββββ| 3069/3125 [1:23:10<02:15, 2.42s/it]
98%|ββββββββββ| 3070/3125 [1:23:12<02:15, 2.46s/it]
{'loss': '0.9363', 'grad_norm': '5.658', 'learning_rate': '1e-05', 'train/loss_infonce': '0.02258', 'train/loss_cot': '1.09', 'epoch': '4.912'} |
|
98%|ββββββββββ| 3070/3125 [1:23:13<02:15, 2.46s/it]
98%|ββββββββββ| 3071/3125 [1:23:15<02:10, 2.42s/it]
98%|ββββββββββ| 3072/3125 [1:23:17<02:06, 2.38s/it]
98%|ββββββββββ| 3073/3125 [1:23:19<01:59, 2.31s/it]
98%|ββββββββββ| 3074/3125 [1:23:21<01:58, 2.32s/it]
98%|ββββββββββ| 3075/3125 [1:23:24<01:56, 2.34s/it]
98%|ββββββββββ| 3076/3125 [1:23:26<01:56, 2.37s/it]
98%|ββββββββββ| 3077/3125 [1:23:29<01:54, 2.39s/it]
98%|ββββββββββ| 3078/3125 [1:23:31<01:50, 2.35s/it]
99%|ββββββββββ| 3079/3125 [1:23:33<01:46, 2.33s/it]
99%|ββββββββββ| 3080/3125 [1:23:36<01:46, 2.36s/it]
{'loss': '1.002', 'grad_norm': '5.522', 'learning_rate': '1e-05', 'train/loss_infonce': '0.08203', 'train/loss_cot': '0.7513', 'epoch': '4.928'} |
|
99%|ββββββββββ| 3080/3125 [1:23:36<01:46, 2.36s/it]
99%|ββββββββββ| 3081/3125 [1:23:38<01:46, 2.42s/it]
99%|ββββββββββ| 3082/3125 [1:23:41<01:43, 2.42s/it]
99%|ββββββββββ| 3083/3125 [1:23:43<01:39, 2.37s/it]
99%|ββββββββββ| 3084/3125 [1:23:45<01:33, 2.29s/it]
99%|ββββββββββ| 3085/3125 [1:23:47<01:33, 2.33s/it]
99%|ββββββββββ| 3086/3125 [1:23:50<01:31, 2.35s/it]
99%|ββββββββββ| 3087/3125 [1:23:52<01:26, 2.28s/it]
99%|ββββββββββ| 3088/3125 [1:23:55<01:34, 2.56s/it]
99%|ββββββββββ| 3089/3125 [1:23:58<01:34, 2.62s/it]
99%|ββββββββββ| 3090/3125 [1:24:00<01:26, 2.48s/it]
{'loss': '0.9326', 'grad_norm': '5.445', 'learning_rate': '1e-05', 'train/loss_infonce': '0.006226', 'train/loss_cot': '1.042', 'epoch': '4.944'} |
|
99%|ββββββββββ| 3090/3125 [1:24:00<01:26, 2.48s/it]
99%|ββββββββββ| 3091/3125 [1:24:02<01:23, 2.47s/it]
99%|ββββββββββ| 3092/3125 [1:24:05<01:20, 2.44s/it]
99%|ββββββββββ| 3093/3125 [1:24:07<01:17, 2.42s/it]
99%|ββββββββββ| 3094/3125 [1:24:10<01:17, 2.51s/it]
99%|ββββββββββ| 3095/3125 [1:24:13<01:16, 2.54s/it]
99%|ββββββββββ| 3096/3125 [1:24:15<01:12, 2.49s/it]
99%|ββββββββββ| 3097/3125 [1:24:17<01:06, 2.38s/it]
99%|ββββββββββ| 3098/3125 [1:24:20<01:06, 2.47s/it]
99%|ββββββββββ| 3099/3125 [1:24:22<01:04, 2.47s/it]
99%|ββββββββββ| 3100/3125 [1:24:25<01:00, 2.44s/it]
{'loss': '0.9142', 'grad_norm': '5.25', 'learning_rate': '1e-05', 'train/loss_infonce': '0.005615', 'train/loss_cot': '1.028', 'epoch': '4.96'} |
|
99%|ββββββββββ| 3100/3125 [1:24:25<01:00, 2.44s/it]
99%|ββββββββββ| 3101/3125 [1:24:27<01:01, 2.55s/it]
99%|ββββββββββ| 3102/3125 [1:24:30<01:01, 2.66s/it]
99%|ββββββββββ| 3103/3125 [1:24:33<00:56, 2.59s/it]
99%|ββββββββββ| 3104/3125 [1:24:35<00:52, 2.51s/it]
99%|ββββββββββ| 3105/3125 [1:24:38<00:54, 2.74s/it]
99%|ββββββββββ| 3106/3125 [1:24:41<00:49, 2.60s/it]
99%|ββββββββββ| 3107/3125 [1:24:43<00:44, 2.48s/it]
99%|ββββββββββ| 3108/3125 [1:24:45<00:41, 2.46s/it]
99%|ββββββββββ| 3109/3125 [1:24:47<00:37, 2.36s/it]
100%|ββββββββββ| 3110/3125 [1:24:49<00:33, 2.24s/it]
{'loss': '0.9832', 'grad_norm': '6.69', 'learning_rate': '1e-05', 'train/loss_infonce': '0.04028', 'train/loss_cot': '0.9677', 'epoch': '4.976'} |
|
100%|ββββββββββ| 3110/3125 [1:24:50<00:33, 2.24s/it]
100%|ββββββββββ| 3111/3125 [1:24:52<00:32, 2.31s/it]
100%|ββββββββββ| 3112/3125 [1:24:54<00:29, 2.27s/it]
100%|ββββββββββ| 3113/3125 [1:24:56<00:27, 2.27s/it]
100%|ββββββββββ| 3114/3125 [1:24:59<00:25, 2.31s/it]
100%|ββββββββββ| 3115/3125 [1:25:01<00:22, 2.27s/it]
100%|ββββββββββ| 3116/3125 [1:25:03<00:20, 2.24s/it]
100%|ββββββββββ| 3117/3125 [1:25:05<00:17, 2.25s/it]
100%|ββββββββββ| 3118/3125 [1:25:07<00:15, 2.21s/it]
100%|ββββββββββ| 3119/3125 [1:25:09<00:12, 2.14s/it]
100%|ββββββββββ| 3120/3125 [1:25:13<00:12, 2.58s/it]
{'loss': '0.9366', 'grad_norm': '5.909', 'learning_rate': '1e-05', 'train/loss_infonce': '0.03296', 'train/loss_cot': '0.9476', 'epoch': '4.992'} |
|
100%|ββββββββββ| 3120/3125 [1:25:13<00:12, 2.58s/it]
100%|ββββββββββ| 3121/3125 [1:25:16<00:10, 2.73s/it]
100%|ββββββββββ| 3122/3125 [1:25:19<00:08, 2.88s/it]
100%|ββββββββββ| 3123/3125 [1:25:21<00:05, 2.68s/it]
100%|ββββββββββ| 3124/3125 [1:25:24<00:02, 2.56s/it]
100%|ββββββββββ| 3125/3125 [1:25:25<00:00, 2.05s/it]
{'train_runtime': '5148', 'train_samples_per_second': '9.712', 'train_steps_per_second': '0.607', 'train_loss': '0.664', 'train/loss_infonce': '0.04126', 'train/loss_cot': '0.7905', 'epoch': '5'} |
|
100%|ββββββββββ| 3125/3125 [1:25:48<00:00, 2.05s/it]
100%|ββββββββββ| 3125/3125 [1:25:48<00:00, 1.65s/it] |
| notebook-622b7fa0e0f7-worker-0:259325:259325 [1] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault |
| notebook-622b7fa0e0f7-worker-0:259326:259326 [2] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault |
| notebook-622b7fa0e0f7-worker-0:259327:259824 [3] NCCL INFO [Service thread] Connection closed by localRank 1 |
| notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Service thread] Connection closed by localRank 1 |
| notebook-622b7fa0e0f7-worker-0:259327:261485 [3] NCCL INFO [Service thread] Connection closed by localRank 1 |
| notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Service thread] Connection closed by localRank 1 |
| notebook-622b7fa0e0f7-worker-0:259327:259327 [3] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault |
| notebook-622b7fa0e0f7-worker-0:259327:259824 [3] NCCL INFO [Service thread] Connection closed by localRank 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259826 [0] NCCL INFO [Service thread] Connection closed by localRank 2 |
| notebook-622b7fa0e0f7-worker-0:259327:261485 [3] NCCL INFO [Service thread] Connection closed by localRank 2 |
| notebook-622b7fa0e0f7-worker-0:259324:261486 [0] NCCL INFO [Service thread] Connection closed by localRank 2 |
| notebook-622b7fa0e0f7-worker-0:259324:259324 [0] NCCL INFO ENV/Plugin: Closing env plugin ncclEnvDefault |
|
|
