Training in progress, step 10000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5943e8c30fa544ff09f628b48252853f4960d70627894b945d089a224bd8bfe1
+oid sha256:6326d50fa1d2df348798efd1612e5978f840515ce7481121817e7fcbbfaebdfd
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -2224,3 +2224,1075 @@ Reading metadata...: 1650it [00:00, 9158.08it/s]
 
 
 
+
+Reading metadata...: 2165it [00:00, 13203.44it/s]                                                 | 9063/60000 [7:58:06<148:20:25, 10.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████                                                                               | 9079/60000 [8:00:46<139:41:59,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████                                                                               | 9100/60000 [8:04:13<140:10:03,  9.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▏                                                                              | 9120/60000 [8:07:29<140:26:05,  9.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▏                                                                              | 9139/60000 [8:10:36<138:42:35,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▏                                                                              | 9160/60000 [8:14:05<135:45:22,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▏                                                                              | 9179/60000 [8:17:11<139:31:44,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▎                                                                              | 9199/60000 [8:20:26<138:00:53,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▎                                                                              | 9219/60000 [8:23:42<141:21:29, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▎                                                                              | 9240/60000 [8:27:12<136:53:31,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▎                                                                              | 9260/60000 [8:30:27<139:31:26,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 15%|██████████████▍                                                                              | 9280/60000 [8:33:47<138:07:12,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▍                                                                              | 9300/60000 [8:37:18<155:43:49, 11.06s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▍                                                                              | 9320/60000 [8:40:33<137:56:38,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▍                                                                              | 9340/60000 [8:43:54<135:14:12,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▌                                                                              | 9359/60000 [8:46:57<135:55:16,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▌                                                                              | 9380/60000 [8:50:24<138:56:53,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▌                                                                              | 9399/60000 [8:53:30<136:26:44,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▌                                                                              | 9420/60000 [8:57:02<136:17:10,  9.70s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▋                                                                              | 9439/60000 [9:00:08<137:23:45,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▋                                                                              | 9460/60000 [9:03:34<135:37:24,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▋                                                                              | 9479/60000 [9:06:42<138:02:13,  9.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▋                                                                              | 9500/60000 [9:10:32<141:53:41, 10.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▊                                                                              | 9520/60000 [9:13:48<136:25:05,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▊                                                                              | 9539/60000 [9:17:20<139:20:31,  9.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▊                                                                              | 9560/60000 [9:20:43<135:20:50,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▊                                                                              | 9579/60000 [9:24:10<152:37:33, 10.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▉                                                                              | 9600/60000 [9:28:05<157:02:48, 11.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▉                                                                              | 9620/60000 [9:31:57<159:55:36, 11.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▉                                                                              | 9639/60000 [9:35:42<159:54:30, 11.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▉                                                                              | 9660/60000 [9:39:39<157:39:24, 11.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                              | 9680/60000 [9:43:30<161:29:53, 11.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                              | 9700/60000 [9:47:15<155:47:13, 11.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                              | 9720/60000 [9:51:08<160:21:19, 11.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                              | 9740/60000 [9:54:54<150:39:54, 10.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████▏                                                                             | 9760/60000 [9:58:46<159:52:31, 11.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|██████████████▉                                                                             | 9780/60000 [10:02:31<154:33:53, 11.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                             | 9799/60000 [10:05:55<147:58:40, 10.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                             | 9819/60000 [10:09:21<144:26:43, 10.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                             | 9840/60000 [10:12:51<138:48:21,  9.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████                                                                             | 9860/60000 [10:16:20<144:17:09, 10.36s/it]
+
+
+
+
+
+
+
+
+
+ 16%|███████████████▏                                                                            | 9870/60000 [10:18:03<141:11:11, 10.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████▏                                                                            | 9880/60000 [10:19:43<140:36:51, 10.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 16%|███████████████▏                                                                            | 9900/60000 [10:23:07<144:03:17, 10.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▏                                                                            | 9920/60000 [10:26:37<145:24:32, 10.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▏                                                                            | 9932/60000 [10:28:36<126:49:14,  9.12s/it]
+
+
+
+
+
+
+
+
+ 17%|███████████████▏                                                                            | 9940/60000 [10:30:03<149:14:38, 10.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▎                                                                            | 9960/60000 [10:33:45<146:49:27, 10.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████���███▎                                                                            | 9979/60000 [10:37:06<138:30:05,  9.97s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▎                                                                            | 9999/60000 [10:40:30<145:31:57, 10.48s/it]
+ 17%|███████████████▏                                                                           | 10000/60000 [10:40:41<148:35:05, 10.70s/it][INFO|trainer.py:3173] 2023-11-19 21:39:56,447 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-19 21:39:56,448 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-19 21:39:56,448 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 7814.29it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-19 21:39:57,946 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+ 17%|███████████████▏                                                                           | 10000/60000 [10:52:32<148:35:05, 10.70s/it]
+ 17%|███████████████▏                                                                           | 10000/60000 [10:52:32<148:35:05, 10.70s/it][INFO|trainer.py:2896] 2023-11-19 21:52:11,864 >> Saving model checkpoint to ./checkpoint-10000
+[INFO|configuration_utils.py:462] 2023-11-19 21:52:11,875 >> Configuration saved in ./checkpoint-10000/config.json
+[INFO|configuration_utils.py:568] 2023-11-19 21:52:11,881 >> Configuration saved in ./checkpoint-10000/generation_config.json
+[2023-11-19 21:52:53,458] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step10000 is about to be saved!
+[2023-11-19 21:52:53,512] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-10000/global_step10000/mp_rank_00_model_states.pt
+[2023-11-19 21:52:53,512] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-10000/global_step10000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-19 21:52:53,418 >> Model weights saved in ./checkpoint-10000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-19 21:52:53,425 >> Feature extractor saved in ./checkpoint-10000/preprocessor_config.json
+[2023-11-19 21:52:59,540] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-10000/global_step10000/mp_rank_00_model_states.pt.
+[2023-11-19 21:52:59,549] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-10000/global_step10000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-19 21:53:22,734] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-10000/global_step10000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-19 21:53:22,753] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-10000/global_step10000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-19 21:53:22,753] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step10000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-19 21:54:32,178 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▏                                                                           | 10019/60000 [10:58:44<165:01:51, 11.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▏                                                                           | 10039/60000 [11:02:23<146:52:26, 10.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 17%|███████████████▎                                                                           | 10059/60000 [11:05:55<158:39:37, 11.44s/it]
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0721, "train/learning_rate": 2.5909830508474577e-06, "train/epoch": 0.15, "train/global_step": 9060, "_timestamp": 1700413010.009219, "_runtime": 28660.975131988525, "_step": 104, "eval/loss": 0.1575927734375, "eval/wer": 8.839050131926122, "eval/runtime": 625.1584, "eval/samples_per_second": 2.726, "eval/steps_per_second": 0.681}
+{"train/loss": 0.0811, "train/learning_rate": 2.5401864406779662e-06, "train/epoch": 0.17, "train/global_step": 10060, "_timestamp": 1700424322.005514, "_runtime": 39972.971426963806, "_step": 155, "eval/loss": 0.1591796875, "eval/wer": 8.84847342630984, "eval/runtime": 710.5652, "eval/samples_per_second": 2.398, "eval/steps_per_second": 0.6}

wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2934cff68c20a64087ef6ac4105471f9a4cc52d2fb7000104b15270fdab34e59
-size 1605851
+oid sha256:6e0be4b9f699e918cc3f27a0a488b94ff548d1c0a2491cc931b47313d9be3733
+size 2315432