Model save

README.md

@@ -16,8 +16,8 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.4646
-- Accuracy: 0.3759
+- Loss: 3.4648
+- Accuracy: 0.3758
 
 ## Model description
 
@@ -143,8 +143,8 @@ The following hyperparameters were used during training:
 | 3.1324        | 9.8113  | 91000 | 0.3944   | 3.3028          |
 | 3.1419        | 9.9191  | 92000 | 0.3946   | 3.3012          |
 | 3.2899        | 10.0270 | 93000 | 3.4552   | 0.3784          |
-| 3.3629        | 10.1348 | 94000 | 3.4698   | 0.3757          |
-| 3.3727        | 10.2426 | 95000 | 3.4646   | 0.3759          |
+| 3.3628        | 10.1348 | 94000 | 3.4695   | 0.3757          |
+| 3.3727        | 10.2426 | 95000 | 3.4648   | 0.3758          |
 
 
 ### Framework versions

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 10.0,
-    "eval_accuracy": 0.3941857232061164,
-    "eval_loss": 3.3042373657226562,
-    "eval_runtime": 183.0156,
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.39419332890826336,
+    "eval_loss": 3.304239273071289,
+    "eval_runtime": 88.5119,
     "eval_samples": 18011,
-    "eval_samples_per_second": 98.412,
-    "eval_steps_per_second": 6.152,
-    "perplexity": 27.227768838990173,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4577919183201544,
-    "train_runtime": 79361.548,
+    "eval_samples_per_second": 203.487,
+    "eval_steps_per_second": 12.721,
+    "perplexity": 27.22782077188737,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07937381800601356,
+    "train_runtime": 805.3493,
     "train_samples": 296775,
-    "train_samples_per_second": 37.395,
-    "train_steps_per_second": 1.169
+    "train_samples_per_second": 18425.235,
+    "train_steps_per_second": 575.837
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 10.0,
-    "eval_accuracy": 0.3941857232061164,
-    "eval_loss": 3.3042373657226562,
-    "eval_runtime": 183.0156,
+    "epoch": 10.242587601078167,
+    "eval_accuracy": 0.39419332890826336,
+    "eval_loss": 3.304239273071289,
+    "eval_runtime": 88.5119,
     "eval_samples": 18011,
-    "eval_samples_per_second": 98.412,
-    "eval_steps_per_second": 6.152,
-    "perplexity": 27.227768838990173
+    "eval_samples_per_second": 203.487,
+    "eval_steps_per_second": 12.721,
+    "perplexity": 27.22782077188737
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 10.0,
-    "total_flos": 7.75449427968e+17,
-    "train_loss": 3.4577919183201544,
-    "train_runtime": 79361.548,
+    "epoch": 10.242587601078167,
+    "total_flos": 7.94262454272e+17,
+    "train_loss": 0.07937381800601356,
+    "train_runtime": 805.3493,
     "train_samples": 296775,
-    "train_samples_per_second": 37.395,
-    "train_steps_per_second": 1.169
+    "train_samples_per_second": 18425.235,
+    "train_steps_per_second": 575.837
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 3.3042373657226562,
   "best_model_checkpoint": "/scratch/cl5625/exceptions/models/100M__6910/checkpoint-90000",
-  "epoch": 10.0,
+  "epoch": 10.242587601078167,
   "eval_steps": 1000,
-  "global_step": 92750,
+  "global_step": 95000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -13822,19 +13822,361 @@
       "step": 92750
     },
     {
-      "epoch": 10.0,
-      "step": 92750,
-      "total_flos": 7.75449427968e+17,
-      "train_loss": 3.4577919183201544,
-      "train_runtime": 79361.548,
-      "train_samples_per_second": 37.395,
-      "train_steps_per_second": 1.169
+      "epoch": 10.005390835579515,
+      "grad_norm": 0.991041898727417,
+      "learning_rate": 0.0004800867033322549,
+      "loss": 3.1823,
+      "step": 92800
+    },
+    {
+      "epoch": 10.01078167115903,
+      "grad_norm": 1.0173295736312866,
+      "learning_rate": 0.00048002199935296016,
+      "loss": 3.254,
+      "step": 92850
+    },
+    {
+      "epoch": 10.016172506738544,
+      "grad_norm": 1.002999186515808,
+      "learning_rate": 0.00047995729537366545,
+      "loss": 3.2655,
+      "step": 92900
+    },
+    {
+      "epoch": 10.021563342318059,
+      "grad_norm": 0.9992165565490723,
+      "learning_rate": 0.0004798925913943707,
+      "loss": 3.2905,
+      "step": 92950
+    },
+    {
+      "epoch": 10.026954177897574,
+      "grad_norm": 0.971848726272583,
+      "learning_rate": 0.000479827887415076,
+      "loss": 3.2899,
+      "step": 93000
+    },
+    {
+      "epoch": 10.026954177897574,
+      "eval_accuracy": 0.3784375736462855,
+      "eval_loss": 3.4551994800567627,
+      "eval_runtime": 89.8017,
+      "eval_samples_per_second": 200.564,
+      "eval_steps_per_second": 12.539,
+      "step": 93000
+    },
+    {
+      "epoch": 10.032345013477089,
+      "grad_norm": 1.0214179754257202,
+      "learning_rate": 0.0004797631834357813,
+      "loss": 3.306,
+      "step": 93050
+    },
+    {
+      "epoch": 10.037735849056604,
+      "grad_norm": 0.9353964328765869,
+      "learning_rate": 0.00047969847945648653,
+      "loss": 3.3177,
+      "step": 93100
+    },
+    {
+      "epoch": 10.04312668463612,
+      "grad_norm": 0.9428226947784424,
+      "learning_rate": 0.00047963377547719177,
+      "loss": 3.2998,
+      "step": 93150
+    },
+    {
+      "epoch": 10.048517520215633,
+      "grad_norm": 1.005053162574768,
+      "learning_rate": 0.00047956907149789707,
+      "loss": 3.3267,
+      "step": 93200
+    },
+    {
+      "epoch": 10.053908355795148,
+      "grad_norm": 0.9448357820510864,
+      "learning_rate": 0.00047950436751860237,
+      "loss": 3.3126,
+      "step": 93250
+    },
+    {
+      "epoch": 10.059299191374663,
+      "grad_norm": 0.9822508096694946,
+      "learning_rate": 0.0004794396635393076,
+      "loss": 3.3302,
+      "step": 93300
+    },
+    {
+      "epoch": 10.064690026954178,
+      "grad_norm": 0.9297692775726318,
+      "learning_rate": 0.0004793749595600129,
+      "loss": 3.3601,
+      "step": 93350
+    },
+    {
+      "epoch": 10.070080862533693,
+      "grad_norm": 0.8432241082191467,
+      "learning_rate": 0.0004793102555807182,
+      "loss": 3.357,
+      "step": 93400
+    },
+    {
+      "epoch": 10.075471698113208,
+      "grad_norm": 0.9269798994064331,
+      "learning_rate": 0.00047924555160142344,
+      "loss": 3.3448,
+      "step": 93450
+    },
+    {
+      "epoch": 10.080862533692722,
+      "grad_norm": 0.8403341174125671,
+      "learning_rate": 0.0004791808476221287,
+      "loss": 3.3547,
+      "step": 93500
+    },
+    {
+      "epoch": 10.086253369272237,
+      "grad_norm": 0.8937442302703857,
+      "learning_rate": 0.00047911614364283404,
+      "loss": 3.3602,
+      "step": 93550
+    },
+    {
+      "epoch": 10.091644204851752,
+      "grad_norm": 0.8770005106925964,
+      "learning_rate": 0.0004790514396635393,
+      "loss": 3.3676,
+      "step": 93600
+    },
+    {
+      "epoch": 10.097035040431267,
+      "grad_norm": 0.8686730265617371,
+      "learning_rate": 0.0004789867356842445,
+      "loss": 3.3638,
+      "step": 93650
+    },
+    {
+      "epoch": 10.102425876010782,
+      "grad_norm": 0.9371838569641113,
+      "learning_rate": 0.00047892203170494976,
+      "loss": 3.3615,
+      "step": 93700
+    },
+    {
+      "epoch": 10.107816711590296,
+      "grad_norm": 0.8879263401031494,
+      "learning_rate": 0.0004788573277256551,
+      "loss": 3.3525,
+      "step": 93750
+    },
+    {
+      "epoch": 10.11320754716981,
+      "grad_norm": 0.8846799731254578,
+      "learning_rate": 0.00047879262374636035,
+      "loss": 3.3659,
+      "step": 93800
+    },
+    {
+      "epoch": 10.118598382749326,
+      "grad_norm": 0.8981801271438599,
+      "learning_rate": 0.0004787279197670656,
+      "loss": 3.3628,
+      "step": 93850
+    },
+    {
+      "epoch": 10.123989218328841,
+      "grad_norm": 0.8389490842819214,
+      "learning_rate": 0.00047866321578777095,
+      "loss": 3.3669,
+      "step": 93900
+    },
+    {
+      "epoch": 10.129380053908356,
+      "grad_norm": 0.8326027393341064,
+      "learning_rate": 0.0004785985118084762,
+      "loss": 3.3665,
+      "step": 93950
+    },
+    {
+      "epoch": 10.134770889487871,
+      "grad_norm": 0.845197319984436,
+      "learning_rate": 0.00047853380782918143,
+      "loss": 3.3629,
+      "step": 94000
+    },
+    {
+      "epoch": 10.134770889487871,
+      "eval_accuracy": 0.37573385518590996,
+      "eval_loss": 3.469757080078125,
+      "eval_runtime": 88.8318,
+      "eval_samples_per_second": 202.754,
+      "eval_steps_per_second": 12.676,
+      "step": 94000
+    },
+    {
+      "epoch": 10.140161725067385,
+      "grad_norm": 0.8682674169540405,
+      "learning_rate": 0.00047846910384988673,
+      "loss": 3.3696,
+      "step": 94050
+    },
+    {
+      "epoch": 10.1455525606469,
+      "grad_norm": 0.8294532299041748,
+      "learning_rate": 0.000478404399870592,
+      "loss": 3.3659,
+      "step": 94100
+    },
+    {
+      "epoch": 10.150943396226415,
+      "grad_norm": 0.8355298638343811,
+      "learning_rate": 0.00047833969589129727,
+      "loss": 3.3727,
+      "step": 94150
+    },
+    {
+      "epoch": 10.15633423180593,
+      "grad_norm": 0.8717609643936157,
+      "learning_rate": 0.00047827499191200256,
+      "loss": 3.3758,
+      "step": 94200
+    },
+    {
+      "epoch": 10.161725067385445,
+      "grad_norm": 0.9847083687782288,
+      "learning_rate": 0.0004782102879327078,
+      "loss": 3.3753,
+      "step": 94250
+    },
+    {
+      "epoch": 10.167115902964959,
+      "grad_norm": 0.9330268502235413,
+      "learning_rate": 0.0004781455839534131,
+      "loss": 3.3701,
+      "step": 94300
+    },
+    {
+      "epoch": 10.172506738544474,
+      "grad_norm": 0.8347208499908447,
+      "learning_rate": 0.00047808087997411834,
+      "loss": 3.3764,
+      "step": 94350
+    },
+    {
+      "epoch": 10.177897574123989,
+      "grad_norm": 0.8661344051361084,
+      "learning_rate": 0.00047801617599482364,
+      "loss": 3.382,
+      "step": 94400
+    },
+    {
+      "epoch": 10.183288409703504,
+      "grad_norm": 0.9129799604415894,
+      "learning_rate": 0.0004779514720155289,
+      "loss": 3.3697,
+      "step": 94450
+    },
+    {
+      "epoch": 10.18867924528302,
+      "grad_norm": 0.8427007794380188,
+      "learning_rate": 0.0004778867680362342,
+      "loss": 3.3838,
+      "step": 94500
+    },
+    {
+      "epoch": 10.194070080862534,
+      "grad_norm": 0.8299038410186768,
+      "learning_rate": 0.0004778220640569395,
+      "loss": 3.3869,
+      "step": 94550
+    },
+    {
+      "epoch": 10.199460916442048,
+      "grad_norm": 0.8165901303291321,
+      "learning_rate": 0.0004777573600776447,
+      "loss": 3.3775,
+      "step": 94600
+    },
+    {
+      "epoch": 10.204851752021563,
+      "grad_norm": 0.7907843589782715,
+      "learning_rate": 0.00047769265609835,
+      "loss": 3.3772,
+      "step": 94650
+    },
+    {
+      "epoch": 10.210242587601078,
+      "grad_norm": 0.840810239315033,
+      "learning_rate": 0.0004776279521190553,
+      "loss": 3.3774,
+      "step": 94700
+    },
+    {
+      "epoch": 10.215633423180593,
+      "grad_norm": 0.7826108932495117,
+      "learning_rate": 0.00047756324813976055,
+      "loss": 3.394,
+      "step": 94750
+    },
+    {
+      "epoch": 10.221024258760108,
+      "grad_norm": 0.8025938868522644,
+      "learning_rate": 0.0004774985441604658,
+      "loss": 3.3775,
+      "step": 94800
+    },
+    {
+      "epoch": 10.226415094339623,
+      "grad_norm": 0.7517758011817932,
+      "learning_rate": 0.00047743384018117114,
+      "loss": 3.3966,
+      "step": 94850
+    },
+    {
+      "epoch": 10.231805929919137,
+      "grad_norm": 0.8382764458656311,
+      "learning_rate": 0.0004773691362018764,
+      "loss": 3.3913,
+      "step": 94900
+    },
+    {
+      "epoch": 10.237196765498652,
+      "grad_norm": 0.8257710337638855,
+      "learning_rate": 0.00047730443222258163,
+      "loss": 3.3955,
+      "step": 94950
+    },
+    {
+      "epoch": 10.242587601078167,
+      "grad_norm": 0.7660510540008545,
+      "learning_rate": 0.00047723972824328687,
+      "loss": 3.3727,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "eval_accuracy": 0.3759389918381037,
+      "eval_loss": 3.4646387100219727,
+      "eval_runtime": 88.8637,
+      "eval_samples_per_second": 202.681,
+      "eval_steps_per_second": 12.671,
+      "step": 95000
+    },
+    {
+      "epoch": 10.242587601078167,
+      "step": 95000,
+      "total_flos": 7.94262454272e+17,
+      "train_loss": 0.07937381800601356,
+      "train_runtime": 805.3493,
+      "train_samples_per_second": 18425.235,
+      "train_steps_per_second": 575.837
     }
   ],
   "logging_steps": 50,
-  "max_steps": 92750,
+  "max_steps": 463750,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 50,
   "save_steps": 10000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -13848,7 +14190,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.75449427968e+17,
+  "total_flos": 7.94262454272e+17,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5370cb9d79e4d219f2c8f35a18d33b03650be712ffdb19623c3e2a34c403257
+oid sha256:4837377791b4f3c368c2a5da720e8e58192cb7c92682fc87b58c1daf6580744d
 size 5304