End of training

README.md

@@ -16,14 +16,14 @@ This student model is distilled from the teacher model [gpt2](https://huggingfac
 The [Distily](https://github.com/lapp0/distily) library was used for this distillation.
 
 It achieves the following results on the evaluation set:
-- eval_enwikippl: 756.0
+- eval_enwikippl: 210.0
 - eval_frwikippl: 776.0
-- eval_zhwikippl: 185.0
-- eval_tinystoriesppl: 342.0
-- eval_loss: 0.6940
-- eval_runtime: 25.4979
-- eval_samples_per_second: 98.047
-- eval_steps_per_second: 12.276
+- eval_zhwikippl: 169.0
+- eval_tinystoriesppl: 166.0
+- eval_loss: 1.2670
+- eval_runtime: 25.479
+- eval_samples_per_second: 98.12
+- eval_steps_per_second: 12.285
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment.
@@ -53,7 +53,7 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: cosine
+- lr_scheduler_type: inverse_sqrt
 - lr_scheduler_warmup_ratio: 0.5
 - num_epochs: 1.0
 
@@ -64,69 +64,69 @@ Peak GPU Memory: 7.2012 GB
 | step | epoch | enwikippl | frwikippl | loss | runtime | samples_per_second | steps_per_second | tinystoriesppl | zhwikippl |
 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
 | **teacher eval** |  | 43.25 | 61.25 |  |  |  |  | 11.6875 | 19.125 |
-| 0 | 0 | 468151435264.0 | 23914377904128.0 | 20.7618 | 25.4265 | 98.322 | 12.31 | 2315255808.0 | 20615843020800.0 |
-| 1000 | 0.0162 | 164864.0 | 696320.0 | 4.5118 | 25.4542 | 98.216 | 12.297 | 35584.0 | 2621440.0 |
-| 2000 | 0.0323 | 16256.0 | 9408.0 | 2.0635 | 25.4676 | 98.164 | 12.29 | 12288.0 | 163840.0 |
-| 3000 | 0.0485 | 3632.0 | 3168.0 | 1.3162 | 25.4251 | 98.328 | 12.311 | 3200.0 | 2272.0 |
-| 4000 | 0.0646 | 2080.0 | 2304.0 | 1.1214 | 25.4493 | 98.235 | 12.299 | 1216.0 | 1136.0 |
-| 5000 | 0.0808 | 1584.0 | 1768.0 | 0.9676 | 25.4407 | 98.268 | 12.303 | 852.0 | 604.0 |
-| 6000 | 0.0970 | 1304.0 | 1328.0 | 0.8357 | 25.4819 | 98.109 | 12.283 | 668.0 | 340.0 |
-| 7000 | 0.1131 | 1000.0 | 1008.0 | 0.7855 | 25.491 | 98.074 | 12.279 | 532.0 | 346.0 |
-| 8000 | 0.1293 | 844.0 | 832.0 | 0.7307 | 25.4698 | 98.155 | 12.289 | 362.0 | 270.0 |
-| 9000 | 0.1455 | 756.0 | 776.0 | 0.6940 | 25.4979 | 98.047 | 12.276 | 342.0 | 185.0 |
-| 10000 | 0.1616 | 700.0 | 764.0 | 0.6619 | 25.4292 | 98.312 | 12.309 | 332.0 | 171.0 |
-| 11000 | 0.1778 | 608.0 | 696.0 | 0.6211 | 25.4297 | 98.31 | 12.308 | 280.0 | 157.0 |
-| 12000 | 0.1939 | 540.0 | 620.0 | 0.6029 | 25.5013 | 98.034 | 12.274 | 241.0 | 121.5 |
-| 13000 | 0.2101 | 524.0 | 612.0 | 0.5846 | 25.4964 | 98.053 | 12.276 | 237.0 | 187.0 |
-| 14000 | 0.2263 | 478.0 | 568.0 | 0.5704 | 25.5243 | 97.946 | 12.263 | 235.0 | 120.5 |
-| 15000 | 0.2424 | 490.0 | 548.0 | 0.5667 | 25.7907 | 96.934 | 12.136 | 227.0 | 122.5 |
-| 16000 | 0.2586 | 492.0 | 560.0 | 0.5578 | 25.462 | 98.185 | 12.293 | 212.0 | 144.0 |
-| 17000 | 0.2747 | 466.0 | 572.0 | 0.5592 | 25.4854 | 98.095 | 12.282 | 193.0 | 138.0 |
-| 18000 | 0.2909 | 462.0 | 536.0 | 0.5471 | 25.5487 | 97.852 | 12.251 | 204.0 | 118.0 |
-| 19000 | 0.3071 | 504.0 | 560.0 | 0.5687 | 25.5071 | 98.012 | 12.271 | 227.0 | 231.0 |
-| 20000 | 0.3232 | 474.0 | 544.0 | 0.5414 | 25.5202 | 97.961 | 12.265 | 209.0 | 179.0 |
-| 21000 | 0.3394 | 446.0 | 510.0 | 0.4969 | 25.4669 | 98.167 | 12.29 | 208.0 | 128.0 |
-| 22000 | 0.3556 | 406.0 | 492.0 | 0.4527 | 25.4429 | 98.259 | 12.302 | 192.0 | 127.0 |
-| 23000 | 0.3717 | 384.0 | 478.0 | 0.4195 | 25.4289 | 98.313 | 12.309 | 187.0 | 182.0 |
-| 24000 | 0.3879 | 360.0 | 416.0 | 0.3870 | 25.4524 | 98.223 | 12.297 | 185.0 | 132.0 |
-| 25000 | 0.4040 | 336.0 | 394.0 | 0.3674 | 25.4298 | 98.31 | 12.308 | 166.0 | 151.0 |
-| 26000 | 0.4202 | 342.0 | 382.0 | 0.3416 | 25.4247 | 98.33 | 12.311 | 183.0 | 115.5 |
-| 27000 | 0.4364 | 312.0 | 346.0 | 0.3249 | 25.4434 | 98.257 | 12.302 | 166.0 | 119.0 |
-| 28000 | 0.4525 | 300.0 | 328.0 | 0.3045 | 25.626 | 97.557 | 12.214 | 160.0 | 133.0 |
-| 29000 | 0.4687 | 314.0 | 328.0 | 0.3185 | 25.453 | 98.22 | 12.297 | 177.0 | 101.0 |
-| 30000 | 0.4848 | 306.0 | 350.0 | 0.3136 | 25.4849 | 98.097 | 12.282 | 157.0 | 135.0 |
-| 31000 | 0.5010 | 298.0 | 326.0 | 0.3085 | 25.4984 | 98.045 | 12.275 | 154.0 | 122.5 |
-| 32000 | 0.5172 | 310.0 | 314.0 | 0.3171 | 25.4799 | 98.117 | 12.284 | 160.0 | 90.0 |
-| 33000 | 0.5333 | 292.0 | 304.0 | 0.2967 | 25.4868 | 98.09 | 12.281 | 163.0 | 110.0 |
-| 34000 | 0.5495 | 276.0 | 280.0 | 0.2787 | 25.467 | 98.166 | 12.29 | 155.0 | 166.0 |
-| 35000 | 0.5657 | 266.0 | 272.0 | 0.2809 | 25.478 | 98.124 | 12.285 | 150.0 | 161.0 |
-| 36000 | 0.5818 | 272.0 | 264.0 | 0.2744 | 25.4961 | 98.054 | 12.276 | 155.0 | 124.0 |
-| 37000 | 0.5980 | 258.0 | 260.0 | 0.2563 | 25.4747 | 98.137 | 12.287 | 144.0 | 79.0 |
-| 38000 | 0.6141 | 248.0 | 233.0 | 0.2556 | 25.5269 | 97.936 | 12.262 | 130.0 | 95.5 |
-| 39000 | 0.6303 | 243.0 | 232.0 | 0.2485 | 25.4742 | 98.139 | 12.287 | 133.0 | 88.0 |
-| 40000 | 0.6465 | 237.0 | 224.0 | 0.2389 | 25.7279 | 97.171 | 12.166 | 126.5 | 70.0 |
-| 41000 | 0.6626 | 238.0 | 227.0 | 0.2311 | 25.4394 | 98.273 | 12.304 | 132.0 | 100.0 |
-| 42000 | 0.6788 | 240.0 | 224.0 | 0.2434 | 25.477 | 98.128 | 12.286 | 124.0 | 72.5 |
-| 43000 | 0.6949 | 227.0 | 233.0 | 0.2378 | 25.4885 | 98.083 | 12.28 | 120.5 | 72.5 |
-| 44000 | 0.7111 | 220.0 | 216.0 | 0.2322 | 25.464 | 98.178 | 12.292 | 119.0 | 89.5 |
-| 45000 | 0.7273 | 200.0 | 191.0 | 0.2055 | 25.4558 | 98.209 | 12.296 | 114.5 | 51.75 |
-| 46000 | 0.7434 | 191.0 | 186.0 | 0.1916 | 25.4687 | 98.16 | 12.29 | 108.0 | 44.75 |
-| 47000 | 0.7596 | 186.0 | 192.0 | 0.1882 | 25.4606 | 98.191 | 12.294 | 100.0 | 43.75 |
-| 48000 | 0.7758 | 181.0 | 173.0 | 0.1804 | 25.4867 | 98.09 | 12.281 | 100.0 | 61.75 |
-| 49000 | 0.7919 | 180.0 | 176.0 | 0.1798 | 25.4404 | 98.269 | 12.303 | 101.5 | 39.5 |
-| 50000 | 0.8081 | 179.0 | 175.0 | 0.1795 | 25.4625 | 98.184 | 12.293 | 100.0 | 38.75 |
-| 51000 | 0.8242 | 177.0 | 178.0 | 0.1757 | 25.4489 | 98.236 | 12.299 | 101.0 | 36.75 |
-| 52000 | 0.8404 | 175.0 | 176.0 | 0.1721 | 25.476 | 98.132 | 12.286 | 98.0 | 35.5 |
-| 53000 | 0.8566 | 173.0 | 174.0 | 0.1696 | 25.465 | 98.174 | 12.291 | 96.5 | 35.25 |
-| 54000 | 0.8727 | 171.0 | 171.0 | 0.1688 | 25.4252 | 98.327 | 12.311 | 96.0 | 34.25 |
-| 55000 | 0.8889 | 170.0 | 172.0 | 0.1683 | 25.4198 | 98.349 | 12.313 | 96.5 | 34.5 |
-| 56000 | 0.9051 | 170.0 | 171.0 | 0.1674 | 25.4164 | 98.361 | 12.315 | 95.0 | 34.25 |
-| 57000 | 0.9212 | 170.0 | 172.0 | 0.1674 | 25.4607 | 98.191 | 12.293 | 94.5 | 34.25 |
-| 58000 | 0.9374 | 170.0 | 172.0 | 0.1672 | 25.4467 | 98.245 | 12.3 | 94.5 | 34.25 |
-| 59000 | 0.9535 | 170.0 | 172.0 | 0.1672 | 25.539 | 97.89 | 12.256 | 94.5 | 34.25 |
-| 60000 | 0.9697 | 170.0 | 172.0 | 0.1674 | 25.4689 | 98.159 | 12.29 | 94.5 | 34.25 |
-| 61000 | 0.9859 | 170.0 | 172.0 | 0.1672 | 25.5445 | 97.868 | 12.253 | 94.5 | 34.25 |
-| 61875 | 1.0 | 170.0 | 172.0 | 0.1672 | 25.499 | 98.043 | 12.275 | 94.5 | 34.25 |
+| 0 | 0 | 1477468749824.0 | 128093104635904.0 | 19.0496 | 25.4251 | 98.328 | 12.311 | 7985954816.0 | 67345087201280.0 |
+| 1000 | 0.0162 | 73728.0 | 1736704.0 | 5.1613 | 25.443 | 98.259 | 12.302 | 15872.0 | 3735552.0 |
+| 2000 | 0.0323 | 4000.0 | 42752.0 | 3.2656 | 25.4471 | 98.243 | 12.3 | 2688.0 | 544768.0 |
+| 3000 | 0.0485 | 1120.0 | 8160.0 | 2.3670 | 25.4374 | 98.28 | 12.305 | 728.0 | 16192.0 |
+| 4000 | 0.0646 | 620.0 | 4512.0 | 1.9891 | 25.4378 | 98.279 | 12.305 | 410.0 | 1464.0 |
+| 5000 | 0.0808 | 422.0 | 2080.0 | 1.7081 | 25.4876 | 98.087 | 12.28 | 286.0 | 372.0 |
+| 6000 | 0.0970 | 312.0 | 1296.0 | 1.4979 | 25.4407 | 98.268 | 12.303 | 231.0 | 237.0 |
+| 7000 | 0.1131 | 262.0 | 996.0 | 1.4032 | 25.4618 | 98.186 | 12.293 | 209.0 | 193.0 |
+| 8000 | 0.1293 | 226.0 | 860.0 | 1.3139 | 25.4752 | 98.135 | 12.286 | 179.0 | 221.0 |
+| 9000 | 0.1455 | 210.0 | 776.0 | 1.2670 | 25.479 | 98.12 | 12.285 | 166.0 | 169.0 |
+| 10000 | 0.1616 | 186.0 | 652.0 | 1.1738 | 25.4466 | 98.245 | 12.3 | 153.0 | 147.0 |
+| 11000 | 0.1778 | 167.0 | 568.0 | 1.0682 | 25.5929 | 97.683 | 12.23 | 145.0 | 149.0 |
+| 12000 | 0.1939 | 205.0 | 576.0 | 0.9983 | 25.4381 | 98.278 | 12.304 | 190.0 | 141.0 |
+| 13000 | 0.2101 | 141.0 | 472.0 | 0.9487 | 25.4558 | 98.21 | 12.296 | 118.0 | 132.0 |
+| 14000 | 0.2263 | 139.0 | 478.0 | 0.8964 | 25.4863 | 98.092 | 12.281 | 113.5 | 145.0 |
+| 15000 | 0.2424 | 129.0 | 458.0 | 0.8617 | 25.5441 | 97.87 | 12.253 | 98.5 | 119.0 |
+| 16000 | 0.2586 | 116.0 | 430.0 | 0.8348 | 25.4844 | 98.099 | 12.282 | 94.5 | 109.0 |
+| 17000 | 0.2747 | 120.0 | 452.0 | 0.8054 | 25.4588 | 98.198 | 12.294 | 86.5 | 110.0 |
+| 18000 | 0.2909 | 102.0 | 380.0 | 0.7811 | 25.4713 | 98.15 | 12.288 | 82.5 | 104.0 |
+| 19000 | 0.3071 | 108.5 | 408.0 | 0.8030 | 25.4558 | 98.21 | 12.296 | 86.0 | 111.5 |
+| 20000 | 0.3232 | 94.0 | 366.0 | 0.7253 | 25.5141 | 97.985 | 12.268 | 74.0 | 123.5 |
+| 21000 | 0.3394 | 87.5 | 322.0 | 0.6815 | 25.4799 | 98.117 | 12.284 | 70.0 | 142.0 |
+| 22000 | 0.3556 | 78.5 | 282.0 | 0.6363 | 25.4302 | 98.308 | 12.308 | 64.5 | 128.0 |
+| 23000 | 0.3717 | 77.0 | 270.0 | 0.6005 | 25.4991 | 98.043 | 12.275 | 61.25 | 110.5 |
+| 24000 | 0.3879 | 75.0 | 264.0 | 0.5879 | 25.4595 | 98.195 | 12.294 | 65.5 | 109.5 |
+| 25000 | 0.4040 | 70.5 | 235.0 | 0.5705 | 25.5194 | 97.965 | 12.265 | 58.0 | 140.0 |
+| 26000 | 0.4202 | 72.5 | 229.0 | 0.5606 | 25.5376 | 97.895 | 12.256 | 57.5 | 131.0 |
+| 27000 | 0.4364 | 71.5 | 236.0 | 0.5460 | 25.5324 | 97.915 | 12.259 | 56.25 | 120.0 |
+| 28000 | 0.4525 | 70.0 | 238.0 | 0.5403 | 25.5379 | 97.894 | 12.256 | 56.0 | 111.0 |
+| 29000 | 0.4687 | 76.5 | 249.0 | 0.5596 | 25.5341 | 97.908 | 12.258 | 58.25 | 113.5 |
+| 30000 | 0.4848 | 72.0 | 222.0 | 0.5487 | 25.5124 | 97.992 | 12.269 | 54.0 | 162.0 |
+| 31000 | 0.5010 | 70.5 | 253.0 | 0.5503 | 25.4372 | 98.281 | 12.305 | 56.0 | 136.0 |
+| 32000 | 0.5172 | 72.5 | 247.0 | 0.5404 | 25.5107 | 97.998 | 12.269 | 53.75 | 117.0 |
+| 33000 | 0.5333 | 70.0 | 235.0 | 0.5380 | 25.5177 | 97.971 | 12.266 | 51.25 | 127.0 |
+| 34000 | 0.5495 | 71.5 | 251.0 | 0.5315 | 25.5159 | 97.978 | 12.267 | 54.25 | 141.0 |
+| 35000 | 0.5657 | 73.5 | 247.0 | 0.5319 | 25.4405 | 98.268 | 12.303 | 55.75 | 98.0 |
+| 36000 | 0.5818 | 71.5 | 231.0 | 0.5224 | 25.4323 | 98.3 | 12.307 | 53.0 | 147.0 |
+| 37000 | 0.5980 | 69.0 | 237.0 | 0.5154 | 25.4606 | 98.191 | 12.294 | 53.0 | 115.5 |
+| 38000 | 0.6141 | 69.5 | 224.0 | 0.5031 | 25.5405 | 97.884 | 12.255 | 51.0 | 98.0 |
+| 39000 | 0.6303 | 64.5 | 219.0 | 0.4864 | 25.4803 | 98.115 | 12.284 | 50.0 | 458.0 |
+| 40000 | 0.6465 | 67.0 | 214.0 | 0.4865 | 25.5475 | 97.857 | 12.252 | 53.0 | 110.5 |
+| 41000 | 0.6626 | 67.5 | 218.0 | 0.4886 | 25.4575 | 98.203 | 12.295 | 47.5 | 95.0 |
+| 42000 | 0.6788 | 63.25 | 210.0 | 0.4788 | 25.43 | 98.309 | 12.308 | 51.0 | 92.0 |
+| 43000 | 0.6949 | 63.0 | 203.0 | 0.4746 | 25.4807 | 98.113 | 12.284 | 48.25 | 84.0 |
+| 44000 | 0.7111 | 62.75 | 211.0 | 0.4785 | 25.4599 | 98.194 | 12.294 | 48.5 | 90.0 |
+| 45000 | 0.7273 | 63.5 | 205.0 | 0.4745 | 25.4377 | 98.279 | 12.305 | 47.25 | 92.5 |
+| 46000 | 0.7434 | 63.25 | 204.0 | 0.4697 | 25.5287 | 97.929 | 12.261 | 45.5 | 116.0 |
+| 47000 | 0.7596 | 63.75 | 229.0 | 0.4705 | 25.4923 | 98.069 | 12.278 | 49.0 | 101.5 |
+| 48000 | 0.7758 | 63.75 | 205.0 | 0.4669 | 25.4951 | 98.058 | 12.277 | 46.0 | 75.5 |
+| 49000 | 0.7919 | 62.75 | 200.0 | 0.4651 | 25.5061 | 98.016 | 12.272 | 49.75 | 87.0 |
+| 50000 | 0.8081 | 63.25 | 200.0 | 0.4666 | 25.5442 | 97.869 | 12.253 | 46.5 | 149.0 |
+| 51000 | 0.8242 | 62.75 | 191.0 | 0.4644 | 25.4188 | 98.353 | 12.314 | 46.5 | 85.0 |
+| 52000 | 0.8404 | 61.25 | 187.0 | 0.4531 | 25.5074 | 98.011 | 12.271 | 47.25 | 62.75 |
+| 53000 | 0.8566 | 60.25 | 207.0 | 0.4613 | 25.4158 | 98.364 | 12.315 | 44.5 | 129.0 |
+| 54000 | 0.8727 | 65.0 | 209.0 | 0.4588 | 25.4519 | 98.225 | 12.298 | 45.5 | 95.0 |
+| 55000 | 0.8889 | 60.25 | 196.0 | 0.4571 | 25.4656 | 98.172 | 12.291 | 45.5 | 75.5 |
+| 56000 | 0.9051 | 63.75 | 195.0 | 0.4519 | 25.5712 | 97.766 | 12.24 | 46.0 | 82.0 |
+| 57000 | 0.9212 | 65.5 | 212.0 | 0.4563 | 25.4339 | 98.294 | 12.306 | 46.0 | 100.0 |
+| 58000 | 0.9374 | 62.5 | 204.0 | 0.4474 | 25.4328 | 98.298 | 12.307 | 45.25 | 80.5 |
+| 59000 | 0.9535 | 62.0 | 214.0 | 0.4461 | 25.4201 | 98.347 | 12.313 | 44.0 | 85.0 |
+| 60000 | 0.9697 | 62.0 | 211.0 | 0.4494 | 25.5124 | 97.992 | 12.269 | 44.25 | 85.0 |
+| 61000 | 0.9859 | 62.0 | 219.0 | 0.4483 | 25.4698 | 98.155 | 12.289 | 46.0 | 62.75 |
+| 61875 | 1.0 | 61.25 | 203.0 | 0.4444 | 25.5209 | 97.959 | 12.264 | 45.5 | 100.0 |
 
 ### Framework versions
 - Distily 0.2.0

logs/lr_scheduler_type=inverse_sqrt, warmup_ratio=0.5/events.out.tfevents.1724187476.f383272e719b

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4538f4c6cdaacdc1cd047c19ca528f739cfdc3fa8eacb6891952317ee1b9bd38
+size 312