diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6b95679fc57e3367f262d28f8d11547ecd720dde --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# algomaster99/pretrain-2.2e18 + +Pretrain run `pretrain_2.2e18_bf16_qknorm` (see `src/model/train.py`). + +## Model config + +``` +GPTConfig(vocab_size=32000, block_size=1024, n_layer=11, n_head=16, n_embd=1024, dropout=0.0, bias=True, qk_norm=True) +``` + +## Final training state + +- iter: 259653 +- final_train_loss: 3.1813 +- final_val_loss: 2.6673 + +## Files + +- `ckpt_final.pt` -- final trained weights (`model_state_dict`, `model_cfg`, `iter_num`, `final_train_loss`, `final_val_loss`) +- `ckpt.pt` -- best-val-loss checkpoint seen during training +- `ckpt_iter*.pt` -- periodic snapshots (no optimizer state) + +Load with `torch.load(path, weights_only=False)` and `src.model.gpt.GPTConfig`/`GPT`. diff --git a/ckpt.pt b/ckpt.pt new file mode 100644 index 0000000000000000000000000000000000000000..dca3cec3fa294ad68f5ffa5bc98da68bc26bf88a --- /dev/null +++ b/ckpt.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b69366e97f357c40fb811723ede3bd64d7de7cf7487bb89d4d2c99f702a95b +size 689574313 diff --git a/ckpt_final.pt b/ckpt_final.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a2ff181aa55aeaa4b1159615fa8467be942e418 --- /dev/null +++ b/ckpt_final.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd98c094ef2fb0c41efcd4d98d70be12832d20bb5dd26ccc848bb3d1cf4dacd5 +size 689587013 diff --git a/ckpt_iter0003246.pt b/ckpt_iter0003246.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f8f82b61fd82ba94a6b71f1993b08b7af6177eb --- /dev/null +++ b/ckpt_iter0003246.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:067c969c2684c7e87833a01084128f8fbc51bf32dda338da5a293cc677542c2a +size 689588065 diff --git a/ckpt_iter0006492.pt b/ckpt_iter0006492.pt new file mode 100644 index 0000000000000000000000000000000000000000..362387a6d315ee79e264916eeb75038dd214c7b6 --- /dev/null +++ b/ckpt_iter0006492.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50baa42935589aea1420a36ea953274958c5d8d8feebea6c57e747081c9efe82 +size 689588065 diff --git a/ckpt_iter0009738.pt b/ckpt_iter0009738.pt new file mode 100644 index 0000000000000000000000000000000000000000..6deb2d6dbcdcee2e53df13219f945e013a36b400 --- /dev/null +++ b/ckpt_iter0009738.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e5776ae2d51b5f6c637842ef75c0e1b5179c57929edb52a52022fd2a167732 +size 689588065 diff --git a/ckpt_iter0012984.pt b/ckpt_iter0012984.pt new file mode 100644 index 0000000000000000000000000000000000000000..950e390cded01e30b5aaba9de4d2f4b7ed93a5ac --- /dev/null +++ b/ckpt_iter0012984.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8561a2bdaabb7024f69fbdafc60eab56c42817c30809d9114625705fb71f0080 +size 689588065 diff --git a/ckpt_iter0016230.pt b/ckpt_iter0016230.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e6d49da6b825416ae838fb0a1e7ee47d7976863 --- /dev/null +++ b/ckpt_iter0016230.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45270034713f4959969d34a329fdfe0afc2e76fed5368c6811d5933e7d3c255 +size 689588065 diff --git a/ckpt_iter0019476.pt b/ckpt_iter0019476.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdae432c85e5ab68b717bb04b173390432081489 --- /dev/null +++ b/ckpt_iter0019476.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a91ed563d4e1274d2c2054bc213d9bdd835f6e869f93dc692b486c0cd87e9cb2 +size 689588065 diff --git a/ckpt_iter0022722.pt b/ckpt_iter0022722.pt new file mode 100644 index 0000000000000000000000000000000000000000..218bd5986225bcbd98d8f46071ebdf4755d31232 --- /dev/null +++ b/ckpt_iter0022722.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4692775068ec5ad4bd3c8346b62a66b619a3e4d0c243d14447387f5e11771efc +size 689588065 diff --git a/ckpt_iter0025968.pt b/ckpt_iter0025968.pt new file mode 100644 index 0000000000000000000000000000000000000000..969205bb30bfb96873593ceee4719d6706b6b2a7 --- /dev/null +++ b/ckpt_iter0025968.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6696d6cab52762bdff47d9da418d78832f41f614f1e9ca9001b9abc399a70f7 +size 689588065 diff --git a/ckpt_iter0029214.pt b/ckpt_iter0029214.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6307f97b8a3ef24901e05a604c13538c1f31ecf --- /dev/null +++ b/ckpt_iter0029214.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8955c279e688f3c97ab3e10612bd256985f5404eede08546f6ad0f982132215e +size 689588065 diff --git a/ckpt_iter0032460.pt b/ckpt_iter0032460.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5389ecc197f1bd2db7a10a2e4bdedd7eddf8310 --- /dev/null +++ b/ckpt_iter0032460.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4165e6eb9f679770c9597225f455caac8e25baceb471a4fd67e43e27f2ce1da0 +size 689588065 diff --git a/ckpt_iter0035706.pt b/ckpt_iter0035706.pt new file mode 100644 index 0000000000000000000000000000000000000000..a689886c3ae79fb439b995143f2972a77c3e3c7e --- /dev/null +++ b/ckpt_iter0035706.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2771dafadea41976bbe8b749cdfe7bb5e63a9b0d4cbf5035d88c47bf8ee31b +size 689588065 diff --git a/ckpt_iter0038952.pt b/ckpt_iter0038952.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfcee92aba8db26235b0e68fbc7ebb2933be8f68 --- /dev/null +++ b/ckpt_iter0038952.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f202091ba38599e828949695f2cb1df7f5eae5992d339b8751e603a24dcb42f1 +size 689588065 diff --git a/ckpt_iter0042198.pt b/ckpt_iter0042198.pt new file mode 100644 index 0000000000000000000000000000000000000000..162142a50b46e2a212754b6fba65c849854e3094 --- /dev/null +++ b/ckpt_iter0042198.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f631bb2b885d3b6f75bcda0ae9c6d1c76c48541be3a346e00febe000aa2bfd +size 689588065 diff --git a/ckpt_iter0045444.pt b/ckpt_iter0045444.pt new file mode 100644 index 0000000000000000000000000000000000000000..f738deb487e69399ba700c7974b48cac3d600468 --- /dev/null +++ b/ckpt_iter0045444.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dbe20ab9396bbd209f7cf5fb35196713d14d40a96c6f6d6228390341bc3d915 +size 689588065 diff --git a/ckpt_iter0048690.pt b/ckpt_iter0048690.pt new file mode 100644 index 0000000000000000000000000000000000000000..7773dfe6cce89f9bd6ac9b279493c46066d6a516 --- /dev/null +++ b/ckpt_iter0048690.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb6ca357207b7f50432003d37d1d0ab2b92baefd3e73a162c28b0d87ee757f7b +size 689588065 diff --git a/ckpt_iter0051936.pt b/ckpt_iter0051936.pt new file mode 100644 index 0000000000000000000000000000000000000000..9acd56f3f505fec521d9ea1c9a4cbda138b87b26 --- /dev/null +++ b/ckpt_iter0051936.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b160c5662c8bcb3b70d3dbbbc6c09f0fe94081ff2e58af65f127b5a540d1ca +size 689588065 diff --git a/ckpt_iter0055182.pt b/ckpt_iter0055182.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2f965abd6eb653a023a3c3dc2088f66eb40b033 --- /dev/null +++ b/ckpt_iter0055182.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:983a8a60bd6fdc10d085bf9cff62e2c69bde857336d62c1768b44b999bcd7e3e +size 689588065 diff --git a/ckpt_iter0058428.pt b/ckpt_iter0058428.pt new file mode 100644 index 0000000000000000000000000000000000000000..23ec0ee3b23a1aff8d98ce834bd478717f996274 --- /dev/null +++ b/ckpt_iter0058428.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a597987cbed154e7ab56078e15e1de4daccaed2380ed8c3a75ef06923bc66c +size 689588065 diff --git a/ckpt_iter0061674.pt b/ckpt_iter0061674.pt new file mode 100644 index 0000000000000000000000000000000000000000..c13215ec9b5a2dd72e7c6a21cf3ae290919343e4 --- /dev/null +++ b/ckpt_iter0061674.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1616e643c84a8912eb69f28cb3e4ca4b367f346656d03cf70fb18fa0b768cbe8 +size 689588065 diff --git a/ckpt_iter0064920.pt b/ckpt_iter0064920.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f2651e05b722d849b21ae61cc9be342235f1d6b --- /dev/null +++ b/ckpt_iter0064920.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c00de502e1fdebcb040c0428cb7aba059260f014d1ed67e9c3d7da628529739 +size 689588065 diff --git a/ckpt_iter0068166.pt b/ckpt_iter0068166.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dbb1ca33b4379987624a306b30f6c5ae4cc7b91 --- /dev/null +++ b/ckpt_iter0068166.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8d4ee21081061fad768573fad5ac012aa7ea712d7c642d3d677c1fdc7f2bba +size 689588065 diff --git a/ckpt_iter0071412.pt b/ckpt_iter0071412.pt new file mode 100644 index 0000000000000000000000000000000000000000..f25a43fd98465c210ae9418d00ee3dadcf70ead7 --- /dev/null +++ b/ckpt_iter0071412.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c49c0d764e53c9191815dac2197531bd00abc8bd8fcac1138f22598a1fd87eb +size 689588065 diff --git a/ckpt_iter0074658.pt b/ckpt_iter0074658.pt new file mode 100644 index 0000000000000000000000000000000000000000..91c28eedf1963f5973eb67592c62f6fd676bb7a1 --- /dev/null +++ b/ckpt_iter0074658.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c199c302d04d91762b72df28f4634049ab9f108352198ad766d9fdbea7159f2 +size 689588065 diff --git a/ckpt_iter0077904.pt b/ckpt_iter0077904.pt new file mode 100644 index 0000000000000000000000000000000000000000..4294485ee96ebea7f24fe93522752a5e12491876 --- /dev/null +++ b/ckpt_iter0077904.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1c8139ea85d5bc605541089140d1944dacaee24e7276050ef089a0f4dabc95 +size 689588065 diff --git a/ckpt_iter0081150.pt b/ckpt_iter0081150.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b5c340ebc08091b06e0a51ca278566830ff02ef --- /dev/null +++ b/ckpt_iter0081150.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9e57e42093e1502faed96cffce93d9a0548bb1f59e6616566af51fcd4011af +size 689588065 diff --git a/ckpt_iter0084396.pt b/ckpt_iter0084396.pt new file mode 100644 index 0000000000000000000000000000000000000000..a98654ba9117b3811df879098a36798541a3d6cf --- /dev/null +++ b/ckpt_iter0084396.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32efcd3e8e28d53fea5a96936622c8f958ff636d9e52282ed9fbdb2ab3ce8d8e +size 689588065 diff --git a/ckpt_iter0087642.pt b/ckpt_iter0087642.pt new file mode 100644 index 0000000000000000000000000000000000000000..85ffbb8d18d90862a15dd170aae549d7f68f7933 --- /dev/null +++ b/ckpt_iter0087642.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:225f9bb488d5cd7d9627b6c8fedafa36b5fb3c880f0fdc48946bba49cf75114a +size 689588065 diff --git a/ckpt_iter0090888.pt b/ckpt_iter0090888.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0472052973916ddfaab6ddae33ff88a01464f43 --- /dev/null +++ b/ckpt_iter0090888.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c66f9e6edf5b6865d13cb6111c62f58514730f9200accd786e0fc8a4eb519e3 +size 689588065 diff --git a/ckpt_iter0094134.pt b/ckpt_iter0094134.pt new file mode 100644 index 0000000000000000000000000000000000000000..599db13f19ad4110de195218bef97401e77ff2c1 --- /dev/null +++ b/ckpt_iter0094134.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cbf42c2b9a72ede2444c8f568fe45b78ce2ae45385ae41e5c1f706e1b647a13 +size 689588065 diff --git a/ckpt_iter0097380.pt b/ckpt_iter0097380.pt new file mode 100644 index 0000000000000000000000000000000000000000..34d38fab0adb4ca7130f05ce01ca32d718a89513 --- /dev/null +++ b/ckpt_iter0097380.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33df769767ccdb755364b92a0302757571826d29594e250938473ab0c82b60ef +size 689588065 diff --git a/ckpt_iter0100626.pt b/ckpt_iter0100626.pt new file mode 100644 index 0000000000000000000000000000000000000000..e372362a279a000940edaed3550fa3e362581e1f --- /dev/null +++ b/ckpt_iter0100626.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebbc80debac7da89def4fd448ec60e12a3f03c5e93ef970afdb43c27cb0adb21 +size 689588065 diff --git a/ckpt_iter0103872.pt b/ckpt_iter0103872.pt new file mode 100644 index 0000000000000000000000000000000000000000..510f5d17292ded10fee3f07493911598ce340373 --- /dev/null +++ b/ckpt_iter0103872.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfa99c308f58288aa4cce0172d3ca6effbaeecd8d3a88bcbd7f61dc212d37e3e +size 689588065 diff --git a/ckpt_iter0107118.pt b/ckpt_iter0107118.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ce1649c4dda9d2b25e0cbca17141cca14e34d41 --- /dev/null +++ b/ckpt_iter0107118.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ce854b1931710504f320486c9aba996ed7fdac39ad45a8485ccc276e65e234 +size 689588065 diff --git a/ckpt_iter0110364.pt b/ckpt_iter0110364.pt new file mode 100644 index 0000000000000000000000000000000000000000..25b146ef0c01a995296faeee2c8fea0b79f0b7c7 --- /dev/null +++ b/ckpt_iter0110364.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4047e2e954c5f541e4b0d79a3af430558c3e19c31881809b4dbb189779d11ff8 +size 689588065 diff --git a/ckpt_iter0113610.pt b/ckpt_iter0113610.pt new file mode 100644 index 0000000000000000000000000000000000000000..20abb14a98d087a485dc84a663fe35c58a05036a --- /dev/null +++ b/ckpt_iter0113610.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ea0b28da5b9993661231baf2dd05fc2768fa27577274405dd68ae21c43cd384 +size 689588065 diff --git a/ckpt_iter0116856.pt b/ckpt_iter0116856.pt new file mode 100644 index 0000000000000000000000000000000000000000..0eee2b7fbcfc2081c40ba9dda3843dab16c557d4 --- /dev/null +++ b/ckpt_iter0116856.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb5ce96b27961db8c5462a0e1f2c332dc27cec09dc3471b7a82b117133fa359 +size 689588065 diff --git a/ckpt_iter0120102.pt b/ckpt_iter0120102.pt new file mode 100644 index 0000000000000000000000000000000000000000..25c406ffe14fada5e07b99e4f9ff1d166e1dbcf4 --- /dev/null +++ b/ckpt_iter0120102.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ab49311864383967159512e3b1ac65a8dee7c1664f0513ee207e3c30223a74 +size 689588065 diff --git a/ckpt_iter0123348.pt b/ckpt_iter0123348.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e848efbb1369cc690a0340e4a8de1c47d5b43d1 --- /dev/null +++ b/ckpt_iter0123348.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac0223812f37472d380422dad6b27b5311ce1e3bc114404e7eea8d095d0eafa1 +size 689588065 diff --git a/ckpt_iter0126594.pt b/ckpt_iter0126594.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b8a24779545f431aaf000bb461a24f56960453b --- /dev/null +++ b/ckpt_iter0126594.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5c53d243c774aae7fe79ccb6fa0257c816dffaa4316632e675a48331aac952 +size 689588065 diff --git a/ckpt_iter0129840.pt b/ckpt_iter0129840.pt new file mode 100644 index 0000000000000000000000000000000000000000..335e721016b604a7ad42955778564e8d60657b46 --- /dev/null +++ b/ckpt_iter0129840.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:995a0106dadfb0b5fb69389d6e82951984a71041abb4dc4491cd90cabbe4247f +size 689588065 diff --git a/ckpt_iter0133086.pt b/ckpt_iter0133086.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1d724c511576415dc4fe81ba721d0a8140b2aa --- /dev/null +++ b/ckpt_iter0133086.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1892965065f18e3f2df44813dc6dfc07c3d8fb1b14b5838833b8e6099c1b1c0 +size 689588065 diff --git a/ckpt_iter0136332.pt b/ckpt_iter0136332.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5769ef52e2927d9a29652818adebb6903604798 --- /dev/null +++ b/ckpt_iter0136332.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e3bf7aa646fe9512fdc67982be9b70ab00a6a01fafc0d574ca5dc0049a429e3 +size 689588065 diff --git a/ckpt_iter0139578.pt b/ckpt_iter0139578.pt new file mode 100644 index 0000000000000000000000000000000000000000..757b1c1a9555e933f167c7f61573090340e9c100 --- /dev/null +++ b/ckpt_iter0139578.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95767e3e67ca62c7d9cc488abe7594d9579facea6801af5a1171710ec1f69d9a +size 689588065 diff --git a/ckpt_iter0142824.pt b/ckpt_iter0142824.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd12c92e9d630b484c6d58c54c99f691c809c161 --- /dev/null +++ b/ckpt_iter0142824.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917c1ace0d00de5bffb4ea789e95062aae9693ea28df1982dd547ca72c68830f +size 689588065 diff --git a/ckpt_iter0146070.pt b/ckpt_iter0146070.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a617cb25e68fb276d6aea97c739157dac89abd4 --- /dev/null +++ b/ckpt_iter0146070.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:413010c07e113c8d8d67449e60066121312d60bf6f7a4950399cbb47503c89d0 +size 689588065 diff --git a/ckpt_iter0149316.pt b/ckpt_iter0149316.pt new file mode 100644 index 0000000000000000000000000000000000000000..808f3a5e9d6c4584442f51a86e3049b7ed8442e5 --- /dev/null +++ b/ckpt_iter0149316.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbea4764dc22ed1455fd9df143e0fa2ab895ee5f0251ae6ca0399df8b8d19f1 +size 689588065 diff --git a/ckpt_iter0152562.pt b/ckpt_iter0152562.pt new file mode 100644 index 0000000000000000000000000000000000000000..c49ecbf82f1d1b471706dd322cf2afe4f59d695d --- /dev/null +++ b/ckpt_iter0152562.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:699a849519a25fc40afc959cb2fc3461e1a3f926ee02205ff92b36568aa1f80e +size 689588065 diff --git a/ckpt_iter0155808.pt b/ckpt_iter0155808.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbbb0abfefd90498beb88e143bcca8008e2d6c8f --- /dev/null +++ b/ckpt_iter0155808.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:566b6f2ffaeac258c9ccaaf9a166f68fb32936e109947900f6cc9e79066d010e +size 689588065 diff --git a/ckpt_iter0159054.pt b/ckpt_iter0159054.pt new file mode 100644 index 0000000000000000000000000000000000000000..519ea6402ef312c122b5a234e4a0a0cc2e5a400f --- /dev/null +++ b/ckpt_iter0159054.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb1c29c23ba3abdc4fba1b8b3b27f40e98e419e1807f49daa56548ef9a4b235 +size 689588065 diff --git a/ckpt_iter0162300.pt b/ckpt_iter0162300.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab81bc4c4204dfe77eab4c45847def1034557013 --- /dev/null +++ b/ckpt_iter0162300.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:313eb9dc04e80aa52dc014a569b6c7c6f9b700f245ff8332985a9e79f89c3a66 +size 689588065 diff --git a/ckpt_iter0165546.pt b/ckpt_iter0165546.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc549deee6495b95ced7621bfea34ecf026de046 --- /dev/null +++ b/ckpt_iter0165546.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:967621336ae41f9c08f0b7bfd522450c2ccbacf644cc309377b76a684c4c54bc +size 689588065 diff --git a/ckpt_iter0168792.pt b/ckpt_iter0168792.pt new file mode 100644 index 0000000000000000000000000000000000000000..57bbc3b58cf529ac4c9f9b4c200405db18b40aa2 --- /dev/null +++ b/ckpt_iter0168792.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ea7cdfe40f6023434f53a890ff560f14c7c0262e5279ff7ba57975503f20ae0 +size 689588065 diff --git a/ckpt_iter0172038.pt b/ckpt_iter0172038.pt new file mode 100644 index 0000000000000000000000000000000000000000..bed14fd49d919f07eee0bdeb266a625aaea1c5f7 --- /dev/null +++ b/ckpt_iter0172038.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd17fd2670c324cebb5e3373e98cfe0f0873360e636489224edb18ae2bcf4b73 +size 689588065 diff --git a/ckpt_iter0175284.pt b/ckpt_iter0175284.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ca87897e1c02ba80b594f8993cec7dfd5cb419c --- /dev/null +++ b/ckpt_iter0175284.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0624685d847f80bcd26ac783df50daab0057e2bdeede99d778bfab10e81d560c +size 689588065 diff --git a/ckpt_iter0178530.pt b/ckpt_iter0178530.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fa7801e2631291632128232520ed5f76381a209 --- /dev/null +++ b/ckpt_iter0178530.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a0f7714f9344a4de0ed233f9f1d8fd95eb890790b94378bd74882d6d082ffb +size 689588065 diff --git a/ckpt_iter0181776.pt b/ckpt_iter0181776.pt new file mode 100644 index 0000000000000000000000000000000000000000..568901a69df30917d6083b80a061b1b9a5986d6b --- /dev/null +++ b/ckpt_iter0181776.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83e6a3283698276530ac234de9b20f2d05dbafbffa5bcd0c7e2190c44c9e5498 +size 689588065 diff --git a/ckpt_iter0185022.pt b/ckpt_iter0185022.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a94d83d631d86093a09d942324792b4007a22a1 --- /dev/null +++ b/ckpt_iter0185022.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c4bcfdffa575e3824fba5ab5e54f507514daa4ac4df434d67318edf07dda060 +size 689588065 diff --git a/ckpt_iter0188268.pt b/ckpt_iter0188268.pt new file mode 100644 index 0000000000000000000000000000000000000000..72a8d349fbe71d38f19405dd5ee3d85933cce36a --- /dev/null +++ b/ckpt_iter0188268.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f8512e5a501bc2ffcb2c554d7e49cd8aa012bca938124e53ceb95524e45493f +size 689588065 diff --git a/ckpt_iter0191514.pt b/ckpt_iter0191514.pt new file mode 100644 index 0000000000000000000000000000000000000000..99515f48ceea2e6640dc03be74cfdca665676f37 --- /dev/null +++ b/ckpt_iter0191514.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a84fb9630bb146707cd832543f8525757196cf03652813ee165fd3d288f41e +size 689588065 diff --git a/ckpt_iter0194760.pt b/ckpt_iter0194760.pt new file mode 100644 index 0000000000000000000000000000000000000000..85f7dc03f003fa3399effb25bd9d3c6b4bd6ff52 --- /dev/null +++ b/ckpt_iter0194760.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e63f28fd666bd14660491b994a0dd9b793060b6bd482ccb4fecbdded79f515ed +size 689588065 diff --git a/ckpt_iter0198006.pt b/ckpt_iter0198006.pt new file mode 100644 index 0000000000000000000000000000000000000000..8777b0f3b7890d20a93181b6edd3b09a343f382e --- /dev/null +++ b/ckpt_iter0198006.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffda7ead25be49677635610ad927614a0d074e299aae7eff548d1ea53c16b661 +size 689588065 diff --git a/ckpt_iter0201252.pt b/ckpt_iter0201252.pt new file mode 100644 index 0000000000000000000000000000000000000000..67e9c3f98d74a9d22cc3c2b94ea85b0f6292588a --- /dev/null +++ b/ckpt_iter0201252.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2a6e7507b82b0dbba32ce5551aefc2a1fac1bab749c59a7992f1b9767ba222c +size 689588065 diff --git a/ckpt_iter0204498.pt b/ckpt_iter0204498.pt new file mode 100644 index 0000000000000000000000000000000000000000..89f540d1637da436b791a58db3c8fc3518f69c31 --- /dev/null +++ b/ckpt_iter0204498.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:332534732c0778722233fe7a6d6088bac92b33fa0c65440d821f190139f902a2 +size 689588065 diff --git a/ckpt_iter0207744.pt b/ckpt_iter0207744.pt new file mode 100644 index 0000000000000000000000000000000000000000..653613551058af2e4c2e04107a1638263aa16462 --- /dev/null +++ b/ckpt_iter0207744.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e4d433f91a5ae3e2fb7685affba73a9333ff1732be85324a1376937ad99fe4 +size 689588065 diff --git a/ckpt_iter0210990.pt b/ckpt_iter0210990.pt new file mode 100644 index 0000000000000000000000000000000000000000..814f9f6254da499db6f94219963651277c45b986 --- /dev/null +++ b/ckpt_iter0210990.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f981d8401b55018504d110d5b2fb537eeef3c266a2496686416ac71c05faaf5 +size 689588065 diff --git a/ckpt_iter0214236.pt b/ckpt_iter0214236.pt new file mode 100644 index 0000000000000000000000000000000000000000..49e6977e6cf515105fe989c03a89204daedbaa86 --- /dev/null +++ b/ckpt_iter0214236.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c4ab40732a31f31b322c164927c1e51294e181d27629586321a89a94eb00aec +size 689588065 diff --git a/ckpt_iter0217482.pt b/ckpt_iter0217482.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c60d06d8c4f5cc767e012cf79b48b72e179ec59 --- /dev/null +++ b/ckpt_iter0217482.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef32ca1309774e5ae5e1c79270756cb05d7bb5c53e5537c6b3d7d34d6a74405 +size 689588065 diff --git a/ckpt_iter0220728.pt b/ckpt_iter0220728.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdb473628be7ddb339cb23f540474863b483f86d --- /dev/null +++ b/ckpt_iter0220728.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c9d25ab43fb17d269dc2e1c464653ae87f04ebb547bb083066260d46094f8d +size 689588065 diff --git a/ckpt_iter0223974.pt b/ckpt_iter0223974.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d065ee64de17f131303e8ba28bfb68044a24f89 --- /dev/null +++ b/ckpt_iter0223974.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610cbf2d8b6e58359b6539aa2fe132956a5feef416053d1bb9c303a009b4f585 +size 689588065 diff --git a/ckpt_iter0227220.pt b/ckpt_iter0227220.pt new file mode 100644 index 0000000000000000000000000000000000000000..5473bcd97f7c694363ae8f1174fc7afd6b5c6689 --- /dev/null +++ b/ckpt_iter0227220.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5812603104b9a7fd5bf8d0b5dbd611f68ea856456d5fdda42a289a59bae36e +size 689588065 diff --git a/ckpt_iter0230466.pt b/ckpt_iter0230466.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8afb28710ffeb374039b06451a230fb328cea2a --- /dev/null +++ b/ckpt_iter0230466.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d036909500977ef544bd59067ed7f9d25395966982aaab3fb3062ce7c8f1ee72 +size 689588065 diff --git a/ckpt_iter0233712.pt b/ckpt_iter0233712.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ca0d6861d5ff1425694658ee1a7f6bd4c516edb --- /dev/null +++ b/ckpt_iter0233712.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1aacdab63fb88a385e0c8f3e5612b63cd12db7bc3fab332b7bd50a3fc92a941 +size 689588065 diff --git a/ckpt_iter0236958.pt b/ckpt_iter0236958.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8f9d210444c4d857e5136fb29f78eb4f7b2b079 --- /dev/null +++ b/ckpt_iter0236958.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea1d286082960af7c4448eed2371e59418ae5f83e7fa35a74be4fd11d48bcc9 +size 689588065 diff --git a/ckpt_iter0240204.pt b/ckpt_iter0240204.pt new file mode 100644 index 0000000000000000000000000000000000000000..b55c74cf98704089c6ec4a28c8f9f3c37b7d4b24 --- /dev/null +++ b/ckpt_iter0240204.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e644ba1376bdff39f24ec631613e015bc8ce96bcae2e55b3cc7ba830d77f1e +size 689588065 diff --git a/ckpt_iter0243450.pt b/ckpt_iter0243450.pt new file mode 100644 index 0000000000000000000000000000000000000000..255f901ea5af6dc9d5ff216cdb226494e3a1c3c5 --- /dev/null +++ b/ckpt_iter0243450.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb61db4c1217af8200b83e99e2d65fbe00b8a40d71f193aff563bc672d1a6d8 +size 689588065 diff --git a/ckpt_iter0246696.pt b/ckpt_iter0246696.pt new file mode 100644 index 0000000000000000000000000000000000000000..b739ca88b62215fd20edb6ed0a5f456755a34997 --- /dev/null +++ b/ckpt_iter0246696.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94352921a59ddbc01a87c7173ccfcade3bfb0cdce86efc37dc94685213110029 +size 689588065 diff --git a/ckpt_iter0249942.pt b/ckpt_iter0249942.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa490884beaf7bf12d773c7c8ab90d05131c5bc3 --- /dev/null +++ b/ckpt_iter0249942.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:268f3a9a90ee222f5bc2b425eeeeb0379fc9b82219d079598d8e154de087f707 +size 689588065 diff --git a/ckpt_iter0253188.pt b/ckpt_iter0253188.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed0dc2671240cb7f7a4768264685b50c4de6d8e7 --- /dev/null +++ b/ckpt_iter0253188.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae8eea2a75d2a75d0600c46ff73f4677cc26117eea9160381d996eb6e56d9f48 +size 689588065 diff --git a/ckpt_iter0256434.pt b/ckpt_iter0256434.pt new file mode 100644 index 0000000000000000000000000000000000000000..70c611798db461dbd748e03501ef201fc176fa48 --- /dev/null +++ b/ckpt_iter0256434.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a1ee58ee586638020bac0c1e1835db9b932dffb0f16ed33bdd5f1b1fbacb4f9 +size 689588065 diff --git a/log.jsonl b/log.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4f9acf86353030761faf53fd20a64d5f63ef13c0 --- /dev/null +++ b/log.jsonl @@ -0,0 +1,81 @@ +{"iter": 0, "train_loss": 10.581914901733398, "val_loss": 10.600311279296875, "lr": 4.6217840086273296e-08, "tokens_seen": 0, "elapsed_s": 17.76690173149109} +{"iter": 3246, "train_loss": 5.375073432922363, "val_loss": 4.705842018127441, "lr": 0.00015006932676012941, "tokens_seen": 26591232, "elapsed_s": 409.71820974349976} +{"iter": 6492, "train_loss": 4.497573375701904, "val_loss": 3.811401128768921, "lr": 0.0002999999999896055, "tokens_seen": 53182464, "elapsed_s": 798.4643449783325} +{"iter": 9738, "train_loss": 4.209537506103516, "val_loss": 3.6260745525360107, "lr": 0.00029989042565840173, "tokens_seen": 79773696, "elapsed_s": 1187.4371802806854} +{"iter": 12984, "train_loss": 4.076352596282959, "val_loss": 3.457029104232788, "lr": 0.0002995620153553775, "tokens_seen": 106364928, "elapsed_s": 1576.680532693863} +{"iter": 16230, "train_loss": 3.9825599193573, "val_loss": 3.3896665573120117, "lr": 0.00029901530186844183, "tokens_seen": 132956160, "elapsed_s": 1965.7778820991516} +{"iter": 19476, "train_loss": 3.9212944507598877, "val_loss": 3.3226213455200195, "lr": 0.0002982511721439931, "tokens_seen": 159547392, "elapsed_s": 2354.6651570796967} +{"iter": 22722, "train_loss": 3.779982089996338, "val_loss": 3.2340052127838135, "lr": 0.0002972708658480048, "tokens_seen": 186138624, "elapsed_s": 2743.343657016754} +{"iter": 25968, "train_loss": 3.8384053707122803, "val_loss": 3.271618366241455, "lr": 0.0002960759733548859, "tokens_seen": 212729856, "elapsed_s": 3132.0662574768066} +{"iter": 29214, "train_loss": 3.7781074047088623, "val_loss": 3.1929574012756348, "lr": 0.0002946684331673779, "tokens_seen": 239321088, "elapsed_s": 3518.9962816238403} +{"iter": 32460, "train_loss": 3.7145934104919434, "val_loss": 3.1775624752044678, "lr": 0.00029305052877167564, "tokens_seen": 265912320, "elapsed_s": 3907.2831752300262} +{"iter": 35706, "train_loss": 3.70058536529541, "val_loss": 3.209263563156128, "lr": 0.0002912248849328729, "tokens_seen": 292503552, "elapsed_s": 4295.23272562027} +{"iter": 38952, "train_loss": 3.728814125061035, "val_loss": 3.150474786758423, "lr": 0.0002891944634367428, "tokens_seen": 319094784, "elapsed_s": 4681.817955255508} +{"iter": 42198, "train_loss": 3.7317419052124023, "val_loss": 3.0795416831970215, "lr": 0.00028696255828476226, "tokens_seen": 345686016, "elapsed_s": 5068.912436008453} +{"iter": 45444, "train_loss": 3.703348398208618, "val_loss": 3.1266896724700928, "lr": 0.0002845327903501737, "tokens_seen": 372277248, "elapsed_s": 5455.955375671387} +{"iter": 48690, "train_loss": 3.728621244430542, "val_loss": 3.152674913406372, "lr": 0.0002819091015037561, "tokens_seen": 398868480, "elapsed_s": 5841.689475774765} +{"iter": 51936, "train_loss": 3.7084076404571533, "val_loss": 3.041043758392334, "lr": 0.00027909574821883334, "tokens_seen": 425459712, "elapsed_s": 6228.090328216553} +{"iter": 55182, "train_loss": 3.6928341388702393, "val_loss": 3.138061761856079, "lr": 0.0002760972946658954, "tokens_seen": 452050944, "elapsed_s": 6620.438642501831} +{"iter": 58428, "train_loss": 3.664003372192383, "val_loss": 3.1727945804595947, "lr": 0.00027291860530803534, "tokens_seen": 478642176, "elapsed_s": 7017.370945215225} +{"iter": 61674, "train_loss": 3.6775317192077637, "val_loss": 3.065586805343628, "lr": 0.0002695648370092141, "tokens_seen": 505233408, "elapsed_s": 7416.363888263702} +{"iter": 64920, "train_loss": 3.6276235580444336, "val_loss": 3.1685380935668945, "lr": 0.00026604143066815615, "tokens_seen": 531824640, "elapsed_s": 7815.244849920273} +{"iter": 68166, "train_loss": 3.683591842651367, "val_loss": 3.008053779602051, "lr": 0.00026235410239144966, "tokens_seen": 558415872, "elapsed_s": 8213.673540830612} +{"iter": 71412, "train_loss": 3.652228355407715, "val_loss": 3.0844790935516357, "lr": 0.0002585088342201691, "tokens_seen": 585007104, "elapsed_s": 8610.022660255432} +{"iter": 74658, "train_loss": 3.592236280441284, "val_loss": 2.992539644241333, "lr": 0.00025451186442506697, "tokens_seen": 611598336, "elapsed_s": 9005.840159654617} +{"iter": 77904, "train_loss": 3.6231398582458496, "val_loss": 3.0135445594787598, "lr": 0.0002503696773860774, "tokens_seen": 638189568, "elapsed_s": 9402.070857524872} +{"iter": 81150, "train_loss": 3.61862850189209, "val_loss": 2.961634874343872, "lr": 0.0002460889930725512, "tokens_seen": 664780800, "elapsed_s": 9800.644268274307} +{"iter": 84396, "train_loss": 3.568911075592041, "val_loss": 3.0869228839874268, "lr": 0.00024167675614128833, "tokens_seen": 691372032, "elapsed_s": 10209.137510061264} +{"iter": 87642, "train_loss": 3.615424871444702, "val_loss": 3.023510694503784, "lr": 0.00023714012467005493, "tokens_seen": 717963264, "elapsed_s": 10607.348737716675} +{"iter": 90888, "train_loss": 3.5905284881591797, "val_loss": 3.0703125, "lr": 0.00023248645854486173, "tokens_seen": 744554496, "elapsed_s": 11004.023602247238} +{"iter": 94134, "train_loss": 3.482330560684204, "val_loss": 2.9946448802948, "lr": 0.0002277233075198449, "tokens_seen": 771145728, "elapsed_s": 11401.12715959549} +{"iter": 97380, "train_loss": 3.5333425998687744, "val_loss": 3.018442392349243, "lr": 0.00022285839896911855, "tokens_seen": 797736960, "elapsed_s": 11797.211772203445} +{"iter": 100626, "train_loss": 3.5357277393341064, "val_loss": 3.0929079055786133, "lr": 0.00021789962535047076, "tokens_seen": 824328192, "elapsed_s": 12193.576053142548} +{"iter": 103872, "train_loss": 3.558652877807617, "val_loss": 2.9821386337280273, "lr": 0.00021285503140123984, "tokens_seen": 850919424, "elapsed_s": 12589.411823749542} +{"iter": 107118, "train_loss": 3.5311083793640137, "val_loss": 3.0164098739624023, "lr": 0.0002077328010871441, "tokens_seen": 877510656, "elapsed_s": 12985.430166959763} +{"iter": 110364, "train_loss": 3.544306755065918, "val_loss": 2.9715845584869385, "lr": 0.00020254124432523806, "tokens_seen": 904101888, "elapsed_s": 13380.9542491436} +{"iter": 113610, "train_loss": 3.54836368560791, "val_loss": 2.975806951522827, "lr": 0.00019728878350253482, "tokens_seen": 930693120, "elapsed_s": 13780.561975717545} +{"iter": 116856, "train_loss": 3.494091272354126, "val_loss": 2.9469220638275146, "lr": 0.00019198393981216563, "tokens_seen": 957284352, "elapsed_s": 14179.889274597168} +{"iter": 120102, "train_loss": 3.5346028804779053, "val_loss": 2.9467151165008545, "lr": 0.00018663531942924373, "tokens_seen": 983875584, "elapsed_s": 14578.00502872467} +{"iter": 123348, "train_loss": 3.577099323272705, "val_loss": 2.9489409923553467, "lr": 0.00018125159954886065, "tokens_seen": 1010466816, "elapsed_s": 14972.863373279572} +{"iter": 126594, "train_loss": 3.491459369659424, "val_loss": 2.9584884643554688, "lr": 0.00017584151430886398, "tokens_seen": 1037058048, "elapsed_s": 15367.147650718689} +{"iter": 129840, "train_loss": 3.462223529815674, "val_loss": 2.9434103965759277, "lr": 0.0001704138406202565, "tokens_seen": 1063649280, "elapsed_s": 15761.51324391365} +{"iter": 133086, "train_loss": 3.4455323219299316, "val_loss": 2.9253101348876953, "lr": 0.00016497738392820283, "tokens_seen": 1090240512, "elapsed_s": 16158.703459739685} +{"iter": 136332, "train_loss": 3.5261361598968506, "val_loss": 2.9316606521606445, "lr": 0.00015954096392674466, "tokens_seen": 1116831744, "elapsed_s": 16553.807311296463} +{"iter": 139578, "train_loss": 3.4169921875, "val_loss": 2.9899818897247314, "lr": 0.00015411340025039982, "tokens_seen": 1143422976, "elapsed_s": 16946.26268863678} +{"iter": 142824, "train_loss": 3.4065358638763428, "val_loss": 2.877406597137451, "lr": 0.00014870349816585726, "tokens_seen": 1170014208, "elapsed_s": 17336.261150836945} +{"iter": 146070, "train_loss": 3.479498863220215, "val_loss": 2.8717873096466064, "lr": 0.0001433200342869823, "tokens_seen": 1196605440, "elapsed_s": 17725.82638812065} +{"iter": 149316, "train_loss": 3.4614174365997314, "val_loss": 2.919450283050537, "lr": 0.00013797174233630524, "tokens_seen": 1223196672, "elapsed_s": 18119.39487528801} +{"iter": 152562, "train_loss": 3.3986704349517822, "val_loss": 2.9236016273498535, "lr": 0.000132667298976094, "tokens_seen": 1249787904, "elapsed_s": 18516.443214416504} +{"iter": 155808, "train_loss": 3.418363332748413, "val_loss": 2.8491899967193604, "lr": 0.00012741530973199664, "tokens_seen": 1276379136, "elapsed_s": 18913.7375164032} +{"iter": 159054, "train_loss": 3.403559684753418, "val_loss": 2.9037880897521973, "lr": 0.00012222429503209114, "tokens_seen": 1302970368, "elapsed_s": 19311.084295511246} +{"iter": 162300, "train_loss": 3.3902008533477783, "val_loss": 2.839219570159912, "lr": 0.0001171026763839904, "tokens_seen": 1329561600, "elapsed_s": 19706.191299915314} +{"iter": 165546, "train_loss": 3.3977162837982178, "val_loss": 2.875446081161499, "lr": 0.00011205876271242858, "tokens_seen": 1356152832, "elapsed_s": 20102.504667520523} +{"iter": 168792, "train_loss": 3.3581061363220215, "val_loss": 2.885312795639038, "lr": 0.00010710073687949316, "tokens_seen": 1382744064, "elapsed_s": 20499.22602367401} +{"iter": 172038, "train_loss": 3.334460496902466, "val_loss": 2.8073480129241943, "lr": 0.00010223664240937181, "tokens_seen": 1409335296, "elapsed_s": 20896.007513046265} +{"iter": 175284, "train_loss": 3.359037160873413, "val_loss": 2.767348289489746, "lr": 9.747437043915003e-05, "tokens_seen": 1435926528, "elapsed_s": 21290.919981241226} +{"iter": 178530, "train_loss": 3.330073833465576, "val_loss": 2.8427131175994873, "lr": 9.282164691683035e-05, "tokens_seen": 1462517760, "elapsed_s": 21684.66718149185} +{"iter": 181776, "train_loss": 3.2883856296539307, "val_loss": 2.7900028228759766, "lr": 8.828602006734164e-05, "tokens_seen": 1489108992, "elapsed_s": 22078.475178956985} +{"iter": 185022, "train_loss": 3.3039779663085938, "val_loss": 2.7671658992767334, "lr": 8.38748481468728e-05, "tokens_seen": 1515700224, "elapsed_s": 22469.67812180519} +{"iter": 188268, "train_loss": 3.3069698810577393, "val_loss": 2.7552013397216797, "lr": 7.95952875053973e-05, "tokens_seen": 1542291456, "elapsed_s": 22862.731419563293} +{"iter": 191514, "train_loss": 3.3404059410095215, "val_loss": 2.8055148124694824, "lr": 7.545428097675506e-05, "tokens_seen": 1568882688, "elapsed_s": 23256.098614692688} +{"iter": 194760, "train_loss": 3.2542057037353516, "val_loss": 2.7680041790008545, "lr": 7.145854661512653e-05, "tokens_seen": 1595473920, "elapsed_s": 23648.27381157875} +{"iter": 198006, "train_loss": 3.2535359859466553, "val_loss": 2.7795395851135254, "lr": 6.761456679617264e-05, "tokens_seen": 1622065152, "elapsed_s": 24040.111388206482} +{"iter": 201252, "train_loss": 3.224721670150757, "val_loss": 2.7252328395843506, "lr": 6.392857770052093e-05, "tokens_seen": 1648656384, "elapsed_s": 24434.91019296646} +{"iter": 204498, "train_loss": 3.3069686889648438, "val_loss": 2.7585625648498535, "lr": 6.040655919666048e-05, "tokens_seen": 1675247616, "elapsed_s": 24828.67792034149} +{"iter": 207744, "train_loss": 3.203245162963867, "val_loss": 2.8056564331054688, "lr": 5.705422513965764e-05, "tokens_seen": 1701838848, "elapsed_s": 25220.907521247864} +{"iter": 210990, "train_loss": 3.2357778549194336, "val_loss": 2.7875421047210693, "lr": 5.387701410143218e-05, "tokens_seen": 1728430080, "elapsed_s": 25613.663235902786} +{"iter": 214236, "train_loss": 3.258211612701416, "val_loss": 2.7365469932556152, "lr": 5.0880080547631635e-05, "tokens_seen": 1755021312, "elapsed_s": 26005.971933841705} +{"iter": 217482, "train_loss": 3.281918525695801, "val_loss": 2.6761887073516846, "lr": 4.806828647541797e-05, "tokens_seen": 1781612544, "elapsed_s": 26397.99613571167} +{"iter": 220728, "train_loss": 3.2263457775115967, "val_loss": 2.698730707168579, "lr": 4.544619352573298e-05, "tokens_seen": 1808203776, "elapsed_s": 26792.957167625427} +{"iter": 223974, "train_loss": 3.1748874187469482, "val_loss": 2.7305846214294434, "lr": 4.3018055582839086e-05, "tokens_seen": 1834795008, "elapsed_s": 27186.7781727314} +{"iter": 227220, "train_loss": 3.2497973442077637, "val_loss": 2.6917898654937744, "lr": 4.0787811873140555e-05, "tokens_seen": 1861386240, "elapsed_s": 27579.863702058792} +{"iter": 230466, "train_loss": 3.181405544281006, "val_loss": 2.754448175430298, "lr": 3.875908057448241e-05, "tokens_seen": 1887977472, "elapsed_s": 27974.329090833664} +{"iter": 233712, "train_loss": 3.1910016536712646, "val_loss": 2.644343852996826, "lr": 3.693515294629337e-05, "tokens_seen": 1914568704, "elapsed_s": 28368.707402944565} +{"iter": 236958, "train_loss": 3.224911689758301, "val_loss": 2.6826670169830322, "lr": 3.5318987990097086e-05, "tokens_seen": 1941159936, "elapsed_s": 28762.860654354095} +{"iter": 240204, "train_loss": 3.1701831817626953, "val_loss": 2.6258621215820312, "lr": 3.391320764905286e-05, "tokens_seen": 1967751168, "elapsed_s": 29156.332152605057} +{"iter": 243450, "train_loss": 3.1385345458984375, "val_loss": 2.732184410095215, "lr": 3.272009255431443e-05, "tokens_seen": 1994342400, "elapsed_s": 29551.647617816925} +{"iter": 246696, "train_loss": 3.157072067260742, "val_loss": 2.686042547225952, "lr": 3.17415783251074e-05, "tokens_seen": 2020933632, "elapsed_s": 29944.447645902634} +{"iter": 249942, "train_loss": 3.1293420791625977, "val_loss": 2.7044787406921387, "lr": 3.0979252428527675e-05, "tokens_seen": 2047524864, "elapsed_s": 30337.687045812607} +{"iter": 253188, "train_loss": 3.1720211505889893, "val_loss": 2.6855907440185547, "lr": 3.0434351604155618e-05, "tokens_seen": 2074116096, "elapsed_s": 30731.93634748459} +{"iter": 256434, "train_loss": 3.20706844329834, "val_loss": 2.6418163776397705, "lr": 3.0107759857663707e-05, "tokens_seen": 2100707328, "elapsed_s": 31124.59182691574} +{"iter": 259653, "train_loss": 3.2143514156341553, "val_loss": 2.717590570449829, "lr": 3.0000000010394488e-05, "tokens_seen": 2127077376, "elapsed_s": 31514.58386850357}