diff --git a/.gitattributes b/.gitattributes index 56763615aacd3220a4d31de65bdd8f945036dd27..2227b854cb27fd665000e2e930c93cecca76bec1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -245,3 +245,100 @@ model-ladders/count-1b/1B/step16268/model_and_optim/__12_0.distcp filter=lfs dif model-ladders/count-1b/1B/step16268/model_and_optim/__13_0.distcp filter=lfs diff=lfs merge=lfs -text model-ladders/count-1b/1B/step16268/model_and_optim/__14_0.distcp filter=lfs diff=lfs merge=lfs -text model-ladders/count-1b/1B/step16268/model_and_optim/__15_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__4_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__5_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__6_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__7_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__8_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step16268/model_and_optim/__9_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__10_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__11_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__12_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__13_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__14_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__15_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__4_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__5_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__6_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__7_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__8_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step32537/model_and_optim/__9_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__10_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__11_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__12_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__13_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__14_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__15_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__4_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__5_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__6_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__7_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__8_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-1b/1B/step8134/model_and_optim/__9_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step18883/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step18883/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step18883/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step18883/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step18883/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step4721/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step4721/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step4721/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step4721/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step4721/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step9442/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step9442/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step9442/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step9442/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-370m/370M/step9442/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step11148/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step11148/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step11148/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step11148/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step11148/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step22297/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step22297/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step22297/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step22297/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step22297/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step5574/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step5574/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step5574/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step5574/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-600m/600M/step5574/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step2190/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step2190/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step2190/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step4381/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step4381/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step4381/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step8762/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step8762/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-60m/60M/step8762/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step11570/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step11570/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step11570/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step11570/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step11570/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step23139/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step23139/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step23139/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step23139/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step23139/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step5785/model_and_optim/.metadata filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step5785/model_and_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step5785/model_and_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step5785/model_and_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +model-ladders/count-760m/760M/step5785/model_and_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__1_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b55b29d4d52a31c64f586b54ae6b6ce07d522c93 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522cf8767c0aa3baa5ec37e04da91f0ed2a3edc9f6db1ab65b455946518019f2 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__2_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eb99d74f19c1d7ee9bd87c14566f1de01de3d7b1 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f50f99929c7c7a86dcde28ede47c785379a04121505336b75eb00d7aec5c8add +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__3_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dbc99248ec1f97cef238a1d79f70d53b6412f7dd --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:888716e1f60048e7652ede411ebd8ae789c4717aa80d1905e53a60f4fd29289d +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__4_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__4_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9d0783eac37813ddd190a979b71d9f9accf50177 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__4_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:367988326492117313a5a9e4da0a35766a432071a92565b0a3f6fe8e286c0008 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__5_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__5_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1b92072110e273ccf394aea811670e80ca3897d8 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__5_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33317f9a0d659cf234c1fe2a9beea1bda0f2b3e2eaea22db4f201fe30bf5d71f +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__6_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__6_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7f1b3992b0738f3edf7c12a8df4b99e1d3082e5d --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__6_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56257ea3568f7bea6678b686f8e48677d377de6f127d6344cd1140f6b9d5936d +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__7_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__7_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4cd513f73d0400dc194b2319063d6d3287e7a76b --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__7_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f6cba72f7b9a7d16c8f1c76720ab5e4e074bbb4d74543449428583521810be +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__8_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__8_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6e11316f75a256a4bcf55f18b742fa880e3cf1a6 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__8_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed623bfea53d87c5d3a1809393d1aed06548327b3d9668e9166aba61f82e204 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/model_and_optim/__9_0.distcp b/model-ladders/count-1b/1B/step16268/model_and_optim/__9_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a7c738cbcfbb72bd54db14dc4cc46c3f6baa65f9 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/model_and_optim/__9_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7f493b2d1e85ffa5da3c1830fcee5bcd428c4105622a3c4036da3976265757f +size 1114546935 diff --git a/model-ladders/count-1b/1B/step16268/release_args.json b/model-ladders/count-1b/1B/step16268/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..7dbdce2d191faf48524633e43e121fdab0a913ef --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "1B", + "seed": 1042, + "fixed_global_batch_size": 1572864, + "sequence_length": 8192, + "trained_world_size": 16, + "d_over_n": 20, + "step": 16268, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-1b/1B/step16268/train/rank0.pt b/model-ladders/count-1b/1B/step16268/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c946cf3cef51f0944ab1736a8830be9f13d539e --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42832ead1d165df3d0fd45ba8aa74673d8b545a42b1e444b2c0204897d023401 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank1.pt b/model-ladders/count-1b/1B/step16268/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ded7c1722f599e0aea477bbba3b95ae6e6deb5b5 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25fb2dfcfa2dfcf535cbb12f9eb208ad3888133ce40ec950f67ce21faa8975d3 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank10.pt b/model-ladders/count-1b/1B/step16268/train/rank10.pt new file mode 100644 index 0000000000000000000000000000000000000000..38ade7406e7f05b1971780759e501aff0f04aef2 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank10.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d468dc101764996bdf790bd2096e1a34ec8cae5267529d7bd73704890d512b3 +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank11.pt b/model-ladders/count-1b/1B/step16268/train/rank11.pt new file mode 100644 index 0000000000000000000000000000000000000000..39e6831dc0f1558d8f877549168f80ab16093f08 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank11.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afb37bfd48e806516e631a374d86876d0639f670ca084c70927181defab5bbda +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank12.pt b/model-ladders/count-1b/1B/step16268/train/rank12.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcae383ec93a89ee4acd175c5febe72279264af8 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank12.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b3643c4a730e899ae64a8482cfa24a0cf00f5b380b6e11796b3c4a75ca1f0c8 +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank13.pt b/model-ladders/count-1b/1B/step16268/train/rank13.pt new file mode 100644 index 0000000000000000000000000000000000000000..f696696bf25572a8778049c895ac4be2107ed96a --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank13.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d464aea1c622744d3cf5ef0ae34c21ab132376170b22e3a3e8e48860fa82bd7 +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank14.pt b/model-ladders/count-1b/1B/step16268/train/rank14.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0707a7e5d6c24a23916b1096b0e07ce6eeff70f --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank14.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7beb24b619c8d7ae39d1917a3471d63d8772311072d66899373ba769abd30fd +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank15.pt b/model-ladders/count-1b/1B/step16268/train/rank15.pt new file mode 100644 index 0000000000000000000000000000000000000000..08db3b24822b1d2f3067508b142195e5b017e9a6 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank15.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c04126af9e3a19b9e24f2dacabfbffe87d7a87658be1b0d2d530514ee0f1e935 +size 15645 diff --git a/model-ladders/count-1b/1B/step16268/train/rank2.pt b/model-ladders/count-1b/1B/step16268/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbdd435361b1e625b9586d5e7744d9da37144f0d --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:814b504058f9c4dc6c156ab594a388e5a42ad97a7573c0d3bf609ab55c1ec1ed +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank3.pt b/model-ladders/count-1b/1B/step16268/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2cfc318d6a8e8b403351e9c4d7eb5cad4a17cc5 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93cd9d2da2d5759f95edfb90f01f261a1705405e9de9dae2a12a7ab95020a0e2 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank4.pt b/model-ladders/count-1b/1B/step16268/train/rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..5125785c78c0cbc9b4265e92bf654ff84b50fdfb --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34386ae7d6e4f6e6f6e6f0d7f4064a50c2339e3b41803016f468acec281f9f12 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank5.pt b/model-ladders/count-1b/1B/step16268/train/rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..f02c3163674881f1f0cc95f6517e6007be1d8891 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a79cd991a0be6adca4368aba6cbba7196615a8963731d587f06d7226972aec9 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank6.pt b/model-ladders/count-1b/1B/step16268/train/rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a8687e463c8a087d6a66c3db9a73a330f09d8bb --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0ff9137f8b9226498de696f2b49c64bf3d26dd6252a42759f80e113b833b2d +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank7.pt b/model-ladders/count-1b/1B/step16268/train/rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7124f49cac97537aaf7f8f2b1a5fb492a8ad21e --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04153f529b4fe903379d7edcbfc5c701f358b100e99dae415c674fea1ed5a5d1 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank8.pt b/model-ladders/count-1b/1B/step16268/train/rank8.pt new file mode 100644 index 0000000000000000000000000000000000000000..62e7c5c3540d5e4fc5393f2af4def85e3968a4af --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank8.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a59240366d78a170e0224bd97190cea341bd68fd190cc6c351ab27962f9650a1 +size 15637 diff --git a/model-ladders/count-1b/1B/step16268/train/rank9.pt b/model-ladders/count-1b/1B/step16268/train/rank9.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cd431cb9aa5f8569e6d2a552d54e062e9efc8c3 --- /dev/null +++ b/model-ladders/count-1b/1B/step16268/train/rank9.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82c57cc6c715b6916d6af784e2694cd5ce33b97f1c69ec405f72d12782d55d7c +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/.metadata.json b/model-ladders/count-1b/1B/step32537/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-1b/1B/step32537/config.json b/model-ladders/count-1b/1B/step32537/config.json new file mode 100644 index 0000000000000000000000000000000000000000..992bfd5b2b0e7109a6f9b76c1c7107eb340ff134 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "1B", + "model": { + "d_model": 2048, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 8192, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0010308784697907792, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 12793675776, + 12793675776, + 25588924416 + ], + "warmup": 1279395840, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/.metadata b/model-ladders/count-1b/1B/step32537/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..961e537b09f7630e42df46fc8eec92685756d02d --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1b67b1cbabf26b35c27923de71d56218a35aa0d00d3683762d8aada37daa157 +size 1961340 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__0_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3aa63e20f4bdaf47d3fc65069879880cd8527bc2 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5307e60b32df3e543c53021c8da6b82d61070ea472d67c4378e872c453d3d704 +size 1115973577 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__10_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__10_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..92860180fe321199c47ffe60f77ae84ddf60ff50 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__10_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dbc2556684bf874d7f2639da168380a10d7a8a0d229281828bf13a361e63165 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__11_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__11_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eb1b97635205d22a3044c8cc8e46ae660fbc0af6 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__11_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eef597e1313145251f2da9486cf7b452a76c39e054ef039a335a563f20a0acbb +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__12_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__12_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d711de8a98d4e5dc969ab9030bf0f136349fc893 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__12_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef34efd885ebc1bc86f539dfe43ae8a8549d7172b88008963195a226c59d311c +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__13_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__13_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bb05b17d12588e11e8f95307f9db8f5e73151f60 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__13_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c11c65225f9f11fb382c58f6a166706992adb62f635590ed33e0cd7b8d50de0 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__14_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__14_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..03cd20fdc5f49618b7d1c4e817f5be8ff6f43d1f --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__14_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4120b424e9542ea8d3a95919aa5b04eafe3eb046c50619591838e119891d26e6 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__15_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__15_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..80e9d310bc5dccac405bef8a4a9b9b58c03b9bdb --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__15_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5ebd8eff07c3a5d5da6bfd16e269dfdd3260c74ed18a031df059bf1b5fa1f00 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__1_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b2e83286b288ff2f26a1397c32bf03df56c20d99 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1647e0f97ba5f7f799de011b72f6c4b1dcffa66d1b04fa1c565f81b7346bca7f +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__2_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8cb03ebc68dc7ae635945d437d53ea99e9e8ec35 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb691764c2bf41c80cae2c183896ca2ff83b2ce95593ff08041eaf1546ff53d4 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__3_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..589f5de7a44c31b90e5914e63fceae7baa44392e --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca6f555ad37b141203625f4814337fd3d60578bbeae8cd58c1f1f3b920b0f72 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__4_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__4_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e335f163ed75c7ab9e4ca13e12518724fbcf657a --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__4_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55eff1ce44b386f8ca3b44b5bb95d23dfb1cee5cb97da3de209a7eafed9a2c42 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__5_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__5_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2d746c1b689eab5d5b8cb47cd60f55ecf3c2a097 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__5_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d010deeecff2da2d81dee790698d2ce43de8f7b030b740c273720d39c0c122 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__6_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__6_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e962ac4c644dcc62a481af5a83a4224789d4ab0d --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__6_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45cb07440221a467be85ac6da81cdaeb0d4feef4087670201a0d08a97b8159a0 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__7_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__7_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..815d795fe7c86252fef2964cc38702e82cf0e272 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__7_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00596d7b52e21b9265c431ca877e98ec7df35b5462c08ead87a63c03e3b99e21 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__8_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__8_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4ce28f2c8034a4850c5f85d8afec34d5dad8aa9c --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__8_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e63ac5717f4af5fc458133b5fc8ae629954127be9ee15652226ef535a40c98a +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/model_and_optim/__9_0.distcp b/model-ladders/count-1b/1B/step32537/model_and_optim/__9_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..78de8c37d940610eb364a863586d85e961ce35aa --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/model_and_optim/__9_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6940926d976a2716271174fd983ece3b2d50ac1fb477d357aad46b5f4857acf8 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step32537/release_args.json b/model-ladders/count-1b/1B/step32537/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..0f725a70c72c4e5cfd79f9ea2752938a5099b1ff --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "1B", + "seed": 1042, + "fixed_global_batch_size": 1572864, + "sequence_length": 8192, + "trained_world_size": 16, + "d_over_n": 40, + "step": 32537, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-1b/1B/step32537/train/rank0.pt b/model-ladders/count-1b/1B/step32537/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a804be013346f22d18f3bb873daed221a295b7e --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62288d59054b3bb379a7e45eccf4dc9d1a70813c2e9bb8e8b4cb30d601b48bab +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank1.pt b/model-ladders/count-1b/1B/step32537/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2573f7dded3d15569b9e3831280be2cca10cc06 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca166a4797f2918aef37ac26e0b70dcce4fcf5f6b3cc79ebf0ff5d2a58efd3e3 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank10.pt b/model-ladders/count-1b/1B/step32537/train/rank10.pt new file mode 100644 index 0000000000000000000000000000000000000000..e886f6ee13c7da53e6ea8992f414040b06c5e9d7 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank10.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:686605674979701602b6117e2608018be9d7e250d7a981dd1060b860bc430c92 +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank11.pt b/model-ladders/count-1b/1B/step32537/train/rank11.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e45e0165d879c6b611b0891d43e0b37561e5cea --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank11.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a333f2f2fbcf1b4a1c1dfc7926bf4b210cf4ab65f9139da68015821bc820e3 +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank12.pt b/model-ladders/count-1b/1B/step32537/train/rank12.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b4c2afbd7623576ec1347caaec420c96a20a10a --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank12.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f25cfb47163d9636f42ce462d30b07bb46c111e6860c27c772cac17dd4c6b9c7 +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank13.pt b/model-ladders/count-1b/1B/step32537/train/rank13.pt new file mode 100644 index 0000000000000000000000000000000000000000..6643727e284acf6ed02a85507f48c7aa26432421 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank13.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e61adeeb971e3b6a2b4cf478d38b501929281057781a7fffbc63db233fda6603 +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank14.pt b/model-ladders/count-1b/1B/step32537/train/rank14.pt new file mode 100644 index 0000000000000000000000000000000000000000..86e712b7d0448cf444703282bcd3810483290669 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank14.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a614923b2a9b1ad2412074382e2641bba3800f7fb2fd60287d4323d8af91d3dc +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank15.pt b/model-ladders/count-1b/1B/step32537/train/rank15.pt new file mode 100644 index 0000000000000000000000000000000000000000..820c3d65c7a534aaa3cd2de7de600b3406686da5 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank15.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c472feaf7e4ad6c330f01faf9fe7ccb4b5abd645abd47480d6cf528b41eeee7b +size 15645 diff --git a/model-ladders/count-1b/1B/step32537/train/rank2.pt b/model-ladders/count-1b/1B/step32537/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..8227dfe80d0118ccb19ced13709954d32de7590e --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9378b73b43fefc8474bdeef0e48c826fcc7662e808f21654a7efd4c5570aaeff +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank3.pt b/model-ladders/count-1b/1B/step32537/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..f98d4538efb10ebb8f7ffa786ae4fa174deeaef2 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a43a8148b7fb95186733435998e68b0583dbb0680aec19bd515064ed4ecff7c1 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank4.pt b/model-ladders/count-1b/1B/step32537/train/rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..14bea44158a2f380b2e1e289e9ab6bd1fed14a65 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d847907959399a771fb20e185a10ab0454c448de0bcbc320c0a09a522fafcc7 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank5.pt b/model-ladders/count-1b/1B/step32537/train/rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5553d54d2bd525da28f36e867abbc70b79c08e7 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7c942788129fcf00719de74820c8e9c984873c383aec5d7e3a0f0ee113164c1 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank6.pt b/model-ladders/count-1b/1B/step32537/train/rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..1efa42328f3136adcb2e5ed6db686c049a446769 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e57ada05e708a4b747d2200ee1f2dd8ec72cb09c56f65c3a965d2b1789032d03 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank7.pt b/model-ladders/count-1b/1B/step32537/train/rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..f206e5c714cad8f8231db96fa74fd5e11b79c1da --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79882ba195928cf0b68ece69a4b89eda29c2eb97351937a3db5c3aba01f60aa0 +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank8.pt b/model-ladders/count-1b/1B/step32537/train/rank8.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a2f7ccb51d0f6c378625ce8adf290dde340b404 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank8.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2afadcb18c0b781824219b7944a446f7c422807c96f7804b45e3e674496a549e +size 15637 diff --git a/model-ladders/count-1b/1B/step32537/train/rank9.pt b/model-ladders/count-1b/1B/step32537/train/rank9.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3d716723fc8cc48357194f5581a7d025d995931 --- /dev/null +++ b/model-ladders/count-1b/1B/step32537/train/rank9.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c51e08784d4ed8c61a46fa3ab38ca94912227787e3f09202e31cfd9591ee7b2 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/.metadata.json b/model-ladders/count-1b/1B/step8134/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-1b/1B/step8134/config.json b/model-ladders/count-1b/1B/step8134/config.json new file mode 100644 index 0000000000000000000000000000000000000000..992bfd5b2b0e7109a6f9b76c1c7107eb340ff134 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "1B", + "model": { + "d_model": 2048, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 8192, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0010308784697907792, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 12793675776, + 12793675776, + 25588924416 + ], + "warmup": 1279395840, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/.metadata b/model-ladders/count-1b/1B/step8134/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..cbb653ed8ee5b1e057346bb0c15b45a02650ca59 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c267f6bcd00525e49e2fbdfd86e1d8100f63c8eaee202e3d9a8158a5997a41e4 +size 1961340 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__0_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1c7147cd33bcdf8ac21fae7469e03c2248cb28d3 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f58a8d0142ea4ec9912d3f7e9dc9e7b38612e7f81db0c4d83097133c30543de8 +size 1115973577 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__10_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__10_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..149d1b858d350853c7739f52466cd8bb1cc9ea85 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__10_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd3b7ee140f7b13b31ec53bb8018ea563706ac1c408fddb7e58a861782834514 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__11_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__11_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a5b1d78cf13d1869d955d7a0a4fe622a14486520 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__11_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9738739981e544853d143eb5edae4fb1c250089672eadd21744fc02493a9f49d +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__12_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__12_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..52417ecea106b2f731855ee6636a23be37f68654 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__12_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49db5f001251b2a20a8a08ee4285899f1a1850f6e154612265f28a76d3615f6d +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__13_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__13_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..13065395d089d63602bef6895e20541df60921a1 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__13_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41d4c9725e10985d944adf59f40a172bbbdd95a0685ebd15a6254a987d087bc7 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__14_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__14_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..85bce40cdca2af9d388acb0b916d7743e71ea62a --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__14_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1430f418fe18789b7a0bd124898e7cd03bbb606e0ed0a0536f8b23b31d5e9ba +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__15_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__15_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..512c1e71ad8e071fffe8eb6c632829ec7b7b5bd5 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__15_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f3c68b933a0994aa1a61e2e688ce68a86f79a102516bf91e348ab42e77b103 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__1_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..aaa779c7309775e8be48c8209c764fc90aca6ed2 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d1d53fb8fe29b0f609651194516f242c0382aeaea2b00920251187650c26d93 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__2_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e9663ba04fb622e48aa35e7e216bdc3550c3e8f5 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86ab8b0ed3a7306a52733058a9e92ea173203826d6313781ff3a0a6f5f5c3141 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__3_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..eeee06f3d0cb38a6f5aea2534e1ac500c6c1e9a8 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e4f7d20c95bb5179f6a2da2f699f516b9cdde6089287412c715148feaae14fe +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__4_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__4_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3e4a25e78f4aefc9737917e37cb833cc8d104574 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__4_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:303953fc86aca9facc7f664c046c5b825ca9679e38dd644b653fefd399c1947c +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__5_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__5_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..575d76d906fc6f22975a13fb56d7bc1cb1885e7f --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__5_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d14413fb444b7977a617e50ee94629f093a7147da51315014095cb5002fe46dd +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__6_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__6_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..989fd228ccf954dedea7d97f0888fcde03df30f6 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__6_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa7d406269595540639e9684bf0e818b0287e427656d89770ae40463b184172a +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__7_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__7_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..84269e362b1af38d09c9989633e07d295b8ab75e --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__7_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e7d578227b8c9d86f17653fc0b91e7c5717724b044ec11356df68011e3b095 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__8_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__8_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3a9c7807a7aebb57142bafe79d1505aeca0bbe05 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__8_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f9f9061bf123edde6f4546f8b128f1da933472c89be77e2424e0a6422f398f2 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/model_and_optim/__9_0.distcp b/model-ladders/count-1b/1B/step8134/model_and_optim/__9_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d35989b5253d8e2e4dd925295bf1fdb8077cc974 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/model_and_optim/__9_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9685b879a87e2b2a6b2aa9540996e0b3b59e1750aa62704822fb3e8ca6d4583 +size 1114546935 diff --git a/model-ladders/count-1b/1B/step8134/release_args.json b/model-ladders/count-1b/1B/step8134/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..454155e43e803f0c1e586d270d06c1ae5428e8c5 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "1B", + "seed": 1042, + "fixed_global_batch_size": 1572864, + "sequence_length": 8192, + "trained_world_size": 16, + "d_over_n": 10, + "step": 8134, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-1b/1B/step8134/train/rank0.pt b/model-ladders/count-1b/1B/step8134/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fc095e68dc9700a61a3cdee20846edbd1b4f83b --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2bd2b5dade6a59063b55b4ab1d909865e09d26fbdcef08eb2f1d474dfe27bd3 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank1.pt b/model-ladders/count-1b/1B/step8134/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fb625ea0d47ebc9048c306b77a5f95813d92fe2 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc9092a537720cfe4c119a29417cc8389129e93cc3dd8bda14d4d7770298a3ad +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank10.pt b/model-ladders/count-1b/1B/step8134/train/rank10.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a190147b000406cdb9411b0686a175a8e902a04 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank10.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3bfe2a71e837f7ef55067a67f2043968010f686401483e38b1812ad2ac363ca +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank11.pt b/model-ladders/count-1b/1B/step8134/train/rank11.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fa83b2e846949c1036216a6f13edf432599e8e6 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank11.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5853ee287f4b790cb87cf559ad2ca6406aec8cb506e02f632095779722667045 +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank12.pt b/model-ladders/count-1b/1B/step8134/train/rank12.pt new file mode 100644 index 0000000000000000000000000000000000000000..25fb52ec933236bfa11a42422f2b32d395fa0b2f --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank12.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65e8d909f32f8f7b7a092ffcce8bda74dbc04efaaad3d1bdffefaf9e980821d4 +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank13.pt b/model-ladders/count-1b/1B/step8134/train/rank13.pt new file mode 100644 index 0000000000000000000000000000000000000000..6547ec009a8c7e9bf241d0792415b4f5d02a4b71 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank13.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fddae6b6c174d3f5d794dd50ba58172c7a67496c6f976e079d2249d0a7fea4c4 +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank14.pt b/model-ladders/count-1b/1B/step8134/train/rank14.pt new file mode 100644 index 0000000000000000000000000000000000000000..636075c60b7f5065fa22ecbcac405655ffc13cc5 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank14.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10b97e916c6695be604e5fcfb6ada4b27ccd72807e1b9774743fcc10b9606a4a +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank15.pt b/model-ladders/count-1b/1B/step8134/train/rank15.pt new file mode 100644 index 0000000000000000000000000000000000000000..27c264d167d6e85e65e7fd9a677c1fefdc870234 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank15.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ee336d3beb656bddf63143b4bcb30bb6dff24a1e7d56fa4c4e3bcaffb46f581 +size 15645 diff --git a/model-ladders/count-1b/1B/step8134/train/rank2.pt b/model-ladders/count-1b/1B/step8134/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5255653c08b911d2ee70f16dea323c3b4d779439 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd3f513ff6a69a4828d583895c6fbfc59c328c8f2ed02f398c1de1c523a0b7b +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank3.pt b/model-ladders/count-1b/1B/step8134/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f32dcd99b53426819cfb9f96ae19accff3fa5ed --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:908403df3d1a2e0f4faba4b4703a4966ad2e72f3915a72abd84a88a30e3fb84b +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank4.pt b/model-ladders/count-1b/1B/step8134/train/rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..282a9cb392d40c2aed108c626fdf7348dccf9d1d --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1ee2e12e8d1095cdcf1b3d3aa14c8240e8f0e36ef741d04f8a8f98a0763069 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank5.pt b/model-ladders/count-1b/1B/step8134/train/rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bfd9929b95bb94fa55f7f8a23dcf3cf48d091b7 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c027a2258d4a1450188018a5db4a058fb85a32b0a833f6f15d2160456d7515 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank6.pt b/model-ladders/count-1b/1B/step8134/train/rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b43101df5579471f0ffc0306d5b64723aafee56 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78cb29d2ca3e533a27f118279031dcab6eb0d1291516b83ad22264d77d24cfc9 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank7.pt b/model-ladders/count-1b/1B/step8134/train/rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..76d742329f448098208e1b512a0645a0f8ca4d19 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3056332e6ebcdc3aeb9821e108eaf7900fcdaa7bdd23df63c53ba09f8b8793 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank8.pt b/model-ladders/count-1b/1B/step8134/train/rank8.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb0f1c08aefbae5a9ccb38397743434c0e281532 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank8.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8babcdf64ce8c0f3305c62353569d0cb11d31591fac893b77c2e06b91f1f7884 +size 15637 diff --git a/model-ladders/count-1b/1B/step8134/train/rank9.pt b/model-ladders/count-1b/1B/step8134/train/rank9.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f226b2219ae1c72ff9f0e1ea6efac97926b24c8 --- /dev/null +++ b/model-ladders/count-1b/1B/step8134/train/rank9.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17251f5e38b2118121f15520d015b6c124edacb7b1f3b63ce8482ea76ec47054 +size 15637 diff --git a/model-ladders/count-370m/370M/step18883/.metadata.json b/model-ladders/count-370m/370M/step18883/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-370m/370M/step18883/config.json b/model-ladders/count-370m/370M/step18883/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bacde3324aabe380ba154a76a1aa77ecfdaec5e --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "370M", + "model": { + "d_model": 1024, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 4096, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0015570961583169151, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 3712745472, + 3712745472, + 7424704512 + ], + "warmup": 371262464, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-370m/370M/step18883/model_and_optim/.metadata b/model-ladders/count-370m/370M/step18883/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..95ab371030ffdd957558bf765c25bcaf4eb98c1d --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15e1d203db0a9ff64c3687851f9f748ebf78c47a07a29a01a611a24262bad5da +size 835959 diff --git a/model-ladders/count-370m/370M/step18883/model_and_optim/__0_0.distcp b/model-ladders/count-370m/370M/step18883/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b7df589e9c23bfabcd0fa72d33dc8a624b2ed7df --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b327b1097d225555be2169bd597d27782822ce4133c6e218b9a27df0742e7f19 +size 1424357833 diff --git a/model-ladders/count-370m/370M/step18883/model_and_optim/__1_0.distcp b/model-ladders/count-370m/370M/step18883/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f0a6d10437facef80447719d2a1339d4fd36509e --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb09f562ea0f7e7efd09f7c3c40f63d6a2d1ab2bf2c599cde19c490e8bb5eb97 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step18883/model_and_optim/__2_0.distcp b/model-ladders/count-370m/370M/step18883/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2efac73610e3b4425580c15ec81aba151d2fa07d --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15bfbcc3e27ad984a0b5c43586b161f052bbd8615a616476766a53b76d6de0a8 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step18883/model_and_optim/__3_0.distcp b/model-ladders/count-370m/370M/step18883/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c9cd4ae1f672aa6e88f8cfa230c5a0bb43b0195b --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90cf1142a2b9d855b34093230331c93801c4834b13d2774969b0eb97740b957b +size 1422931191 diff --git a/model-ladders/count-370m/370M/step18883/release_args.json b/model-ladders/count-370m/370M/step18883/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..aa6b4785277dc869d71aafad075bc87a97190f2b --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "370M", + "seed": 1042, + "fixed_global_batch_size": 786432, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 40, + "step": 18883, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-370m/370M/step18883/train/rank0.pt b/model-ladders/count-370m/370M/step18883/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..262332a93abfb34f0780cc26de36224a9722f39a --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43cc7e14fd5a2000550b049f30a98bf45121a110ae083bfd3235eb746faa0d0d +size 15637 diff --git a/model-ladders/count-370m/370M/step18883/train/rank1.pt b/model-ladders/count-370m/370M/step18883/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..3626bbc7f755f122b47cfeec94aad9b4edd0b7f0 --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f8f12dc15c8a246deaeeed3581865ea95625d7ec220aaec32bac2d1aacc84c0 +size 15637 diff --git a/model-ladders/count-370m/370M/step18883/train/rank2.pt b/model-ladders/count-370m/370M/step18883/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..8201213138e8f51878f6e4f23462fad86bab2b34 --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40806b576a1e17f6dfeeb75a73b0eaaca0faf42a3dbe50b9ee7d38ddd3cc822 +size 15637 diff --git a/model-ladders/count-370m/370M/step18883/train/rank3.pt b/model-ladders/count-370m/370M/step18883/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b4a31da0b7af498d0c89deaa1c3fda66b892d26 --- /dev/null +++ b/model-ladders/count-370m/370M/step18883/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6ff6d3eca9c8b52199b8558f9f43ccfe217e72c67ae93c0a627dbfb15d7505 +size 15637 diff --git a/model-ladders/count-370m/370M/step4721/.metadata.json b/model-ladders/count-370m/370M/step4721/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-370m/370M/step4721/config.json b/model-ladders/count-370m/370M/step4721/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bacde3324aabe380ba154a76a1aa77ecfdaec5e --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "370M", + "model": { + "d_model": 1024, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 4096, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0015570961583169151, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 3712745472, + 3712745472, + 7424704512 + ], + "warmup": 371262464, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-370m/370M/step4721/model_and_optim/.metadata b/model-ladders/count-370m/370M/step4721/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..33780c46e37d817cb605ae9901a732e6c423321e --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fabffea44e10654444b6a41791a2b3c696b2c2b6a52ea4cd44464f9ee2b719a +size 835959 diff --git a/model-ladders/count-370m/370M/step4721/model_and_optim/__0_0.distcp b/model-ladders/count-370m/370M/step4721/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0c27f7d7b7227eb6095a7ff4a9e9d21d6a905e55 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9668e04f0c4470e64e31dc4df636dea6ba21ba304c8ee0693bacaa7d29b0b00c +size 1424357833 diff --git a/model-ladders/count-370m/370M/step4721/model_and_optim/__1_0.distcp b/model-ladders/count-370m/370M/step4721/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a8902d1c08dd8bec7d8f4002082e73ca6f94ded5 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b0c0b0f24abb70318e6d5e980d777f27ba9315557d3e87d2268479a8efb9138 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step4721/model_and_optim/__2_0.distcp b/model-ladders/count-370m/370M/step4721/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0077d158df9a3ea5b0122b15e6dcea220b5d215a --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aaf1cd6b90cfaf6dc0245c0420b87a9d97ba46c68c808a1a723a1d08019d7a5 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step4721/model_and_optim/__3_0.distcp b/model-ladders/count-370m/370M/step4721/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..beb299668b227d3321725f7ca8a65f5af0232682 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f83c86b0cdbdcad586b48f7fd9152e97f4c2c64f31f321dcb7da66a749b0202 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step4721/release_args.json b/model-ladders/count-370m/370M/step4721/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..1eec088c1c38807fbbae5c725350f2975c5f2c19 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "370M", + "seed": 1042, + "fixed_global_batch_size": 786432, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 10, + "step": 4721, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-370m/370M/step4721/train/rank0.pt b/model-ladders/count-370m/370M/step4721/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..292423532ba06f840efcd6e0e76d9fe4b87f6a03 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f61a3cf421024f1fd4f13c78f3ab27166a061a865156a67976db23ee1700775 +size 15637 diff --git a/model-ladders/count-370m/370M/step4721/train/rank1.pt b/model-ladders/count-370m/370M/step4721/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..35e08de504be7c2a1b3ac5f718ed18a320938383 --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69cda08d69808e12bc52f11b2ee1625da4e5b24fddd72dcaf94cd2c976500f1b +size 15637 diff --git a/model-ladders/count-370m/370M/step4721/train/rank2.pt b/model-ladders/count-370m/370M/step4721/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..857000e8733665ddd337bf76ffc00770d91eb8af --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b86eef7a12b394168980d42b2e4baccecd56628ddfc516642587a7856c952ba8 +size 15637 diff --git a/model-ladders/count-370m/370M/step4721/train/rank3.pt b/model-ladders/count-370m/370M/step4721/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c945e1d00baec98d95fd5a4d1b12a37dc8d930e --- /dev/null +++ b/model-ladders/count-370m/370M/step4721/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf66db9c8127695f79a6498395dc61addb9f503f4dc588c1923dca4cff87bcd +size 15637 diff --git a/model-ladders/count-370m/370M/step9442/.metadata.json b/model-ladders/count-370m/370M/step9442/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-370m/370M/step9442/config.json b/model-ladders/count-370m/370M/step9442/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bacde3324aabe380ba154a76a1aa77ecfdaec5e --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "370M", + "model": { + "d_model": 1024, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 4096, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0015570961583169151, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 3712745472, + 3712745472, + 7424704512 + ], + "warmup": 371262464, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-370m/370M/step9442/model_and_optim/.metadata b/model-ladders/count-370m/370M/step9442/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2677942b91615c0bb820b067c68faec38d641c56 --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31f38dafa37ef7fe8df1bd4ead2b16f48be3f2ebd0eb74a92824c27bbde55bc0 +size 835959 diff --git a/model-ladders/count-370m/370M/step9442/model_and_optim/__0_0.distcp b/model-ladders/count-370m/370M/step9442/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e4ea2865b321a9f5a3a797f86d9be8f81accc1bc --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026b9636ace5dde8064f762e63d8c59f87c8baf13d819d16e698ce685de4ae74 +size 1424357833 diff --git a/model-ladders/count-370m/370M/step9442/model_and_optim/__1_0.distcp b/model-ladders/count-370m/370M/step9442/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..46a705f40630f3c4a3028de36bef92c499e1d9cc --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49067be555cf65df306f7ac7306c9911784bfeea634d860d21d951097dc9075f +size 1422931191 diff --git a/model-ladders/count-370m/370M/step9442/model_and_optim/__2_0.distcp b/model-ladders/count-370m/370M/step9442/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..5aa67a228378114451a5a3c0b0dea280a9474b2b --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:276c6d91508c53d9a4334494bc1f26285aa780d96ed5fa0bca2f759b42d3804f +size 1422931191 diff --git a/model-ladders/count-370m/370M/step9442/model_and_optim/__3_0.distcp b/model-ladders/count-370m/370M/step9442/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2aad5f8fe1194e55a9e51200e3e5281d5540c4f4 --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3469a23763355ba9320a48a36fdaa8f6e9f2250f940e171e39a5b279ca7862 +size 1422931191 diff --git a/model-ladders/count-370m/370M/step9442/release_args.json b/model-ladders/count-370m/370M/step9442/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..29edace55c71bd923e41859bcaa80cd53940cfc4 --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "370M", + "seed": 1042, + "fixed_global_batch_size": 786432, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 20, + "step": 9442, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-370m/370M/step9442/train/rank0.pt b/model-ladders/count-370m/370M/step9442/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f611d4877e1d26b8c15befd0e2b623cb771d599d --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91fafb5e7ebfeae422377c9d9400e896b660577737451405092eb3f54e47fc80 +size 15637 diff --git a/model-ladders/count-370m/370M/step9442/train/rank1.pt b/model-ladders/count-370m/370M/step9442/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4987275bce165dda44c9d140dccabf1d70d7c76e --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d221312e1b5d69f056cdbbe09eca5d870a4bdc975cdeec4511af995fed26a223 +size 15637 diff --git a/model-ladders/count-370m/370M/step9442/train/rank2.pt b/model-ladders/count-370m/370M/step9442/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b97bdedb0ee780e97660dfb02e1a6a9ba0ff7cd --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb0a16160d7251f5c79bf40ad205e32db936dab4e96dc3ee077306e27dfb007f +size 15637 diff --git a/model-ladders/count-370m/370M/step9442/train/rank3.pt b/model-ladders/count-370m/370M/step9442/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..56cb5840d7d5e1a1435afb274ffce1d89e369046 --- /dev/null +++ b/model-ladders/count-370m/370M/step9442/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363c36737e7d8cb755a8b2285a0eaa243ee97f1fb2060863eb2a3bca97fe0662 +size 15637 diff --git a/model-ladders/count-600m/600M/step11148/.metadata.json b/model-ladders/count-600m/600M/step11148/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-600m/600M/step11148/config.json b/model-ladders/count-600m/600M/step11148/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c39d97993352ff90b9fc8f1ed5d6f8964706d43f --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "600M", + "model": { + "d_model": 1280, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 5120, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0013675978081465023, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 5479464960, + 5479464960, + 10959912960 + ], + "warmup": 547964160, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-600m/600M/step11148/model_and_optim/.metadata b/model-ladders/count-600m/600M/step11148/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e15a73b9aced7a93835f4d2f05dcb8fe580314b7 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd0756b09f1b7ff65f0682811a6dacf0ea4d5f686edb2efb3219f0c2b6c471b5 +size 835959 diff --git a/model-ladders/count-600m/600M/step11148/model_and_optim/__0_0.distcp b/model-ladders/count-600m/600M/step11148/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3774aa166becc5b337e48f4e3239d32f80d56301 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62771a096e39b9ca27611f1e0df00adfceebe14f6cb31979ed1e7665854be54e +size 2031534793 diff --git a/model-ladders/count-600m/600M/step11148/model_and_optim/__1_0.distcp b/model-ladders/count-600m/600M/step11148/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..92103109527729fb9978ee340e79bba8b8d1ef5f --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cb5058afb59ef62d393a8a0904f3601584bd5f27bc9f9d892d130e60e755998 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step11148/model_and_optim/__2_0.distcp b/model-ladders/count-600m/600M/step11148/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..35e17540672d75db9a83b3733cf6b0d2204ca727 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a0428d37ece7341d8e5e7fcc5367d6835f38a97eab0adf9055d06a10f9aac8 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step11148/model_and_optim/__3_0.distcp b/model-ladders/count-600m/600M/step11148/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4362554f3ffc1bbcc664d82e1e172ad227d233e9 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a593ba71c9c9ece5af90ebf8d4963f92a9a7a8a52362e7164eaa9958fe2e5ec +size 2030108151 diff --git a/model-ladders/count-600m/600M/step11148/release_args.json b/model-ladders/count-600m/600M/step11148/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..3f4e1f27b0cc6897f79350c5bcc7df64439efa15 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "600M", + "seed": 1042, + "fixed_global_batch_size": 983040, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 20, + "step": 11148, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-600m/600M/step11148/train/rank0.pt b/model-ladders/count-600m/600M/step11148/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..92a22d7d52ce6a0bf9eeccc0ed6a2911ce8a7fff --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e2696b0f90704324066bd353bdf5a1d0c1b34baa2ca5676bbbdef0be7c93cab +size 15637 diff --git a/model-ladders/count-600m/600M/step11148/train/rank1.pt b/model-ladders/count-600m/600M/step11148/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..57114b40f65ce5d5b0a1b372b88261c22dfce5e6 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0177113dac32202cfb72eb4f9440545324a13a90ff0dd008187fe045b23ebeba +size 15637 diff --git a/model-ladders/count-600m/600M/step11148/train/rank2.pt b/model-ladders/count-600m/600M/step11148/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..49281f9486917392b721f014246d676e210f3201 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1bb49377b0294b8919b903be36eb16bb6369ed20d76f8e3e722c5fd2f9acbb +size 15637 diff --git a/model-ladders/count-600m/600M/step11148/train/rank3.pt b/model-ladders/count-600m/600M/step11148/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..267014bc6c3fabdd1864c02ba9e7718f4488de27 --- /dev/null +++ b/model-ladders/count-600m/600M/step11148/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfad10043cc7a5be4d8d215f5b9d7fa26beb4143fb3ef3ac6a58dae5a316afe8 +size 15637 diff --git a/model-ladders/count-600m/600M/step22297/.metadata.json b/model-ladders/count-600m/600M/step22297/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-600m/600M/step22297/config.json b/model-ladders/count-600m/600M/step22297/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c39d97993352ff90b9fc8f1ed5d6f8964706d43f --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "600M", + "model": { + "d_model": 1280, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 5120, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0013675978081465023, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 5479464960, + 5479464960, + 10959912960 + ], + "warmup": 547964160, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-600m/600M/step22297/model_and_optim/.metadata b/model-ladders/count-600m/600M/step22297/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8e603d1e3954b22b9fe2c754872516161ffc7be0 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faa9ad44d0f667a6d4bff349ae8f0d709ac6159b58e1d4c4ce0e3ae7d4459ff2 +size 835959 diff --git a/model-ladders/count-600m/600M/step22297/model_and_optim/__0_0.distcp b/model-ladders/count-600m/600M/step22297/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c05d7d8532218197528cc25006113d4b5d5001fa --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d22e1fffe8d4b31745b63815c9bccc41fd6cb86d16e4f676e95cfaf75a35e6b +size 2031534793 diff --git a/model-ladders/count-600m/600M/step22297/model_and_optim/__1_0.distcp b/model-ladders/count-600m/600M/step22297/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9239a609895c3f13976f401926814e1362fe7731 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:216a18642ab3aa5219974c0ddf6d30ee50b320e9291503fe7288160b909f985f +size 2030108151 diff --git a/model-ladders/count-600m/600M/step22297/model_and_optim/__2_0.distcp b/model-ladders/count-600m/600M/step22297/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1fcb0f72b8a7064f7231c2ab7503f4be55481be0 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7934090a1b145d20e2e16252e4d1e44e8ab8699111beef6fca5158dddb3a9b2a +size 2030108151 diff --git a/model-ladders/count-600m/600M/step22297/model_and_optim/__3_0.distcp b/model-ladders/count-600m/600M/step22297/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..01fc07e1bc4337a1c034644f0613a55b097f56a5 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c86d5bc9cfb570c8e184564fc0b5cc763a7c76986a676029c98715f42e6d65 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step22297/release_args.json b/model-ladders/count-600m/600M/step22297/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..26d6901a8887b41393a9edbfdf1e21ce37c99a5b --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "600M", + "seed": 1042, + "fixed_global_batch_size": 983040, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 40, + "step": 22297, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-600m/600M/step22297/train/rank0.pt b/model-ladders/count-600m/600M/step22297/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cfd554c803bca2af72446d327680470050c4bd9 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e05af1a881f93717bcd72c505af251a0163d14acc05e7abaeebe9c8bd69512f +size 15637 diff --git a/model-ladders/count-600m/600M/step22297/train/rank1.pt b/model-ladders/count-600m/600M/step22297/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..f99d8cc667131288e541d34ffb03c9854cc0fffa --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab4193b457d278dbcce62e2147edf7999a7f33e0d62e79dfaf315744359348c +size 15637 diff --git a/model-ladders/count-600m/600M/step22297/train/rank2.pt b/model-ladders/count-600m/600M/step22297/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e30a60fbf779cf56796e4292a225db93a2984a0 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8922e97b427cd95275e35cc590267b9b30ce56dea13127bc82cd069c7965c32 +size 15637 diff --git a/model-ladders/count-600m/600M/step22297/train/rank3.pt b/model-ladders/count-600m/600M/step22297/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb7f9dc9bbaf94cc76f5f7304618fc0bc1c03a35 --- /dev/null +++ b/model-ladders/count-600m/600M/step22297/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:266e989129bfee31457bb263e6df1c98f5866f877f10977cdf17eaf150d267bd +size 15637 diff --git a/model-ladders/count-600m/600M/step5574/.metadata.json b/model-ladders/count-600m/600M/step5574/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-600m/600M/step5574/config.json b/model-ladders/count-600m/600M/step5574/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c39d97993352ff90b9fc8f1ed5d6f8964706d43f --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "600M", + "model": { + "d_model": 1280, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 5120, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0013675978081465023, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 5479464960, + 5479464960, + 10959912960 + ], + "warmup": 547964160, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-600m/600M/step5574/model_and_optim/.metadata b/model-ladders/count-600m/600M/step5574/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..fe111ca9010cf1654b6037ae51f423ef668f3823 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f998b9908787463376532bfc5846c000e3133b88d7e3adbea8b01373928f843 +size 835959 diff --git a/model-ladders/count-600m/600M/step5574/model_and_optim/__0_0.distcp b/model-ladders/count-600m/600M/step5574/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ae0dd4225a7c8d5eaf586e432761712c3a5f678d --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff373d5e9f6aece08150d73a155280d20a38ad2331541497850acd7aaef4d5b0 +size 2031534793 diff --git a/model-ladders/count-600m/600M/step5574/model_and_optim/__1_0.distcp b/model-ladders/count-600m/600M/step5574/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8242eec89f0b545353ef55cafa47e60938c63561 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0c5469378587ec3651d7d25fd8798c18461179aac583abd3a3d5d6a373a07d6 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step5574/model_and_optim/__2_0.distcp b/model-ladders/count-600m/600M/step5574/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c29ec094ba6139cf067f23420f19cb7eb9534752 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a8384d76816e5c530d75c5dd677e8faff8df85279242548b1272968262bc11 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step5574/model_and_optim/__3_0.distcp b/model-ladders/count-600m/600M/step5574/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d2bac4e5e228d1498c335b61953da72cc6e04002 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c02600adff49cb95fc3f25d6a0fa98d0b04ca912effcbfdf2298b9fcd552043 +size 2030108151 diff --git a/model-ladders/count-600m/600M/step5574/release_args.json b/model-ladders/count-600m/600M/step5574/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..e471166ddd38bcbc88d0ee019f5891ae04fd8c90 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "600M", + "seed": 1042, + "fixed_global_batch_size": 983040, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 10, + "step": 5574, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-600m/600M/step5574/train/rank0.pt b/model-ladders/count-600m/600M/step5574/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..530cf7ba487826e09e6ac6f65384d2e4bbda426e --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d90a3efdfcd2c59105bdcf72c529d56205ecfb1b9db824c4aa99865bf2ccc754 +size 15637 diff --git a/model-ladders/count-600m/600M/step5574/train/rank1.pt b/model-ladders/count-600m/600M/step5574/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..a86ab3a1be1ac1e77d429bcf9e46574afe91857a --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20cc82a15c05f5f06eff05f8f97ce6c21ff88e5e275a4dc3a3dac5f7ea37c37 +size 15637 diff --git a/model-ladders/count-600m/600M/step5574/train/rank2.pt b/model-ladders/count-600m/600M/step5574/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..676ba93a246d640afc4fc6fcad24b0ca901e5629 --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb9349a9a11991c36bdba4c6a53b63b9bcc80f5eafa5932bf2e1b415e8c19ade +size 15637 diff --git a/model-ladders/count-600m/600M/step5574/train/rank3.pt b/model-ladders/count-600m/600M/step5574/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f94f2b4d703d49a36903898a05539bbf161059a --- /dev/null +++ b/model-ladders/count-600m/600M/step5574/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:838318269260d5cc3b6fd1df005b1f2b2ab115a2bac74c773e244dda4c860929 +size 15637 diff --git a/model-ladders/count-60m/60M/step2190/.metadata.json b/model-ladders/count-60m/60M/step2190/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-60m/60M/step2190/config.json b/model-ladders/count-60m/60M/step2190/config.json new file mode 100644 index 0000000000000000000000000000000000000000..26f044fc651fc4c7bea4ff7b0f901f1c9eae2bc7 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "60M", + "model": { + "d_model": 384, + "vocab_size": 100352, + "n_layers": 8, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 8, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 1536, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.00290078685078361, + "betas": [ + 0.9, + 0.99 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 574095360, + 574357504, + 1148452864 + ], + "warmup": 57422208, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-60m/60M/step2190/model_and_optim/.metadata b/model-ladders/count-60m/60M/step2190/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..eb4b841d8e4e565ccbcd5c70ccb84479aa42bd23 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef9c8d82b74e6c863996991740aa83d8160db6dab0424823413729ae1de2639c +size 329788 diff --git a/model-ladders/count-60m/60M/step2190/model_and_optim/__0_0.distcp b/model-ladders/count-60m/60M/step2190/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b7ff89d768ad4881302f1a891dcd277fa747c794 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1236b8e1e67c8bd5ba9f1bd5ccf61cc32b7b3e27e9d8b111696e678a4673d9 +size 576921873 diff --git a/model-ladders/count-60m/60M/step2190/model_and_optim/__1_0.distcp b/model-ladders/count-60m/60M/step2190/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dfb0314b95b129310d2228c2f709c5d23c55baf6 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c67a2113e22c320da7f29ab640806a2c43ba42b85b8ea5a7fb328494b27a61a +size 576188847 diff --git a/model-ladders/count-60m/60M/step2190/release_args.json b/model-ladders/count-60m/60M/step2190/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..37f4e6090481d93113ffb29ccebe5ce7290cf434 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "60M", + "seed": 1042, + "fixed_global_batch_size": 262144, + "sequence_length": 8192, + "trained_world_size": 2, + "d_over_n": 10, + "step": 2190, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-17gb" +} diff --git a/model-ladders/count-60m/60M/step2190/train/rank0.pt b/model-ladders/count-60m/60M/step2190/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bc087e363b411dc2a84aa8871449fbb8a3078e1 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5997b375eec9fbffd41c598f975c6fd6cbafa0bacfe397fbbf002cab7462ee +size 15637 diff --git a/model-ladders/count-60m/60M/step2190/train/rank1.pt b/model-ladders/count-60m/60M/step2190/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f818156280f50f64d156b6061fcacff035f4552 --- /dev/null +++ b/model-ladders/count-60m/60M/step2190/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d42fa7b7dfcbc8a25c3997e280e1278fbbbb300f282e9755ad0ba06ce610e1 +size 15637 diff --git a/model-ladders/count-60m/60M/step4381/.metadata.json b/model-ladders/count-60m/60M/step4381/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-60m/60M/step4381/config.json b/model-ladders/count-60m/60M/step4381/config.json new file mode 100644 index 0000000000000000000000000000000000000000..26f044fc651fc4c7bea4ff7b0f901f1c9eae2bc7 --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "60M", + "model": { + "d_model": 384, + "vocab_size": 100352, + "n_layers": 8, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 8, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 1536, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.00290078685078361, + "betas": [ + 0.9, + 0.99 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 574095360, + 574357504, + 1148452864 + ], + "warmup": 57422208, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-60m/60M/step4381/model_and_optim/.metadata b/model-ladders/count-60m/60M/step4381/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b36895d7f04319fbfc7a7ccca35ace18126413ce --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65c10ecd838a372cbc1971668a0ea1d1d22ffbe5c27231a128ea6431a6a1003e +size 329788 diff --git a/model-ladders/count-60m/60M/step4381/model_and_optim/__0_0.distcp b/model-ladders/count-60m/60M/step4381/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..02caef17567faa6f25d9f58d253167224ddd311d --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be22c76a026d4bc18540e58beb336f9b7276e8d6fe3a2d955465008640e53a1f +size 576921873 diff --git a/model-ladders/count-60m/60M/step4381/model_and_optim/__1_0.distcp b/model-ladders/count-60m/60M/step4381/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c845a3f971d33b8dbc22dc5dcef0947eba9cc4bc --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5652d0426861c1557a365eeff0c66c22890dd8ac2be853689cc9240e939a49bf +size 576188847 diff --git a/model-ladders/count-60m/60M/step4381/release_args.json b/model-ladders/count-60m/60M/step4381/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..a75256aa7962ae5395f53813bef44f1eb42c0058 --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "60M", + "seed": 1042, + "fixed_global_batch_size": 262144, + "sequence_length": 8192, + "trained_world_size": 2, + "d_over_n": 20, + "step": 4381, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-17gb" +} diff --git a/model-ladders/count-60m/60M/step4381/train/rank0.pt b/model-ladders/count-60m/60M/step4381/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c40c1b9f580581a63cdf8d43f176603c5a95f7ee --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a4f012f8d9a6bea78a774d56df3bd7862cc2d13e2f3cd6f1c1b8cd87e8bf3f +size 15637 diff --git a/model-ladders/count-60m/60M/step4381/train/rank1.pt b/model-ladders/count-60m/60M/step4381/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..56c84acb65522d75bc87c353bf8270f37a2e2c90 --- /dev/null +++ b/model-ladders/count-60m/60M/step4381/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50ed341b50271d859714b4c6a2f6314396479676851f73d9e3947fc35b6950b1 +size 15637 diff --git a/model-ladders/count-60m/60M/step8762/.metadata.json b/model-ladders/count-60m/60M/step8762/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-60m/60M/step8762/config.json b/model-ladders/count-60m/60M/step8762/config.json new file mode 100644 index 0000000000000000000000000000000000000000..26f044fc651fc4c7bea4ff7b0f901f1c9eae2bc7 --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "60M", + "model": { + "d_model": 384, + "vocab_size": 100352, + "n_layers": 8, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 8, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 1536, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.00290078685078361, + "betas": [ + 0.9, + 0.99 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 574095360, + 574357504, + 1148452864 + ], + "warmup": 57422208, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-60m/60M/step8762/model_and_optim/.metadata b/model-ladders/count-60m/60M/step8762/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..bda1a2b9c893960b5bb2a0a47c267ecc09eb0e37 --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f9b978c41f737521dfea66d3ae65b1d07e2976910a53519e2a608b287f646f +size 329788 diff --git a/model-ladders/count-60m/60M/step8762/model_and_optim/__0_0.distcp b/model-ladders/count-60m/60M/step8762/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4f8ea7d7ca7f4348949a52c96b91e9d197e9cd1c --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611c431b354ccdae87e98982021b4832e1d623b78f210d5a8b55471efde5825c +size 576921873 diff --git a/model-ladders/count-60m/60M/step8762/model_and_optim/__1_0.distcp b/model-ladders/count-60m/60M/step8762/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..42c8125e27062ba0298c052bc3dfa065d75ec2d5 --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6becaa1af78fd0193d4b0567bb2880546b8adf429a8560420e845276d408b5b1 +size 576188847 diff --git a/model-ladders/count-60m/60M/step8762/release_args.json b/model-ladders/count-60m/60M/step8762/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..bfee06012463eaae8a5115117598513e6d92f5a8 --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "60M", + "seed": 1042, + "fixed_global_batch_size": 262144, + "sequence_length": 8192, + "trained_world_size": 2, + "d_over_n": 40, + "step": 8762, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-17gb" +} diff --git a/model-ladders/count-60m/60M/step8762/train/rank0.pt b/model-ladders/count-60m/60M/step8762/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3935565d38b9322be15c0ca60333d222fcd9d9a --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c33d4883aadbd236a6c6503d0de206e2751c43473ea50d662e15e5744564075 +size 15637 diff --git a/model-ladders/count-60m/60M/step8762/train/rank1.pt b/model-ladders/count-60m/60M/step8762/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b4286bea068280d519f290593f60701dec47ed --- /dev/null +++ b/model-ladders/count-60m/60M/step8762/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faba384e933807b614fe3646015d8e5a2bfe06d22835f372bf6d229901527ac0 +size 15637 diff --git a/model-ladders/count-760m/760M/step11570/.metadata.json b/model-ladders/count-760m/760M/step11570/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-760m/760M/step11570/config.json b/model-ladders/count-760m/760M/step11570/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e84fde908c4fc84bd77f65475b32b69528d96b49 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "760M", + "model": { + "d_model": 1536, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 6144, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0012272809143622813, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 7582515200, + 7582515200, + 15163719680 + ], + "warmup": 758220288, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-760m/760M/step11570/model_and_optim/.metadata b/model-ladders/count-760m/760M/step11570/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e8a272e40b74b3bfb455ae6669f09e25ef634691 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c3879f04b7fd09fc63b8daa4c86a1a8ac401f91ce8412a707932e251e48ef5 +size 838461 diff --git a/model-ladders/count-760m/760M/step11570/model_and_optim/__0_0.distcp b/model-ladders/count-760m/760M/step11570/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b2e1ab68dc20db6c7ba293b2a18acab240101333 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229d6415a7bbe8a67b4edc76a8aa0570a1a70d706c9ad958ce9a8953bba4c047 +size 2739375049 diff --git a/model-ladders/count-760m/760M/step11570/model_and_optim/__1_0.distcp b/model-ladders/count-760m/760M/step11570/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1520967c32b66aac9c2c9ddfef84df6411cf808e --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8bac166608bec7cfee4a5d7bead23e379c93ff3e559f7586ccc8b99231abb8a +size 2737948407 diff --git a/model-ladders/count-760m/760M/step11570/model_and_optim/__2_0.distcp b/model-ladders/count-760m/760M/step11570/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fce9098ca684f3b82de413a33b8280c38bca3ccc --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f06d98cafe46a87a7779a93eb67e162e137a52a9c3214c2450e80489e7db2aa +size 2737948407 diff --git a/model-ladders/count-760m/760M/step11570/model_and_optim/__3_0.distcp b/model-ladders/count-760m/760M/step11570/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..57b8631daf4bcc0037b49f9b13ea06d51c0be999 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16deee68088e04351aab7187f0d103404b29f4c130761a7cd4b77864b5f7d4fe +size 2737948407 diff --git a/model-ladders/count-760m/760M/step11570/release_args.json b/model-ladders/count-760m/760M/step11570/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd6f1f6cd821cd89aa8d6578632347ad2fc002a --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "760M", + "seed": 1042, + "fixed_global_batch_size": 1310720, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 20, + "step": 11570, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-760m/760M/step11570/train/rank0.pt b/model-ladders/count-760m/760M/step11570/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c59b144007c69ff4c1ae62e009f6284d7d591074 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfec70832c7a27f50f7239fc7b2eac2445440d2ec1c32af91a60ff3e44929f4b +size 15637 diff --git a/model-ladders/count-760m/760M/step11570/train/rank1.pt b/model-ladders/count-760m/760M/step11570/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed9468098d1a18a5cb86756f253ca0f05826a31c --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74388d6c1d9559a70276213e2c8c4bf1aa0518fb829fce0c341615846d7aa02e +size 15637 diff --git a/model-ladders/count-760m/760M/step11570/train/rank2.pt b/model-ladders/count-760m/760M/step11570/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..91cc3f235c4a1e625db76a171ed6550537cb5190 --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a821c1aaafe074a03b5426e0aa97fd0bea2bfeb78a2951472e76574a1a57e2d3 +size 15637 diff --git a/model-ladders/count-760m/760M/step11570/train/rank3.pt b/model-ladders/count-760m/760M/step11570/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..85ad04345c82c8027cf5944628ed822fd0c6c80d --- /dev/null +++ b/model-ladders/count-760m/760M/step11570/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e881b5f484008672e7ff72bfdaa6bdd38ebd909e3c9c537928d89a79b17b9262 +size 15637 diff --git a/model-ladders/count-760m/760M/step23139/.metadata.json b/model-ladders/count-760m/760M/step23139/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-760m/760M/step23139/config.json b/model-ladders/count-760m/760M/step23139/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e84fde908c4fc84bd77f65475b32b69528d96b49 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "760M", + "model": { + "d_model": 1536, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 6144, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0012272809143622813, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 7582515200, + 7582515200, + 15163719680 + ], + "warmup": 758220288, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-760m/760M/step23139/model_and_optim/.metadata b/model-ladders/count-760m/760M/step23139/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..fc042afe2e36b77d54d0eba7fb5758811a136dcf --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf04a200ba6dc7544ff7127f1b57da76bfc01fde8f9f422ce02fcac7a5c7e6e8 +size 838461 diff --git a/model-ladders/count-760m/760M/step23139/model_and_optim/__0_0.distcp b/model-ladders/count-760m/760M/step23139/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..86ebde3abde0f82e0689523dd87174479e589723 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6c3f93c35a6f0d96cc0ca778b845290d5cd80d598aed5726850724740c6654 +size 2739375049 diff --git a/model-ladders/count-760m/760M/step23139/model_and_optim/__1_0.distcp b/model-ladders/count-760m/760M/step23139/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1784b9405ce5916eab1161e85f807f5ce64e4a6d --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48f8130d7d08c5e26310111d8e6ab8197f83e55e9abc01c74a24a85978af0f6a +size 2737948407 diff --git a/model-ladders/count-760m/760M/step23139/model_and_optim/__2_0.distcp b/model-ladders/count-760m/760M/step23139/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4d84e2182ced1e3218a21fa9c76da73bf879a962 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49e99c87338f16f538281da04b4b352886fa3b09c5267bafa205e51b30a4ac6e +size 2737948407 diff --git a/model-ladders/count-760m/760M/step23139/model_and_optim/__3_0.distcp b/model-ladders/count-760m/760M/step23139/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7aca59cf72957cec6fdb089a0b20db62ab4ae5f4 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03f3e1b69d58717d7eedd3f54f0453c235abf2fe196d6c4fcb4adefafe3d30b +size 2737948407 diff --git a/model-ladders/count-760m/760M/step23139/release_args.json b/model-ladders/count-760m/760M/step23139/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..d40d51addc2294489d83264b524b0e76ed3293c2 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "760M", + "seed": 1042, + "fixed_global_batch_size": 1310720, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 40, + "step": 23139, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-760m/760M/step23139/train/rank0.pt b/model-ladders/count-760m/760M/step23139/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cdf1d1b1e454168bb0cf733349a267a9063a906 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cc1408be421c69343fc2376b06b3ed51aa494bd11afbcdc654701b1b64baa28 +size 15637 diff --git a/model-ladders/count-760m/760M/step23139/train/rank1.pt b/model-ladders/count-760m/760M/step23139/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..2317a4cacfecc2bdad2c40eaf0127c0cab451282 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae7f33c4a26708955562cd946b060f5dfe8cc8a6fa01f5af93dcce4276039c97 +size 15637 diff --git a/model-ladders/count-760m/760M/step23139/train/rank2.pt b/model-ladders/count-760m/760M/step23139/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ad382390a07f463681c35a9b1dd9d7ad5443d57 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc0558e34f20685c4f61072622e6b479c0f3884ff972af5eefa53a4e98e8aff1 +size 15637 diff --git a/model-ladders/count-760m/760M/step23139/train/rank3.pt b/model-ladders/count-760m/760M/step23139/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..e001d3bf997a6053539d318406b50e6f07d37b36 --- /dev/null +++ b/model-ladders/count-760m/760M/step23139/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b77b06fd1df0050b4a5412aa6e91b7f9da0a255201501193a8ebcc2ecb32428f +size 15637 diff --git a/model-ladders/count-760m/760M/step5785/.metadata.json b/model-ladders/count-760m/760M/step5785/.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f7af4574fecaee7b32ac08ab5fa8f66591bd179a --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/.metadata.json @@ -0,0 +1 @@ +{"ephemeral": false, "version": "2.5.0"} \ No newline at end of file diff --git a/model-ladders/count-760m/760M/step5785/config.json b/model-ladders/count-760m/760M/step5785/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e84fde908c4fc84bd77f65475b32b69528d96b49 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/config.json @@ -0,0 +1,162 @@ +{ + "seed": 1042, + "size": "760M", + "model": { + "d_model": 1536, + "vocab_size": 100352, + "n_layers": 16, + "block": { + "sequence_mixer": { + "name": "default", + "n_heads": 16, + "bias": false, + "rope": { + "name": "default", + "theta": 500000, + "full_precision": true, + "no_global_rope": false, + "_CLASS_": "olmo_core.nn.rope.RoPEConfig" + }, + "qk_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "backend": "flash_3", + "dtype": "float32", + "use_head_qk_norm": false, + "_CLASS_": "olmo_core.nn.attention.AttentionConfig", + "type": "attention" + }, + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "feed_forward": { + "hidden_size": 6144, + "name": "default", + "bias": false, + "dtype": "float32", + "activation": "silu", + "_CLASS_": "olmo_core.nn.feed_forward.FeedForwardConfig" + }, + "name": "reordered_norm", + "_CLASS_": "olmo_core.nn.transformer.config.TransformerBlockConfig" + }, + "lm_head": { + "name": "default", + "layer_norm": { + "name": "rms", + "eps": 1e-06, + "bias": false, + "dtype": "float32", + "_CLASS_": "olmo_core.nn.layer_norm.LayerNormConfig" + }, + "bias": false, + "dtype": "float32", + "loss_implementation": "default", + "_CLASS_": "olmo_core.nn.lm_head.LMHeadConfig" + }, + "name": "default", + "dtype": "float32", + "init_method": "normal", + "init_seed": 0, + "init_std": 0.02, + "_CLASS_": "olmo_core.nn.transformer.config.TransformerConfig" + }, + "optim": { + "group_overrides": [ + { + "params": [ + "embeddings.weight" + ], + "opts": { + "weight_decay": 0.0 + }, + "_CLASS_": "olmo_core.optim.config.OptimGroupOverride" + } + ], + "compile": false, + "fixed_fields": [ + "initial_lr" + ], + "lr": 0.0012272809143622813, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.1, + "foreach": true, + "step_increment_bugfix": true, + "rolling_interval_length": 128, + "sigma_factor": 6, + "_CLASS_": "olmo_core.optim.adamw.SkipStepAdamWConfig", + "type": "skip_step_adamw" + }, + "scheduler": { + "lr_field": "lr", + "initial_lr_field": "initial_lr", + "units": "tokens", + "period_lengths": [ + 7582515200, + 7582515200, + 15163719680 + ], + "warmup": 758220288, + "decay_fraction": 0.1, + "warmup_min_lr": 0.0, + "decay_min_lr": 0.0, + "_CLASS_": "olmo_core.optim.scheduler.WSDS", + "type": "wsds" + }, + "data_loader": { + "seed": 0, + "shuffle": true, + "sources_per_epoch": -1, + "num_workers": 16, + "generate_doc_lengths": false, + "instance_filter_config": { + "repetition_max_period": 13, + "repetition_min_period": 1, + "repetition_max_count": 32, + "_CLASS_": "olmo_core.data.composable.data_loader.InstanceFilterConfig" + }, + "display_source_visualization": true, + "_CLASS_": "olmo_core.data.composable.data_loader.ComposableDataLoaderConfig", + "type": "composable" + }, + "instance_sources": [ + { + "sources": [ + { + "tokenizer": { + "vocab_size": 100278, + "eos_token_id": 100257, + "pad_token_id": 100277, + "identifier": "allenai/dolma2-tokenizer", + "_CLASS_": "olmo_core.data.tokenizer.TokenizerConfig" + }, + "source_group_size": 1, + "long_doc_strategy": "truncate", + "source_paths": [ + "/all-dressed-snazzy2-fixed/**/*.npy", + "/arxiv/**/*.npy", + "/finemath-3plus/**/*.npy", + "/s2pdf/**/*.npy", + "/stack-edu/**/*.npy", + "/wikipedia/**/*.npy" + ], + "_CLASS_": "olmo_core.data.composable.numpy_document_source.NumpyDocumentSourceConfig" + } + ], + "sequence_length": 8192, + "_CLASS_": "olmo_core.data.composable.concat_and_chunk_instance_source.ConcatAndChunkInstanceSourceConfig" + } + ] +} diff --git a/model-ladders/count-760m/760M/step5785/model_and_optim/.metadata b/model-ladders/count-760m/760M/step5785/model_and_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1e8887fe492ddc772d22dbe072adb2eeab19afac --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/model_and_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22888d35c6dddefd3ea339f9e9a31ee3dd36ff49f4a2d37e8cc38a7b134533f2 +size 838461 diff --git a/model-ladders/count-760m/760M/step5785/model_and_optim/__0_0.distcp b/model-ladders/count-760m/760M/step5785/model_and_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2a948fe375c79ae6b89b9193e692c3d8858168b0 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/model_and_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a71ed022e48722e706d2e06c5c8d8cfdba6fbb407074add3c4b1c748796c49 +size 2739375049 diff --git a/model-ladders/count-760m/760M/step5785/model_and_optim/__1_0.distcp b/model-ladders/count-760m/760M/step5785/model_and_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3bb008b6c9e6456ec15657b328b126c5c9a65480 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/model_and_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c891b8043f11746540ace9275782e71ae121479d6af88e1907b067855a75edf4 +size 2737948407 diff --git a/model-ladders/count-760m/760M/step5785/model_and_optim/__2_0.distcp b/model-ladders/count-760m/760M/step5785/model_and_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c64b6b2b80a5070bed264a03f441899b1327ccc4 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/model_and_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74fdfd2e5c8438d9f5090bf9c8562d36b86a53b0520b17f64d1756bc8571a9c5 +size 2737948407 diff --git a/model-ladders/count-760m/760M/step5785/model_and_optim/__3_0.distcp b/model-ladders/count-760m/760M/step5785/model_and_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..72659c38c4c61ebc4c563a1d2c564994e5c57041 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/model_and_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611f8233cb6d0ceff0e780719b4b65fb67415750b8e6fa11b79712399294872e +size 2737948407 diff --git a/model-ladders/count-760m/760M/step5785/release_args.json b/model-ladders/count-760m/760M/step5785/release_args.json new file mode 100644 index 0000000000000000000000000000000000000000..82458dbc0fb5c0fdb4a3990053ef83b3a0a647e0 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/release_args.json @@ -0,0 +1,13 @@ +{ + "size": "760M", + "seed": 1042, + "fixed_global_batch_size": 1310720, + "sequence_length": 8192, + "trained_world_size": 4, + "d_over_n": 10, + "step": 5785, + "count_k": 8, + "count_max_order": 6, + "count_num_orders": 6, + "count_table": "general-797gb" +} diff --git a/model-ladders/count-760m/760M/step5785/train/rank0.pt b/model-ladders/count-760m/760M/step5785/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5c0b55e71bc3d2eec22aa65ca341ce14ee56b57 --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e930c6854bc4aa1d8586447ae6bea280c5119e45594cc637d4f59cdefd69d84 +size 15637 diff --git a/model-ladders/count-760m/760M/step5785/train/rank1.pt b/model-ladders/count-760m/760M/step5785/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..37aad7804da993a681bfb34e76870feae4d6a5df --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5939c1dbb3e65536244ed2b3732d4ad322c7b6b634f01aca8adab5d771ed1066 +size 15637 diff --git a/model-ladders/count-760m/760M/step5785/train/rank2.pt b/model-ladders/count-760m/760M/step5785/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..15214771a0c5540a66e5809bef83a51f187cd19b --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e30f1a95b0beb5732b0ffd44e331abdb8c3a36afdf39c22aae74c222828f1e58 +size 15637 diff --git a/model-ladders/count-760m/760M/step5785/train/rank3.pt b/model-ladders/count-760m/760M/step5785/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..b148d865ff7cd06134b63febac83a1bccd1f1ffa --- /dev/null +++ b/model-ladders/count-760m/760M/step5785/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18e6829b14c55ef72e836f17e89ff9b023630402bc31c91276a95bbf5210f694 +size 15637