Add datasets/FCP-20B artifacts 488-495
Browse files
datasets/FCP-20B/packing/source2b/pack_ids.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b4eaddd4247a59f5f2be82fc42b33b63957ace0c5c2f751b3eb3464b9abff5a2
|
| 3 |
+
size 393051776
|
datasets/FCP-20B/packing/source2b/pack_starts.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:67e66804a4b7bb745ceea6669babf4e481e9c41cb0e6518ab502118433f4a860
|
| 3 |
+
size 97437888
|
datasets/FCP-20B/packing/source2b/packing_READY.json
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"passed": true,
|
| 3 |
+
"rows": 49131472,
|
| 4 |
+
"packs": 12179735,
|
| 5 |
+
"sha256": "67e66804a4b7bb745ceea6669babf4e481e9c41cb0e6518ab502118433f4a860",
|
| 6 |
+
"member_sha256": "b4eaddd4247a59f5f2be82fc42b33b63957ace0c5c2f751b3eb3464b9abff5a2",
|
| 7 |
+
"code_hashes": {
|
| 8 |
+
"bfd.py": "ba37e45bfe6882b7c1747bfaafbaf55611f0cdfa2971ad16cd58d88a4009640d",
|
| 9 |
+
"config.py": "6b617d0a45048a8851b617f29608602dc7e290166749dd1551433033c44ec214",
|
| 10 |
+
"encoding.py": "3aed0431713870e35595f922f22be6a4cce123877f493de5bb1f848809683ab4",
|
| 11 |
+
"fcp_format.py": "bef4a540107f949af0e9a2149419ea05cdc4f52385b66eaed84ef25dc40a8305",
|
| 12 |
+
"gate.py": "6c862bfac6e1bd38988cd161804ff9ecb16418e08a031992dfa2f1f78a99ef37",
|
| 13 |
+
"layout.py": "f30a8a77cb97a4bf8d33248a14bbc9112c358b0a0162255d7a77abb15f8f7e2b",
|
| 14 |
+
"model.py": "f0fb8a8789fbf0cd2d7033cf813fa40dc4ebf47617f1f3b70dafa419842eeb5e",
|
| 15 |
+
"prepare_packs.py": "9111f8372105c8381c3b46cd6f31e38ba8d0c4a2c7afde28f4b5a8215c0a813e",
|
| 16 |
+
"runtime.py": "628f7ab759bc4e1047e86dbe71f143fdffa54f16ed498217e7abe82c114c1f7a",
|
| 17 |
+
"train.py": "ea3a7991373ca107118011db93c15e3c18d8f94695909ec26d686ab228e292b3",
|
| 18 |
+
"upload.py": "747a9bce54eafc35b1fa14ba5aa653908bf8f81f24b5e526bce9c947b4ef3182",
|
| 19 |
+
"validate_data.py": "f8dec33d7fc01b54b562ede8cfb87ecbf529dd68be8b48cf9147e6bd3a9324fa",
|
| 20 |
+
"verify.py": "cec4349e3b7d5adcab0f4377b0416cc87b6d810c749833e7b455dbd7e9462a97",
|
| 21 |
+
"checkpoint_files.py": "3a4cd9b1cc858e9c48dc5c00b92a1d4e86330643d03fdb9df361c6ed85788155"
|
| 22 |
+
},
|
| 23 |
+
"algorithm": "BFD",
|
| 24 |
+
"buffer_records": 1000,
|
| 25 |
+
"global_packs_per_update": 128,
|
| 26 |
+
"segments": [
|
| 27 |
+
{
|
| 28 |
+
"segment": 0,
|
| 29 |
+
"end_row": 4898104,
|
| 30 |
+
"source_tokens": 2000000031,
|
| 31 |
+
"target_tokens": 2000000000,
|
| 32 |
+
"start_pack": 0,
|
| 33 |
+
"end_pack": 1218688
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"segment": 1,
|
| 37 |
+
"end_row": 9802903,
|
| 38 |
+
"source_tokens": 4000000388,
|
| 39 |
+
"target_tokens": 4000000000,
|
| 40 |
+
"start_pack": 1218688,
|
| 41 |
+
"end_pack": 2437128
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"segment": 2,
|
| 45 |
+
"end_row": 14714284,
|
| 46 |
+
"source_tokens": 6000000190,
|
| 47 |
+
"target_tokens": 6000000000,
|
| 48 |
+
"start_pack": 2437128,
|
| 49 |
+
"end_pack": 3655801
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"segment": 3,
|
| 53 |
+
"end_row": 19637704,
|
| 54 |
+
"source_tokens": 8000000397,
|
| 55 |
+
"target_tokens": 8000000000,
|
| 56 |
+
"start_pack": 3655801,
|
| 57 |
+
"end_pack": 4874591
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"segment": 4,
|
| 61 |
+
"end_row": 24567228,
|
| 62 |
+
"source_tokens": 10000000091,
|
| 63 |
+
"target_tokens": 10000000000,
|
| 64 |
+
"start_pack": 4874591,
|
| 65 |
+
"end_pack": 6092853
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"segment": 5,
|
| 69 |
+
"end_row": 29489067,
|
| 70 |
+
"source_tokens": 12000000382,
|
| 71 |
+
"target_tokens": 12000000000,
|
| 72 |
+
"start_pack": 6092853,
|
| 73 |
+
"end_pack": 7310504
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"segment": 6,
|
| 77 |
+
"end_row": 34384959,
|
| 78 |
+
"source_tokens": 14000000003,
|
| 79 |
+
"target_tokens": 14000000000,
|
| 80 |
+
"start_pack": 7310504,
|
| 81 |
+
"end_pack": 8527473
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"segment": 7,
|
| 85 |
+
"end_row": 39291912,
|
| 86 |
+
"source_tokens": 16000000433,
|
| 87 |
+
"target_tokens": 16000000000,
|
| 88 |
+
"start_pack": 8527473,
|
| 89 |
+
"end_pack": 9745065
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"segment": 8,
|
| 93 |
+
"end_row": 44208957,
|
| 94 |
+
"source_tokens": 18000000424,
|
| 95 |
+
"target_tokens": 18000000000,
|
| 96 |
+
"start_pack": 9745065,
|
| 97 |
+
"end_pack": 10962290
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"segment": 9,
|
| 101 |
+
"end_row": 49131472,
|
| 102 |
+
"source_tokens": 20000000040,
|
| 103 |
+
"target_tokens": 20000000000,
|
| 104 |
+
"start_pack": 10962290,
|
| 105 |
+
"end_pack": 12179735
|
| 106 |
+
}
|
| 107 |
+
],
|
| 108 |
+
"source_manifest_sha256": "3467ec6846c9a44f645a2d387ed158f5f5f8f1898e5e5945a318f829037ee6f0",
|
| 109 |
+
"updates": 95158
|
| 110 |
+
}
|