Update tokenizer, configuration and training reports
Browse files
reports/sources/dclm_edu/manifest.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"repo": "HuggingFaceTB/dclm-edu",
|
| 3 |
+
"revision": "dbad8ad71224482740cd9c9d353591adbf62fe04",
|
| 4 |
+
"license": "cc-by-4.0",
|
| 5 |
+
"used_files": [
|
| 6 |
+
{
|
| 7 |
+
"path": "data/155_00003.parquet",
|
| 8 |
+
"sha256": "06ce7d4bc521f205a11d6ed2ac1789af126f4b038fccde3835f2f7fbd1860858"
|
| 9 |
+
}
|
| 10 |
+
],
|
| 11 |
+
"tokens": 1000049267,
|
| 12 |
+
"validation_tokens": 500425,
|
| 13 |
+
"docs": 731346,
|
| 14 |
+
"overlap_rejected": 190,
|
| 15 |
+
"sha256": "a4b6f76d3b58ad88fdaf28268fb4da7939878da3389ceedd78a6fc7b4df1e88f",
|
| 16 |
+
"validation_sha256": "b5903e2d512bc0fe079f39e48595efbbd812f83044874c491cf840de6905fd40",
|
| 17 |
+
"tokenizer_sha256": "d1ffe47f6fd8cc19ba7c3e5d2537699d6a99d09b584d3319944f4e40fe6c44ea",
|
| 18 |
+
"split": "document hash 1% held out",
|
| 19 |
+
"decontamination": "normalized 13-word exact overlap and exact short sentences; not semantic guarantee"
|
| 20 |
+
}
|