Update tokenizer, configuration and training reports
Browse files
reports/sources/cosmopedia/manifest.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"repo": "HuggingFaceTB/smollm-corpus",
|
| 3 |
+
"revision": "3ba9d605774198c5868892d7a8deda78031a781f",
|
| 4 |
+
"license": "odc-by",
|
| 5 |
+
"used_files": [
|
| 6 |
+
{
|
| 7 |
+
"path": "cosmopedia-v2/train-00102-of-00104.parquet",
|
| 8 |
+
"sha256": "6493d76632745f86c8407593bb07724f88422a3aeace3599758f8e6fd80341f1"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"path": "cosmopedia-v2/train-00058-of-00104.parquet",
|
| 12 |
+
"sha256": "1a672a44e013caf7989deab521406570d80df8b4af5944d95c5fa648097be787"
|
| 13 |
+
}
|
| 14 |
+
],
|
| 15 |
+
"tokens": 500073748,
|
| 16 |
+
"validation_tokens": 504286,
|
| 17 |
+
"docs": 681402,
|
| 18 |
+
"overlap_rejected": 23,
|
| 19 |
+
"sha256": "dd32ad84db946897bdcc3730a550ef936cf481d9ea6daf392a9193f8582be9e5",
|
| 20 |
+
"validation_sha256": "b4a0bb0228ad2849845e59d6d0e2a3a7c4ec4b804f59f5acf02c514486a9f0fa",
|
| 21 |
+
"tokenizer_sha256": "d1ffe47f6fd8cc19ba7c3e5d2537699d6a99d09b584d3319944f4e40fe6c44ea",
|
| 22 |
+
"split": "document hash 1% held out",
|
| 23 |
+
"decontamination": "normalized 13-word exact overlap and exact short sentences; not semantic guarantee"
|
| 24 |
+
}
|