Update tokenizer, configuration and training reports

reports/DATA_READY.json

@@ -0,0 +1,10 @@
+{
+  "status": "ready",
+  "sources": {
+    "fineweb_edu": 3000000000,
+    "dclm_edu": 1000000000,
+    "cosmopedia": 500000000,
+    "finemath": 250000000
+  },
+  "tokenizer_sha256": "d1ffe47f6fd8cc19ba7c3e5d2537699d6a99d09b584d3319944f4e40fe6c44ea"
+}

reports/sources/fineweb_edu/manifest.json

@@ -0,0 +1,32 @@
+{
+  "repo": "HuggingFaceFW/fineweb-edu",
+  "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+  "license": "odc-by",
+  "used_files": [
+    {
+      "path": "sample/10BT/007_00000.parquet",
+      "sha256": "4b7d1f697d4afff7f5f65cb7aa4d83acc6db716da4d46ea23399d0c50608c6e3"
+    },
+    {
+      "path": "sample/10BT/003_00000.parquet",
+      "sha256": "22184e6eb25759ddd97783751ffc73e1705dfa2542e630dae1f2a8bac8ee6ddb"
+    },
+    {
+      "path": "sample/10BT/008_00000.parquet",
+      "sha256": "a6d9dcc0c72ecd7c7f0173c2d06b3a4249a2ccaa94961469d1a522ca689bbf1a"
+    },
+    {
+      "path": "sample/10BT/012_00000.parquet",
+      "sha256": "08b47a3e1c25161f796d2f8dbf99ccf60affdebdcea4910833d0d5783315551f"
+    }
+  ],
+  "tokens": 3000103735,
+  "validation_tokens": 500445,
+  "docs": 2838956,
+  "overlap_rejected": 1016,
+  "sha256": "f99a4db7edf2ac21609c022767962454710eff7b4c123d67230403578249908d",
+  "validation_sha256": "927c4974c3f72e559e6cc367d5999695ffa9b3ac32962a60c0604019dd4e282f",
+  "tokenizer_sha256": "d1ffe47f6fd8cc19ba7c3e5d2537699d6a99d09b584d3319944f4e40fe6c44ea",
+  "split": "document hash 1% held out",
+  "decontamination": "normalized 13-word exact overlap and exact short sentences; not semantic guarantee"
+}