Spaces:
Running
Running
portal: data page of repository commit 714ed1f4afaa
Browse filesFiles of catalog/portal/ (29), digest 2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221.
- first.json +61 -61
- funnel.json +20 -20
- llms.txt +1 -1
- manual/manual.en.md +63 -63
- manual/manual.zh.md +63 -63
- portal.json +2 -2
first.json
CHANGED
|
@@ -2,8 +2,8 @@
|
|
| 2 |
"collections": 14,
|
| 3 |
"date": "2026-11-15",
|
| 4 |
"format": "portal-first/5",
|
| 5 |
-
"index": "
|
| 6 |
-
"raw_rows":
|
| 7 |
"raw_rows_not_counted": 0,
|
| 8 |
"raw_sources": 139,
|
| 9 |
"ready_sources": 1845,
|
|
@@ -11,30 +11,30 @@
|
|
| 11 |
"ready_units_not_counted": 0,
|
| 12 |
"sizes": {
|
| 13 |
"corpus": {
|
| 14 |
-
"bytes":
|
| 15 |
"check": {
|
| 16 |
-
"compared":
|
| 17 |
"error": 2,
|
| 18 |
-
"gone":
|
| 19 |
"hash_mismatch": 14,
|
| 20 |
-
"match":
|
| 21 |
-
"needs_auth":
|
| 22 |
"no_length": 2022,
|
| 23 |
-
"rows":
|
| 24 |
"size_mismatch": 139,
|
| 25 |
-
"unreachable":
|
| 26 |
},
|
| 27 |
"counts": {
|
| 28 |
"corpora": 139,
|
| 29 |
-
"rows":
|
| 30 |
"rows_not_counted": 0
|
| 31 |
},
|
| 32 |
-
"distinct_bytes":
|
| 33 |
"estimate": {
|
| 34 |
-
"bytes":
|
| 35 |
-
"high":
|
| 36 |
-
"low":
|
| 37 |
-
"readings":
|
| 38 |
"rows": 277856236
|
| 39 |
},
|
| 40 |
"facts": {
|
|
@@ -62,95 +62,95 @@
|
|
| 62 |
},
|
| 63 |
"shown": {
|
| 64 |
"bytes": "4.45 PB",
|
| 65 |
-
"compared": "23,
|
| 66 |
"corpora": "139",
|
| 67 |
"distinct_bytes": "4.13 PB",
|
| 68 |
"estimate": "466 TB",
|
| 69 |
-
"match": "23,
|
| 70 |
"range": "323 TB to 695 TB",
|
| 71 |
-
"read": "26,
|
| 72 |
-
"rows": "1,189,
|
| 73 |
-
"unsized_rows": "288,989,
|
| 74 |
},
|
| 75 |
"shown_zh": {
|
| 76 |
"bytes": "4.45 PB",
|
| 77 |
-
"compared": "23,
|
| 78 |
"corpora": "139",
|
| 79 |
"distinct_bytes": "4.13 PB",
|
| 80 |
"estimate": "466 TB",
|
| 81 |
-
"match": "23,
|
| 82 |
"range": "323 TB \u5230 695 TB",
|
| 83 |
-
"read": "26,
|
| 84 |
-
"rows": "1,189,
|
| 85 |
-
"unsized_rows": "288,989,
|
| 86 |
},
|
| 87 |
-
"unsized_rows":
|
| 88 |
},
|
| 89 |
"index": {
|
| 90 |
-
"bytes":
|
| 91 |
"check": {
|
| 92 |
-
"compared":
|
| 93 |
"error": 5,
|
| 94 |
-
"gone":
|
| 95 |
"hash_mismatch": 47,
|
| 96 |
-
"match":
|
| 97 |
-
"needs_auth":
|
| 98 |
"no_length": 2022,
|
| 99 |
-
"rows":
|
| 100 |
"size_mismatch": 160,
|
| 101 |
-
"unreachable":
|
| 102 |
},
|
| 103 |
"counts": {
|
| 104 |
"corpora": 139,
|
| 105 |
"material_sets": 1845
|
| 106 |
},
|
| 107 |
-
"distinct_bytes":
|
| 108 |
"estimate": {
|
| 109 |
-
"bytes":
|
| 110 |
-
"high":
|
| 111 |
-
"low":
|
| 112 |
-
"readings":
|
| 113 |
"rows": 277856236
|
| 114 |
},
|
| 115 |
"shown": {
|
| 116 |
"bytes": "4.45 PB",
|
| 117 |
-
"compared": "35,
|
| 118 |
"corpora": "139",
|
| 119 |
"distinct_bytes": "4.13 PB",
|
| 120 |
"estimate": "466 TB",
|
| 121 |
-
"match": "35,
|
| 122 |
"material_sets": "1,845",
|
| 123 |
"range": "323 TB to 695 TB",
|
| 124 |
-
"read": "38,
|
| 125 |
-
"unsized_rows": "288,989,
|
| 126 |
},
|
| 127 |
"shown_zh": {
|
| 128 |
"bytes": "4.45 PB",
|
| 129 |
-
"compared": "35,
|
| 130 |
"corpora": "139",
|
| 131 |
"distinct_bytes": "4.13 PB",
|
| 132 |
"estimate": "466 TB",
|
| 133 |
-
"match": "35,
|
| 134 |
"material_sets": "1,845",
|
| 135 |
"range": "323 TB \u5230 695 TB",
|
| 136 |
-
"read": "38,
|
| 137 |
-
"unsized_rows": "288,989,
|
| 138 |
},
|
| 139 |
-
"unsized_rows":
|
| 140 |
},
|
| 141 |
"material_set": {
|
| 142 |
-
"bytes":
|
| 143 |
"check": {
|
| 144 |
-
"compared":
|
| 145 |
"error": 3,
|
| 146 |
-
"gone":
|
| 147 |
"hash_mismatch": 33,
|
| 148 |
-
"match":
|
| 149 |
-
"needs_auth":
|
| 150 |
"no_length": 0,
|
| 151 |
-
"rows":
|
| 152 |
"size_mismatch": 21,
|
| 153 |
-
"unreachable":
|
| 154 |
},
|
| 155 |
"counts": {
|
| 156 |
"collections": 14,
|
|
@@ -162,7 +162,7 @@
|
|
| 162 |
"units": 4244,
|
| 163 |
"units_not_counted": 0
|
| 164 |
},
|
| 165 |
-
"distinct_bytes":
|
| 166 |
"estimate": null,
|
| 167 |
"facts": {
|
| 168 |
"collecting": 0,
|
|
@@ -180,32 +180,32 @@
|
|
| 180 |
"shown": {
|
| 181 |
"bytes": "2.89 GB",
|
| 182 |
"collections": "14",
|
| 183 |
-
"compared": "11,
|
| 184 |
"distinct_bytes": "2.82 GB",
|
| 185 |
"empty_collections": "1",
|
| 186 |
"estimate": "none",
|
| 187 |
"files": "11,955",
|
| 188 |
"in_no_collection": "1",
|
| 189 |
-
"match": "11,
|
| 190 |
"material_sets": "1,845",
|
| 191 |
"range": "none",
|
| 192 |
-
"read": "11,
|
| 193 |
"units": "4,244",
|
| 194 |
"unsized_rows": "0"
|
| 195 |
},
|
| 196 |
"shown_zh": {
|
| 197 |
"bytes": "2.89 GB",
|
| 198 |
"collections": "14",
|
| 199 |
-
"compared": "11,
|
| 200 |
"distinct_bytes": "2.82 GB",
|
| 201 |
"empty_collections": "1",
|
| 202 |
"estimate": "\u65e0",
|
| 203 |
"files": "11,955",
|
| 204 |
"in_no_collection": "1",
|
| 205 |
-
"match": "11,
|
| 206 |
"material_sets": "1,845",
|
| 207 |
"range": "\u65e0",
|
| 208 |
-
"read": "11,
|
| 209 |
"units": "4,244",
|
| 210 |
"unsized_rows": "0"
|
| 211 |
},
|
|
|
|
| 2 |
"collections": 14,
|
| 3 |
"date": "2026-11-15",
|
| 4 |
"format": "portal-first/5",
|
| 5 |
+
"index": "89d4297c25bb7bf352b1874eca8f5c6f86adc7e1",
|
| 6 |
+
"raw_rows": 1189479628,
|
| 7 |
"raw_rows_not_counted": 0,
|
| 8 |
"raw_sources": 139,
|
| 9 |
"ready_sources": 1845,
|
|
|
|
| 11 |
"ready_units_not_counted": 0,
|
| 12 |
"sizes": {
|
| 13 |
"corpus": {
|
| 14 |
+
"bytes": 4445624070340900,
|
| 15 |
"check": {
|
| 16 |
+
"compared": 23848,
|
| 17 |
"error": 2,
|
| 18 |
+
"gone": 190,
|
| 19 |
"hash_mismatch": 14,
|
| 20 |
+
"match": 23695,
|
| 21 |
+
"needs_auth": 516,
|
| 22 |
"no_length": 2022,
|
| 23 |
+
"rows": 26722,
|
| 24 |
"size_mismatch": 139,
|
| 25 |
+
"unreachable": 144
|
| 26 |
},
|
| 27 |
"counts": {
|
| 28 |
"corpora": 139,
|
| 29 |
+
"rows": 1189479628,
|
| 30 |
"rows_not_counted": 0
|
| 31 |
},
|
| 32 |
+
"distinct_bytes": 4132631988631039,
|
| 33 |
"estimate": {
|
| 34 |
+
"bytes": 465697354954429,
|
| 35 |
+
"high": 695045945289888,
|
| 36 |
+
"low": 322590912679608,
|
| 37 |
+
"readings": 4388,
|
| 38 |
"rows": 277856236
|
| 39 |
},
|
| 40 |
"facts": {
|
|
|
|
| 62 |
},
|
| 63 |
"shown": {
|
| 64 |
"bytes": "4.45 PB",
|
| 65 |
+
"compared": "23,848",
|
| 66 |
"corpora": "139",
|
| 67 |
"distinct_bytes": "4.13 PB",
|
| 68 |
"estimate": "466 TB",
|
| 69 |
+
"match": "23,695",
|
| 70 |
"range": "323 TB to 695 TB",
|
| 71 |
+
"read": "26,722",
|
| 72 |
+
"rows": "1,189,479,628",
|
| 73 |
+
"unsized_rows": "288,989,937"
|
| 74 |
},
|
| 75 |
"shown_zh": {
|
| 76 |
"bytes": "4.45 PB",
|
| 77 |
+
"compared": "23,848",
|
| 78 |
"corpora": "139",
|
| 79 |
"distinct_bytes": "4.13 PB",
|
| 80 |
"estimate": "466 TB",
|
| 81 |
+
"match": "23,695",
|
| 82 |
"range": "323 TB \u5230 695 TB",
|
| 83 |
+
"read": "26,722",
|
| 84 |
+
"rows": "1,189,479,628",
|
| 85 |
+
"unsized_rows": "288,989,937"
|
| 86 |
},
|
| 87 |
+
"unsized_rows": 288989937
|
| 88 |
},
|
| 89 |
"index": {
|
| 90 |
+
"bytes": 4445626955419580,
|
| 91 |
"check": {
|
| 92 |
+
"compared": 35601,
|
| 93 |
"error": 5,
|
| 94 |
+
"gone": 193,
|
| 95 |
"hash_mismatch": 47,
|
| 96 |
+
"match": 35394,
|
| 97 |
+
"needs_auth": 526,
|
| 98 |
"no_length": 2022,
|
| 99 |
+
"rows": 38491,
|
| 100 |
"size_mismatch": 160,
|
| 101 |
+
"unreachable": 144
|
| 102 |
},
|
| 103 |
"counts": {
|
| 104 |
"corpora": 139,
|
| 105 |
"material_sets": 1845
|
| 106 |
},
|
| 107 |
+
"distinct_bytes": 4132634808526109,
|
| 108 |
"estimate": {
|
| 109 |
+
"bytes": 465697354954429,
|
| 110 |
+
"high": 695045945289888,
|
| 111 |
+
"low": 322590912679608,
|
| 112 |
+
"readings": 4388,
|
| 113 |
"rows": 277856236
|
| 114 |
},
|
| 115 |
"shown": {
|
| 116 |
"bytes": "4.45 PB",
|
| 117 |
+
"compared": "35,601",
|
| 118 |
"corpora": "139",
|
| 119 |
"distinct_bytes": "4.13 PB",
|
| 120 |
"estimate": "466 TB",
|
| 121 |
+
"match": "35,394",
|
| 122 |
"material_sets": "1,845",
|
| 123 |
"range": "323 TB to 695 TB",
|
| 124 |
+
"read": "38,491",
|
| 125 |
+
"unsized_rows": "288,989,937"
|
| 126 |
},
|
| 127 |
"shown_zh": {
|
| 128 |
"bytes": "4.45 PB",
|
| 129 |
+
"compared": "35,601",
|
| 130 |
"corpora": "139",
|
| 131 |
"distinct_bytes": "4.13 PB",
|
| 132 |
"estimate": "466 TB",
|
| 133 |
+
"match": "35,394",
|
| 134 |
"material_sets": "1,845",
|
| 135 |
"range": "323 TB \u5230 695 TB",
|
| 136 |
+
"read": "38,491",
|
| 137 |
+
"unsized_rows": "288,989,937"
|
| 138 |
},
|
| 139 |
+
"unsized_rows": 288989937
|
| 140 |
},
|
| 141 |
"material_set": {
|
| 142 |
+
"bytes": 2885078680,
|
| 143 |
"check": {
|
| 144 |
+
"compared": 11753,
|
| 145 |
"error": 3,
|
| 146 |
+
"gone": 3,
|
| 147 |
"hash_mismatch": 33,
|
| 148 |
+
"match": 11699,
|
| 149 |
+
"needs_auth": 10,
|
| 150 |
"no_length": 0,
|
| 151 |
+
"rows": 11769,
|
| 152 |
"size_mismatch": 21,
|
| 153 |
+
"unreachable": 0
|
| 154 |
},
|
| 155 |
"counts": {
|
| 156 |
"collections": 14,
|
|
|
|
| 162 |
"units": 4244,
|
| 163 |
"units_not_counted": 0
|
| 164 |
},
|
| 165 |
+
"distinct_bytes": 2819895070,
|
| 166 |
"estimate": null,
|
| 167 |
"facts": {
|
| 168 |
"collecting": 0,
|
|
|
|
| 180 |
"shown": {
|
| 181 |
"bytes": "2.89 GB",
|
| 182 |
"collections": "14",
|
| 183 |
+
"compared": "11,753",
|
| 184 |
"distinct_bytes": "2.82 GB",
|
| 185 |
"empty_collections": "1",
|
| 186 |
"estimate": "none",
|
| 187 |
"files": "11,955",
|
| 188 |
"in_no_collection": "1",
|
| 189 |
+
"match": "11,699",
|
| 190 |
"material_sets": "1,845",
|
| 191 |
"range": "none",
|
| 192 |
+
"read": "11,769",
|
| 193 |
"units": "4,244",
|
| 194 |
"unsized_rows": "0"
|
| 195 |
},
|
| 196 |
"shown_zh": {
|
| 197 |
"bytes": "2.89 GB",
|
| 198 |
"collections": "14",
|
| 199 |
+
"compared": "11,753",
|
| 200 |
"distinct_bytes": "2.82 GB",
|
| 201 |
"empty_collections": "1",
|
| 202 |
"estimate": "\u65e0",
|
| 203 |
"files": "11,955",
|
| 204 |
"in_no_collection": "1",
|
| 205 |
+
"match": "11,699",
|
| 206 |
"material_sets": "1,845",
|
| 207 |
"range": "\u65e0",
|
| 208 |
+
"read": "11,769",
|
| 209 |
"units": "4,244",
|
| 210 |
"unsized_rows": "0"
|
| 211 |
},
|
funnel.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
{
|
| 2 |
"format": "portal-funnel/1",
|
| 3 |
"funnel": {
|
| 4 |
-
"counted_on": "release-2026-11-15.
|
| 5 |
"domain_names": {
|
| 6 |
"D1": "software-engineering",
|
| 7 |
"D10": "misc",
|
|
@@ -52,21 +52,21 @@
|
|
| 52 |
},
|
| 53 |
"corpus": {
|
| 54 |
"by_domain": {
|
| 55 |
-
"D1":
|
| 56 |
"D10": 241140,
|
| 57 |
-
"D2":
|
| 58 |
"D3": 761632806,
|
| 59 |
"D4": 9646616,
|
| 60 |
"D5": 45635385,
|
| 61 |
"D6": 39426757,
|
| 62 |
"D7": 8719015,
|
| 63 |
-
"D8":
|
| 64 |
"D9": 12753055,
|
| 65 |
"multi": 109275447,
|
| 66 |
-
"unclassified":
|
| 67 |
},
|
| 68 |
"corpora": 139,
|
| 69 |
-
"rows":
|
| 70 |
"rows_not_counted": 0
|
| 71 |
}
|
| 72 |
},
|
|
@@ -127,7 +127,7 @@
|
|
| 127 |
"zh": "后训练"
|
| 128 |
},
|
| 129 |
{
|
| 130 |
-
"before":
|
| 131 |
"by_domain": {
|
| 132 |
"D1": {
|
| 133 |
"bytes": 2968634067,
|
|
@@ -138,8 +138,8 @@
|
|
| 138 |
"kept": 102989
|
| 139 |
},
|
| 140 |
"D2": {
|
| 141 |
-
"bytes":
|
| 142 |
-
"kept":
|
| 143 |
},
|
| 144 |
"D3": {
|
| 145 |
"bytes": 429176613679,
|
|
@@ -174,15 +174,15 @@
|
|
| 174 |
"kept": 16193621
|
| 175 |
},
|
| 176 |
"unclassified": {
|
| 177 |
-
"bytes":
|
| 178 |
-
"kept":
|
| 179 |
}
|
| 180 |
},
|
| 181 |
-
"bytes":
|
| 182 |
"domains": [],
|
| 183 |
"forms": [],
|
| 184 |
"id": "mid-training",
|
| 185 |
-
"kept":
|
| 186 |
"kind": "corpus",
|
| 187 |
"measure": "rows",
|
| 188 |
"name": "Mid-training",
|
|
@@ -190,7 +190,7 @@
|
|
| 190 |
"zh": "中训练"
|
| 191 |
},
|
| 192 |
{
|
| 193 |
-
"before":
|
| 194 |
"by_domain": {
|
| 195 |
"D1": {
|
| 196 |
"bytes": 181098708,
|
|
@@ -237,15 +237,15 @@
|
|
| 237 |
"kept": 8262018
|
| 238 |
},
|
| 239 |
"unclassified": {
|
| 240 |
-
"bytes":
|
| 241 |
-
"kept":
|
| 242 |
}
|
| 243 |
},
|
| 244 |
-
"bytes":
|
| 245 |
"domains": [],
|
| 246 |
"forms": [],
|
| 247 |
"id": "pre-training",
|
| 248 |
-
"kept":
|
| 249 |
"kind": "corpus",
|
| 250 |
"measure": "rows",
|
| 251 |
"name": "Pre-training",
|
|
@@ -255,6 +255,6 @@
|
|
| 255 |
],
|
| 256 |
"table": 3
|
| 257 |
},
|
| 258 |
-
"index": "
|
| 259 |
-
"release": "release-2026-11-15.
|
| 260 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"format": "portal-funnel/1",
|
| 3 |
"funnel": {
|
| 4 |
+
"counted_on": "release-2026-11-15.6",
|
| 5 |
"domain_names": {
|
| 6 |
"D1": "software-engineering",
|
| 7 |
"D10": "misc",
|
|
|
|
| 52 |
},
|
| 53 |
"corpus": {
|
| 54 |
"by_domain": {
|
| 55 |
+
"D1": 29959198,
|
| 56 |
"D10": 241140,
|
| 57 |
+
"D2": 161289873,
|
| 58 |
"D3": 761632806,
|
| 59 |
"D4": 9646616,
|
| 60 |
"D5": 45635385,
|
| 61 |
"D6": 39426757,
|
| 62 |
"D7": 8719015,
|
| 63 |
+
"D8": 4535660,
|
| 64 |
"D9": 12753055,
|
| 65 |
"multi": 109275447,
|
| 66 |
+
"unclassified": 6364676
|
| 67 |
},
|
| 68 |
"corpora": 139,
|
| 69 |
+
"rows": 1189479628,
|
| 70 |
"rows_not_counted": 0
|
| 71 |
}
|
| 72 |
},
|
|
|
|
| 127 |
"zh": "后训练"
|
| 128 |
},
|
| 129 |
{
|
| 130 |
+
"before": 34629789,
|
| 131 |
"by_domain": {
|
| 132 |
"D1": {
|
| 133 |
"bytes": 2968634067,
|
|
|
|
| 138 |
"kept": 102989
|
| 139 |
},
|
| 140 |
"D2": {
|
| 141 |
+
"bytes": 1268594860300,
|
| 142 |
+
"kept": 11396027
|
| 143 |
},
|
| 144 |
"D3": {
|
| 145 |
"bytes": 429176613679,
|
|
|
|
| 174 |
"kept": 16193621
|
| 175 |
},
|
| 176 |
"unclassified": {
|
| 177 |
+
"bytes": 4259294359,
|
| 178 |
+
"kept": 477858
|
| 179 |
}
|
| 180 |
},
|
| 181 |
+
"bytes": 3864859030859,
|
| 182 |
"domains": [],
|
| 183 |
"forms": [],
|
| 184 |
"id": "mid-training",
|
| 185 |
+
"kept": 34231698,
|
| 186 |
"kind": "corpus",
|
| 187 |
"measure": "rows",
|
| 188 |
"name": "Mid-training",
|
|
|
|
| 190 |
"zh": "中训练"
|
| 191 |
},
|
| 192 |
{
|
| 193 |
+
"before": 155528248,
|
| 194 |
"by_domain": {
|
| 195 |
"D1": {
|
| 196 |
"bytes": 181098708,
|
|
|
|
| 237 |
"kept": 8262018
|
| 238 |
},
|
| 239 |
"unclassified": {
|
| 240 |
+
"bytes": 41361755124,
|
| 241 |
+
"kept": 472000
|
| 242 |
}
|
| 243 |
},
|
| 244 |
+
"bytes": 30709390159493,
|
| 245 |
"domains": [],
|
| 246 |
"forms": [],
|
| 247 |
"id": "pre-training",
|
| 248 |
+
"kept": 150610889,
|
| 249 |
"kind": "corpus",
|
| 250 |
"measure": "rows",
|
| 251 |
"name": "Pre-training",
|
|
|
|
| 255 |
],
|
| 256 |
"table": 3
|
| 257 |
},
|
| 258 |
+
"index": "89d4297c25bb7bf352b1874eca8f5c6f86adc7e1",
|
| 259 |
+
"release": "release-2026-11-15.6"
|
| 260 |
}
|
llms.txt
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
- The index holds two layers over the same public web, kept apart and never added together. Each corpus is one whole upstream read in bulk and counted in rows; a material set is admitted for building tasks from works the search found upstream or from a group of corpus rows, its files checked and grouped into units, and it is counted in units, apart from any corpus its rows came from. Task material comes first: material sets, grouped into collections, one per task family. A material set (task-ready) is the units admitted from one source, which reads one upstream (one dataset or one data catalog), for building tasks (the catalog column `kind` is `material_set`, and `collection` names its collection). A unit is a group of files that its upstream publishes as one thing, delivered whole; its key `unit_id` is the `unit_group` its rows share (`extra.source_meta.unit_group`), defined once, in the glossary of the data page's manual, https://battam-training-data-foundation.static.hf.space/index.html?lng=en#manual/glossary . A unit is material, not a task: one unit may give several tasks, and one task may use several units. Corpora come second, to search when no material set fits: a corpus (raw; plural corpora) is one whole upstream that a collector keeps reading, such as Common Crawl's office documents, PyPI's release files, Mendeley Data or NVD: its rows are pointers to files or API responses, and it is counted in rows (`kind` is `corpus`); `tools/fetch.py` takes a corpus row by row, each under its own key as its `unit_id`.
|
| 4 |
- Pick material sets or corpora from `meta/catalog.parquet` (one row per material set or corpus, about 50 KB; do not download `meta/` whole). Columns: `source`; `kind`, `material_set` or `corpus` (select one layer: units and rows are never added together); `collection`, the code of a material set's collection (every code with its name is in `meta/vocabulary.json`), or of the collection that lists a corpus as holding its kind of material (`corpora` in `meta/vocabulary.json`; empty for any other corpus); `domain`, a list of codes D1 to D10 or leaves such as D3.5, or `multi`, `unclassified`; `form`, such as `test-suites` or `spreadsheets`; `upstream_kind`, `upstream_agent` and `upstream_entry`, the source's upstream (`dataset` or `catalog`, its agent, and the address its reading starts from; a source that still reads several upstreams gives each, joined by ` | `); `units` (a material set's distinct `unit_id` values: a test case, one workbook, one package; a corpus is counted in rows, and gives the same number as `files`) and `files` (its rows: for a material set, the files of its units and its README.md and LICENSE); `publishers` (a material set's distinct publisher names over all its rows; empty when its rows name none); `language` (the language codes of the rows' labels, what the default profile filters on; `not recorded` when a row has none); `contamination.level` (by the rule of the default profile of `tools/fetch.py` over `meta/bench_overlap.parquet` and `meta/bench_checked.parquet`: `none` when every data file is compared and kept and no row is removed, the hit class with the most removed rows when some are removed, `not scanned` otherwise); `license.ship`; `pii.level`; `distinct_bytes` (deduplicated size) and `bytes` (recorded size, the sum over rows); `size_check` (the rows read in `meta/sizes.parquet`, `compared` those that compared a size or hash, and how many have each status, `match` when the measured size agrees; the check's share is `match` over `compared`), `unsized_rows` (rows without a recorded size) and `unsized_estimate` (their estimated size, with its 95% range); `fetch_check` (whether the upstream still serves the files: `rows`, the sampled rows read in `meta/sizes.parquet`; `fetched`, those that fetched and matched their recorded size and hash; `read_at`, the day read; `level`, `all`, `some`, `none`, `needs auth` (no row read fetched, and the source's rows need the paying or signed-in reader its `needs_auth` names, which the check, reading anonymously, does not have) or `not read`); the same figures for the whole index, each layer and each collection are in `meta/size_figures.json`. Who published each file of a material set, and under which license and license URL, is in `meta/attribution.parquet` (one row per source, publisher, license and license URL, with its number of files). The vocabulary is `meta/vocabulary.json`: every domain code with its name (and `zh`, its Chinese name), what it covers and its parent, every form with its Chinese name and what it means, every category of `meta/not_included.parquet` with its requirement in English and in Chinese, the two layers with their relation, and every collection with its name, what it holds, its campaigns (take a collection with `--tag campaign:<name>` for each of its campaigns) and the corpora it lists, if any (take one with `--source`), and the criteria the default profile of `tools/fetch.py` applies, in its order (`profile`, each with the option that keeps what it removes).
|
| 5 |
-
- Size of this release: Whole index: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (35,
|
| 6 |
- Fetch with one command: `uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/main/tools/fetch.py --kind material_set --domain D3 --form spreadsheets --out ./data` takes those material sets whole, as the data page does unless told otherwise; take a corpus, whose whole can be petabytes, with `--sample N` (the data page takes 3 units of each by default) or `--limit N`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--kind` (`material_set` or `corpus`: one layer), `--domain`, `--form`, `--tag`, `--ext`, or `--selection <file>` (a selection file the data page saves: the ids of its sources and its commit); `--sample N` takes at most N units per source, `--limit N` takes N units over the whole selection, `--unit <unit_id>` takes the unit of that `unit_id` whole (with `--source`); before its first download the run prints the units, files and total size it selected.
|
| 7 |
- It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
|
| 8 |
- The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, rows of groupings awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), rows recorded in a language other than English, files above 20 MB, and rows that the release's fetch check read and found refused by their site (HTTP 401 or 403, a login, or an empty or short body). Before downloading it prints how many units each criterion removed; each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-language`, `--any-size`, `--allow-refused`; `--allow-benchmark` also keeps rows not compared yet), and `--all` turns off every criterion except benchmark and not compared, so rows that matched a benchmark or were not compared yet come in only with their own options. The license and the personal-data mark are labels of every row, not filters: by default every row comes, and two options leave rows out only when given, `--training-only` the files whose license does not allow shipping (license.ship yes and conditional stay) and `--no-personal-data` personal data (catalog pii.level present or suspected, rows marked sensitive). Compared with the benchmark list (keys of 2026-11-08): 1,845 of 1,845 material sets; 139 of 139 corpora. The profile also removes every row whose data file was not compared (`--allow-benchmark` keeps them, unchecked).
|
|
|
|
| 2 |
|
| 3 |
- The index holds two layers over the same public web, kept apart and never added together. Each corpus is one whole upstream read in bulk and counted in rows; a material set is admitted for building tasks from works the search found upstream or from a group of corpus rows, its files checked and grouped into units, and it is counted in units, apart from any corpus its rows came from. Task material comes first: material sets, grouped into collections, one per task family. A material set (task-ready) is the units admitted from one source, which reads one upstream (one dataset or one data catalog), for building tasks (the catalog column `kind` is `material_set`, and `collection` names its collection). A unit is a group of files that its upstream publishes as one thing, delivered whole; its key `unit_id` is the `unit_group` its rows share (`extra.source_meta.unit_group`), defined once, in the glossary of the data page's manual, https://battam-training-data-foundation.static.hf.space/index.html?lng=en#manual/glossary . A unit is material, not a task: one unit may give several tasks, and one task may use several units. Corpora come second, to search when no material set fits: a corpus (raw; plural corpora) is one whole upstream that a collector keeps reading, such as Common Crawl's office documents, PyPI's release files, Mendeley Data or NVD: its rows are pointers to files or API responses, and it is counted in rows (`kind` is `corpus`); `tools/fetch.py` takes a corpus row by row, each under its own key as its `unit_id`.
|
| 4 |
- Pick material sets or corpora from `meta/catalog.parquet` (one row per material set or corpus, about 50 KB; do not download `meta/` whole). Columns: `source`; `kind`, `material_set` or `corpus` (select one layer: units and rows are never added together); `collection`, the code of a material set's collection (every code with its name is in `meta/vocabulary.json`), or of the collection that lists a corpus as holding its kind of material (`corpora` in `meta/vocabulary.json`; empty for any other corpus); `domain`, a list of codes D1 to D10 or leaves such as D3.5, or `multi`, `unclassified`; `form`, such as `test-suites` or `spreadsheets`; `upstream_kind`, `upstream_agent` and `upstream_entry`, the source's upstream (`dataset` or `catalog`, its agent, and the address its reading starts from; a source that still reads several upstreams gives each, joined by ` | `); `units` (a material set's distinct `unit_id` values: a test case, one workbook, one package; a corpus is counted in rows, and gives the same number as `files`) and `files` (its rows: for a material set, the files of its units and its README.md and LICENSE); `publishers` (a material set's distinct publisher names over all its rows; empty when its rows name none); `language` (the language codes of the rows' labels, what the default profile filters on; `not recorded` when a row has none); `contamination.level` (by the rule of the default profile of `tools/fetch.py` over `meta/bench_overlap.parquet` and `meta/bench_checked.parquet`: `none` when every data file is compared and kept and no row is removed, the hit class with the most removed rows when some are removed, `not scanned` otherwise); `license.ship`; `pii.level`; `distinct_bytes` (deduplicated size) and `bytes` (recorded size, the sum over rows); `size_check` (the rows read in `meta/sizes.parquet`, `compared` those that compared a size or hash, and how many have each status, `match` when the measured size agrees; the check's share is `match` over `compared`), `unsized_rows` (rows without a recorded size) and `unsized_estimate` (their estimated size, with its 95% range); `fetch_check` (whether the upstream still serves the files: `rows`, the sampled rows read in `meta/sizes.parquet`; `fetched`, those that fetched and matched their recorded size and hash; `read_at`, the day read; `level`, `all`, `some`, `none`, `needs auth` (no row read fetched, and the source's rows need the paying or signed-in reader its `needs_auth` names, which the check, reading anonymously, does not have) or `not read`); the same figures for the whole index, each layer and each collection are in `meta/size_figures.json`. Who published each file of a material set, and under which license and license URL, is in `meta/attribution.parquet` (one row per source, publisher, license and license URL, with its number of files). The vocabulary is `meta/vocabulary.json`: every domain code with its name (and `zh`, its Chinese name), what it covers and its parent, every form with its Chinese name and what it means, every category of `meta/not_included.parquet` with its requirement in English and in Chinese, the two layers with their relation, and every collection with its name, what it holds, its campaigns (take a collection with `--tag campaign:<name>` for each of its campaigns) and the corpora it lists, if any (take one with `--source`), and the criteria the default profile of `tools/fetch.py` applies, in its order (`profile`, each with the option that keeps what it removes).
|
| 5 |
+
- Size of this release: Whole index: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (35,601 of the 38,491 rows read compared); not compared: address unreachable 144, file gone 193, login needed 526, no size from the server 2,022, other error 5; estimated size of 277,856,236 of the 288,989,937 rows without a recorded size: 466 TB (an estimate; 95% range 323 TB to 695 TB, the sum of its sources' ranges, wider than a joint interval; from 4,388 rows read), the other 11,133,701 with too few rows read for an estimate. Task material: Recorded size 2.89 GB; deduplicated size 2.82 GB; size check: 99.5% match (11,753 of the 11,769 rows read compared); not compared: file gone 3, login needed 10, other error 3. Corpora: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (23,848 of the 26,722 rows read compared); not compared: address unreachable 144, file gone 190, login needed 516, no size from the server 2,022, other error 2; estimated size of 277,856,236 of the 288,989,937 rows without a recorded size: 466 TB (an estimate; 95% range 323 TB to 695 TB, the sum of its sources' ranges, wider than a joint interval; from 4,388 rows read), the other 11,133,701 with too few rows read for an estimate. Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-15.6`. The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
|
| 6 |
- Fetch with one command: `uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/main/tools/fetch.py --kind material_set --domain D3 --form spreadsheets --out ./data` takes those material sets whole, as the data page does unless told otherwise; take a corpus, whose whole can be petabytes, with `--sample N` (the data page takes 3 units of each by default) or `--limit N`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--kind` (`material_set` or `corpus`: one layer), `--domain`, `--form`, `--tag`, `--ext`, or `--selection <file>` (a selection file the data page saves: the ids of its sources and its commit); `--sample N` takes at most N units per source, `--limit N` takes N units over the whole selection, `--unit <unit_id>` takes the unit of that `unit_id` whole (with `--source`); before its first download the run prints the units, files and total size it selected.
|
| 7 |
- It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
|
| 8 |
- The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, rows of groupings awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), rows recorded in a language other than English, files above 20 MB, and rows that the release's fetch check read and found refused by their site (HTTP 401 or 403, a login, or an empty or short body). Before downloading it prints how many units each criterion removed; each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-language`, `--any-size`, `--allow-refused`; `--allow-benchmark` also keeps rows not compared yet), and `--all` turns off every criterion except benchmark and not compared, so rows that matched a benchmark or were not compared yet come in only with their own options. The license and the personal-data mark are labels of every row, not filters: by default every row comes, and two options leave rows out only when given, `--training-only` the files whose license does not allow shipping (license.ship yes and conditional stay) and `--no-personal-data` personal data (catalog pii.level present or suspected, rows marked sensitive). Compared with the benchmark list (keys of 2026-11-08): 1,845 of 1,845 material sets; 139 of 139 corpora. The profile also removes every row whose data file was not compared (`--allow-benchmark` keeps them, unchecked).
|
manual/manual.en.md
CHANGED
|
@@ -12,7 +12,7 @@ Each row records one file's address, size, hash (a fingerprint of its content) a
|
|
| 12 |
<h2 id="manual/guides">Guides</h2>
|
| 13 |
|
| 14 |
One goal each;
|
| 15 |
-
commands are pinned to `release-2026-11-15.
|
| 16 |
|
| 17 |
<h3 id="manual/take">Take the files a need calls for</h3>
|
| 18 |
|
|
@@ -23,9 +23,9 @@ commands are pinned to `release-2026-11-15.5`.
|
|
| 23 |
3. In that folder, run the command of "Take it again from the file" with [uv](https://docs.astral.sh/uv/):
|
| 24 |
|
| 25 |
```sh
|
| 26 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 27 |
-
--selection training-data-foundation-selection-
|
| 28 |
-
--revision release-2026-11-15.
|
| 29 |
--out ./data
|
| 30 |
```
|
| 31 |
|
|
@@ -41,7 +41,7 @@ commands are pinned to `release-2026-11-15.5`.
|
|
| 41 |
```sh
|
| 42 |
uvx --from duckdb-cli duckdb -c "
|
| 43 |
SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
|
| 44 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 45 |
WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
|
| 46 |
ORDER BY collection, units DESC"
|
| 47 |
```
|
|
@@ -55,9 +55,9 @@ commands are pinned to `release-2026-11-15.5`.
|
|
| 55 |
to leave out the files whose license does not let them be shipped and the rows marked as carrying personal data, add two options:
|
| 56 |
|
| 57 |
```sh
|
| 58 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 59 |
-
--selection training-data-foundation-selection-
|
| 60 |
-
--revision release-2026-11-15.
|
| 61 |
--training-only \
|
| 62 |
--no-personal-data \
|
| 63 |
--out ./data
|
|
@@ -129,7 +129,7 @@ it follows your filter changes, so copying it shares the state.
|
|
| 129 |
2 when the index could not be read or a name is not in the catalog, such as a misspelt source:
|
| 130 |
|
| 131 |
```text
|
| 132 |
-
error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
|
| 133 |
```
|
| 134 |
|
| 135 |
</details>
|
|
@@ -137,11 +137,11 @@ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
|
|
| 137 |
<details id="manual/pin">
|
| 138 |
<summary>Pin a release</summary>
|
| 139 |
|
| 140 |
-
The page's commands pin the script URL and `--revision` to `release-2026-11-15.
|
| 141 |
the script's first line names the commit:
|
| 142 |
|
| 143 |
```text
|
| 144 |
-
reading hf://datasets/Battam/training-data-foundation@
|
| 145 |
```
|
| 146 |
|
| 147 |
Later, `--revision <commit>`, the same tag or the selection list take the same files.
|
|
@@ -214,7 +214,7 @@ Rows per class:
|
|
| 214 |
|
| 215 |
```sql
|
| 216 |
SELECT hit_class, scope, count(*) AS rows
|
| 217 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 218 |
GROUP BY ALL ORDER BY hit_class, scope;
|
| 219 |
```
|
| 220 |
|
|
@@ -251,16 +251,16 @@ the dataset https://huggingface.co/datasets/Battam/training-data-foundation .
|
|
| 251 |
|
| 252 |
| To | Read |
|
| 253 |
|---|---|
|
| 254 |
-
| Read every field of a row and every column of the catalog | [docs/fields.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 255 |
-
| See every material set and every corpus, by domain, with its counts, size, license tier and login | [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 256 |
-
| See what was not included and why, and which sources need a login | [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 257 |
-
| Build tasks from the files, and keep what an agent must not see out of its environment | [docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 258 |
-
| See every release and what it changed | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 259 |
-
| Read the dataset as Croissant metadata | [Croissant generated by the Hub](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant); [`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 260 |
| Look through the catalog, one row per material set or corpus with its `kind` and `collection` | [The `catalog` subset in the Dataset Viewer](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
|
| 261 |
| Query the catalog in the browser; it lists every source as published, without the default profile of `tools/fetch.py`, which the command applies | [SQL Console](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
|
| 262 |
| Watch the index in use, in the page's language, with subtitles | [Usage video on the data page](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
|
| 263 |
-
| Check every number of the dataset card and of the tables here | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 264 |
|
| 265 |
Each row of the data files under `data/` points to one file or record (Row, in the [glossary](#manual/glossary));
|
| 266 |
the catalog's column `data_files` names the files of each source, and one material set or corpus is read from exactly those files, as in the queries of the dataset card and with `tools/fetch.py`.
|
|
@@ -310,9 +310,9 @@ Each row also has a training and a redistribution permit (yes, conditional, no;
|
|
| 310 |
- This catalog itself (addresses, sizes, hashes and descriptive text) is released under CC0 1.0; that dedication does not extend to the files it points to.
|
| 311 |
- The files under `media/` (the usage example video and its subtitles) and under `examples/` (the worked examples) are released under CC0 1.0 as well.
|
| 312 |
- Each file keeps the license of its source: see the license columns of each row (a row without its own license takes the license of its batch, `batches[].license` in `meta/sources.json`).
|
| 313 |
-
- The tier of every source: on the page of its domain, which [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 314 |
- License tiers and reason codes are computed from the license categories of ScanCode LicenseDB, licensed CC-BY-4.0;
|
| 315 |
-
its attribution: [`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 316 |
|
| 317 |
<h4 id="manual/personal">Personal and sensitive information</h4>
|
| 318 |
|
|
@@ -358,9 +358,9 @@ Each row also has a training and a redistribution permit (yes, conditional, no;
|
|
| 358 |
|
| 359 |
<h4 id="manual/releases">Releases</h4>
|
| 360 |
|
| 361 |
-
- Latest release: `release-2026-11-15.
|
| 362 |
-
- Releases are listed in [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 363 |
-
each one is a commit with its own release tag (this one is [`release-2026-11-15.
|
| 364 |
so any earlier state stays readable at its tag.
|
| 365 |
- `meta/projection.json` records the release tag and the version of the program and templates that generated the dataset card and its pages.
|
| 366 |
|
|
@@ -395,7 +395,7 @@ to cite it:
|
|
| 395 |
@misc{training_data_foundation,
|
| 396 |
title = {training-data-foundation: an index of public materials for building agent environments},
|
| 397 |
howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
|
| 398 |
-
note = {Release release-2026-11-15.
|
| 399 |
year = {2026}
|
| 400 |
}
|
| 401 |
```
|
|
@@ -430,7 +430,7 @@ to cite it:
|
|
| 430 |
- **Form:** the shape of a source's material, one id such as `spreadsheets` or `test-suites` (the catalog column `form`, the option `--form` of `tools/fetch.py`).
|
| 431 |
- **Training stage:** what the material can feed, by the stage table `catalog/stages.yaml` of the code repository: pre-training and mid-training take corpus rows, post-training the units of material sets ([How the training-stage tables are counted](#manual/stage-counting)).
|
| 432 |
- **Default profile:** the rules `tools/fetch.py` applies unless an option relaxes them: it leaves out duplicate copies, rows that matched a benchmark or were not compared with the benchmark list yet, rows in a language other than English and files above 20 MB;
|
| 433 |
-
rows of any license and rows marked as carrying personal data come unless `--training-only` or `--no-personal-data` leaves them out ([docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 434 |
|
| 435 |
Every row carries a `unit_id`: in a material set, the key of its unit;
|
| 436 |
in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte range, then `!/<member_path>` for an archive member), and `tools/fetch.py` puts the file of a corpus row in a folder named after it.
|
|
@@ -440,8 +440,8 @@ in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte ran
|
|
| 440 |
<details id="manual/numbers">
|
| 441 |
<summary>Where this manual's numbers come from</summary>
|
| 442 |
|
| 443 |
-
The numbers and command outputs of this manual were filled in at deploy time, from the index commit `
|
| 444 |
-
This release holds 1,845 material sets with 4,244 units in 14 collections and 139 corpora with 1,189,
|
| 445 |
|
| 446 |
</details>
|
| 447 |
|
|
@@ -452,7 +452,7 @@ Each table below gives what a training stage can take, counted before the rules
|
|
| 452 |
its last two rows count how much of the stage's count is allowed for training (every file's training permit yes or conditional) and how much is marked as carrying personal data.
|
| 453 |
How they are counted: [How the training-stage tables are counted](#manual/stage-counting).
|
| 454 |
|
| 455 |
-
The stage fields (`meta/stage_fields.parquet`) were counted over the data files of release `release-2026-11-15.
|
| 456 |
|
| 457 |
##### Post-training (in use)
|
| 458 |
|
|
@@ -464,7 +464,7 @@ Material set units for building tasks: every file of the unit in English or no n
|
|
| 464 |
| After the benchmark rule | 1,826 | 4,202 | 1,879 |
|
| 465 |
| After the rule on rows not compared with the benchmark list | 1,826 | 4,202 | 1,879 |
|
| 466 |
| After deduplication: the stage's count | 1,766 | 4,059 | 1,823 |
|
| 467 |
-
| Of the stage's count, allowed for training (yes or conditional) | 1,
|
| 468 |
| Of the stage's count, marked as carrying personal data | 357 | 631 | 400 |
|
| 469 |
|
| 470 |
##### Mid-training (provisional: conditions and counts only)
|
|
@@ -473,12 +473,12 @@ Corpus rows of code, documentation and the terminal side: conditions and counts
|
|
| 473 |
|
| 474 |
| Step | Rows | Size |
|
| 475 |
|---|---:|---:|
|
| 476 |
-
| In the stage's condition, before the rules | 34,629,
|
| 477 |
-
| After the benchmark rule | 34,
|
| 478 |
-
| After the rule on rows not compared with the benchmark list | 34,
|
| 479 |
-
| After deduplication: the stage's count | 34,231,
|
| 480 |
| Of the stage's count, allowed for training (yes or conditional) | 9,937,384 | 1.42 TB |
|
| 481 |
-
| Of the stage's count, marked as carrying personal data | 6,104,
|
| 482 |
|
| 483 |
##### Pre-training (provisional: conditions and counts only)
|
| 484 |
|
|
@@ -486,10 +486,10 @@ Corpus rows of plain text, the mid-training types not counted here: conditions a
|
|
| 486 |
|
| 487 |
| Step | Rows | Size |
|
| 488 |
|---|---:|---:|
|
| 489 |
-
| In the stage's condition, before the rules | 155,
|
| 490 |
-
| After the benchmark rule | 155,
|
| 491 |
-
| After the rule on rows not compared with the benchmark list | 155,
|
| 492 |
-
| After deduplication: the stage's count | 150,
|
| 493 |
| Of the stage's count, allowed for training (yes or conditional) | 119,168,563 | 30.0 TB |
|
| 494 |
| Of the stage's count, marked as carrying personal data | 40,477,415 | 676 GB |
|
| 495 |
|
|
@@ -550,7 +550,7 @@ or add `&open=zenodo-workbooks-cc-by` to the address.
|
|
| 550 |
<h4 id="manual/workbooks-5">5. Go to "Get the data" and keep the selection</h4>
|
| 551 |
|
| 552 |
**Do**: click "Get the data" at the top;
|
| 553 |
-
under "Keep this selection" click "Download the selection list (JSON)" and put `training-data-foundation-selection-
|
| 554 |
|
| 555 |
**See**: the heading "Take every unit of each of 51 material sets".
|
| 556 |
The list records release, filters and sources:
|
|
@@ -559,7 +559,7 @@ The list records release, filters and sources:
|
|
| 559 |
{
|
| 560 |
"format": "training-data-foundation-selection/1",
|
| 561 |
"dataset": "Battam/training-data-foundation",
|
| 562 |
-
"revision": "release-2026-11-15.
|
| 563 |
"filters": {
|
| 564 |
"collection": "A",
|
| 565 |
"train": "yes",
|
|
@@ -583,9 +583,9 @@ for 200 in all, use `--limit`.
|
|
| 583 |
**Do**: in the list's folder, run the command of "Take it again from the file" with `--limit 220` added (220 units in all, in a fixed order: a margin over 200, since some sites fail) and `--out ./data` replaced by `--out ./take`:
|
| 584 |
|
| 585 |
```sh
|
| 586 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 587 |
-
--selection training-data-foundation-selection-
|
| 588 |
-
--revision release-2026-11-15.
|
| 589 |
--ext xlsx \
|
| 590 |
--ext xlsm \
|
| 591 |
--ext xls \
|
|
@@ -614,7 +614,7 @@ ok austender-ict-procurement-statistics workbooks/2016-17-austender-ict-contract
|
|
| 614 |
...
|
| 615 |
310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
|
| 616 |
files of units: 220 ok; files of the sources themselves: 90 ok
|
| 617 |
-
run_id
|
| 618 |
```
|
| 619 |
|
| 620 |
Before the first download, `removed by` gives the units the default profile removed and the option that keeps them, counted in the data files read as far as `--limit` needed;
|
|
@@ -630,8 +630,8 @@ failed western-australia-government-workbooks workbooks/wa-gross-regional-produc
|
|
| 630 |
incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
|
| 631 |
5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
|
| 632 |
files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
|
| 633 |
-
run_id
|
| 634 |
-
its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '
|
| 635 |
```
|
| 636 |
|
| 637 |
**Get**: every `ok` file matched its recorded size and hash.
|
|
@@ -702,7 +702,7 @@ a workbook stores the cached results of its formulas, so a task that asks for co
|
|
| 702 |
**See** (truncated):
|
| 703 |
|
| 704 |
```text
|
| 705 |
-
Dataset commit
|
| 706 |
...
|
| 707 |
- Source: `austender-ict-procurement-statistics`
|
| 708 |
- License: CC-BY-3.0-AU
|
|
@@ -735,7 +735,7 @@ Let DuckDB write the `--source` options, say for the disk image material sets wi
|
|
| 735 |
|
| 736 |
```sql
|
| 737 |
SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
|
| 738 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 739 |
WHERE kind = 'material_set' AND collection = 'B'
|
| 740 |
AND license.tier = 'share-alike';
|
| 741 |
```
|
|
@@ -744,7 +744,7 @@ WHERE kind = 'material_set' AND collection = 'B'
|
|
| 744 |
--source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
|
| 745 |
```
|
| 746 |
|
| 747 |
-
Put them after `fetch.py --revision release-2026-11-15.
|
| 748 |
Read only the catalog, not all of `meta/`.
|
| 749 |
|
| 750 |
</details>
|
|
@@ -758,8 +758,8 @@ Run "1 Take" (output truncated):
|
|
| 758 |

|
| 759 |
|
| 760 |
```sh
|
| 761 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 762 |
-
--revision release-2026-11-15.
|
| 763 |
--source dosfstools-fsck-cases \
|
| 764 |
--source e2fsprogs-fsck-cases \
|
| 765 |
--source exfatprogs-fsck-cases \
|
|
@@ -774,7 +774,7 @@ ok dosfstools-fsck-cases README.md: 3.32kB
|
|
| 774 |
ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
|
| 775 |
59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
|
| 776 |
files of units: 52 ok; files of the sources themselves: 7 ok
|
| 777 |
-
run_id
|
| 778 |
```
|
| 779 |
|
| 780 |
An exFAT image is `exfat.img.tar.xz` upstream;
|
|
@@ -818,7 +818,7 @@ One, the catalog's `contamination.level` (values in [Benchmark overlap marks](#m
|
|
| 818 |
|
| 819 |
```sql
|
| 820 |
SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
|
| 821 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 822 |
WHERE kind = 'material_set'
|
| 823 |
AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
|
| 824 |
GROUP BY ALL;
|
|
@@ -838,8 +838,8 @@ GROUP BY ALL;
|
|
| 838 |
Two, take the picked sources and read `removed by benchmark` and `removed by not compared`, printed before the first download (output truncated):
|
| 839 |
|
| 840 |
```sh
|
| 841 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 842 |
-
--revision release-2026-11-15.
|
| 843 |
--source python-docx-acceptance-test-files \
|
| 844 |
--source python-pptx-acceptance-test-files \
|
| 845 |
--source docling-docx-groundtruth \
|
|
@@ -867,7 +867,7 @@ Three, match file by file against the overlap table;
|
|
| 867 |
```sql
|
| 868 |
SELECT o.bench, o.hit_class, count(*) AS files
|
| 869 |
FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
|
| 870 |
-
JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 871 |
ON o.source = f.source AND o.uri = f.uri
|
| 872 |
WHERE f.status = 'ok'
|
| 873 |
GROUP BY ALL;
|
|
@@ -963,7 +963,7 @@ Each corpus is one whole upstream read in bulk and counted in rows; a material s
|
|
| 963 |
- **Admission** makes the material sets.
|
| 964 |
Its candidates are the works the search found upstream for the task family of a collection, and the units of a group of corpus rows chosen with `tools/fetch.py` (`--where` and `--sample`), which the fetch tool fetched and checked against their recorded size and sha256.
|
| 965 |
It takes a candidate when its pointer facts hold: a download address, a recorded size and sha256, a recorded license, no legal block, no fact found wrong, and content not already in the index;
|
| 966 |
-
every candidate it does not take is listed with its reason in `meta/not_included.parquet` ([docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 967 |
- **What admission adds:** the candidate's files as one unit with its `unit_id`, the material set's README.md and LICENSE, its collection (the one of its task family), its form, its upstream, and labels such as language, robots and terms.
|
| 968 |
The collect step then fetches every file of every unit again and checks its sha256 before the material set is published.
|
| 969 |
- A material set made from corpus rows is a source of its own, apart from the corpus: the same file is a row of the corpus and a file of a unit, so the two layers' counts are never added;
|
|
@@ -980,7 +980,7 @@ How the recorded size and its check are measured, and which files have the same
|
|
| 980 |
|
| 981 |
The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
|
| 982 |
|
| 983 |
-
Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-15.
|
| 984 |
|
| 985 |
</details>
|
| 986 |
|
|
@@ -991,10 +991,10 @@ The stage table `catalog/stages.yaml` of the code repository gives each training
|
|
| 991 |
The release records those fields for every unit of a material set and every row of a corpus, with how many of the rules every stage shares it passes, in this order: the benchmark rule (a row that matches a public benchmark task is left out), the rule on rows not compared with the benchmark list yet and deduplication (`meta/stage_fields.parquet`);
|
| 992 |
each stage counts what meets its condition before those rules and after each of them, and its count after the last one is what it can take;
|
| 993 |
the data page shows these steps under each stage's tile.
|
| 994 |
-
What no stage takes is counted per layer by reason, in `no_stage` of [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 995 |
so post-training's units and material sets plus those in no stage are every material set's units and material sets, as the data page shows under its tiles.
|
| 996 |
The training permit and the personal-data mark are labels of every row, neither a condition nor a rule of any stage.
|
| 997 |
-
The dataset card shows each stage's count, how much of it is allowed for training, and what it keeps under each top-level domain (`by_domain` in [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 998 |
The tables of this release: [What each training stage can take](#manual/stage-tables).
|
| 999 |
|
| 1000 |
</details>
|
|
@@ -1005,10 +1005,10 @@ The tables of this release: [What each training stage can take](#manual/stage-ta
|
|
| 1005 |
- **Data collection:** rows are collected from the public listings, APIs and archives of each source and checked when added:
|
| 1006 |
the address must answer, and size and hash are fetched again and compared.
|
| 1007 |
- **Checks before release:** every source has its license recorded and grouped into a tier, personal information is read from sample files, and rows are compared with public benchmarks;
|
| 1008 |
-
rows that copy a public benchmark are removed, and units that do not meet a requirement are listed with the reason: [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 1009 |
- **Release:** each release is one commit with its own tag;
|
| 1010 |
the dataset card, its figures under `docs/figures/`, the pages under `docs/`, `cards/*.md`, `meta/catalog.parquet`, `meta/croissant.json`, `meta/size_figures.json` and `meta/stage_figures.json` are generated from the same commit, so their numbers agree, and this manual is filled from that commit when the data page is deployed.
|
| 1011 |
-
Every number and figure of the card is read from [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 1012 |
- **Chinese version:** every name and description the Chinese card, its pages and the Chinese manual's tables show is translated by a language model once per text and kept, by the sha256 of its English text, in `meta/translations.parquet`.
|
| 1013 |
|
| 1014 |
</details>
|
|
|
|
| 12 |
<h2 id="manual/guides">Guides</h2>
|
| 13 |
|
| 14 |
One goal each;
|
| 15 |
+
commands are pinned to `release-2026-11-15.6`.
|
| 16 |
|
| 17 |
<h3 id="manual/take">Take the files a need calls for</h3>
|
| 18 |
|
|
|
|
| 23 |
3. In that folder, run the command of "Take it again from the file" with [uv](https://docs.astral.sh/uv/):
|
| 24 |
|
| 25 |
```sh
|
| 26 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 27 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 28 |
+
--revision release-2026-11-15.6 \
|
| 29 |
--out ./data
|
| 30 |
```
|
| 31 |
|
|
|
|
| 41 |
```sh
|
| 42 |
uvx --from duckdb-cli duckdb -c "
|
| 43 |
SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
|
| 44 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 45 |
WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
|
| 46 |
ORDER BY collection, units DESC"
|
| 47 |
```
|
|
|
|
| 55 |
to leave out the files whose license does not let them be shipped and the rows marked as carrying personal data, add two options:
|
| 56 |
|
| 57 |
```sh
|
| 58 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 59 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 60 |
+
--revision release-2026-11-15.6 \
|
| 61 |
--training-only \
|
| 62 |
--no-personal-data \
|
| 63 |
--out ./data
|
|
|
|
| 129 |
2 when the index could not be read or a name is not in the catalog, such as a misspelt source:
|
| 130 |
|
| 131 |
```text
|
| 132 |
+
error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1; closest: dosfstools-fsck-cases
|
| 133 |
```
|
| 134 |
|
| 135 |
</details>
|
|
|
|
| 137 |
<details id="manual/pin">
|
| 138 |
<summary>Pin a release</summary>
|
| 139 |
|
| 140 |
+
The page's commands pin the script URL and `--revision` to `release-2026-11-15.6`;
|
| 141 |
the script's first line names the commit:
|
| 142 |
|
| 143 |
```text
|
| 144 |
+
reading hf://datasets/Battam/training-data-foundation@89d4297c25bb7bf352b1874eca8f5c6f86adc7e1 (commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1); pass --revision 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1 to select the same rows later
|
| 145 |
```
|
| 146 |
|
| 147 |
Later, `--revision <commit>`, the same tag or the selection list take the same files.
|
|
|
|
| 214 |
|
| 215 |
```sql
|
| 216 |
SELECT hit_class, scope, count(*) AS rows
|
| 217 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/bench_overlap.parquet'
|
| 218 |
GROUP BY ALL ORDER BY hit_class, scope;
|
| 219 |
```
|
| 220 |
|
|
|
|
| 251 |
|
| 252 |
| To | Read |
|
| 253 |
|---|---|
|
| 254 |
+
| Read every field of a row and every column of the catalog | [docs/fields.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/fields.md) |
|
| 255 |
+
| See every material set and every corpus, by domain, with its counts, size, license tier and login | [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/sources.md) and the domain pages it lists |
|
| 256 |
+
| See what was not included and why, and which sources need a login | [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.md) |
|
| 257 |
+
| Build tasks from the files, and keep what an agent must not see out of its environment | [docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/for-agents.md) |
|
| 258 |
+
| See every release and what it changed | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/CHANGELOG.md) |
|
| 259 |
+
| Read the dataset as Croissant metadata | [Croissant generated by the Hub](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant); [`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/croissant.json) describes the catalog and every source |
|
| 260 |
| Look through the catalog, one row per material set or corpus with its `kind` and `collection` | [The `catalog` subset in the Dataset Viewer](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
|
| 261 |
| Query the catalog in the browser; it lists every source as published, without the default profile of `tools/fetch.py`, which the command applies | [SQL Console](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
|
| 262 |
| Watch the index in use, in the page's language, with subtitles | [Usage video on the data page](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
|
| 263 |
+
| Check every number of the dataset card and of the tables here | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json) and [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/size_figures.json) |
|
| 264 |
|
| 265 |
Each row of the data files under `data/` points to one file or record (Row, in the [glossary](#manual/glossary));
|
| 266 |
the catalog's column `data_files` names the files of each source, and one material set or corpus is read from exactly those files, as in the queries of the dataset card and with `tools/fetch.py`.
|
|
|
|
| 310 |
- This catalog itself (addresses, sizes, hashes and descriptive text) is released under CC0 1.0; that dedication does not extend to the files it points to.
|
| 311 |
- The files under `media/` (the usage example video and its subtitles) and under `examples/` (the worked examples) are released under CC0 1.0 as well.
|
| 312 |
- Each file keeps the license of its source: see the license columns of each row (a row without its own license takes the license of its batch, `batches[].license` in `meta/sources.json`).
|
| 313 |
+
- The tier of every source: on the page of its domain, which [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/sources.md) lists.
|
| 314 |
- License tiers and reason codes are computed from the license categories of ScanCode LicenseDB, licensed CC-BY-4.0;
|
| 315 |
+
its attribution: [`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/NOTICE).
|
| 316 |
|
| 317 |
<h4 id="manual/personal">Personal and sensitive information</h4>
|
| 318 |
|
|
|
|
| 358 |
|
| 359 |
<h4 id="manual/releases">Releases</h4>
|
| 360 |
|
| 361 |
+
- Latest release: `release-2026-11-15.6` (its newest CHANGELOG entry is dated 2026-11-15).
|
| 362 |
+
- Releases are listed in [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/CHANGELOG.md);
|
| 363 |
+
each one is a commit with its own release tag (this one is [`release-2026-11-15.6`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-15.6)),
|
| 364 |
so any earlier state stays readable at its tag.
|
| 365 |
- `meta/projection.json` records the release tag and the version of the program and templates that generated the dataset card and its pages.
|
| 366 |
|
|
|
|
| 395 |
@misc{training_data_foundation,
|
| 396 |
title = {training-data-foundation: an index of public materials for building agent environments},
|
| 397 |
howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
|
| 398 |
+
note = {Release release-2026-11-15.6},
|
| 399 |
year = {2026}
|
| 400 |
}
|
| 401 |
```
|
|
|
|
| 430 |
- **Form:** the shape of a source's material, one id such as `spreadsheets` or `test-suites` (the catalog column `form`, the option `--form` of `tools/fetch.py`).
|
| 431 |
- **Training stage:** what the material can feed, by the stage table `catalog/stages.yaml` of the code repository: pre-training and mid-training take corpus rows, post-training the units of material sets ([How the training-stage tables are counted](#manual/stage-counting)).
|
| 432 |
- **Default profile:** the rules `tools/fetch.py` applies unless an option relaxes them: it leaves out duplicate copies, rows that matched a benchmark or were not compared with the benchmark list yet, rows in a language other than English and files above 20 MB;
|
| 433 |
+
rows of any license and rows marked as carrying personal data come unless `--training-only` or `--no-personal-data` leaves them out ([docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/for-agents.md)).
|
| 434 |
|
| 435 |
Every row carries a `unit_id`: in a material set, the key of its unit;
|
| 436 |
in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte range, then `!/<member_path>` for an archive member), and `tools/fetch.py` puts the file of a corpus row in a folder named after it.
|
|
|
|
| 440 |
<details id="manual/numbers">
|
| 441 |
<summary>Where this manual's numbers come from</summary>
|
| 442 |
|
| 443 |
+
The numbers and command outputs of this manual were filled in at deploy time, from the index commit `89d4297c25bb7bf352b1874eca8f5c6f86adc7e1` (release tag `release-2026-11-15.6`).
|
| 444 |
+
This release holds 1,845 material sets with 4,244 units in 14 collections and 139 corpora with 1,189,479,628 rows, which point to 4.13 PB of deduplicated content, and of the 35,601 rows whose size was compared with their file, 99.4% matched.
|
| 445 |
|
| 446 |
</details>
|
| 447 |
|
|
|
|
| 452 |
its last two rows count how much of the stage's count is allowed for training (every file's training permit yes or conditional) and how much is marked as carrying personal data.
|
| 453 |
How they are counted: [How the training-stage tables are counted](#manual/stage-counting).
|
| 454 |
|
| 455 |
+
The stage fields (`meta/stage_fields.parquet`) were counted over the data files of release `release-2026-11-15.6`, after it was published.
|
| 456 |
|
| 457 |
##### Post-training (in use)
|
| 458 |
|
|
|
|
| 464 |
| After the benchmark rule | 1,826 | 4,202 | 1,879 |
|
| 465 |
| After the rule on rows not compared with the benchmark list | 1,826 | 4,202 | 1,879 |
|
| 466 |
| After deduplication: the stage's count | 1,766 | 4,059 | 1,823 |
|
| 467 |
+
| Of the stage's count, allowed for training (yes or conditional) | 1,412 | 3,395 | 1,339 |
|
| 468 |
| Of the stage's count, marked as carrying personal data | 357 | 631 | 400 |
|
| 469 |
|
| 470 |
##### Mid-training (provisional: conditions and counts only)
|
|
|
|
| 473 |
|
| 474 |
| Step | Rows | Size |
|
| 475 |
|---|---:|---:|
|
| 476 |
+
| In the stage's condition, before the rules | 34,629,789 | 4.01 TB |
|
| 477 |
+
| After the benchmark rule | 34,620,507 | 4.01 TB |
|
| 478 |
+
| After the rule on rows not compared with the benchmark list | 34,620,507 | 4.01 TB |
|
| 479 |
+
| After deduplication: the stage's count | 34,231,698 | 3.86 TB |
|
| 480 |
| Of the stage's count, allowed for training (yes or conditional) | 9,937,384 | 1.42 TB |
|
| 481 |
+
| Of the stage's count, marked as carrying personal data | 6,104,380 | 341 GB |
|
| 482 |
|
| 483 |
##### Pre-training (provisional: conditions and counts only)
|
| 484 |
|
|
|
|
| 486 |
|
| 487 |
| Step | Rows | Size |
|
| 488 |
|---|---:|---:|
|
| 489 |
+
| In the stage's condition, before the rules | 155,528,248 | 30.9 TB |
|
| 490 |
+
| After the benchmark rule | 155,527,231 | 30.9 TB |
|
| 491 |
+
| After the rule on rows not compared with the benchmark list | 155,527,231 | 30.9 TB |
|
| 492 |
+
| After deduplication: the stage's count | 150,610,889 | 30.7 TB |
|
| 493 |
| Of the stage's count, allowed for training (yes or conditional) | 119,168,563 | 30.0 TB |
|
| 494 |
| Of the stage's count, marked as carrying personal data | 40,477,415 | 676 GB |
|
| 495 |
|
|
|
|
| 550 |
<h4 id="manual/workbooks-5">5. Go to "Get the data" and keep the selection</h4>
|
| 551 |
|
| 552 |
**Do**: click "Get the data" at the top;
|
| 553 |
+
under "Keep this selection" click "Download the selection list (JSON)" and put `training-data-foundation-selection-89d4297.json` in your fetch folder.
|
| 554 |
|
| 555 |
**See**: the heading "Take every unit of each of 51 material sets".
|
| 556 |
The list records release, filters and sources:
|
|
|
|
| 559 |
{
|
| 560 |
"format": "training-data-foundation-selection/1",
|
| 561 |
"dataset": "Battam/training-data-foundation",
|
| 562 |
+
"revision": "release-2026-11-15.6",
|
| 563 |
"filters": {
|
| 564 |
"collection": "A",
|
| 565 |
"train": "yes",
|
|
|
|
| 583 |
**Do**: in the list's folder, run the command of "Take it again from the file" with `--limit 220` added (220 units in all, in a fixed order: a margin over 200, since some sites fail) and `--out ./data` replaced by `--out ./take`:
|
| 584 |
|
| 585 |
```sh
|
| 586 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 587 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 588 |
+
--revision release-2026-11-15.6 \
|
| 589 |
--ext xlsx \
|
| 590 |
--ext xlsm \
|
| 591 |
--ext xls \
|
|
|
|
| 614 |
...
|
| 615 |
310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
|
| 616 |
files of units: 220 ok; files of the sources themselves: 90 ok
|
| 617 |
+
run_id 20261115T040737.805810Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
|
| 618 |
```
|
| 619 |
|
| 620 |
Before the first download, `removed by` gives the units the default profile removed and the option that keeps them, counted in the data files read as far as `--limit` needed;
|
|
|
|
| 630 |
incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
|
| 631 |
5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
|
| 632 |
files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
|
| 633 |
+
run_id 20261115T041058.860782Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
|
| 634 |
+
its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261115T041058.860782Z' AND json_extract_string(json, 'status') <> 'ok'
|
| 635 |
```
|
| 636 |
|
| 637 |
**Get**: every `ok` file matched its recorded size and hash.
|
|
|
|
| 702 |
**See** (truncated):
|
| 703 |
|
| 704 |
```text
|
| 705 |
+
Dataset commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1: https://huggingface.co/datasets/Battam/training-data-foundation/tree/89d4297c25bb7bf352b1874eca8f5c6f86adc7e1
|
| 706 |
...
|
| 707 |
- Source: `austender-ict-procurement-statistics`
|
| 708 |
- License: CC-BY-3.0-AU
|
|
|
|
| 735 |
|
| 736 |
```sql
|
| 737 |
SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
|
| 738 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 739 |
WHERE kind = 'material_set' AND collection = 'B'
|
| 740 |
AND license.tier = 'share-alike';
|
| 741 |
```
|
|
|
|
| 744 |
--source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
|
| 745 |
```
|
| 746 |
|
| 747 |
+
Put them after `fetch.py --revision release-2026-11-15.6`.
|
| 748 |
Read only the catalog, not all of `meta/`.
|
| 749 |
|
| 750 |
</details>
|
|
|
|
| 758 |

|
| 759 |
|
| 760 |
```sh
|
| 761 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 762 |
+
--revision release-2026-11-15.6 \
|
| 763 |
--source dosfstools-fsck-cases \
|
| 764 |
--source e2fsprogs-fsck-cases \
|
| 765 |
--source exfatprogs-fsck-cases \
|
|
|
|
| 774 |
ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
|
| 775 |
59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
|
| 776 |
files of units: 52 ok; files of the sources themselves: 7 ok
|
| 777 |
+
run_id 20261115T041105.290223Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
|
| 778 |
```
|
| 779 |
|
| 780 |
An exFAT image is `exfat.img.tar.xz` upstream;
|
|
|
|
| 818 |
|
| 819 |
```sql
|
| 820 |
SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
|
| 821 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 822 |
WHERE kind = 'material_set'
|
| 823 |
AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
|
| 824 |
GROUP BY ALL;
|
|
|
|
| 838 |
Two, take the picked sources and read `removed by benchmark` and `removed by not compared`, printed before the first download (output truncated):
|
| 839 |
|
| 840 |
```sh
|
| 841 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 842 |
+
--revision release-2026-11-15.6 \
|
| 843 |
--source python-docx-acceptance-test-files \
|
| 844 |
--source python-pptx-acceptance-test-files \
|
| 845 |
--source docling-docx-groundtruth \
|
|
|
|
| 867 |
```sql
|
| 868 |
SELECT o.bench, o.hit_class, count(*) AS files
|
| 869 |
FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
|
| 870 |
+
JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/bench_overlap.parquet' AS o
|
| 871 |
ON o.source = f.source AND o.uri = f.uri
|
| 872 |
WHERE f.status = 'ok'
|
| 873 |
GROUP BY ALL;
|
|
|
|
| 963 |
- **Admission** makes the material sets.
|
| 964 |
Its candidates are the works the search found upstream for the task family of a collection, and the units of a group of corpus rows chosen with `tools/fetch.py` (`--where` and `--sample`), which the fetch tool fetched and checked against their recorded size and sha256.
|
| 965 |
It takes a candidate when its pointer facts hold: a download address, a recorded size and sha256, a recorded license, no legal block, no fact found wrong, and content not already in the index;
|
| 966 |
+
every candidate it does not take is listed with its reason in `meta/not_included.parquet` ([docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.md)).
|
| 967 |
- **What admission adds:** the candidate's files as one unit with its `unit_id`, the material set's README.md and LICENSE, its collection (the one of its task family), its form, its upstream, and labels such as language, robots and terms.
|
| 968 |
The collect step then fetches every file of every unit again and checks its sha256 before the material set is published.
|
| 969 |
- A material set made from corpus rows is a source of its own, apart from the corpus: the same file is a row of the corpus and a file of a unit, so the two layers' counts are never added;
|
|
|
|
| 980 |
|
| 981 |
The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
|
| 982 |
|
| 983 |
+
Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-15.6`.
|
| 984 |
|
| 985 |
</details>
|
| 986 |
|
|
|
|
| 991 |
The release records those fields for every unit of a material set and every row of a corpus, with how many of the rules every stage shares it passes, in this order: the benchmark rule (a row that matches a public benchmark task is left out), the rule on rows not compared with the benchmark list yet and deduplication (`meta/stage_fields.parquet`);
|
| 992 |
each stage counts what meets its condition before those rules and after each of them, and its count after the last one is what it can take;
|
| 993 |
the data page shows these steps under each stage's tile.
|
| 994 |
+
What no stage takes is counted per layer by reason, in `no_stage` of [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json): what meets no stage's condition, what a rule removes, and what the catalog holds but the stage fields do not, since a release counts its stage fields over an earlier release's files (not counted yet);
|
| 995 |
so post-training's units and material sets plus those in no stage are every material set's units and material sets, as the data page shows under its tiles.
|
| 996 |
The training permit and the personal-data mark are labels of every row, neither a condition nor a rule of any stage.
|
| 997 |
+
The dataset card shows each stage's count, how much of it is allowed for training, and what it keeps under each top-level domain (`by_domain` in [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json)).
|
| 998 |
The tables of this release: [What each training stage can take](#manual/stage-tables).
|
| 999 |
|
| 1000 |
</details>
|
|
|
|
| 1005 |
- **Data collection:** rows are collected from the public listings, APIs and archives of each source and checked when added:
|
| 1006 |
the address must answer, and size and hash are fetched again and compared.
|
| 1007 |
- **Checks before release:** every source has its license recorded and grouped into a tier, personal information is read from sample files, and rows are compared with public benchmarks;
|
| 1008 |
+
rows that copy a public benchmark are removed, and units that do not meet a requirement are listed with the reason: [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.md).
|
| 1009 |
- **Release:** each release is one commit with its own tag;
|
| 1010 |
the dataset card, its figures under `docs/figures/`, the pages under `docs/`, `cards/*.md`, `meta/catalog.parquet`, `meta/croissant.json`, `meta/size_figures.json` and `meta/stage_figures.json` are generated from the same commit, so their numbers agree, and this manual is filled from that commit when the data page is deployed.
|
| 1011 |
+
Every number and figure of the card is read from [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json), and every other count and size of the dataset's pages and of this manual's tables from [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/size_figures.json), where each is computed once for the release.
|
| 1012 |
- **Chinese version:** every name and description the Chinese card, its pages and the Chinese manual's tables show is translated by a language model once per text and kept, by the sha256 of its English text, in `meta/translations.parquet`.
|
| 1013 |
|
| 1014 |
</details>
|
manual/manual.zh.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
| 12 |
|
| 13 |
<h2 id="manual/guides">指南</h2>
|
| 14 |
|
| 15 |
-
每条一个目标,命令都钉在 `release-2026-11-15.
|
| 16 |
|
| 17 |
<h3 id="manual/take">按需求取回文件</h3>
|
| 18 |
|
|
@@ -23,9 +23,9 @@
|
|
| 23 |
3. 在那个目录里用 [uv](https://docs.astral.sh/uv/) 跑「按文件再取一次」那条命令:
|
| 24 |
|
| 25 |
```sh
|
| 26 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 27 |
-
--selection training-data-foundation-selection-
|
| 28 |
-
--revision release-2026-11-15.
|
| 29 |
--out ./data
|
| 30 |
```
|
| 31 |
|
|
@@ -41,7 +41,7 @@
|
|
| 41 |
```sh
|
| 42 |
uvx --from duckdb-cli duckdb -c "
|
| 43 |
SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
|
| 44 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 45 |
WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
|
| 46 |
ORDER BY collection, units DESC"
|
| 47 |
```
|
|
@@ -55,9 +55,9 @@
|
|
| 55 |
要去掉许可不允许随附分发的文件和标为带有个人数据的行,加两个选项:
|
| 56 |
|
| 57 |
```sh
|
| 58 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 59 |
-
--selection training-data-foundation-selection-
|
| 60 |
-
--revision release-2026-11-15.
|
| 61 |
--training-only \
|
| 62 |
--no-personal-data \
|
| 63 |
--out ./data
|
|
@@ -128,7 +128,7 @@
|
|
| 128 |
读不了索引或名字不在目录表里为 2,例如来源名写错时:
|
| 129 |
|
| 130 |
```text
|
| 131 |
-
error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
|
| 132 |
```
|
| 133 |
|
| 134 |
</details>
|
|
@@ -136,10 +136,10 @@ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
|
|
| 136 |
<details id="manual/pin">
|
| 137 |
<summary>钉住发布版本取</summary>
|
| 138 |
|
| 139 |
-
页面给的命令把脚本网址和 `--revision` 都钉在 `release-2026-11-15.
|
| 140 |
|
| 141 |
```text
|
| 142 |
-
reading hf://datasets/Battam/training-data-foundation@
|
| 143 |
```
|
| 144 |
|
| 145 |
日后用 `--revision <提交号>`、同一个标签或选择清单,取到的就是同样的文件。
|
|
@@ -211,7 +211,7 @@ scope 为 `container` 时命中整个分组(只在分组是一个完整单位
|
|
| 211 |
|
| 212 |
```sql
|
| 213 |
SELECT hit_class, scope, count(*) AS rows
|
| 214 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 215 |
GROUP BY ALL ORDER BY hit_class, scope;
|
| 216 |
```
|
| 217 |
|
|
@@ -248,16 +248,16 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 248 |
|
| 249 |
| 想做的事 | 去读 |
|
| 250 |
|---|---|
|
| 251 |
-
| 查看每个行字段和目录的每一列 | [docs/fields.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 252 |
-
| 按领域查看每个材料集和每个语料库,以及它的计数、大小、许可证等级和登录要求 | [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 253 |
-
| 查看哪些没有收录、为什么,以及哪些来源需要登录 | [docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 254 |
-
| 用这些文件构建任务,并把智能体不该看到的内容挡在它的环境之外 | [docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 255 |
-
| 查看每次发布及其改动 | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 256 |
-
| 以 Croissant 元数据读取本数据
|
| 257 |
| 浏览目录,每个材料集或语料库一行,带有它的 `kind` 和 `collection` | [数据集查看器中的 `catalog` 子集](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
|
| 258 |
| 在浏览器里查询目录;它按发布原样列出每个来源,不应用 `tools/fetch.py` 的默认配置,命令才应用它 | [SQL 控制台](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
|
| 259 |
| 观看本索引的使用示例视频,用页面的语言,带字幕 | [数据页上的使用示例视频](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
|
| 260 |
-
| 核对数据集卡片和这里各表的每个数字 | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 261 |
|
| 262 |
`data/` 下数据文件的每一行指向一个文件或一条记录(见[术语](#manual/glossary)中的「行」);
|
| 263 |
目录的 `data_files` 列列出每个来源的文件,单个材料集或语料库就从这些文件读取,见数据集卡片中的查询和 `tools/fetch.py`。
|
|
@@ -308,9 +308,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 308 |
这一声明不延及它所指向的文件。
|
| 309 |
- `media/` 下的文件(使用示例视频及其字幕)和 `examples/` 下的文件(完整示例)同样以 CC0 1.0 发布。
|
| 310 |
- 每个文件保留其来源的许可证:见每一行的许可证列(没有自己许可证的行沿用所在批次的许可证,即 `meta/sources.json` 中的 `batches[].license`)。
|
| 311 |
-
- 每个来源的等级:在它所属领域的页面上,这些页面由 [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 312 |
- 许可证等级和原因码由 ScanCode LicenseDB 的许可证类别算出,它以 CC-BY-4.0 许可;
|
| 313 |
-
它的署名:[`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 314 |
|
| 315 |
<h4 id="manual/personal">个人与敏感信息</h4>
|
| 316 |
|
|
@@ -356,9 +356,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 356 |
|
| 357 |
<h4 id="manual/releases">发布</h4>
|
| 358 |
|
| 359 |
-
- 最新发布:`release-2026-11-15.
|
| 360 |
-
- 各次发布列在 [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 361 |
-
每次发布是一个带有自己发布标签的提交(本次为 [`release-2026-11-15.
|
| 362 |
因此任何较早的状态都可以在它的标签处读到。
|
| 363 |
- `meta/projection.json` 记录发布标签,以及生成数据集卡片及其页面的程序和模板的版本。
|
| 364 |
|
|
@@ -401,7 +401,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 401 |
@misc{training_data_foundation,
|
| 402 |
title = {training-data-foundation: an index of public materials for building agent environments},
|
| 403 |
howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
|
| 404 |
-
note = {Release release-2026-11-15.
|
| 405 |
year = {2026}
|
| 406 |
}
|
| 407 |
```
|
|
@@ -436,7 +436,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 436 |
- **形式:** 一个来源的材料的形态,一个标识,例如 `spreadsheets` 或 `test-suites`(目录列 `form`,`tools/fetch.py` 的选项 `--form`)。
|
| 437 |
- **训练阶段:** 材料能用于哪种训练,由代码仓库的阶段表 `catalog/stages.yaml` 决定:预训练和中训练用语料库的行,后训练用材料集的单元(见[训练阶段表怎样计数](#manual/stage-counting))。
|
| 438 |
- **默认配置:** `tools/fetch.py` 在没有选项放宽时应用的规则:去掉重复的副本、与基准测试相符或还没有与基准测试清单比对的行、记录为英语以外语言的行,以及大于 20 MB 的文件;
|
| 439 |
-
任何许可证的行和标为带有个人数据的行都会进来,除非 `--training-only` 或 `--no-personal-data` 把它们去掉([docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 440 |
|
| 441 |
每一行都带有 `unit_id`:在材料集中,是它所属单元的键;
|
| 442 |
在语料库中,是该行自己的键(它的 `uri`,字节范围再接 `@<range_offset>`,存档成员再接 `!/<member_path>`),`tools/fetch.py` 把语料库一行的文件放进以它命名的文件夹。
|
|
@@ -446,8 +446,8 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 446 |
<details id="manual/numbers">
|
| 447 |
<summary>本手册的数字从哪来</summary>
|
| 448 |
|
| 449 |
-
本手册的数字和命令输出是部署时填入的,取自索引提交 `
|
| 450 |
-
本次发布有 1,845 个材料集,共 4,244 个单元,分在 14 个合集,以及 139 个语料库,共 1,189,
|
| 451 |
|
| 452 |
</details>
|
| 453 |
|
|
@@ -458,7 +458,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 458 |
每张表的最后两行统计该阶段的计数中有多少允许训练(每个文件的训练许可为 yes 或 conditional),有多少标为带有个人数据。
|
| 459 |
怎样计数:[训练阶段表怎样计数](#manual/stage-counting)。
|
| 460 |
|
| 461 |
-
阶段字段(`meta/stage_fields.parquet`)是按发布 `release-2026-11-15.
|
| 462 |
|
| 463 |
##### 后训练(在用)
|
| 464 |
|
|
@@ -470,7 +470,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 470 |
| 基准测试规则之后 | 1,826 | 4,202 | 1,879 |
|
| 471 |
| 未与基准测试清单比对的行的规则之后 | 1,826 | 4,202 | 1,879 |
|
| 472 |
| 去重之后:该阶段的计数 | 1,766 | 4,059 | 1,823 |
|
| 473 |
-
| 该阶段的计数中,允许训练的(yes 或 conditional) | 1,
|
| 474 |
| 该阶段的计数中,标为带有个人数据的 | 357 | 631 | 400 |
|
| 475 |
|
| 476 |
##### 中训练(暂定:只给条件和计数)
|
|
@@ -479,12 +479,12 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 479 |
|
| 480 |
| 步骤 | 行数 | 大小 |
|
| 481 |
|---|---:|---:|
|
| 482 |
-
| 符合
|
| 483 |
-
| 基准测试规则之后 | 34,
|
| 484 |
-
| 未与基准测试清单比对的行的规则之后 | 34,
|
| 485 |
-
| 去重之后:该阶段的计数 | 34,231,
|
| 486 |
| 该阶段的计数中,允许训练的(yes 或 conditional) | 9,937,384 | 1.42 TB |
|
| 487 |
-
| 该阶段的计数中,标为带有个人数据的 | 6,104,
|
| 488 |
|
| 489 |
##### 预训练(暂定:只给条件和计数)
|
| 490 |
|
|
@@ -492,10 +492,10 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 492 |
|
| 493 |
| 步骤 | 行数 | 大小 |
|
| 494 |
|---|---:|---:|
|
| 495 |
-
| 符合该阶段的条件,在各条规则之前 | 155,
|
| 496 |
-
| 基准测试规则之后 | 155,
|
| 497 |
-
| 未与基准测试清单比对的行的规则之后 | 155,
|
| 498 |
-
| 去重之后:该阶段的计数 | 150,
|
| 499 |
| 该阶段的计数中,允许训练的(yes 或 conditional) | 119,168,563 | 30.0 TB |
|
| 500 |
| 该阶段的计数中,标为带有个人数据的 | 40,477,415 | 676 GB |
|
| 501 |
|
|
@@ -554,7 +554,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 554 |
|
| 555 |
<h4 id="manual/workbooks-5">5. 到「获取数据」,保存选择清单</h4>
|
| 556 |
|
| 557 |
-
**做**:点顶部「获取数据」,在「保存这次选择」里点「下载选择清单(JSON)」,把 `training-data-foundation-selection-
|
| 558 |
|
| 559 |
**看到**:标题「从 51 个材料集中获取全部单元」。
|
| 560 |
清单记着版本、筛选和来源:
|
|
@@ -563,7 +563,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 563 |
{
|
| 564 |
"format": "training-data-foundation-selection/1",
|
| 565 |
"dataset": "Battam/training-data-foundation",
|
| 566 |
-
"revision": "release-2026-11-15.
|
| 567 |
"filters": {
|
| 568 |
"collection": "A",
|
| 569 |
"train": "yes",
|
|
@@ -587,9 +587,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
|
|
| 587 |
**做**:在清单所在目录跑「按文件再取一次」那条命令,加上 `--limit 220`(一共 220 个单元,固定顺序;比 200 多取一些留余量,有的原站会失效),`--out ./data` 换成 `--out ./take`:
|
| 588 |
|
| 589 |
```sh
|
| 590 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 591 |
-
--selection training-data-foundation-selection-
|
| 592 |
-
--revision release-2026-11-15.
|
| 593 |
--ext xlsx \
|
| 594 |
--ext xlsm \
|
| 595 |
--ext xls \
|
|
@@ -618,7 +618,7 @@ ok austender-ict-procurement-statistics workbooks/2016-17-austender-ict-contract
|
|
| 618 |
...
|
| 619 |
310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
|
| 620 |
files of units: 220 ok; files of the sources themselves: 90 ok
|
| 621 |
-
run_id
|
| 622 |
```
|
| 623 |
|
| 624 |
第一次下载之前,`removed by` 是默认配置去掉的单元和保留它们的开关,数的是 `--limit` 需要读到的那些数据文件;
|
|
@@ -634,8 +634,8 @@ failed western-australia-government-workbooks workbooks/wa-gross-regional-produc
|
|
| 634 |
incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
|
| 635 |
5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
|
| 636 |
files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
|
| 637 |
-
run_id
|
| 638 |
-
its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '
|
| 639 |
```
|
| 640 |
|
| 641 |
**得到**:`ok` 的文件已按记录的大小和哈希核对。
|
|
@@ -704,7 +704,7 @@ GROUP BY ALL ORDER BY source;
|
|
| 704 |
**看到**(截短):
|
| 705 |
|
| 706 |
```text
|
| 707 |
-
Dataset commit
|
| 708 |
...
|
| 709 |
- Source: `austender-ict-procurement-statistics`
|
| 710 |
- License: CC-BY-3.0-AU
|
|
@@ -737,7 +737,7 @@ Dataset commit a75f8672208d4529d5f03fcec66359cfbf4e8717: https://huggingface.co/
|
|
| 737 |
|
| 738 |
```sql
|
| 739 |
SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
|
| 740 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 741 |
WHERE kind = 'material_set' AND collection = 'B'
|
| 742 |
AND license.tier = 'share-alike';
|
| 743 |
```
|
|
@@ -746,7 +746,7 @@ WHERE kind = 'material_set' AND collection = 'B'
|
|
| 746 |
--source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
|
| 747 |
```
|
| 748 |
|
| 749 |
-
接在 `fetch.py --revision release-2026-11-15.
|
| 750 |
只读目录表,别整个下载 `meta/`。
|
| 751 |
|
| 752 |
</details>
|
|
@@ -760,8 +760,8 @@ WHERE kind = 'material_set' AND collection = 'B'
|
|
| 760 |

|
| 761 |
|
| 762 |
```sh
|
| 763 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 764 |
-
--revision release-2026-11-15.
|
| 765 |
--source dosfstools-fsck-cases \
|
| 766 |
--source e2fsprogs-fsck-cases \
|
| 767 |
--source exfatprogs-fsck-cases \
|
|
@@ -776,7 +776,7 @@ ok dosfstools-fsck-cases README.md: 3.32kB
|
|
| 776 |
ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
|
| 777 |
59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
|
| 778 |
files of units: 52 ok; files of the sources themselves: 7 ok
|
| 779 |
-
run_id
|
| 780 |
```
|
| 781 |
|
| 782 |
exFAT 镜像在原站是 `exfat.img.tar.xz`,脚本解出后再按单元自己的 SHA-256 核对,盘上只留镜像。
|
|
@@ -819,7 +819,7 @@ Pass 3: Checking directory connectivity
|
|
| 819 |
|
| 820 |
```sql
|
| 821 |
SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
|
| 822 |
-
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 823 |
WHERE kind = 'material_set'
|
| 824 |
AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
|
| 825 |
GROUP BY ALL;
|
|
@@ -839,8 +839,8 @@ GROUP BY ALL;
|
|
| 839 |
二,取回挑中的来源,看第一次下载之前打印的 `removed by benchmark` 和 `removed by not compared`(输出截短):
|
| 840 |
|
| 841 |
```sh
|
| 842 |
-
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.
|
| 843 |
-
--revision release-2026-11-15.
|
| 844 |
--source python-docx-acceptance-test-files \
|
| 845 |
--source python-pptx-acceptance-test-files \
|
| 846 |
--source docling-docx-groundtruth \
|
|
@@ -867,7 +867,7 @@ fetching: 48 units: 56 files (8 files of the sources themselves among them), 25.
|
|
| 867 |
```sql
|
| 868 |
SELECT o.bench, o.hit_class, count(*) AS files
|
| 869 |
FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
|
| 870 |
-
JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.
|
| 871 |
ON o.source = f.source AND o.uri = f.uri
|
| 872 |
WHERE f.status = 'ok'
|
| 873 |
GROUP BY ALL;
|
|
@@ -959,7 +959,7 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
|
|
| 959 |
- **收录**产生材料集。
|
| 960 |
它的候选是检索为某个合集的任务类别在上游找到的作品,以及用 `tools/fetch.py`(`--where` 和 `--sample`)选出的一组语料库行中的单元,这些单元由下载工具取回,并按记录的大小和 sha256 校验过。
|
| 961 |
候选的指针事实成立时才收录:有下载地址,记录了大小和 sha256,记录了许可证,没有法律封锁,没有发现事实有误,内容也不在索引中;
|
| 962 |
-
未收录的每个候选都连同原因列在 `meta/not_included.parquet` 中([docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 963 |
- **收录添加的内容:** 把候选的文件作为一个单元并给出它的 `unit_id`,材料集的 README.md 和 LICENSE,它的合集(它的任务类别所属的合集),它的形式,它的上游,以及语言(`language`)、`robots` 和使用条款(`terms`)等标签。
|
| 964 |
之后采集步骤再次取回每个单元的每个文件,并在材料集发布之前校验它的 sha256。
|
| 965 |
- 由语料库行做成的材料集是一个独立的来源,与那个语料库分开:同一个文件既是语料库的一行,又是一个单元的一个文件,所以两层的计数从不相加;
|
|
@@ -976,7 +976,7 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
|
|
| 976 |
|
| 977 |
记录的大小是有大小的各行 `size_bytes` 之和(目录列 `bytes`)。核对读取每个数据文件中抽取的一组行(`meta/sizes.parquet`),每天读取有某个来源的行尚未在其中读过的文件,以及最久没读的那些来源的文件,约占全部读数的七分之一,所以每个来源一周之内都会再读一次:对其中有行的每个来源,有记录大小的行各抽 20 行;没有记录大小的行按其所在来源的样本分摊,样本量按目前读到的大小的离散程度定,使其平均大小在 95% 置信度下误差不超过 25%(每个来源每轮最多 400 行,还没有读数的来源取 20 行);其比例是比对过的行(状态 `match`、`size_mismatch` 或 `hash_mismatch`)中与记录的大小一致的行所占的比例,没能比对的行(没有取到文件)按状态另行计数(`unreachable`、`gone`、`needs_auth`、`no_length`、`error`;目录列 `size_check`)。一个来源没有记录大小的行的估计(目录列 `unsized_rows` 和 `unsized_estimate`)是这些行的行数乘以读过的这类行的平均实测大小,并给出 95% BCa 自助法范围(`scipy.stats.bootstrap`,用其默认重抽样次数);读过的这类行少于 10 行的来源没有估计。在来源之上的层级,估计是其各来源估计之和,范围是各来源范围之和,比联合区间更宽。每个层级的数字都在 `meta/size_figures.json` 中。
|
| 978 |
|
| 979 |
-
去重后内容是各行 `size_bytes` 之和,减去与另一行内容重复的那些行的大小(sha256 相同;没有 sha256 时 sha1 相同;再没有时 md5 和大小都相同;并且哈希范围相同;每种内容只保留一份,留在按注册顺序排在最前的来源中),在每次发布之后按它发布的数据文件重新计算,每个来源的值在目录列 `distinct_bytes` 中;同一来源也整个列出的存档,其成员的哈希与存档不同,所以不扣除。`tools/fetch.py` 读取你自己的选择时会去掉其中的重复。 这里的去重后大小是按发布 `release-2026-11-15.
|
| 980 |
|
| 981 |
</details>
|
| 982 |
|
|
@@ -987,10 +987,10 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
|
|
| 987 |
发布为材料集的每个单元和语料库的每一行记录这些字段,并记录它按以下顺序通过了各阶段共用规则中的几条:基准测试规则(与公开基准测试任务相符的行被去掉)、还没有与基准测试清单比对的行的规则和去重(`meta/stage_fields.parquet`);
|
| 988 |
每个阶段在这些规则之前和每条规则之后统计符合它的条件的材料,最后一条规则之后的计数就是它能用的材料;
|
| 989 |
数据页在每个阶段那一块下面显示这几步。
|
| 990 |
-
没有任何阶段取用的部分按层、按原因统计,在 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 991 |
所以后训练的单元数和材料集数加上未归入任何阶段的,就是全部材料集的单元数和材料集数,数据页在各阶段下面就是这样显示的。
|
| 992 |
训练许可和个人数据标记是每一行的标签,既不是任何阶段的条件,也不是规则。
|
| 993 |
-
数据集卡片显示每个阶段的计数、其中有多少允许训练,以及它在每个顶层领域下保留的部分([`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 994 |
这次发布的各张表:[每个训练阶段能用多少](#manual/stage-tables)。
|
| 995 |
|
| 996 |
</details>
|
|
@@ -1001,10 +1001,10 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
|
|
| 1001 |
- **数据采集:** 行从每个来源的公开列表、API 和存档中采集,加入时即校验:
|
| 1002 |
地址必须能访问,大小和哈希值会重新取回并比对。
|
| 1003 |
- **发布前的检查:** 每个来源都记录了许可证并归入一个等级,个人信息从样本文件中判读,各行与公开基准测试比对;
|
| 1004 |
-
复制了公开基准测试的行会被移除,不满足某项要求的单元会连同原因一起列出:[docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 1005 |
- **发布:** 每次发布是一个带有自己标签的提交;
|
| 1006 |
数据集卡片、`docs/figures/` 下它的图、`docs/` 下的页面、`cards/*.md`、`meta/catalog.parquet`、`meta/croissant.json`、`meta/size_figures.json` 和 `meta/stage_figures.json` 由同一个提交生成,因此数字彼此一致,本手册则在部署数据页时从这个提交填入。
|
| 1007 |
-
卡片的每个数字和每张图都读自 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.
|
| 1008 |
- **中文版本:** 中文卡片、它的页面和中文手册各表显示的每个名称和描述,都由语言模型对每段文字翻译一次,按英文原文的 sha256 保存在 `meta/translations.parquet` 中。
|
| 1009 |
|
| 1010 |
</details>
|
|
|
|
| 12 |
|
| 13 |
<h2 id="manual/guides">指南</h2>
|
| 14 |
|
| 15 |
+
每条一个目标,命令都钉在 `release-2026-11-15.6`。
|
| 16 |
|
| 17 |
<h3 id="manual/take">按需求取回文件</h3>
|
| 18 |
|
|
|
|
| 23 |
3. 在那个目录里用 [uv](https://docs.astral.sh/uv/) 跑「按文件再取一次」那条命令:
|
| 24 |
|
| 25 |
```sh
|
| 26 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 27 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 28 |
+
--revision release-2026-11-15.6 \
|
| 29 |
--out ./data
|
| 30 |
```
|
| 31 |
|
|
|
|
| 41 |
```sh
|
| 42 |
uvx --from duckdb-cli duckdb -c "
|
| 43 |
SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
|
| 44 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 45 |
WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
|
| 46 |
ORDER BY collection, units DESC"
|
| 47 |
```
|
|
|
|
| 55 |
要去掉许可不允许随附分发的文件和标为带有个人数据的行,加两个选项:
|
| 56 |
|
| 57 |
```sh
|
| 58 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 59 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 60 |
+
--revision release-2026-11-15.6 \
|
| 61 |
--training-only \
|
| 62 |
--no-personal-data \
|
| 63 |
--out ./data
|
|
|
|
| 128 |
读不了索引或名字不在目录表里为 2,例如来源名写错时:
|
| 129 |
|
| 130 |
```text
|
| 131 |
+
error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1; closest: dosfstools-fsck-cases
|
| 132 |
```
|
| 133 |
|
| 134 |
</details>
|
|
|
|
| 136 |
<details id="manual/pin">
|
| 137 |
<summary>钉住发布版本取</summary>
|
| 138 |
|
| 139 |
+
页面给的命令把脚本网址和 `--revision` 都钉在 `release-2026-11-15.6`,脚本首行打印对应的提交:
|
| 140 |
|
| 141 |
```text
|
| 142 |
+
reading hf://datasets/Battam/training-data-foundation@89d4297c25bb7bf352b1874eca8f5c6f86adc7e1 (commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1); pass --revision 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1 to select the same rows later
|
| 143 |
```
|
| 144 |
|
| 145 |
日后用 `--revision <提交号>`、同一个标签或选择清单,取到的就是同样的文件。
|
|
|
|
| 211 |
|
| 212 |
```sql
|
| 213 |
SELECT hit_class, scope, count(*) AS rows
|
| 214 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/bench_overlap.parquet'
|
| 215 |
GROUP BY ALL ORDER BY hit_class, scope;
|
| 216 |
```
|
| 217 |
|
|
|
|
| 248 |
|
| 249 |
| 想做的事 | 去读 |
|
| 250 |
|---|---|
|
| 251 |
+
| 查看每个行字段和目录的每一列 | [docs/fields.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/fields.zh.md) |
|
| 252 |
+
| 按领域查看每个材料集和每个语料库,以及它的计数、大小、许可证等级和登录要求 | [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/sources.zh.md) 及其列出的领域页面 |
|
| 253 |
+
| 查看哪些没有收录、为什么,以及哪些来源需要登录 | [docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.zh.md) |
|
| 254 |
+
| 用这些文件构建任务,并把智能体不该看到的内容挡在它的环境之外 | [docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/for-agents.zh.md) |
|
| 255 |
+
| 查看每次发布及其改动 | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/CHANGELOG.md) |
|
| 256 |
+
| 以 Croissant 元数据读取本数据集 | [由 Hub 生成的 Croissant](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant);[`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/croissant.json) 描述目录和每个来源 |
|
| 257 |
| 浏览目录,每个材料集或语料库一行,带有它的 `kind` 和 `collection` | [数据集查看器中的 `catalog` 子集](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
|
| 258 |
| 在浏览器里查询目录;它按发布原样列出每个来源,不应用 `tools/fetch.py` 的默认配置,命令才应用它 | [SQL 控制台](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
|
| 259 |
| 观看本索引的使用示例视频,用页面的语言,带字幕 | [数据页上的使用示例视频](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
|
| 260 |
+
| 核对数据集卡片和这里各表的每个数字 | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json) 和 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/size_figures.json) |
|
| 261 |
|
| 262 |
`data/` 下数据文件的每一行指向一个文件或一条记录(见[术语](#manual/glossary)中的「行」);
|
| 263 |
目录的 `data_files` 列列出每个来源的文件,单个材料集或语料库就从这些文件读取,见数据集卡片中的查询和 `tools/fetch.py`。
|
|
|
|
| 308 |
这一声明不延及它所指向的文件。
|
| 309 |
- `media/` 下的文件(使用示例视频及其字幕)和 `examples/` 下的文件(完整示例)同样以 CC0 1.0 发布。
|
| 310 |
- 每个文件保留其来源的许可证:见每一行的许可证列(没有自己许可证的行沿用所在批次的许可证,即 `meta/sources.json` 中的 `batches[].license`)。
|
| 311 |
+
- 每个来源的等级:在它所属领域的页面上,这些页面由 [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/sources.zh.md) 列出。
|
| 312 |
- 许可证等级和原因码由 ScanCode LicenseDB 的许可证类别算出,它以 CC-BY-4.0 许可;
|
| 313 |
+
它的署名:[`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/NOTICE)。
|
| 314 |
|
| 315 |
<h4 id="manual/personal">个人与敏感信息</h4>
|
| 316 |
|
|
|
|
| 356 |
|
| 357 |
<h4 id="manual/releases">发布</h4>
|
| 358 |
|
| 359 |
+
- 最新发布:`release-2026-11-15.6`(其最新的 CHANGELOG 条目日期为 2026-11-15)。
|
| 360 |
+
- 各次发布列在 [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/CHANGELOG.md) 中;
|
| 361 |
+
每次发布是一个带有自己发布标签的提交(本次为 [`release-2026-11-15.6`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-15.6)),
|
| 362 |
因此任何较早的状态都可以在它的标签处读到。
|
| 363 |
- `meta/projection.json` 记录发布标签,以及生成数据集卡片及其页面的程序和模板的版本。
|
| 364 |
|
|
|
|
| 401 |
@misc{training_data_foundation,
|
| 402 |
title = {training-data-foundation: an index of public materials for building agent environments},
|
| 403 |
howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
|
| 404 |
+
note = {Release release-2026-11-15.6},
|
| 405 |
year = {2026}
|
| 406 |
}
|
| 407 |
```
|
|
|
|
| 436 |
- **形式:** 一个来源的材料的形态,一个标识,例如 `spreadsheets` 或 `test-suites`(目录列 `form`,`tools/fetch.py` 的选项 `--form`)。
|
| 437 |
- **训练阶段:** 材料能用于哪种训练,由代码仓库的阶段表 `catalog/stages.yaml` 决定:预训练和中训练用语料库的行,后训练用材料集的单元(见[训练阶段表怎样计数](#manual/stage-counting))。
|
| 438 |
- **默认配置:** `tools/fetch.py` 在没有选项放宽时应用的规则:去掉重复的副本、与基准测试相符或还没有与基准测试清单比对的行、记录为英语以外语言的行,以及大于 20 MB 的文件;
|
| 439 |
+
任何许可证的行和标为带有个人数据的行都会进来,除非 `--training-only` 或 `--no-personal-data` 把它们去掉([docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/for-agents.zh.md))。
|
| 440 |
|
| 441 |
每一行都带有 `unit_id`:在材料集中,是它所属单元的键;
|
| 442 |
在语料库中,是该行自己的键(它的 `uri`,字节范围再接 `@<range_offset>`,存档成员再接 `!/<member_path>`),`tools/fetch.py` 把语料库一行的文件放进以它命名的文件夹。
|
|
|
|
| 446 |
<details id="manual/numbers">
|
| 447 |
<summary>本手册的数字从哪来</summary>
|
| 448 |
|
| 449 |
+
本手册的数字和命令输出是部署时填入的,取自索引提交 `89d4297c25bb7bf352b1874eca8f5c6f86adc7e1`(发布标签 `release-2026-11-15.6`)。
|
| 450 |
+
本次发布有 1,845 个材料集,共 4,244 个单元,分在 14 个合集,以及 139 个语料库,共 1,189,479,628 行,指向去重后共 4.13 PB 的内容,与文件比对过大小的 35,601 行中有 99.4% 一致。
|
| 451 |
|
| 452 |
</details>
|
| 453 |
|
|
|
|
| 458 |
每张表的最后两行统计该阶段的计数中有多少允许训练(每个文件的训练许可为 yes 或 conditional),有多少标为带有个人数据。
|
| 459 |
怎样计数:[训练阶段表怎样计数](#manual/stage-counting)。
|
| 460 |
|
| 461 |
+
阶段字段(`meta/stage_fields.parquet`)是按发布 `release-2026-11-15.6` 的数据文件、在它发布之后算出的。
|
| 462 |
|
| 463 |
##### 后训练(在用)
|
| 464 |
|
|
|
|
| 470 |
| 基准测试规则之后 | 1,826 | 4,202 | 1,879 |
|
| 471 |
| 未与基准测试清单比对的行的规则之后 | 1,826 | 4,202 | 1,879 |
|
| 472 |
| 去重之后:该阶段的计数 | 1,766 | 4,059 | 1,823 |
|
| 473 |
+
| 该阶段的计数中,允许训练的(yes 或 conditional) | 1,412 | 3,395 | 1,339 |
|
| 474 |
| 该阶段的计数中,标为带有个人数据的 | 357 | 631 | 400 |
|
| 475 |
|
| 476 |
##### 中训练(暂定:只给条件和计数)
|
|
|
|
| 479 |
|
| 480 |
| 步骤 | 行数 | 大小 |
|
| 481 |
|---|---:|---:|
|
| 482 |
+
| 符合该阶段的条件,在各条规则之前 | 34,629,789 | 4.01 TB |
|
| 483 |
+
| 基准测试规则之后 | 34,620,507 | 4.01 TB |
|
| 484 |
+
| 未与基准测试清单比对的行的规则之后 | 34,620,507 | 4.01 TB |
|
| 485 |
+
| 去重之后:该阶段的计数 | 34,231,698 | 3.86 TB |
|
| 486 |
| 该阶段的计数中,允许训练的(yes 或 conditional) | 9,937,384 | 1.42 TB |
|
| 487 |
+
| 该阶段的计数中,标为带有个人数据的 | 6,104,380 | 341 GB |
|
| 488 |
|
| 489 |
##### 预训练(暂定:只给条件和计数)
|
| 490 |
|
|
|
|
| 492 |
|
| 493 |
| 步骤 | 行数 | 大小 |
|
| 494 |
|---|---:|---:|
|
| 495 |
+
| 符合该阶段的条件,在各条规则之前 | 155,528,248 | 30.9 TB |
|
| 496 |
+
| 基准测试规则之后 | 155,527,231 | 30.9 TB |
|
| 497 |
+
| 未与基准测试清单比对的行的规则之后 | 155,527,231 | 30.9 TB |
|
| 498 |
+
| 去重之后:该阶段的计数 | 150,610,889 | 30.7 TB |
|
| 499 |
| 该阶段的计数中,允许训练的(yes 或 conditional) | 119,168,563 | 30.0 TB |
|
| 500 |
| 该阶段的计数中,标为带有个人数据的 | 40,477,415 | 676 GB |
|
| 501 |
|
|
|
|
| 554 |
|
| 555 |
<h4 id="manual/workbooks-5">5. 到「获取数据」,保存选择清单</h4>
|
| 556 |
|
| 557 |
+
**做**:点顶部「获取数据」,在「保存这次选择」里点「下载选择清单(JSON)」,把 `training-data-foundation-selection-89d4297.json` 放进取件目录。
|
| 558 |
|
| 559 |
**看到**:标题「从 51 个材料集中获取全部单元」。
|
| 560 |
清单记着版本、筛选和来源:
|
|
|
|
| 563 |
{
|
| 564 |
"format": "training-data-foundation-selection/1",
|
| 565 |
"dataset": "Battam/training-data-foundation",
|
| 566 |
+
"revision": "release-2026-11-15.6",
|
| 567 |
"filters": {
|
| 568 |
"collection": "A",
|
| 569 |
"train": "yes",
|
|
|
|
| 587 |
**做**:在清单所在目录跑「按文件再取一次」那条命令,加上 `--limit 220`(一共 220 个单元,固定顺序;比 200 多取一些留余量,有的原站会失效),`--out ./data` 换成 `--out ./take`:
|
| 588 |
|
| 589 |
```sh
|
| 590 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 591 |
+
--selection training-data-foundation-selection-89d4297.json \
|
| 592 |
+
--revision release-2026-11-15.6 \
|
| 593 |
--ext xlsx \
|
| 594 |
--ext xlsm \
|
| 595 |
--ext xls \
|
|
|
|
| 618 |
...
|
| 619 |
310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
|
| 620 |
files of units: 220 ok; files of the sources themselves: 90 ok
|
| 621 |
+
run_id 20261115T040737.805810Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
|
| 622 |
```
|
| 623 |
|
| 624 |
第一次下载之前,`removed by` 是默认配置去掉的单元和保留它们的开关,数的是 `--limit` 需要读到的那些数据文件;
|
|
|
|
| 634 |
incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
|
| 635 |
5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
|
| 636 |
files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
|
| 637 |
+
run_id 20261115T041058.860782Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
|
| 638 |
+
its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261115T041058.860782Z' AND json_extract_string(json, 'status') <> 'ok'
|
| 639 |
```
|
| 640 |
|
| 641 |
**得到**:`ok` 的文件已按记录的大小和哈希核对。
|
|
|
|
| 704 |
**看到**(截短):
|
| 705 |
|
| 706 |
```text
|
| 707 |
+
Dataset commit 89d4297c25bb7bf352b1874eca8f5c6f86adc7e1: https://huggingface.co/datasets/Battam/training-data-foundation/tree/89d4297c25bb7bf352b1874eca8f5c6f86adc7e1
|
| 708 |
...
|
| 709 |
- Source: `austender-ict-procurement-statistics`
|
| 710 |
- License: CC-BY-3.0-AU
|
|
|
|
| 737 |
|
| 738 |
```sql
|
| 739 |
SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
|
| 740 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 741 |
WHERE kind = 'material_set' AND collection = 'B'
|
| 742 |
AND license.tier = 'share-alike';
|
| 743 |
```
|
|
|
|
| 746 |
--source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
|
| 747 |
```
|
| 748 |
|
| 749 |
+
接在 `fetch.py --revision release-2026-11-15.6` 后面。
|
| 750 |
只读目录表,别整个下载 `meta/`。
|
| 751 |
|
| 752 |
</details>
|
|
|
|
| 760 |

|
| 761 |
|
| 762 |
```sh
|
| 763 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 764 |
+
--revision release-2026-11-15.6 \
|
| 765 |
--source dosfstools-fsck-cases \
|
| 766 |
--source e2fsprogs-fsck-cases \
|
| 767 |
--source exfatprogs-fsck-cases \
|
|
|
|
| 776 |
ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
|
| 777 |
59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
|
| 778 |
files of units: 52 ok; files of the sources themselves: 7 ok
|
| 779 |
+
run_id 20261115T041105.290223Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
|
| 780 |
```
|
| 781 |
|
| 782 |
exFAT 镜像在原站是 `exfat.img.tar.xz`,脚本解出后再按单元自己的 SHA-256 核对,盘上只留镜像。
|
|
|
|
| 819 |
|
| 820 |
```sql
|
| 821 |
SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
|
| 822 |
+
FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/catalog.parquet'
|
| 823 |
WHERE kind = 'material_set'
|
| 824 |
AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
|
| 825 |
GROUP BY ALL;
|
|
|
|
| 839 |
二,取回挑中的来源,看第一次下载之前打印的 `removed by benchmark` 和 `removed by not compared`(输出截短):
|
| 840 |
|
| 841 |
```sh
|
| 842 |
+
uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-15.6/tools/fetch.py \
|
| 843 |
+
--revision release-2026-11-15.6 \
|
| 844 |
--source python-docx-acceptance-test-files \
|
| 845 |
--source python-pptx-acceptance-test-files \
|
| 846 |
--source docling-docx-groundtruth \
|
|
|
|
| 867 |
```sql
|
| 868 |
SELECT o.bench, o.hit_class, count(*) AS files
|
| 869 |
FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
|
| 870 |
+
JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-15.6/meta/bench_overlap.parquet' AS o
|
| 871 |
ON o.source = f.source AND o.uri = f.uri
|
| 872 |
WHERE f.status = 'ok'
|
| 873 |
GROUP BY ALL;
|
|
|
|
| 959 |
- **收录**产生材料集。
|
| 960 |
它的候选是检索为某个合集的任务类别在上游找到的作品,以及用 `tools/fetch.py`(`--where` 和 `--sample`)选出的一组语料库行中的单元,这些单元由下载工具取回,并按记录的大小和 sha256 校验过。
|
| 961 |
候选的指针事实成立时才收录:有下载地址,记录了大小和 sha256,记录了许可证,没有法律封锁,没有发现事实有误,内容也不在索引中;
|
| 962 |
+
未收录的每个候选都连同原因列在 `meta/not_included.parquet` 中([docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.zh.md))。
|
| 963 |
- **收录添加的内容:** 把候选的文件作为一个单元并给出它的 `unit_id`,材料集的 README.md 和 LICENSE,它的合集(它的任务类别所属的合集),它的形式,它的上游,以及语言(`language`)、`robots` 和使用条款(`terms`)等标签。
|
| 964 |
之后采集步骤再次取回每个单元的每个文件,并在材料集发布之前校验它的 sha256。
|
| 965 |
- 由语料库行做成的材料集是一个独立的来源,与那个语料库分开:同一个文件既是语料库的一行,又是一个单元的一个文件,所以两层的计数从不相加;
|
|
|
|
| 976 |
|
| 977 |
记录的大小是有大小的各行 `size_bytes` 之和(目录列 `bytes`)。核对读取每个数据文件中抽取的一组行(`meta/sizes.parquet`),每天读取有某个来源的行尚未在其中读过的文件,以及最久没读的那些来源的文件,约占全部读数的七分之一,所以每个来源一周之内都会再读一次:对其中有行的每个来源,有记录大小的行各抽 20 行;没有记录大小的行按其所在来源的样本分摊,样本量按目前读到的大小的离散程度定,使其平均大小在 95% 置信度下误差不超过 25%(每个来源每轮最多 400 行,还没有读数的来源取 20 行);其比例是比对过的行(状态 `match`、`size_mismatch` 或 `hash_mismatch`)中与记录的大小一致的行所占的比例,没能比对的行(没有取到文件)按状态另行计数(`unreachable`、`gone`、`needs_auth`、`no_length`、`error`;目录列 `size_check`)。一个来源没有记录大小的行的估计(目录列 `unsized_rows` 和 `unsized_estimate`)是这些行的行数乘以读过的这类行的平均实测大小,并给出 95% BCa 自助法范围(`scipy.stats.bootstrap`,用其默认重抽样次数);读过的这类行少于 10 行的来源没有估计。在来源之上的层级,估计是其各来源估计之和,范围是各来源范围之和,比联合区间更宽。每个层级的数字都在 `meta/size_figures.json` 中。
|
| 978 |
|
| 979 |
+
去重后内容是各行 `size_bytes` 之和,减去与另一行内容重复的那些行的大小(sha256 相同;没有 sha256 时 sha1 相同;再没有时 md5 和大小都相同;并且哈希范围相同;每种内容只保留一份,留在按注册顺序排在最前的来源中),在每次发布之后按它发布的数据文件重新计算,每个来源的值在目录列 `distinct_bytes` 中;同一来源也整个列出的存档,其成员的哈希与存档不同,所以不扣除。`tools/fetch.py` 读取你自己的选择时会去掉其中的重复。 这里的去重后大小是按发布 `release-2026-11-15.6` 的数据文件算出的。
|
| 980 |
|
| 981 |
</details>
|
| 982 |
|
|
|
|
| 987 |
发布为材料集的每个单元和语料库的每一行记录这些字段,并记录它按以下顺序通过了各阶段共用规则中的几条:基准测试规则(与公开基准测试任务相符的行被去掉)、还没有与基准测试清单比对的行的规则和去重(`meta/stage_fields.parquet`);
|
| 988 |
每个阶段在这些规则之前和每条规则之后统计符合它的条件的材料,最后一条规则之后的计数就是它能用的材料;
|
| 989 |
数据页在每个阶段那一块下面显示这几步。
|
| 990 |
+
没有任何阶段取用的部分按层、按原因统计,在 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json) 的 `no_stage` 中:不满足任何阶段条件的、被某条规则去掉的,以及目录里有、阶段字段里还没有的(尚未计数:发布的阶段字段是在更早一次发布的文件上计数的);
|
| 991 |
所以后训练的单元数和材料集数加上未归入任何阶段的,就是全部材料集的单元数和材料集数,数据页在各阶段下面就是这样显示的。
|
| 992 |
训练许可和个人数据标记是每一行的标签,既不是任何阶段的条件,也不是规则。
|
| 993 |
+
数据集卡片显示每个阶段的计数、其中有多少允许训练,以及它在每个顶层领域下保留的部分([`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json) 中的 `by_domain`)。
|
| 994 |
这次发布的各张表:[每个训练阶段能用多少](#manual/stage-tables)。
|
| 995 |
|
| 996 |
</details>
|
|
|
|
| 1001 |
- **数据采集:** 行从每个来源的公开列表、API 和存档中采集,加入时即校验:
|
| 1002 |
地址必须能访问,大小和哈希值会重新取回并比对。
|
| 1003 |
- **发布前的检查:** 每个来源都记录了许可证并归入一个等级,个人信息从样本文件中判读,各行与公开基准测试比对;
|
| 1004 |
+
复制了公开基准测试的行会被移除,不满足某项要求的单元会连同原因一起列出:[docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/docs/access.zh.md)。
|
| 1005 |
- **发布:** 每次发布是一个带有自己标签的提交;
|
| 1006 |
数据集卡片、`docs/figures/` 下它的图、`docs/` 下的页面、`cards/*.md`、`meta/catalog.parquet`、`meta/croissant.json`、`meta/size_figures.json` 和 `meta/stage_figures.json` 由同一个提交生成,因此数字彼此一致,本手册则在部署数据页时从这个提交填入。
|
| 1007 |
+
卡片的每个数字和每张图都读自 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/stage_figures.json),数据集各页面和本手册各表的其他计数和大小读自 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-15.6/meta/size_figures.json),每个数在那里为这次发布只算一次。
|
| 1008 |
- **中文版本:** 中文卡片、它的页面和中文手册各表显示的每个名称和描述,都由语言模型对每段文字翻译一次,按英文原文的 sha256 保存在 `meta/translations.parquet` 中。
|
| 1009 |
|
| 1010 |
</details>
|
portal.json
CHANGED
|
@@ -3,6 +3,6 @@
|
|
| 3 |
"digest": "2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221",
|
| 4 |
"files": 29,
|
| 5 |
"format": "portal-deploy/1",
|
| 6 |
-
"funnel": "
|
| 7 |
-
"index": "
|
| 8 |
}
|
|
|
|
| 3 |
"digest": "2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221",
|
| 4 |
"files": 29,
|
| 5 |
"format": "portal-deploy/1",
|
| 6 |
+
"funnel": "89d4297c25bb7bf352b1874eca8f5c6f86adc7e1",
|
| 7 |
+
"index": "89d4297c25bb7bf352b1874eca8f5c6f86adc7e1"
|
| 8 |
}
|