Battam commited on
Commit
071e45f
·
verified ·
1 Parent(s): f0b8de3

portal: data page of repository commit 714ed1f4afaa

Browse files

Files of catalog/portal/ (29), digest 2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221.

Files changed (6) hide show
  1. first.json +9 -9
  2. funnel.json +16 -16
  3. llms.txt +1 -1
  4. manual/manual.en.md +58 -58
  5. manual/manual.zh.md +58 -58
  6. portal.json +2 -2
first.json CHANGED
@@ -2,8 +2,8 @@
2
  "collections": 14,
3
  "date": "2026-11-14",
4
  "format": "portal-first/5",
5
- "index": "6cfdc9bd5d379409cef7d267960b4ba6b8052367",
6
- "raw_rows": 1189435783,
7
  "raw_rows_not_counted": 0,
8
  "raw_sources": 139,
9
  "ready_sources": 1845,
@@ -11,7 +11,7 @@
11
  "ready_units_not_counted": 0,
12
  "sizes": {
13
  "corpus": {
14
- "bytes": 4445581199218067,
15
  "check": {
16
  "compared": 23852,
17
  "error": 2,
@@ -26,10 +26,10 @@
26
  },
27
  "counts": {
28
  "corpora": 139,
29
- "rows": 1189435783,
30
  "rows_not_counted": 0
31
  },
32
- "distinct_bytes": 4132595894071778,
33
  "estimate": {
34
  "bytes": 464487052504997,
35
  "high": 691974027742576,
@@ -69,7 +69,7 @@
69
  "match": "23,698",
70
  "range": "322 TB to 692 TB",
71
  "read": "26,766",
72
- "rows": "1,189,435,783",
73
  "unsized_rows": "288,986,750"
74
  },
75
  "shown_zh": {
@@ -81,13 +81,13 @@
81
  "match": "23,698",
82
  "range": "322 TB \u5230 692 TB",
83
  "read": "26,766",
84
- "rows": "1,189,435,783",
85
  "unsized_rows": "288,986,750"
86
  },
87
  "unsized_rows": 288986750
88
  },
89
  "index": {
90
- "bytes": 4445584084297674,
91
  "check": {
92
  "compared": 35601,
93
  "error": 5,
@@ -104,7 +104,7 @@
104
  "corpora": 139,
105
  "material_sets": 1845
106
  },
107
- "distinct_bytes": 4132598713967775,
108
  "estimate": {
109
  "bytes": 464487052504997,
110
  "high": 691974027742576,
 
2
  "collections": 14,
3
  "date": "2026-11-14",
4
  "format": "portal-first/5",
5
+ "index": "6632e2e186da270954e3e522f3e70554046e6070",
6
+ "raw_rows": 1189440203,
7
  "raw_rows_not_counted": 0,
8
  "raw_sources": 139,
9
  "ready_sources": 1845,
 
11
  "ready_units_not_counted": 0,
12
  "sizes": {
13
  "corpus": {
14
+ "bytes": 4445584091187323,
15
  "check": {
16
  "compared": 23852,
17
  "error": 2,
 
26
  },
27
  "counts": {
28
  "corpora": 139,
29
+ "rows": 1189440203,
30
  "rows_not_counted": 0
31
  },
32
+ "distinct_bytes": 4132598779469501,
33
  "estimate": {
34
  "bytes": 464487052504997,
35
  "high": 691974027742576,
 
69
  "match": "23,698",
70
  "range": "322 TB to 692 TB",
71
  "read": "26,766",
72
+ "rows": "1,189,440,203",
73
  "unsized_rows": "288,986,750"
74
  },
75
  "shown_zh": {
 
81
  "match": "23,698",
82
  "range": "322 TB \u5230 692 TB",
83
  "read": "26,766",
84
+ "rows": "1,189,440,203",
85
  "unsized_rows": "288,986,750"
86
  },
87
  "unsized_rows": 288986750
88
  },
89
  "index": {
90
+ "bytes": 4445586976266930,
91
  "check": {
92
  "compared": 35601,
93
  "error": 5,
 
104
  "corpora": 139,
105
  "material_sets": 1845
106
  },
107
+ "distinct_bytes": 4132601599365498,
108
  "estimate": {
109
  "bytes": 464487052504997,
110
  "high": 691974027742576,
funnel.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "format": "portal-funnel/1",
3
  "funnel": {
4
- "counted_on": "release-2026-11-14.26",
5
  "domain_names": {
6
  "D1": "software-engineering",
7
  "D10": "misc",
@@ -52,21 +52,21 @@
52
  },
53
  "corpus": {
54
  "by_domain": {
55
- "D1": 29959189,
56
  "D10": 241140,
57
- "D2": 161286697,
58
- "D3": 761632797,
59
  "D4": 9646616,
60
  "D5": 45635385,
61
  "D6": 39426757,
62
  "D7": 8719015,
63
- "D8": 4535647,
64
  "D9": 12753055,
65
  "multi": 109275447,
66
- "unclassified": 6324038
67
  },
68
  "corpora": 139,
69
- "rows": 1189435783,
70
  "rows_not_counted": 0
71
  }
72
  },
@@ -127,7 +127,7 @@
127
  "zh": "后训练"
128
  },
129
  {
130
- "before": 34623038,
131
  "by_domain": {
132
  "D1": {
133
  "bytes": 2968633738,
@@ -138,8 +138,8 @@
138
  "kept": 102989
139
  },
140
  "D2": {
141
- "bytes": 1268593856608,
142
- "kept": 11392875
143
  },
144
  "D3": {
145
  "bytes": 429176613679,
@@ -174,15 +174,15 @@
174
  "kept": 16193621
175
  },
176
  "unclassified": {
177
- "bytes": 4187084004,
178
- "kept": 474260
179
  }
180
  },
181
- "bytes": 3864785816483,
182
  "domains": [],
183
  "forms": [],
184
  "id": "mid-training",
185
- "kept": 34224947,
186
  "kind": "corpus",
187
  "measure": "rows",
188
  "name": "Mid-training",
@@ -255,6 +255,6 @@
255
  ],
256
  "table": 3
257
  },
258
- "index": "6cfdc9bd5d379409cef7d267960b4ba6b8052367",
259
- "release": "release-2026-11-14.26"
260
  }
 
1
  {
2
  "format": "portal-funnel/1",
3
  "funnel": {
4
+ "counted_on": "release-2026-11-14.27",
5
  "domain_names": {
6
  "D1": "software-engineering",
7
  "D10": "misc",
 
52
  },
53
  "corpus": {
54
  "by_domain": {
55
+ "D1": 29959191,
56
  "D10": 241140,
57
+ "D2": 161286703,
58
+ "D3": 761632801,
59
  "D4": 9646616,
60
  "D5": 45635385,
61
  "D6": 39426757,
62
  "D7": 8719015,
63
+ "D8": 4535649,
64
  "D9": 12753055,
65
  "multi": 109275447,
66
+ "unclassified": 6328444
67
  },
68
  "corpora": 139,
69
+ "rows": 1189440203,
70
  "rows_not_counted": 0
71
  }
72
  },
 
127
  "zh": "后训练"
128
  },
129
  {
130
+ "before": 34623576,
131
  "by_domain": {
132
  "D1": {
133
  "bytes": 2968633738,
 
138
  "kept": 102989
139
  },
140
  "D2": {
141
+ "bytes": 1268594068549,
142
+ "kept": 11392878
143
  },
144
  "D3": {
145
  "bytes": 429176613679,
 
174
  "kept": 16193621
175
  },
176
  "unclassified": {
177
+ "bytes": 4189726106,
178
+ "kept": 474795
179
  }
180
  },
181
+ "bytes": 3864788670526,
182
  "domains": [],
183
  "forms": [],
184
  "id": "mid-training",
185
+ "kept": 34225485,
186
  "kind": "corpus",
187
  "measure": "rows",
188
  "name": "Mid-training",
 
255
  ],
256
  "table": 3
257
  },
258
+ "index": "6632e2e186da270954e3e522f3e70554046e6070",
259
+ "release": "release-2026-11-14.27"
260
  }
llms.txt CHANGED
@@ -2,7 +2,7 @@
2
 
3
  - The index holds two layers over the same public web, kept apart and never added together. Each corpus is one whole upstream read in bulk and counted in rows; a material set is admitted for building tasks from works the search found upstream or from a group of corpus rows, its files checked and grouped into units, and it is counted in units, apart from any corpus its rows came from. Task material comes first: material sets, grouped into collections, one per task family. A material set (task-ready) is the units admitted from one source, which reads one upstream (one dataset or one data catalog), for building tasks (the catalog column `kind` is `material_set`, and `collection` names its collection). A unit is a group of files that its upstream publishes as one thing, delivered whole; its key `unit_id` is the `unit_group` its rows share (`extra.source_meta.unit_group`), defined once, in the glossary of the data page's manual, https://battam-training-data-foundation.static.hf.space/index.html?lng=en#manual/glossary . A unit is material, not a task: one unit may give several tasks, and one task may use several units. Corpora come second, to search when no material set fits: a corpus (raw; plural corpora) is one whole upstream that a collector keeps reading, such as Common Crawl's office documents, PyPI's release files, Mendeley Data or NVD: its rows are pointers to files or API responses, and it is counted in rows (`kind` is `corpus`); `tools/fetch.py` takes a corpus row by row, each under its own key as its `unit_id`.
4
  - Pick material sets or corpora from `meta/catalog.parquet` (one row per material set or corpus, about 50 KB; do not download `meta/` whole). Columns: `source`; `kind`, `material_set` or `corpus` (select one layer: units and rows are never added together); `collection`, the code of a material set's collection (every code with its name is in `meta/vocabulary.json`), or of the collection that lists a corpus as holding its kind of material (`corpora` in `meta/vocabulary.json`; empty for any other corpus); `domain`, a list of codes D1 to D10 or leaves such as D3.5, or `multi`, `unclassified`; `form`, such as `test-suites` or `spreadsheets`; `upstream_kind`, `upstream_agent` and `upstream_entry`, the source's upstream (`dataset` or `catalog`, its agent, and the address its reading starts from; a source that still reads several upstreams gives each, joined by ` | `); `units` (a material set's distinct `unit_id` values: a test case, one workbook, one package; a corpus is counted in rows, and gives the same number as `files`) and `files` (its rows: for a material set, the files of its units and its README.md and LICENSE); `publishers` (a material set's distinct publisher names over all its rows; empty when its rows name none); `language` (the language codes of the rows' labels, what the default profile filters on; `not recorded` when a row has none); `contamination.level` (by the rule of the default profile of `tools/fetch.py` over `meta/bench_overlap.parquet` and `meta/bench_checked.parquet`: `none` when every data file is compared and kept and no row is removed, the hit class with the most removed rows when some are removed, `not scanned` otherwise); `license.ship`; `pii.level`; `distinct_bytes` (deduplicated size) and `bytes` (recorded size, the sum over rows); `size_check` (the rows read in `meta/sizes.parquet`, `compared` those that compared a size or hash, and how many have each status, `match` when the measured size agrees; the check's share is `match` over `compared`), `unsized_rows` (rows without a recorded size) and `unsized_estimate` (their estimated size, with its 95% range); `fetch_check` (whether the upstream still serves the files: `rows`, the sampled rows read in `meta/sizes.parquet`; `fetched`, those that fetched and matched their recorded size and hash; `read_at`, the day read; `level`, `all`, `some`, `none`, `needs auth` (no row read fetched, and the source's rows need the paying or signed-in reader its `needs_auth` names, which the check, reading anonymously, does not have) or `not read`); the same figures for the whole index, each layer and each collection are in `meta/size_figures.json`. Who published each file of a material set, and under which license and license URL, is in `meta/attribution.parquet` (one row per source, publisher, license and license URL, with its number of files). The vocabulary is `meta/vocabulary.json`: every domain code with its name (and `zh`, its Chinese name), what it covers and its parent, every form with its Chinese name and what it means, every category of `meta/not_included.parquet` with its requirement in English and in Chinese, the two layers with their relation, and every collection with its name, what it holds, its campaigns (take a collection with `--tag campaign:<name>` for each of its campaigns) and the corpora it lists, if any (take one with `--source`), and the criteria the default profile of `tools/fetch.py` applies, in its order (`profile`, each with the option that keeps what it removes).
5
- - Size of this release: Whole index: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (35,601 of the 38,534 rows read compared); not compared: address unreachable 179, file gone 198, login needed 528, no size from the server 2,023, other error 5; estimated size of 277,856,236 of the 288,986,750 rows without a recorded size: 464 TB (an estimate; 95% range 322 TB to 692 TB, the sum of its sources' ranges, wider than a joint interval; from 4,411 rows read), the other 11,130,514 with too few rows read for an estimate. Task material: Recorded size 2.89 GB; deduplicated size 2.82 GB; size check: 99.5% match (11,749 of the 11,768 rows read compared); not compared: address unreachable 1, file gone 4, login needed 11, other error 3. Corpora: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (23,852 of the 26,766 rows read compared); not compared: address unreachable 178, file gone 194, login needed 517, no size from the server 2,023, other error 2; estimated size of 277,856,236 of the 288,986,750 rows without a recorded size: 464 TB (an estimate; 95% range 322 TB to 692 TB, the sum of its sources' ranges, wider than a joint interval; from 4,411 rows read), the other 11,130,514 with too few rows read for an estimate. Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-14.26`. The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
6
  - Fetch with one command: `uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/main/tools/fetch.py --kind material_set --domain D3 --form spreadsheets --out ./data` takes those material sets whole, as the data page does unless told otherwise; take a corpus, whose whole can be petabytes, with `--sample N` (the data page takes 3 units of each by default) or `--limit N`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--kind` (`material_set` or `corpus`: one layer), `--domain`, `--form`, `--tag`, `--ext`, or `--selection <file>` (a selection file the data page saves: the ids of its sources and its commit); `--sample N` takes at most N units per source, `--limit N` takes N units over the whole selection, `--unit <unit_id>` takes the unit of that `unit_id` whole (with `--source`); before its first download the run prints the units, files and total size it selected.
7
  - It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
8
  - The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, rows of groupings awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), rows recorded in a language other than English, files above 20 MB, and rows that the release's fetch check read and found refused by their site (HTTP 401 or 403, a login, or an empty or short body). Before downloading it prints how many units each criterion removed; each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-language`, `--any-size`, `--allow-refused`; `--allow-benchmark` also keeps rows not compared yet), and `--all` turns off every criterion except benchmark and not compared, so rows that matched a benchmark or were not compared yet come in only with their own options. The license and the personal-data mark are labels of every row, not filters: by default every row comes, and two options leave rows out only when given, `--training-only` the files whose license does not allow shipping (license.ship yes and conditional stay) and `--no-personal-data` personal data (catalog pii.level present or suspected, rows marked sensitive). Compared with the benchmark list (keys of 2026-11-08): 1,845 of 1,845 material sets; 139 of 139 corpora. The profile also removes every row whose data file was not compared (`--allow-benchmark` keeps them, unchecked).
 
2
 
3
  - The index holds two layers over the same public web, kept apart and never added together. Each corpus is one whole upstream read in bulk and counted in rows; a material set is admitted for building tasks from works the search found upstream or from a group of corpus rows, its files checked and grouped into units, and it is counted in units, apart from any corpus its rows came from. Task material comes first: material sets, grouped into collections, one per task family. A material set (task-ready) is the units admitted from one source, which reads one upstream (one dataset or one data catalog), for building tasks (the catalog column `kind` is `material_set`, and `collection` names its collection). A unit is a group of files that its upstream publishes as one thing, delivered whole; its key `unit_id` is the `unit_group` its rows share (`extra.source_meta.unit_group`), defined once, in the glossary of the data page's manual, https://battam-training-data-foundation.static.hf.space/index.html?lng=en#manual/glossary . A unit is material, not a task: one unit may give several tasks, and one task may use several units. Corpora come second, to search when no material set fits: a corpus (raw; plural corpora) is one whole upstream that a collector keeps reading, such as Common Crawl's office documents, PyPI's release files, Mendeley Data or NVD: its rows are pointers to files or API responses, and it is counted in rows (`kind` is `corpus`); `tools/fetch.py` takes a corpus row by row, each under its own key as its `unit_id`.
4
  - Pick material sets or corpora from `meta/catalog.parquet` (one row per material set or corpus, about 50 KB; do not download `meta/` whole). Columns: `source`; `kind`, `material_set` or `corpus` (select one layer: units and rows are never added together); `collection`, the code of a material set's collection (every code with its name is in `meta/vocabulary.json`), or of the collection that lists a corpus as holding its kind of material (`corpora` in `meta/vocabulary.json`; empty for any other corpus); `domain`, a list of codes D1 to D10 or leaves such as D3.5, or `multi`, `unclassified`; `form`, such as `test-suites` or `spreadsheets`; `upstream_kind`, `upstream_agent` and `upstream_entry`, the source's upstream (`dataset` or `catalog`, its agent, and the address its reading starts from; a source that still reads several upstreams gives each, joined by ` | `); `units` (a material set's distinct `unit_id` values: a test case, one workbook, one package; a corpus is counted in rows, and gives the same number as `files`) and `files` (its rows: for a material set, the files of its units and its README.md and LICENSE); `publishers` (a material set's distinct publisher names over all its rows; empty when its rows name none); `language` (the language codes of the rows' labels, what the default profile filters on; `not recorded` when a row has none); `contamination.level` (by the rule of the default profile of `tools/fetch.py` over `meta/bench_overlap.parquet` and `meta/bench_checked.parquet`: `none` when every data file is compared and kept and no row is removed, the hit class with the most removed rows when some are removed, `not scanned` otherwise); `license.ship`; `pii.level`; `distinct_bytes` (deduplicated size) and `bytes` (recorded size, the sum over rows); `size_check` (the rows read in `meta/sizes.parquet`, `compared` those that compared a size or hash, and how many have each status, `match` when the measured size agrees; the check's share is `match` over `compared`), `unsized_rows` (rows without a recorded size) and `unsized_estimate` (their estimated size, with its 95% range); `fetch_check` (whether the upstream still serves the files: `rows`, the sampled rows read in `meta/sizes.parquet`; `fetched`, those that fetched and matched their recorded size and hash; `read_at`, the day read; `level`, `all`, `some`, `none`, `needs auth` (no row read fetched, and the source's rows need the paying or signed-in reader its `needs_auth` names, which the check, reading anonymously, does not have) or `not read`); the same figures for the whole index, each layer and each collection are in `meta/size_figures.json`. Who published each file of a material set, and under which license and license URL, is in `meta/attribution.parquet` (one row per source, publisher, license and license URL, with its number of files). The vocabulary is `meta/vocabulary.json`: every domain code with its name (and `zh`, its Chinese name), what it covers and its parent, every form with its Chinese name and what it means, every category of `meta/not_included.parquet` with its requirement in English and in Chinese, the two layers with their relation, and every collection with its name, what it holds, its campaigns (take a collection with `--tag campaign:<name>` for each of its campaigns) and the corpora it lists, if any (take one with `--source`), and the criteria the default profile of `tools/fetch.py` applies, in its order (`profile`, each with the option that keeps what it removes).
5
+ - Size of this release: Whole index: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (35,601 of the 38,534 rows read compared); not compared: address unreachable 179, file gone 198, login needed 528, no size from the server 2,023, other error 5; estimated size of 277,856,236 of the 288,986,750 rows without a recorded size: 464 TB (an estimate; 95% range 322 TB to 692 TB, the sum of its sources' ranges, wider than a joint interval; from 4,411 rows read), the other 11,130,514 with too few rows read for an estimate. Task material: Recorded size 2.89 GB; deduplicated size 2.82 GB; size check: 99.5% match (11,749 of the 11,768 rows read compared); not compared: address unreachable 1, file gone 4, login needed 11, other error 3. Corpora: Recorded size 4.45 PB; deduplicated size 4.13 PB; size check: 99.4% match (23,852 of the 26,766 rows read compared); not compared: address unreachable 178, file gone 194, login needed 517, no size from the server 2,023, other error 2; estimated size of 277,856,236 of the 288,986,750 rows without a recorded size: 464 TB (an estimate; 95% range 322 TB to 692 TB, the sum of its sources' ranges, wider than a joint interval; from 4,411 rows read), the other 11,130,514 with too few rows read for an estimate. Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-14.27`. The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
6
  - Fetch with one command: `uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/main/tools/fetch.py --kind material_set --domain D3 --form spreadsheets --out ./data` takes those material sets whole, as the data page does unless told otherwise; take a corpus, whose whole can be petabytes, with `--sample N` (the data page takes 3 units of each by default) or `--limit N`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--kind` (`material_set` or `corpus`: one layer), `--domain`, `--form`, `--tag`, `--ext`, or `--selection <file>` (a selection file the data page saves: the ids of its sources and its commit); `--sample N` takes at most N units per source, `--limit N` takes N units over the whole selection, `--unit <unit_id>` takes the unit of that `unit_id` whole (with `--source`); before its first download the run prints the units, files and total size it selected.
7
  - It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
8
  - The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, rows of groupings awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), rows recorded in a language other than English, files above 20 MB, and rows that the release's fetch check read and found refused by their site (HTTP 401 or 403, a login, or an empty or short body). Before downloading it prints how many units each criterion removed; each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-language`, `--any-size`, `--allow-refused`; `--allow-benchmark` also keeps rows not compared yet), and `--all` turns off every criterion except benchmark and not compared, so rows that matched a benchmark or were not compared yet come in only with their own options. The license and the personal-data mark are labels of every row, not filters: by default every row comes, and two options leave rows out only when given, `--training-only` the files whose license does not allow shipping (license.ship yes and conditional stay) and `--no-personal-data` personal data (catalog pii.level present or suspected, rows marked sensitive). Compared with the benchmark list (keys of 2026-11-08): 1,845 of 1,845 material sets; 139 of 139 corpora. The profile also removes every row whose data file was not compared (`--allow-benchmark` keeps them, unchecked).
manual/manual.en.md CHANGED
@@ -12,7 +12,7 @@ Each row records one file's address, size, hash (a fingerprint of its content) a
12
  <h2 id="manual/guides">Guides</h2>
13
 
14
  One goal each;
15
- commands are pinned to `release-2026-11-14.26`.
16
 
17
  <h3 id="manual/take">Take the files a need calls for</h3>
18
 
@@ -23,9 +23,9 @@ commands are pinned to `release-2026-11-14.26`.
23
  3. In that folder, run the command of "Take it again from the file" with [uv](https://docs.astral.sh/uv/):
24
 
25
  ```sh
26
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
27
- --selection training-data-foundation-selection-6cfdc9b.json \
28
- --revision release-2026-11-14.26 \
29
  --out ./data
30
  ```
31
 
@@ -41,7 +41,7 @@ commands are pinned to `release-2026-11-14.26`.
41
  ```sh
42
  uvx --from duckdb-cli duckdb -c "
43
  SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
44
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
45
  WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
46
  ORDER BY collection, units DESC"
47
  ```
@@ -55,9 +55,9 @@ commands are pinned to `release-2026-11-14.26`.
55
  to leave out the files whose license does not let them be shipped and the rows marked as carrying personal data, add two options:
56
 
57
  ```sh
58
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
59
- --selection training-data-foundation-selection-6cfdc9b.json \
60
- --revision release-2026-11-14.26 \
61
  --training-only \
62
  --no-personal-data \
63
  --out ./data
@@ -129,7 +129,7 @@ it follows your filter changes, so copying it shares the state.
129
  2 when the index could not be read or a name is not in the catalog, such as a misspelt source:
130
 
131
  ```text
132
- error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367; closest: dosfstools-fsck-cases
133
  ```
134
 
135
  </details>
@@ -137,11 +137,11 @@ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
137
  <details id="manual/pin">
138
  <summary>Pin a release</summary>
139
 
140
- The page's commands pin the script URL and `--revision` to `release-2026-11-14.26`;
141
  the script's first line names the commit:
142
 
143
  ```text
144
- reading hf://datasets/Battam/training-data-foundation@6cfdc9bd5d379409cef7d267960b4ba6b8052367 (commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367); pass --revision 6cfdc9bd5d379409cef7d267960b4ba6b8052367 to select the same rows later
145
  ```
146
 
147
  Later, `--revision <commit>`, the same tag or the selection list take the same files.
@@ -214,7 +214,7 @@ Rows per class:
214
 
215
  ```sql
216
  SELECT hit_class, scope, count(*) AS rows
217
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/bench_overlap.parquet'
218
  GROUP BY ALL ORDER BY hit_class, scope;
219
  ```
220
 
@@ -251,16 +251,16 @@ the dataset https://huggingface.co/datasets/Battam/training-data-foundation .
251
 
252
  | To | Read |
253
  |---|---|
254
- | Read every field of a row and every column of the catalog | [docs/fields.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/fields.md) |
255
- | See every material set and every corpus, by domain, with its counts, size, license tier and login | [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/sources.md) and the domain pages it lists |
256
- | See what was not included and why, and which sources need a login | [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.md) |
257
- | Build tasks from the files, and keep what an agent must not see out of its environment | [docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/for-agents.md) |
258
- | See every release and what it changed | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/CHANGELOG.md) |
259
- | Read the dataset as Croissant metadata | [Croissant generated by the Hub](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant); [`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/croissant.json) describes the catalog and every source |
260
  | Look through the catalog, one row per material set or corpus with its `kind` and `collection` | [The `catalog` subset in the Dataset Viewer](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
261
  | Query the catalog in the browser; it lists every source as published, without the default profile of `tools/fetch.py`, which the command applies | [SQL Console](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
262
  | Watch the index in use, in the page's language, with subtitles | [Usage video on the data page](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
263
- | Check every number of the dataset card and of the tables here | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json) and [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/size_figures.json) |
264
 
265
  Each row of the data files under `data/` points to one file or record (Row, in the [glossary](#manual/glossary));
266
  the catalog's column `data_files` names the files of each source, and one material set or corpus is read from exactly those files, as in the queries of the dataset card and with `tools/fetch.py`.
@@ -310,9 +310,9 @@ Each row also has a training and a redistribution permit (yes, conditional, no;
310
  - This catalog itself (addresses, sizes, hashes and descriptive text) is released under CC0 1.0; that dedication does not extend to the files it points to.
311
  - The files under `media/` (the usage example video and its subtitles) and under `examples/` (the worked examples) are released under CC0 1.0 as well.
312
  - Each file keeps the license of its source: see the license columns of each row (a row without its own license takes the license of its batch, `batches[].license` in `meta/sources.json`).
313
- - The tier of every source: on the page of its domain, which [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/sources.md) lists.
314
  - License tiers and reason codes are computed from the license categories of ScanCode LicenseDB, licensed CC-BY-4.0;
315
- its attribution: [`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/NOTICE).
316
 
317
  <h4 id="manual/personal">Personal and sensitive information</h4>
318
 
@@ -358,9 +358,9 @@ Each row also has a training and a redistribution permit (yes, conditional, no;
358
 
359
  <h4 id="manual/releases">Releases</h4>
360
 
361
- - Latest release: `release-2026-11-14.26` (its newest CHANGELOG entry is dated 2026-11-14).
362
- - Releases are listed in [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/CHANGELOG.md);
363
- each one is a commit with its own release tag (this one is [`release-2026-11-14.26`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-14.26)),
364
  so any earlier state stays readable at its tag.
365
  - `meta/projection.json` records the release tag and the version of the program and templates that generated the dataset card and its pages.
366
 
@@ -395,7 +395,7 @@ to cite it:
395
  @misc{training_data_foundation,
396
  title = {training-data-foundation: an index of public materials for building agent environments},
397
  howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
398
- note = {Release release-2026-11-14.26},
399
  year = {2026}
400
  }
401
  ```
@@ -430,7 +430,7 @@ to cite it:
430
  - **Form:** the shape of a source's material, one id such as `spreadsheets` or `test-suites` (the catalog column `form`, the option `--form` of `tools/fetch.py`).
431
  - **Training stage:** what the material can feed, by the stage table `catalog/stages.yaml` of the code repository: pre-training and mid-training take corpus rows, post-training the units of material sets ([How the training-stage tables are counted](#manual/stage-counting)).
432
  - **Default profile:** the rules `tools/fetch.py` applies unless an option relaxes them: it leaves out duplicate copies, rows that matched a benchmark or were not compared with the benchmark list yet, rows in a language other than English and files above 20 MB;
433
- rows of any license and rows marked as carrying personal data come unless `--training-only` or `--no-personal-data` leaves them out ([docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/for-agents.md)).
434
 
435
  Every row carries a `unit_id`: in a material set, the key of its unit;
436
  in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte range, then `!/<member_path>` for an archive member), and `tools/fetch.py` puts the file of a corpus row in a folder named after it.
@@ -440,8 +440,8 @@ in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte ran
440
  <details id="manual/numbers">
441
  <summary>Where this manual's numbers come from</summary>
442
 
443
- The numbers and command outputs of this manual were filled in at deploy time, from the index commit `6cfdc9bd5d379409cef7d267960b4ba6b8052367` (release tag `release-2026-11-14.26`).
444
- This release holds 1,845 material sets with 4,244 units in 14 collections and 139 corpora with 1,189,435,783 rows, which point to 4.13 PB of deduplicated content, and of the 35,601 rows whose size was compared with their file, 99.4% matched.
445
 
446
  </details>
447
 
@@ -452,7 +452,7 @@ Each table below gives what a training stage can take, counted before the rules
452
  its last two rows count how much of the stage's count is allowed for training (every file's training permit yes or conditional) and how much is marked as carrying personal data.
453
  How they are counted: [How the training-stage tables are counted](#manual/stage-counting).
454
 
455
- The stage fields (`meta/stage_fields.parquet`) were counted over the data files of release `release-2026-11-14.26`, after it was published.
456
 
457
  ##### Post-training (in use)
458
 
@@ -473,12 +473,12 @@ Corpus rows of code, documentation and the terminal side: conditions and counts
473
 
474
  | Step | Rows | Size |
475
  |---|---:|---:|
476
- | In the stage's condition, before the rules | 34,623,038 | 4.01 TB |
477
- | After the benchmark rule | 34,613,756 | 4.01 TB |
478
- | After the rule on rows not compared with the benchmark list | 34,613,756 | 4.01 TB |
479
- | After deduplication: the stage's count | 34,224,947 | 3.86 TB |
480
  | Of the stage's count, allowed for training (yes or conditional) | 9,937,384 | 1.42 TB |
481
- | Of the stage's count, marked as carrying personal data | 6,104,370 | 341 GB |
482
 
483
  ##### Pre-training (provisional: conditions and counts only)
484
 
@@ -550,7 +550,7 @@ or add `&open=zenodo-workbooks-cc-by` to the address.
550
  <h4 id="manual/workbooks-5">5. Go to "Get the data" and keep the selection</h4>
551
 
552
  **Do**: click "Get the data" at the top;
553
- under "Keep this selection" click "Download the selection list (JSON)" and put `training-data-foundation-selection-6cfdc9b.json` in your fetch folder.
554
 
555
  **See**: the heading "Take every unit of each of 51 material sets".
556
  The list records release, filters and sources:
@@ -559,7 +559,7 @@ The list records release, filters and sources:
559
  {
560
  "format": "training-data-foundation-selection/1",
561
  "dataset": "Battam/training-data-foundation",
562
- "revision": "release-2026-11-14.26",
563
  "filters": {
564
  "collection": "A",
565
  "train": "yes",
@@ -583,9 +583,9 @@ for 200 in all, use `--limit`.
583
  **Do**: in the list's folder, run the command of "Take it again from the file" with `--limit 220` added (220 units in all, in a fixed order: a margin over 200, since some sites fail) and `--out ./data` replaced by `--out ./take`:
584
 
585
  ```sh
586
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
587
- --selection training-data-foundation-selection-6cfdc9b.json \
588
- --revision release-2026-11-14.26 \
589
  --ext xlsx \
590
  --ext xlsm \
591
  --ext xls \
@@ -614,7 +614,7 @@ ok austender-ict-procurement-statistics workbooks/2016-17-austender-ict-contract
614
  ...
615
  310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
616
  files of units: 220 ok; files of the sources themselves: 90 ok
617
- run_id 20261114T214335.805416Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
618
  ```
619
 
620
  Before the first download, `removed by` gives the units the default profile removed and the option that keeps them, counted in the data files read as far as `--limit` needed;
@@ -630,8 +630,8 @@ failed western-australia-government-workbooks workbooks/wa-gross-regional-produc
630
  incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
631
  5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
632
  files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
633
- run_id 20261114T214653.357883Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
634
- its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261114T214653.357883Z' AND json_extract_string(json, 'status') <> 'ok'
635
  ```
636
 
637
  **Get**: every `ok` file matched its recorded size and hash.
@@ -702,7 +702,7 @@ a workbook stores the cached results of its formulas, so a task that asks for co
702
  **See** (truncated):
703
 
704
  ```text
705
- Dataset commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367: https://huggingface.co/datasets/Battam/training-data-foundation/tree/6cfdc9bd5d379409cef7d267960b4ba6b8052367
706
  ...
707
  - Source: `austender-ict-procurement-statistics`
708
  - License: CC-BY-3.0-AU
@@ -735,7 +735,7 @@ Let DuckDB write the `--source` options, say for the disk image material sets wi
735
 
736
  ```sql
737
  SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
738
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
739
  WHERE kind = 'material_set' AND collection = 'B'
740
  AND license.tier = 'share-alike';
741
  ```
@@ -744,7 +744,7 @@ WHERE kind = 'material_set' AND collection = 'B'
744
  --source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
745
  ```
746
 
747
- Put them after `fetch.py --revision release-2026-11-14.26`.
748
  Read only the catalog, not all of `meta/`.
749
 
750
  </details>
@@ -758,8 +758,8 @@ Run "1 Take" (output truncated):
758
  !["Get the data" after picking three disk image material sets](S09-en.png)
759
 
760
  ```sh
761
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
762
- --revision release-2026-11-14.26 \
763
  --source dosfstools-fsck-cases \
764
  --source e2fsprogs-fsck-cases \
765
  --source exfatprogs-fsck-cases \
@@ -774,7 +774,7 @@ ok dosfstools-fsck-cases README.md: 3.32kB
774
  ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
775
  59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
776
  files of units: 52 ok; files of the sources themselves: 7 ok
777
- run_id 20261114T214659.680917Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
778
  ```
779
 
780
  An exFAT image is `exfat.img.tar.xz` upstream;
@@ -818,7 +818,7 @@ One, the catalog's `contamination.level` (values in [Benchmark overlap marks](#m
818
 
819
  ```sql
820
  SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
821
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
822
  WHERE kind = 'material_set'
823
  AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
824
  GROUP BY ALL;
@@ -838,8 +838,8 @@ GROUP BY ALL;
838
  Two, take the picked sources and read `removed by benchmark` and `removed by not compared`, printed before the first download (output truncated):
839
 
840
  ```sh
841
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
842
- --revision release-2026-11-14.26 \
843
  --source python-docx-acceptance-test-files \
844
  --source python-pptx-acceptance-test-files \
845
  --source docling-docx-groundtruth \
@@ -867,7 +867,7 @@ Three, match file by file against the overlap table;
867
  ```sql
868
  SELECT o.bench, o.hit_class, count(*) AS files
869
  FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
870
- JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/bench_overlap.parquet' AS o
871
  ON o.source = f.source AND o.uri = f.uri
872
  WHERE f.status = 'ok'
873
  GROUP BY ALL;
@@ -963,7 +963,7 @@ Each corpus is one whole upstream read in bulk and counted in rows; a material s
963
  - **Admission** makes the material sets.
964
  Its candidates are the works the search found upstream for the task family of a collection, and the units of a group of corpus rows chosen with `tools/fetch.py` (`--where` and `--sample`), which the fetch tool fetched and checked against their recorded size and sha256.
965
  It takes a candidate when its pointer facts hold: a download address, a recorded size and sha256, a recorded license, no legal block, no fact found wrong, and content not already in the index;
966
- every candidate it does not take is listed with its reason in `meta/not_included.parquet` ([docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.md)).
967
  - **What admission adds:** the candidate's files as one unit with its `unit_id`, the material set's README.md and LICENSE, its collection (the one of its task family), its form, its upstream, and labels such as language, robots and terms.
968
  The collect step then fetches every file of every unit again and checks its sha256 before the material set is published.
969
  - A material set made from corpus rows is a source of its own, apart from the corpus: the same file is a row of the corpus and a file of a unit, so the two layers' counts are never added;
@@ -980,7 +980,7 @@ How the recorded size and its check are measured, and which files have the same
980
 
981
  The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
982
 
983
- Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-14.26`.
984
 
985
  </details>
986
 
@@ -991,10 +991,10 @@ The stage table `catalog/stages.yaml` of the code repository gives each training
991
  The release records those fields for every unit of a material set and every row of a corpus, with how many of the rules every stage shares it passes, in this order: the benchmark rule (a row that matches a public benchmark task is left out), the rule on rows not compared with the benchmark list yet and deduplication (`meta/stage_fields.parquet`);
992
  each stage counts what meets its condition before those rules and after each of them, and its count after the last one is what it can take;
993
  the data page shows these steps under each stage's tile.
994
- What no stage takes is counted per layer by reason, in `no_stage` of [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json): what meets no stage's condition, what a rule removes, and what the catalog holds but the stage fields do not, since a release counts its stage fields over an earlier release's files (not counted yet);
995
  so post-training's units and material sets plus those in no stage are every material set's units and material sets, as the data page shows under its tiles.
996
  The training permit and the personal-data mark are labels of every row, neither a condition nor a rule of any stage.
997
- The dataset card shows each stage's count, how much of it is allowed for training, and what it keeps under each top-level domain (`by_domain` in [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json)).
998
  The tables of this release: [What each training stage can take](#manual/stage-tables).
999
 
1000
  </details>
@@ -1005,10 +1005,10 @@ The tables of this release: [What each training stage can take](#manual/stage-ta
1005
  - **Data collection:** rows are collected from the public listings, APIs and archives of each source and checked when added:
1006
  the address must answer, and size and hash are fetched again and compared.
1007
  - **Checks before release:** every source has its license recorded and grouped into a tier, personal information is read from sample files, and rows are compared with public benchmarks;
1008
- rows that copy a public benchmark are removed, and units that do not meet a requirement are listed with the reason: [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.md).
1009
  - **Release:** each release is one commit with its own tag;
1010
  the dataset card, its figures under `docs/figures/`, the pages under `docs/`, `cards/*.md`, `meta/catalog.parquet`, `meta/croissant.json`, `meta/size_figures.json` and `meta/stage_figures.json` are generated from the same commit, so their numbers agree, and this manual is filled from that commit when the data page is deployed.
1011
- Every number and figure of the card is read from [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json), and every other count and size of the dataset's pages and of this manual's tables from [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/size_figures.json), where each is computed once for the release.
1012
  - **Chinese version:** every name and description the Chinese card, its pages and the Chinese manual's tables show is translated by a language model once per text and kept, by the sha256 of its English text, in `meta/translations.parquet`.
1013
 
1014
  </details>
 
12
  <h2 id="manual/guides">Guides</h2>
13
 
14
  One goal each;
15
+ commands are pinned to `release-2026-11-14.27`.
16
 
17
  <h3 id="manual/take">Take the files a need calls for</h3>
18
 
 
23
  3. In that folder, run the command of "Take it again from the file" with [uv](https://docs.astral.sh/uv/):
24
 
25
  ```sh
26
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
27
+ --selection training-data-foundation-selection-6632e2e.json \
28
+ --revision release-2026-11-14.27 \
29
  --out ./data
30
  ```
31
 
 
41
  ```sh
42
  uvx --from duckdb-cli duckdb -c "
43
  SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
44
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
45
  WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
46
  ORDER BY collection, units DESC"
47
  ```
 
55
  to leave out the files whose license does not let them be shipped and the rows marked as carrying personal data, add two options:
56
 
57
  ```sh
58
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
59
+ --selection training-data-foundation-selection-6632e2e.json \
60
+ --revision release-2026-11-14.27 \
61
  --training-only \
62
  --no-personal-data \
63
  --out ./data
 
129
  2 when the index could not be read or a name is not in the catalog, such as a misspelt source:
130
 
131
  ```text
132
+ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 6632e2e186da270954e3e522f3e70554046e6070; closest: dosfstools-fsck-cases
133
  ```
134
 
135
  </details>
 
137
  <details id="manual/pin">
138
  <summary>Pin a release</summary>
139
 
140
+ The page's commands pin the script URL and `--revision` to `release-2026-11-14.27`;
141
  the script's first line names the commit:
142
 
143
  ```text
144
+ reading hf://datasets/Battam/training-data-foundation@6632e2e186da270954e3e522f3e70554046e6070 (commit 6632e2e186da270954e3e522f3e70554046e6070); pass --revision 6632e2e186da270954e3e522f3e70554046e6070 to select the same rows later
145
  ```
146
 
147
  Later, `--revision <commit>`, the same tag or the selection list take the same files.
 
214
 
215
  ```sql
216
  SELECT hit_class, scope, count(*) AS rows
217
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/bench_overlap.parquet'
218
  GROUP BY ALL ORDER BY hit_class, scope;
219
  ```
220
 
 
251
 
252
  | To | Read |
253
  |---|---|
254
+ | Read every field of a row and every column of the catalog | [docs/fields.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/fields.md) |
255
+ | See every material set and every corpus, by domain, with its counts, size, license tier and login | [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/sources.md) and the domain pages it lists |
256
+ | See what was not included and why, and which sources need a login | [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.md) |
257
+ | Build tasks from the files, and keep what an agent must not see out of its environment | [docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/for-agents.md) |
258
+ | See every release and what it changed | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/CHANGELOG.md) |
259
+ | Read the dataset as Croissant metadata | [Croissant generated by the Hub](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant); [`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/croissant.json) describes the catalog and every source |
260
  | Look through the catalog, one row per material set or corpus with its `kind` and `collection` | [The `catalog` subset in the Dataset Viewer](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
261
  | Query the catalog in the browser; it lists every source as published, without the default profile of `tools/fetch.py`, which the command applies | [SQL Console](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
262
  | Watch the index in use, in the page's language, with subtitles | [Usage video on the data page](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
263
+ | Check every number of the dataset card and of the tables here | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json) and [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/size_figures.json) |
264
 
265
  Each row of the data files under `data/` points to one file or record (Row, in the [glossary](#manual/glossary));
266
  the catalog's column `data_files` names the files of each source, and one material set or corpus is read from exactly those files, as in the queries of the dataset card and with `tools/fetch.py`.
 
310
  - This catalog itself (addresses, sizes, hashes and descriptive text) is released under CC0 1.0; that dedication does not extend to the files it points to.
311
  - The files under `media/` (the usage example video and its subtitles) and under `examples/` (the worked examples) are released under CC0 1.0 as well.
312
  - Each file keeps the license of its source: see the license columns of each row (a row without its own license takes the license of its batch, `batches[].license` in `meta/sources.json`).
313
+ - The tier of every source: on the page of its domain, which [docs/sources.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/sources.md) lists.
314
  - License tiers and reason codes are computed from the license categories of ScanCode LicenseDB, licensed CC-BY-4.0;
315
+ its attribution: [`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/NOTICE).
316
 
317
  <h4 id="manual/personal">Personal and sensitive information</h4>
318
 
 
358
 
359
  <h4 id="manual/releases">Releases</h4>
360
 
361
+ - Latest release: `release-2026-11-14.27` (its newest CHANGELOG entry is dated 2026-11-14).
362
+ - Releases are listed in [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/CHANGELOG.md);
363
+ each one is a commit with its own release tag (this one is [`release-2026-11-14.27`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-14.27)),
364
  so any earlier state stays readable at its tag.
365
  - `meta/projection.json` records the release tag and the version of the program and templates that generated the dataset card and its pages.
366
 
 
395
  @misc{training_data_foundation,
396
  title = {training-data-foundation: an index of public materials for building agent environments},
397
  howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
398
+ note = {Release release-2026-11-14.27},
399
  year = {2026}
400
  }
401
  ```
 
430
  - **Form:** the shape of a source's material, one id such as `spreadsheets` or `test-suites` (the catalog column `form`, the option `--form` of `tools/fetch.py`).
431
  - **Training stage:** what the material can feed, by the stage table `catalog/stages.yaml` of the code repository: pre-training and mid-training take corpus rows, post-training the units of material sets ([How the training-stage tables are counted](#manual/stage-counting)).
432
  - **Default profile:** the rules `tools/fetch.py` applies unless an option relaxes them: it leaves out duplicate copies, rows that matched a benchmark or were not compared with the benchmark list yet, rows in a language other than English and files above 20 MB;
433
+ rows of any license and rows marked as carrying personal data come unless `--training-only` or `--no-personal-data` leaves them out ([docs/for-agents.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/for-agents.md)).
434
 
435
  Every row carries a `unit_id`: in a material set, the key of its unit;
436
  in a corpus, the row's own key (its `uri`, then `@<range_offset>` for a byte range, then `!/<member_path>` for an archive member), and `tools/fetch.py` puts the file of a corpus row in a folder named after it.
 
440
  <details id="manual/numbers">
441
  <summary>Where this manual's numbers come from</summary>
442
 
443
+ The numbers and command outputs of this manual were filled in at deploy time, from the index commit `6632e2e186da270954e3e522f3e70554046e6070` (release tag `release-2026-11-14.27`).
444
+ This release holds 1,845 material sets with 4,244 units in 14 collections and 139 corpora with 1,189,440,203 rows, which point to 4.13 PB of deduplicated content, and of the 35,601 rows whose size was compared with their file, 99.4% matched.
445
 
446
  </details>
447
 
 
452
  its last two rows count how much of the stage's count is allowed for training (every file's training permit yes or conditional) and how much is marked as carrying personal data.
453
  How they are counted: [How the training-stage tables are counted](#manual/stage-counting).
454
 
455
+ The stage fields (`meta/stage_fields.parquet`) were counted over the data files of release `release-2026-11-14.27`, after it was published.
456
 
457
  ##### Post-training (in use)
458
 
 
473
 
474
  | Step | Rows | Size |
475
  |---|---:|---:|
476
+ | In the stage's condition, before the rules | 34,623,576 | 4.01 TB |
477
+ | After the benchmark rule | 34,614,294 | 4.01 TB |
478
+ | After the rule on rows not compared with the benchmark list | 34,614,294 | 4.01 TB |
479
+ | After deduplication: the stage's count | 34,225,485 | 3.86 TB |
480
  | Of the stage's count, allowed for training (yes or conditional) | 9,937,384 | 1.42 TB |
481
+ | Of the stage's count, marked as carrying personal data | 6,104,373 | 341 GB |
482
 
483
  ##### Pre-training (provisional: conditions and counts only)
484
 
 
550
  <h4 id="manual/workbooks-5">5. Go to "Get the data" and keep the selection</h4>
551
 
552
  **Do**: click "Get the data" at the top;
553
+ under "Keep this selection" click "Download the selection list (JSON)" and put `training-data-foundation-selection-6632e2e.json` in your fetch folder.
554
 
555
  **See**: the heading "Take every unit of each of 51 material sets".
556
  The list records release, filters and sources:
 
559
  {
560
  "format": "training-data-foundation-selection/1",
561
  "dataset": "Battam/training-data-foundation",
562
+ "revision": "release-2026-11-14.27",
563
  "filters": {
564
  "collection": "A",
565
  "train": "yes",
 
583
  **Do**: in the list's folder, run the command of "Take it again from the file" with `--limit 220` added (220 units in all, in a fixed order: a margin over 200, since some sites fail) and `--out ./data` replaced by `--out ./take`:
584
 
585
  ```sh
586
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
587
+ --selection training-data-foundation-selection-6632e2e.json \
588
+ --revision release-2026-11-14.27 \
589
  --ext xlsx \
590
  --ext xlsm \
591
  --ext xls \
 
614
  ...
615
  310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
616
  files of units: 220 ok; files of the sources themselves: 90 ok
617
+ run_id 20261114T222416.974432Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
618
  ```
619
 
620
  Before the first download, `removed by` gives the units the default profile removed and the option that keeps them, counted in the data files read as far as `--limit` needed;
 
630
  incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
631
  5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
632
  files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
633
+ run_id 20261114T222740.773221Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
634
+ its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261114T222740.773221Z' AND json_extract_string(json, 'status') <> 'ok'
635
  ```
636
 
637
  **Get**: every `ok` file matched its recorded size and hash.
 
702
  **See** (truncated):
703
 
704
  ```text
705
+ Dataset commit 6632e2e186da270954e3e522f3e70554046e6070: https://huggingface.co/datasets/Battam/training-data-foundation/tree/6632e2e186da270954e3e522f3e70554046e6070
706
  ...
707
  - Source: `austender-ict-procurement-statistics`
708
  - License: CC-BY-3.0-AU
 
735
 
736
  ```sql
737
  SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
738
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
739
  WHERE kind = 'material_set' AND collection = 'B'
740
  AND license.tier = 'share-alike';
741
  ```
 
744
  --source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
745
  ```
746
 
747
+ Put them after `fetch.py --revision release-2026-11-14.27`.
748
  Read only the catalog, not all of `meta/`.
749
 
750
  </details>
 
758
  !["Get the data" after picking three disk image material sets](S09-en.png)
759
 
760
  ```sh
761
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
762
+ --revision release-2026-11-14.27 \
763
  --source dosfstools-fsck-cases \
764
  --source e2fsprogs-fsck-cases \
765
  --source exfatprogs-fsck-cases \
 
774
  ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
775
  59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
776
  files of units: 52 ok; files of the sources themselves: 7 ok
777
+ run_id 20261114T222748.411608Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
778
  ```
779
 
780
  An exFAT image is `exfat.img.tar.xz` upstream;
 
818
 
819
  ```sql
820
  SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
821
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
822
  WHERE kind = 'material_set'
823
  AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
824
  GROUP BY ALL;
 
838
  Two, take the picked sources and read `removed by benchmark` and `removed by not compared`, printed before the first download (output truncated):
839
 
840
  ```sh
841
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
842
+ --revision release-2026-11-14.27 \
843
  --source python-docx-acceptance-test-files \
844
  --source python-pptx-acceptance-test-files \
845
  --source docling-docx-groundtruth \
 
867
  ```sql
868
  SELECT o.bench, o.hit_class, count(*) AS files
869
  FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
870
+ JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/bench_overlap.parquet' AS o
871
  ON o.source = f.source AND o.uri = f.uri
872
  WHERE f.status = 'ok'
873
  GROUP BY ALL;
 
963
  - **Admission** makes the material sets.
964
  Its candidates are the works the search found upstream for the task family of a collection, and the units of a group of corpus rows chosen with `tools/fetch.py` (`--where` and `--sample`), which the fetch tool fetched and checked against their recorded size and sha256.
965
  It takes a candidate when its pointer facts hold: a download address, a recorded size and sha256, a recorded license, no legal block, no fact found wrong, and content not already in the index;
966
+ every candidate it does not take is listed with its reason in `meta/not_included.parquet` ([docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.md)).
967
  - **What admission adds:** the candidate's files as one unit with its `unit_id`, the material set's README.md and LICENSE, its collection (the one of its task family), its form, its upstream, and labels such as language, robots and terms.
968
  The collect step then fetches every file of every unit again and checks its sha256 before the material set is published.
969
  - A material set made from corpus rows is a source of its own, apart from the corpus: the same file is a row of the corpus and a file of a unit, so the two layers' counts are never added;
 
980
 
981
  The recorded size is the sum of `size_bytes` over the rows that have one (catalog column `bytes`). The check reads a sample of rows of every data file (`meta/sizes.parquet`), each day the files with rows of a source not read there yet and those of the sources read longest ago, about a seventh of all the readings, so each is read again within a week: 20 of its rows with a recorded size of each source whose rows it holds, and of its rows without one its share of its source's sample, sized for their mean size within 25% at 95% confidence by the spread of the sizes read so far (at most 400 rows per source per round, 20 for a source with none read yet); its share is the rows that match their recorded size out of the rows it compared (status `match`, `size_mismatch` or `hash_mismatch`), and the rows it could not compare, which never reached their file, are counted beside it by status (`unreachable`, `gone`, `needs_auth`, `no_length`, `error`; catalog column `size_check`). The estimate for a source's rows without a recorded size (catalog columns `unsized_rows` and `unsized_estimate`) is their number times the mean measured size of such rows read, with a 95% BCa bootstrap range (`scipy.stats.bootstrap`, its default resample count); a source with fewer than 10 such rows read has none. Above one source, the estimate is the sum of its sources' estimates and its range the sum of their ranges, which is wider than a joint interval. The figures of every level are in `meta/size_figures.json`.
982
 
983
+ Distinct content is the sum of `size_bytes` over the rows minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted after each release over the data files it published, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it. The deduplicated sizes here were counted over the data files of release `release-2026-11-14.27`.
984
 
985
  </details>
986
 
 
991
  The release records those fields for every unit of a material set and every row of a corpus, with how many of the rules every stage shares it passes, in this order: the benchmark rule (a row that matches a public benchmark task is left out), the rule on rows not compared with the benchmark list yet and deduplication (`meta/stage_fields.parquet`);
992
  each stage counts what meets its condition before those rules and after each of them, and its count after the last one is what it can take;
993
  the data page shows these steps under each stage's tile.
994
+ What no stage takes is counted per layer by reason, in `no_stage` of [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json): what meets no stage's condition, what a rule removes, and what the catalog holds but the stage fields do not, since a release counts its stage fields over an earlier release's files (not counted yet);
995
  so post-training's units and material sets plus those in no stage are every material set's units and material sets, as the data page shows under its tiles.
996
  The training permit and the personal-data mark are labels of every row, neither a condition nor a rule of any stage.
997
+ The dataset card shows each stage's count, how much of it is allowed for training, and what it keeps under each top-level domain (`by_domain` in [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json)).
998
  The tables of this release: [What each training stage can take](#manual/stage-tables).
999
 
1000
  </details>
 
1005
  - **Data collection:** rows are collected from the public listings, APIs and archives of each source and checked when added:
1006
  the address must answer, and size and hash are fetched again and compared.
1007
  - **Checks before release:** every source has its license recorded and grouped into a tier, personal information is read from sample files, and rows are compared with public benchmarks;
1008
+ rows that copy a public benchmark are removed, and units that do not meet a requirement are listed with the reason: [docs/access.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.md).
1009
  - **Release:** each release is one commit with its own tag;
1010
  the dataset card, its figures under `docs/figures/`, the pages under `docs/`, `cards/*.md`, `meta/catalog.parquet`, `meta/croissant.json`, `meta/size_figures.json` and `meta/stage_figures.json` are generated from the same commit, so their numbers agree, and this manual is filled from that commit when the data page is deployed.
1011
+ Every number and figure of the card is read from [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json), and every other count and size of the dataset's pages and of this manual's tables from [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/size_figures.json), where each is computed once for the release.
1012
  - **Chinese version:** every name and description the Chinese card, its pages and the Chinese manual's tables show is translated by a language model once per text and kept, by the sha256 of its English text, in `meta/translations.parquet`.
1013
 
1014
  </details>
manual/manual.zh.md CHANGED
@@ -12,7 +12,7 @@
12
 
13
  <h2 id="manual/guides">指南</h2>
14
 
15
- 每条一个目标,命令都钉在 `release-2026-11-14.26`。
16
 
17
  <h3 id="manual/take">按需求取回文件</h3>
18
 
@@ -23,9 +23,9 @@
23
  3. 在那个目录里用 [uv](https://docs.astral.sh/uv/) 跑「按文件再取一次」那条命令:
24
 
25
  ```sh
26
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
27
- --selection training-data-foundation-selection-6cfdc9b.json \
28
- --revision release-2026-11-14.26 \
29
  --out ./data
30
  ```
31
 
@@ -41,7 +41,7 @@
41
  ```sh
42
  uvx --from duckdb-cli duckdb -c "
43
  SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
44
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
45
  WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
46
  ORDER BY collection, units DESC"
47
  ```
@@ -55,9 +55,9 @@
55
  要去掉许可不允许随附分发的文件和标为带有个人数据的行,加两个选项:
56
 
57
  ```sh
58
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
59
- --selection training-data-foundation-selection-6cfdc9b.json \
60
- --revision release-2026-11-14.26 \
61
  --training-only \
62
  --no-personal-data \
63
  --out ./data
@@ -128,7 +128,7 @@
128
  读不了索引或名字不在目录表里为 2,例如来源名写错时:
129
 
130
  ```text
131
- error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367; closest: dosfstools-fsck-cases
132
  ```
133
 
134
  </details>
@@ -136,10 +136,10 @@ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit
136
  <details id="manual/pin">
137
  <summary>钉住发布版本取</summary>
138
 
139
- 页面给的命令把脚本网址和 `--revision` 都钉在 `release-2026-11-14.26`,脚本首行打印对应的提交:
140
 
141
  ```text
142
- reading hf://datasets/Battam/training-data-foundation@6cfdc9bd5d379409cef7d267960b4ba6b8052367 (commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367); pass --revision 6cfdc9bd5d379409cef7d267960b4ba6b8052367 to select the same rows later
143
  ```
144
 
145
  日后用 `--revision <提交号>`、同一个标签或选择清单,取到的就是同样的文件。
@@ -211,7 +211,7 @@ scope 为 `container` 时命中整个分组(只在分组是一个完整单位
211
 
212
  ```sql
213
  SELECT hit_class, scope, count(*) AS rows
214
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/bench_overlap.parquet'
215
  GROUP BY ALL ORDER BY hit_class, scope;
216
  ```
217
 
@@ -248,16 +248,16 @@ GROUP BY ALL ORDER BY hit_class, scope;
248
 
249
  | 想做的事 | 去读 |
250
  |---|---|
251
- | 查看每个行字段和目录的每一列 | [docs/fields.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/fields.zh.md) |
252
- | 按领域查看每个材料集和每个语料库,以及它的计数、大小、许可证等级和登录要求 | [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/sources.zh.md) 及其列出的领域页面 |
253
- | 查看哪些没有收录、为什么,以及哪些来源需要登录 | [docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.zh.md) |
254
- | 用这些文件构建任务,并把智能体不该看到的内容挡在它的环境之外 | [docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/for-agents.zh.md) |
255
- | 查看每次发布及其改动 | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/CHANGELOG.md) |
256
- | 以 Croissant 元���据读取本数据集 | [由 Hub 生成的 Croissant](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant);[`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/croissant.json) 描述目录和每个来源 |
257
  | 浏览目录,每个材料集或语料库一行,带有它的 `kind` 和 `collection` | [数据集查看器中的 `catalog` 子集](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
258
  | 在浏览器里查询目录;它按发布原样列出每个来源,不应用 `tools/fetch.py` 的默认配置,命令才应用它 | [SQL 控制台](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
259
  | 观看本索引的使用示例视频,用页面的语言,带字幕 | [数据页上的使用示例视频](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
260
- | 核对数据集卡片和这里各表的每个数字 | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json) 和 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/size_figures.json) |
261
 
262
  `data/` 下数据文件的每一行指向一个文件或一条记录(见[术语](#manual/glossary)中的「行」);
263
  目录的 `data_files` 列列出每个来源的文件,单个材料集或语料库就从这些文件读取,见数据集卡片中的查询和 `tools/fetch.py`。
@@ -308,9 +308,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
308
  这一声明不延及它所指向的文件。
309
  - `media/` 下的文件(使用示例视频及其字幕)和 `examples/` 下的文件(完整示例)同样以 CC0 1.0 发布。
310
  - 每个文件保留其来源的许可证:见每一行的许可证列(没有自己许可证的行沿用所在批次的许可证,即 `meta/sources.json` 中的 `batches[].license`)。
311
- - 每个来源的等级:在它所属领域的页面上,这些页面由 [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/sources.zh.md) 列出。
312
  - 许可证等级和原因码由 ScanCode LicenseDB 的许可证类别算出,它以 CC-BY-4.0 许可;
313
- 它的署名:[`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/NOTICE)。
314
 
315
  <h4 id="manual/personal">个人与敏感信息</h4>
316
 
@@ -356,9 +356,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
356
 
357
  <h4 id="manual/releases">发布</h4>
358
 
359
- - 最新发布:`release-2026-11-14.26`(其最新的 CHANGELOG 条目日期为 2026-11-14)。
360
- - 各次发布列在 [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/CHANGELOG.md) 中;
361
- 每次发布是一个带有自己发布标签的提交(本次为 [`release-2026-11-14.26`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-14.26)),
362
  因此任何较早的状态都可以在它的标签处读到。
363
  - `meta/projection.json` 记录发布标签,以及生成数据集卡片及其页面的程序和模板的版本。
364
 
@@ -401,7 +401,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
401
  @misc{training_data_foundation,
402
  title = {training-data-foundation: an index of public materials for building agent environments},
403
  howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
404
- note = {Release release-2026-11-14.26},
405
  year = {2026}
406
  }
407
  ```
@@ -436,7 +436,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
436
  - **形式:** 一个来源的材料的形态,一个标识,例如 `spreadsheets` 或 `test-suites`(目录列 `form`,`tools/fetch.py` 的选项 `--form`)。
437
  - **训练阶段:** 材料能用于哪种训练,由代码仓库的阶段表 `catalog/stages.yaml` 决定:预训练和中训练用语料库的行,后训练用材料集的单元(见[训练阶段表怎样计数](#manual/stage-counting))。
438
  - **默认配置:** `tools/fetch.py` 在没有选项放宽时应用的规则:去掉重复的副本、与基准测试相符或还没有与基准测试清单比对的行、记录为英语以外语言的行,以及大于 20 MB 的文件;
439
- 任何许可证的行和标为带有个人数据的行都会进来,除非 `--training-only` 或 `--no-personal-data` 把它们去掉([docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/for-agents.zh.md))。
440
 
441
  每一行都带有 `unit_id`:在材料集中,是它所属单元的键;
442
  在语料库中,是该行自己的键(它的 `uri`,字节范围再接 `@<range_offset>`,存档成员再接 `!/<member_path>`),`tools/fetch.py` 把语料库一行的文件放进以它命名的文件夹。
@@ -446,8 +446,8 @@ GROUP BY ALL ORDER BY hit_class, scope;
446
  <details id="manual/numbers">
447
  <summary>本手册的数字从哪来</summary>
448
 
449
- 本手册的数字和命令输出是部署时填入的,取自索引提交 `6cfdc9bd5d379409cef7d267960b4ba6b8052367`(发布标签 `release-2026-11-14.26`)。
450
- 本次发布有 1,845 个材料集,共 4,244 个单元,分在 14 个合集,以及 139 个语料库,共 1,189,435,783 行,指向去重后共 4.13 PB 的内容,与文件比对过大小的 35,601 行中有 99.4% 一致。
451
 
452
  </details>
453
 
@@ -458,7 +458,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
458
  每张表的最后两行统计该阶段的计数中有多少允许训练(每个文件的训练许可为 yes 或 conditional),有多少标为带有个人数据。
459
  怎样计数:[训练阶段表怎样计数](#manual/stage-counting)。
460
 
461
- 阶段字段(`meta/stage_fields.parquet`)是按发布 `release-2026-11-14.26` 的数据文件、在它发布之后算出的。
462
 
463
  ##### 后训练(在用)
464
 
@@ -479,12 +479,12 @@ GROUP BY ALL ORDER BY hit_class, scope;
479
 
480
  | 步骤 | 行数 | 大小 |
481
  |---|---:|---:|
482
- | 符合该阶段的条件,在各条规则之前 | 34,623,038 | 4.01 TB |
483
- | 基准测试规则之后 | 34,613,756 | 4.01 TB |
484
- | 未与基准测试清单比对的行的规则之后 | 34,613,756 | 4.01 TB |
485
- | 去重之后:该阶段的计数 | 34,224,947 | 3.86 TB |
486
  | 该阶段的计数中,允许训练的(yes 或 conditional) | 9,937,384 | 1.42 TB |
487
- | 该阶段的计数中,标为带有个人数据的 | 6,104,370 | 341 GB |
488
 
489
  ##### 预训练(暂定:只给条件和计数)
490
 
@@ -554,7 +554,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
554
 
555
  <h4 id="manual/workbooks-5">5. 到「获取数据」,保存选择清单</h4>
556
 
557
- **做**:点顶部「获取数据」,在「保存这次选择」里点「下载选择清单(JSON)」,把 `training-data-foundation-selection-6cfdc9b.json` 放进取件目录。
558
 
559
  **看到**:标题「从 51 个材料集中获取全部单元」。
560
  清单记着版本、筛选和来源:
@@ -563,7 +563,7 @@ GROUP BY ALL ORDER BY hit_class, scope;
563
  {
564
  "format": "training-data-foundation-selection/1",
565
  "dataset": "Battam/training-data-foundation",
566
- "revision": "release-2026-11-14.26",
567
  "filters": {
568
  "collection": "A",
569
  "train": "yes",
@@ -587,9 +587,9 @@ GROUP BY ALL ORDER BY hit_class, scope;
587
  **做**:在清单所在目录跑「按文件再取一次」那条命令,加上 `--limit 220`(一共 220 个单元,固定顺序;比 200 多取一些留余量,有的原站会失效),`--out ./data` 换成 `--out ./take`:
588
 
589
  ```sh
590
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
591
- --selection training-data-foundation-selection-6cfdc9b.json \
592
- --revision release-2026-11-14.26 \
593
  --ext xlsx \
594
  --ext xlsm \
595
  --ext xls \
@@ -618,7 +618,7 @@ ok austender-ict-procurement-statistics workbooks/2016-17-austender-ict-contract
618
  ...
619
  310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
620
  files of units: 220 ok; files of the sources themselves: 90 ok
621
- run_id 20261114T214335.805416Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
622
  ```
623
 
624
  第一次下载之前,`removed by` 是默认配置去掉的单元和保留它们的开关,数的是 `--limit` 需要读到的那些数据文件;
@@ -634,8 +634,8 @@ failed western-australia-government-workbooks workbooks/wa-gross-regional-produc
634
  incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
635
  5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
636
  files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
637
- run_id 20261114T214653.357883Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
638
- its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261114T214653.357883Z' AND json_extract_string(json, 'status') <> 'ok'
639
  ```
640
 
641
  **得到**:`ok` 的文件已按记录的大小和哈希核对。
@@ -704,7 +704,7 @@ GROUP BY ALL ORDER BY source;
704
  **看到**(截短):
705
 
706
  ```text
707
- Dataset commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367: https://huggingface.co/datasets/Battam/training-data-foundation/tree/6cfdc9bd5d379409cef7d267960b4ba6b8052367
708
  ...
709
  - Source: `austender-ict-procurement-statistics`
710
  - License: CC-BY-3.0-AU
@@ -737,7 +737,7 @@ Dataset commit 6cfdc9bd5d379409cef7d267960b4ba6b8052367: https://huggingface.co/
737
 
738
  ```sql
739
  SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
740
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
741
  WHERE kind = 'material_set' AND collection = 'B'
742
  AND license.tier = 'share-alike';
743
  ```
@@ -746,7 +746,7 @@ WHERE kind = 'material_set' AND collection = 'B'
746
  --source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
747
  ```
748
 
749
- 接在 `fetch.py --revision release-2026-11-14.26` 后面。
750
  只读目录表,别整个下载 `meta/`。
751
 
752
  </details>
@@ -760,8 +760,8 @@ WHERE kind = 'material_set' AND collection = 'B'
760
  ![挑选三个磁盘镜像材料集后的「获取数据」](S09-zh.png)
761
 
762
  ```sh
763
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
764
- --revision release-2026-11-14.26 \
765
  --source dosfstools-fsck-cases \
766
  --source e2fsprogs-fsck-cases \
767
  --source exfatprogs-fsck-cases \
@@ -776,7 +776,7 @@ ok dosfstools-fsck-cases README.md: 3.32kB
776
  ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
777
  59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
778
  files of units: 52 ok; files of the sources themselves: 7 ok
779
- run_id 20261114T214659.680917Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
780
  ```
781
 
782
  exFAT 镜像在原站是 `exfat.img.tar.xz`,脚本解出后再按单元自己的 SHA-256 核对,盘上只留镜像。
@@ -819,7 +819,7 @@ Pass 3: Checking directory connectivity
819
 
820
  ```sql
821
  SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
822
- FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/catalog.parquet'
823
  WHERE kind = 'material_set'
824
  AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
825
  GROUP BY ALL;
@@ -839,8 +839,8 @@ GROUP BY ALL;
839
  二,取回挑中的来源,看第一次下载之前打印的 `removed by benchmark` 和 `removed by not compared`(输出截短):
840
 
841
  ```sh
842
- uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.26/tools/fetch.py \
843
- --revision release-2026-11-14.26 \
844
  --source python-docx-acceptance-test-files \
845
  --source python-pptx-acceptance-test-files \
846
  --source docling-docx-groundtruth \
@@ -867,7 +867,7 @@ fetching: 48 units: 56 files (8 files of the sources themselves among them), 25.
867
  ```sql
868
  SELECT o.bench, o.hit_class, count(*) AS files
869
  FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
870
- JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.26/meta/bench_overlap.parquet' AS o
871
  ON o.source = f.source AND o.uri = f.uri
872
  WHERE f.status = 'ok'
873
  GROUP BY ALL;
@@ -959,7 +959,7 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
959
  - **收录**产生材料集。
960
  它的候选是检索为某个合集的任务类别在上游找到的作品,以及用 `tools/fetch.py`(`--where` 和 `--sample`)选出的一组语料库行中的单元,这些单元由下载工具取回,并按记录的大小和 sha256 校验过。
961
  候选的指针事实成立时才收录:有下载地址,记录了大小和 sha256,记录了许可证,没有法律封锁,没有发现事实有误,内容也不在索引中;
962
- 未收录的每个候选都连同原因列在 `meta/not_included.parquet` 中([docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.zh.md))。
963
  - **收录添加的内容:** 把候选的文件作为一个单元并给出它的 `unit_id`,材料集的 README.md 和 LICENSE,它的合集(它的任务类别所属的合集),它的形式,它的上游,以及语言(`language`)、`robots` 和使用条款(`terms`)等标签。
964
  之后采集步骤再次取回每个单元的每个文件,并在材料集发布之前校验它的 sha256。
965
  - 由语料库行做成的材料集是一个独立的来源,与那个语料库分开:同一个文件既是语料库的一行,又是一个单元的一个文件,所以两层的计数从不相加;
@@ -976,7 +976,7 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
976
 
977
  记录的大小是有大小的各行 `size_bytes` 之和(目录列 `bytes`)。核对读取每个数据文件中抽取的一组行(`meta/sizes.parquet`),每天读取有某个来源的行尚未在其中读过的文件��以及最久没读的那些来源的文件,约占全部读数的七分之一,所以每个来源一周之内都会再读一次:对其中有行的每个来源,有记录大小的行各抽 20 行;没有记录大小的行按其所在来源的样本分摊,样本量按目前读到的大小的离散程度定,使其平均大小在 95% 置信度下误差不超过 25%(每个来源每轮最多 400 行,还没有读数的来源取 20 行);其比例是比对过的行(状态 `match`、`size_mismatch` 或 `hash_mismatch`)中与记录的大小一致的行所占的比例,没能比对的行(没有取到文件)按状态另行计数(`unreachable`、`gone`、`needs_auth`、`no_length`、`error`;目录列 `size_check`)。一个来源没有记录大小的行的估计(目录列 `unsized_rows` 和 `unsized_estimate`)是这些行的行数乘以读过的这类行的平均实测大小,并给出 95% BCa 自助法范围(`scipy.stats.bootstrap`,用其默认重抽样次数);读过的这类行少于 10 行的来源没有估计。在来源之上的层级,估计是其各来源估计之和,范围是各来源范围之和,比联合区间更宽。每个层级的数字都在 `meta/size_figures.json` 中。
978
 
979
- 去重后内容是各行 `size_bytes` 之和,减去与另一行内容重复的那些行的大小(sha256 相同;没有 sha256 时 sha1 相同;再没有时 md5 和大小都相同;并且哈希范围相同;每种内容只保留一份,留在按注册顺序排在最前的来源中),在每次发布之后按它发布的数据文件重新计算,每个来源的值在目录列 `distinct_bytes` 中;同一来源也整个列出的存档,其成员的哈希与存档不同,所以不扣除。`tools/fetch.py` 读取你自己的选择时会去掉其中的重复。 这里的去重后大小是按发布 `release-2026-11-14.26` 的数据文件算出的。
980
 
981
  </details>
982
 
@@ -987,10 +987,10 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
987
  发布为材料集的每个单元和语料库的每一行记录这些字段,并记录它按以下顺序通过了各阶段共用规则中的几条:基准测试规则(与公开基准测试任务相符的行被去掉)、还没有与基准测试清单比对的行的规则和去重(`meta/stage_fields.parquet`);
988
  每个阶段在这些规则之前和每条规则之后统计符合它的条件的材料,最后一条规则之后的计数就是它能用的材料;
989
  数据页在每个阶段那一块下面显示这几步。
990
- 没有任何阶段取用的部分按层、按原因统计,在 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json) 的 `no_stage` 中:不满足任何阶段条件的、被某条规则去掉的,以及目录里有、阶段字段里还没有的(尚未计数:发布的阶段字段是在更早一次发布的文件上计数的);
991
  所以后训练的单元数和材料集数加上未归入任何阶段的,就是全部材料集的单元数和材料集数,数据页在各阶段下面就是这样显示的。
992
  训练许可和个人数据标记是每一行的标签,既不是任何阶段的条件,也不是规则。
993
- 数据集卡片显示每个阶段的计数、其中有多少允许训练,以及它在每个顶层领域下保留的部分([`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json) 中的 `by_domain`)。
994
  这次发布的各张表:[每个训练阶段能用多少](#manual/stage-tables)。
995
 
996
  </details>
@@ -1001,10 +1001,10 @@ GDPval 在比对清单 `meta/benchmarks.parquet` 里(220 道题),这一版
1001
  - **数据采集:** 行从每个来源的公开列表、API 和存档中采集,加入时即校验:
1002
  地址必须能访问,大小和哈希值会重新取回并比对。
1003
  - **发布前的检查:** 每个来源都记录了许可证并归入一个等级,个人信息从样本文件中判读,各行与公开基准测试比对;
1004
- 复制了公开基准测试的行会被移除,不满足某项要求的单元会连同原因一起列出:[docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/docs/access.zh.md)。
1005
  - **发布:** 每次发布是一个带有自己标签的提交;
1006
  数据集卡片、`docs/figures/` 下它的图、`docs/` 下的页面、`cards/*.md`、`meta/catalog.parquet`、`meta/croissant.json`、`meta/size_figures.json` 和 `meta/stage_figures.json` 由同一个提交生成,因此数字彼此一致,本手册则在部署数据页时从这个提交填入。
1007
- 卡片的每个数字和每张图都读自 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/stage_figures.json),数据集各页面和本手册各表的其他计数和大小读自 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.26/meta/size_figures.json),每个数在那里为这次发布只算一次。
1008
  - **中文版本:** 中文卡片、它的页面和中文手册各表显示的每个名称和描述,都由语言模型对每段文字翻译一次,按英文原文的 sha256 保存在 `meta/translations.parquet` 中。
1009
 
1010
  </details>
 
12
 
13
  <h2 id="manual/guides">指南</h2>
14
 
15
+ 每条一个目标,命令都钉在 `release-2026-11-14.27`。
16
 
17
  <h3 id="manual/take">按需求取回文件</h3>
18
 
 
23
  3. 在那个目录里用 [uv](https://docs.astral.sh/uv/) 跑「按文件再取一次」那条命令:
24
 
25
  ```sh
26
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
27
+ --selection training-data-foundation-selection-6632e2e.json \
28
+ --revision release-2026-11-14.27 \
29
  --out ./data
30
  ```
31
 
 
41
  ```sh
42
  uvx --from duckdb-cli duckdb -c "
43
  SELECT source, collection, title, units, files, top_types, license.tier AS license_tier, needs_auth
44
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
45
  WHERE kind = 'material_set' AND needs_auth = 'no' AND license.tier IN ('permissive', 'share-alike')
46
  ORDER BY collection, units DESC"
47
  ```
 
55
  要去掉许可不允许随附分发的文件和标为带有个人数据的行,加两个选项:
56
 
57
  ```sh
58
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
59
+ --selection training-data-foundation-selection-6632e2e.json \
60
+ --revision release-2026-11-14.27 \
61
  --training-only \
62
  --no-personal-data \
63
  --out ./data
 
128
  读不了索引或名字不在目录表里为 2,例如来源名写错时:
129
 
130
  ```text
131
+ error: --source dosfstools-fsk: no such source in meta/catalog.parquet at commit 6632e2e186da270954e3e522f3e70554046e6070; closest: dosfstools-fsck-cases
132
  ```
133
 
134
  </details>
 
136
  <details id="manual/pin">
137
  <summary>钉住发布版本取</summary>
138
 
139
+ 页面给的命令把脚本网址和 `--revision` 都钉在 `release-2026-11-14.27`,脚本首行打印对应的提交:
140
 
141
  ```text
142
+ reading hf://datasets/Battam/training-data-foundation@6632e2e186da270954e3e522f3e70554046e6070 (commit 6632e2e186da270954e3e522f3e70554046e6070); pass --revision 6632e2e186da270954e3e522f3e70554046e6070 to select the same rows later
143
  ```
144
 
145
  日后用 `--revision <提交号>`、同一个标签或选择清单,取到的就是同样的文件。
 
211
 
212
  ```sql
213
  SELECT hit_class, scope, count(*) AS rows
214
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/bench_overlap.parquet'
215
  GROUP BY ALL ORDER BY hit_class, scope;
216
  ```
217
 
 
248
 
249
  | 想做的事 | 去读 |
250
  |---|---|
251
+ | 查看每个行字段和目录的每一列 | [docs/fields.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/fields.zh.md) |
252
+ | 按领域查看每个材料集和每个语料库,以及它的计数、大小、许可证等级和登录要求 | [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/sources.zh.md) 及其列出的领域页面 |
253
+ | 查看哪些没有收录、为什么,以及哪些来源需要登录 | [docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.zh.md) |
254
+ | 用这些文件构建任务,并把智能体不该看到的内容挡在它的环境之外 | [docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/for-agents.zh.md) |
255
+ | 查看每次发布及其改动 | [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/CHANGELOG.md) |
256
+ | 以 Croissant 元数据读取本数据集 | [由 Hub 生成的 Croissant](https://huggingface.co/api/datasets/Battam/training-data-foundation/croissant);[`meta/croissant.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/croissant.json) 描述目录和每个来源 |
257
  | 浏览目录,每个材料集或语料库一行,带有它的 `kind` 和 `collection` | [数据集查看器中的 `catalog` 子集](https://huggingface.co/datasets/Battam/training-data-foundation/viewer/catalog) |
258
  | 在浏览器里查询目录;它按发布原样列出每个来源,不应用 `tools/fetch.py` 的默认配置,命令才应用它 | [SQL 控制台](https://huggingface.co/datasets/Battam/training-data-foundation?sql_console=true&sql=--+every+source+of+the+catalog+as+published%3A+the+default+profile+of+tools%2Ffetch.py+is+not+applied+here%0ASELECT+source%2C+kind%2C+collection%2C+title%2C+units%2C+files%0AFROM+train%0AORDER+BY+kind%2C+collection%2C+files+DESC%0ALIMIT+50) |
259
  | 观看本索引的使用示例视频,用页面的语言,带字幕 | [数据页上的使用示例视频](https://huggingface.co/spaces/Battam/training-data-foundation#usage-video) |
260
+ | 核对数据集卡片和这里各表的每个数字 | [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json) 和 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/size_figures.json) |
261
 
262
  `data/` 下数据文件的每一行指向一个文件或一条记录(见[术语](#manual/glossary)中的「行」);
263
  目录的 `data_files` 列列出每个来源的文件,单个材料集或语料库就从这些文件读取,见数据集卡片中的查询和 `tools/fetch.py`。
 
308
  这一声明不延及它所指向的文件。
309
  - `media/` 下的文件(使用示例视频及其字幕)和 `examples/` 下的文件(完整示例)同样以 CC0 1.0 发布。
310
  - 每个文件保留其来源的许可证:见每一行的许可证列(没有自己许可证的行沿用所在批次的许可证,即 `meta/sources.json` 中的 `batches[].license`)。
311
+ - 每个来源的等级:在它所属领域的页面上,这些页面由 [docs/sources.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/sources.zh.md) 列出。
312
  - 许可证等级和原因码由 ScanCode LicenseDB 的许可证类别算出,它以 CC-BY-4.0 许可;
313
+ 它的署名:[`NOTICE`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/NOTICE)。
314
 
315
  <h4 id="manual/personal">个人与敏感信息</h4>
316
 
 
356
 
357
  <h4 id="manual/releases">发布</h4>
358
 
359
+ - 最新发布:`release-2026-11-14.27`(其最新的 CHANGELOG 条目日期为 2026-11-14)。
360
+ - 各次发布列在 [CHANGELOG.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/CHANGELOG.md) 中;
361
+ 每次发布是一个带有自己发布标签的提交(本次为 [`release-2026-11-14.27`](https://huggingface.co/datasets/Battam/training-data-foundation/tree/release-2026-11-14.27)),
362
  因此任何较早的状态都可以在它的标签处读到。
363
  - `meta/projection.json` 记录发布标签,以及生成数据集卡片及其页面的程序和模板的版本。
364
 
 
401
  @misc{training_data_foundation,
402
  title = {training-data-foundation: an index of public materials for building agent environments},
403
  howpublished = {https://huggingface.co/datasets/Battam/training-data-foundation},
404
+ note = {Release release-2026-11-14.27},
405
  year = {2026}
406
  }
407
  ```
 
436
  - **形式:** 一个来源的材料的形态,一个标识,例如 `spreadsheets` 或 `test-suites`(目录列 `form`,`tools/fetch.py` 的选项 `--form`)。
437
  - **训练阶段:** 材料能用于哪种训练,由代码仓库的阶段表 `catalog/stages.yaml` 决定:预训练和中训练用语料库的行,后训练用材料集的单元(见[训练阶段表怎样计数](#manual/stage-counting))。
438
  - **默认配置:** `tools/fetch.py` 在没有选项放宽时应用的规则:去掉重复的副本、与基准测试相符或还没有与基准测试清单比对的行、记录为英语以外语言的行,以及大于 20 MB 的文件;
439
+ 任何许可证的行和标为带有个人数据的行都会进来,除非 `--training-only` 或 `--no-personal-data` 把它们去掉([docs/for-agents.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/for-agents.zh.md))。
440
 
441
  每一行都带有 `unit_id`:在材料集中,是它所属单元的键;
442
  在语料库中,是该行自己的键(它的 `uri`,字节范围再接 `@<range_offset>`,存档成员再接 `!/<member_path>`),`tools/fetch.py` 把语料库一行的文件放进以它命名的文件夹。
 
446
  <details id="manual/numbers">
447
  <summary>本手册的数字从哪来</summary>
448
 
449
+ 本手册的数字和命令输出是部署时填入的,取自索引提交 `6632e2e186da270954e3e522f3e70554046e6070`(发布标签 `release-2026-11-14.27`)。
450
+ 本次发布有 1,845 个材料集,共 4,244 个单元,分在 14 个合集,以及 139 个语料库,共 1,189,440,203 行,指向去重后共 4.13 PB 的内容,与文件比对过大小的 35,601 行中有 99.4% 一致。
451
 
452
  </details>
453
 
 
458
  每张表的最后两行统计该阶段的计数中有多少允许训练(每个文件的训练许可为 yes 或 conditional),有多少标为带有个人数据。
459
  怎样计数:[训练阶段表怎样计数](#manual/stage-counting)。
460
 
461
+ 阶段字段(`meta/stage_fields.parquet`)是按发布 `release-2026-11-14.27` 的数据文件、在它发布之后算出的。
462
 
463
  ##### 后训练(在用)
464
 
 
479
 
480
  | 步骤 | 行数 | 大小 |
481
  |---|---:|---:|
482
+ | 符合该阶段的条件,在各条规则之前 | 34,623,576 | 4.01 TB |
483
+ | 基准测试规则之后 | 34,614,294 | 4.01 TB |
484
+ | 未与基准测试清单比对的行的规则之后 | 34,614,294 | 4.01 TB |
485
+ | 去重之后:该阶段的计数 | 34,225,485 | 3.86 TB |
486
  | 该阶段的计数中,允许训练的(yes 或 conditional) | 9,937,384 | 1.42 TB |
487
+ | 该阶段的计数中,标为带有个人数据的 | 6,104,373 | 341 GB |
488
 
489
  ##### 预训练(暂定:只给条件和计数)
490
 
 
554
 
555
  <h4 id="manual/workbooks-5">5. 到「获取数据」,保存选择清单</h4>
556
 
557
+ **做**:点顶部「获取数据」,在「保存这次选择」里点「下载选择清单(JSON)」,把 `training-data-foundation-selection-6632e2e.json` 放进取件目录。
558
 
559
  **看到**:标题「从 51 个材料集中获取全部单元」。
560
  清单记着版本、筛选和来源:
 
563
  {
564
  "format": "training-data-foundation-selection/1",
565
  "dataset": "Battam/training-data-foundation",
566
+ "revision": "release-2026-11-14.27",
567
  "filters": {
568
  "collection": "A",
569
  "train": "yes",
 
587
  **做**:在清单所在目录跑「按文件再取一次」那条命令,加上 `--limit 220`(一共 220 个单元,固定顺序;比 200 多取一些留余量,有的原站会失效),`--out ./data` 换成 `--out ./take`:
588
 
589
  ```sh
590
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
591
+ --selection training-data-foundation-selection-6632e2e.json \
592
+ --revision release-2026-11-14.27 \
593
  --ext xlsx \
594
  --ext xlsm \
595
  --ext xls \
 
618
  ...
619
  310 files fetched and checked (110MB) into take; details in take/_meta/fetch_run.json
620
  files of units: 220 ok; files of the sources themselves: 90 ok
621
+ run_id 20261114T222416.974432Z: 310 ok (lines in take/_meta/fetch_manifest.jsonl)
622
  ```
623
 
624
  第一次下载之前,`removed by` 是默认配置去掉的单元和保留它们的开关,数的是 `--limit` 需要读到的那些数据文件;
 
634
  incomplete: 1 unit miss a file (the files fetched are kept; _meta/fetch_manifest.jsonl says why): ...
635
  5 files fetched and checked (1.35MB) into probe; details in probe/_meta/fetch_run.json
636
  files of units: 1 failed, 1 ok; files of the sources themselves: 4 ok
637
+ run_id 20261114T222740.773221Z: 1 failed, 5 ok (lines in probe/_meta/fetch_manifest.jsonl)
638
+ its lines that are not ok, in DuckDB: SELECT json FROM read_json_objects('probe/_meta/fetch_manifest.jsonl') WHERE json_extract_string(json, 'run_id') = '20261114T222740.773221Z' AND json_extract_string(json, 'status') <> 'ok'
639
  ```
640
 
641
  **得到**:`ok` 的文件已按记录的大小和哈希核对。
 
704
  **看到**(截短):
705
 
706
  ```text
707
+ Dataset commit 6632e2e186da270954e3e522f3e70554046e6070: https://huggingface.co/datasets/Battam/training-data-foundation/tree/6632e2e186da270954e3e522f3e70554046e6070
708
  ...
709
  - Source: `austender-ict-procurement-statistics`
710
  - License: CC-BY-3.0-AU
 
737
 
738
  ```sql
739
  SELECT string_agg('--source ' || source, ' ' ORDER BY source) AS options
740
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
741
  WHERE kind = 'material_set' AND collection = 'B'
742
  AND license.tier = 'share-alike';
743
  ```
 
746
  --source adflib-adf-deleted --source adflib-adf-malformed --source btrfs-progs-convert-needs-recovery --source btrfs-progs-fsck-raw-images --source btrfs-progs-fuzz-images --source dfir-ntfs-fat-deleted --source dftt-deleted-files --source dftt-partition-tables --source dissect-hypervisor-disk-fixtures --source dosfstools-fsck-cases --source e2fsprogs-fsck-cases --source exfatprogs-fsck-cases --source libcdio-iso-malformed --source nist-cfreds-dfr-exfat --source ntfs-corrupted-images --source oe-ext4-hostile-fixtures --source qemu-iotests-sample-images
747
  ```
748
 
749
+ 接在 `fetch.py --revision release-2026-11-14.27` 后面。
750
  只读目录表,别整个下载 `meta/`。
751
 
752
  </details>
 
760
  ![挑选三个磁盘镜像材料集后的「获取数据」](S09-zh.png)
761
 
762
  ```sh
763
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
764
+ --revision release-2026-11-14.27 \
765
  --source dosfstools-fsck-cases \
766
  --source e2fsprogs-fsck-cases \
767
  --source exfatprogs-fsck-cases \
 
776
  ok exfatprogs-fsck-cases cases/bad_bitmap/exfat.img: 5.24MB
777
  59 files fetched and checked (110MB) into data; details in data/_meta/fetch_run.json
778
  files of units: 52 ok; files of the sources themselves: 7 ok
779
+ run_id 20261114T222748.411608Z: 59 ok (lines in data/_meta/fetch_manifest.jsonl)
780
  ```
781
 
782
  exFAT 镜像在原站是 `exfat.img.tar.xz`,脚本解出后再按单元自己的 SHA-256 核对,盘上只留镜像。
 
819
 
820
  ```sql
821
  SELECT contamination.level AS level, count(*) AS material_sets, sum(units) AS units
822
+ FROM 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/catalog.parquet'
823
  WHERE kind = 'material_set'
824
  AND list_has_any(top_types, ['docx', 'xlsx', 'pptx', 'pdf'])
825
  GROUP BY ALL;
 
839
  二,取回挑中的来源,看第一次下载之前打印的 `removed by benchmark` 和 `removed by not compared`(输出截短):
840
 
841
  ```sh
842
+ uv run https://huggingface.co/datasets/Battam/training-data-foundation/resolve/release-2026-11-14.27/tools/fetch.py \
843
+ --revision release-2026-11-14.27 \
844
  --source python-docx-acceptance-test-files \
845
  --source python-pptx-acceptance-test-files \
846
  --source docling-docx-groundtruth \
 
867
  ```sql
868
  SELECT o.bench, o.hit_class, count(*) AS files
869
  FROM read_json_auto('office3/_meta/fetch_manifest.jsonl') AS f
870
+ JOIN 'hf://datasets/Battam/training-data-foundation@release-2026-11-14.27/meta/bench_overlap.parquet' AS o
871
  ON o.source = f.source AND o.uri = f.uri
872
  WHERE f.status = 'ok'
873
  GROUP BY ALL;
 
959
  - **收录**产生材料集。
960
  它的候选是检索为某个合集的任务类别在上游找到的作品,以及用 `tools/fetch.py`(`--where` 和 `--sample`)选出的一组语料库行中的单元,这些单元由下载工具取回,并按记录的大小和 sha256 校验过。
961
  候选的指针事实成立时才收录:有下载地址,记录了大小和 sha256,记录了许可证,没有法律封锁,没有发现事实有误,内容也不在索引中;
962
+ 未收录的每个候选都连同原因列在 `meta/not_included.parquet` 中([docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.zh.md))。
963
  - **收录添加的内容:** 把候选的文件作为一个单元并给出它的 `unit_id`,材料集的 README.md 和 LICENSE,它的合集(它的任务类别所属的合集),它的形式,它的上游,以及语言(`language`)、`robots` 和使用条款(`terms`)等标签。
964
  之后采集步骤再次取回每个单元的每个文件,并在材料集发布之前校验它的 sha256。
965
  - 由语料库行做成的材料集是一个独立的来源,与那个语料库分开:同一个文件既是语料库的一行,又是一个单元的一个文件,所以两层的计数从不相加;
 
976
 
977
  记录的大小是有大小的各行 `size_bytes` 之和(目录列 `bytes`)。核对读取每个数据文件中抽取的一组行(`meta/sizes.parquet`),每天读取有某个来源的行尚未在其中读过的文件��以及最久没读的那些来源的文件,约占全部读数的七分之一,所以每个来源一周之内都会再读一次:对其中有行的每个来源,有记录大小的行各抽 20 行;没有记录大小的行按其所在来源的样本分摊,样本量按目前读到的大小的离散程度定,使其平均大小在 95% 置信度下误差不超过 25%(每个来源每轮最多 400 行,还没有读数的来源取 20 行);其比例是比对过的行(状态 `match`、`size_mismatch` 或 `hash_mismatch`)中与记录的大小一致的行所占的比例,没能比对的行(没有取到文件)按状态另行计数(`unreachable`、`gone`、`needs_auth`、`no_length`、`error`;目录列 `size_check`)。一个来源没有记录大小的行的估计(目录列 `unsized_rows` 和 `unsized_estimate`)是这些行的行数乘以读过的这类行的平均实测大小,并给出 95% BCa 自助法范围(`scipy.stats.bootstrap`,用其默认重抽样次数);读过的这类行少于 10 行的来源没有估计。在来源之上的层级,估计是其各来源估计之和,范围是各来源范围之和,比联合区间更宽。每个层级的数字都在 `meta/size_figures.json` 中。
978
 
979
+ 去重后内容是各行 `size_bytes` 之和,减去与另一行内容重复的那些行的大小(sha256 相同;没有 sha256 时 sha1 相同;再没有时 md5 和大小都相同;并且哈希范围相同;每种内容只保留一份,留在按注册顺序排在最前的来源中),在每次发布之后按它发布的数据文件重新计算,每个来源的值在目录列 `distinct_bytes` 中;同一来源也整个列出的存档,其成员的哈希与存档不同,所以不扣除。`tools/fetch.py` 读取你自己的选择时会去掉其中的重复。 这里的去重后大小是按发布 `release-2026-11-14.27` 的数据文件算出的。
980
 
981
  </details>
982
 
 
987
  发布为材料集的每个单元和语料库的每一行记录这些字段,并记录它按以下顺序通过了各阶段共用规则中的几条:基准测试规则(与公开基准测试任务相符的行被去掉)、还没有与基准测试清单比对的行的规则和去重(`meta/stage_fields.parquet`);
988
  每个阶段在这些规则之前和每条规则之后统计符合它的条件的材料,最后一条规则之后的计数就是它能用的材料;
989
  数据页在每个阶段那一块下面显示这几步。
990
+ 没有任何阶段取用的部分按层、按原因统计,在 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json) 的 `no_stage` 中:不满足任何阶段条件的、被某条规则去掉的,以及目录里有、阶段字段里还没有的(尚未计数:发布的阶段字段是在更早一次发布的文件上计数的);
991
  所以后训练的单元数和材料集数加上未归入任何阶段的,就是全部材料集的单元数和材料集数,数据页在各阶段下面就是这样显示的。
992
  训练许可和个人数据标记是每一行的标签,既不是任何阶段的条件,也不是规则。
993
+ 数据集卡片显示每个阶段的计数、其中有多少允许训练,以及它在每个顶层领域下保留的部分([`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json) 中的 `by_domain`)。
994
  这次发布的各张表:[每个训练阶段能用多少](#manual/stage-tables)。
995
 
996
  </details>
 
1001
  - **数据采集:** 行从每个来源的公开列表、API 和存档中采集,加入时即校验:
1002
  地址必须能访问,大小和哈希值会重新取回并比对。
1003
  - **发布前的检查:** 每个来源都记录了许可证并归入一个等级,个人信息从样本文件中判读,各行与公开基准测试比对;
1004
+ 复制了公开基准测试的行会被移除,不满足某项要求的单元会连同原因一起列出:[docs/access.zh.md](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/docs/access.zh.md)。
1005
  - **发布:** 每次发布是一个带有自己标签的提交;
1006
  数据集卡片、`docs/figures/` 下它的图、`docs/` 下的页面、`cards/*.md`、`meta/catalog.parquet`、`meta/croissant.json`、`meta/size_figures.json` 和 `meta/stage_figures.json` 由同一个提交生成,因此数字彼此一致,本手册则在部署数据页时从这个提交填入。
1007
+ 卡片的每个数字和每张图都读自 [`meta/stage_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/stage_figures.json),数据集各页面和本手册各表的其他计数和大小读自 [`meta/size_figures.json`](https://huggingface.co/datasets/Battam/training-data-foundation/blob/release-2026-11-14.27/meta/size_figures.json),每个数在那里为这次发布只算一次。
1008
  - **中文版本:** 中文卡片、它的页面和中文手册各表显示的每个名称和描述,都由语言模型对每段文字翻译一次,按英文原文的 sha256 保存在 `meta/translations.parquet` 中。
1009
 
1010
  </details>
portal.json CHANGED
@@ -3,6 +3,6 @@
3
  "digest": "2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221",
4
  "files": 29,
5
  "format": "portal-deploy/1",
6
- "funnel": "6cfdc9bd5d379409cef7d267960b4ba6b8052367",
7
- "index": "6cfdc9bd5d379409cef7d267960b4ba6b8052367"
8
  }
 
3
  "digest": "2819f681975cb2bf12b390816894f8ce4980103141fd1e54db9c6657ff057221",
4
  "files": 29,
5
  "format": "portal-deploy/1",
6
+ "funnel": "6632e2e186da270954e3e522f3e70554046e6070",
7
+ "index": "6632e2e186da270954e3e522f3e70554046e6070"
8
  }