Spaces:
Running
Running
data-line commited on
Commit ·
9db9f3d
1
Parent(s): 8aee0db
portal: catalog page of repository commit a8061e9d6cb3
Browse filesFiles of catalog/portal/ (9), digest 01249fb856c54a35a7f899fd95cca8c5829f6ea4f5c922b12406319554fb274f.
- README.md +5 -3
- app.js +6 -5
- data.js +1 -1
- first.json +10 -10
- llms.txt +3 -3
- portal.json +3 -3
README.md
CHANGED
|
@@ -35,12 +35,14 @@ task-ready unit (then labelled "Shippable, raw").
|
|
| 35 |
## Size
|
| 36 |
|
| 37 |
The page leads with distinct content: per source the catalog column `distinct_bytes` (its `bytes` minus the sizes of
|
| 38 |
-
its rows that
|
| 39 |
-
which the daily snapshot
|
|
|
|
| 40 |
of `size_bytes` over the rows, so content that several rows point to counts once per row) stands beside it, with the
|
| 41 |
rows that have no size (`files_without_size`), which are in neither number. The first screen and the note under the
|
| 42 |
source table use the words of the index README, section Size (`index_projection.size_text`); the source table has a
|
| 43 |
-
column for each.
|
|
|
|
| 44 |
are not subtracted. Without `distinct_bytes` (an index commit before it) the page shows referenced alone and says that
|
| 45 |
distinct content is not recorded yet.
|
| 46 |
|
|
|
|
| 35 |
## Size
|
| 36 |
|
| 37 |
The page leads with distinct content: per source the catalog column `distinct_bytes` (its `bytes` minus the sizes of
|
| 38 |
+
its rows that repeat another row's content, from `meta/coverage.json` `projections.duplicate_bytes`,
|
| 39 |
+
which the daily snapshot counts at every release over the hash, hash scope and size columns), in total their sum.
|
| 40 |
+
Referenced (`bytes`, the sum
|
| 41 |
of `size_bytes` over the rows, so content that several rows point to counts once per row) stands beside it, with the
|
| 42 |
rows that have no size (`files_without_size`), which are in neither number. The first screen and the note under the
|
| 43 |
source table use the words of the index README, section Size (`index_projection.size_text`); the source table has a
|
| 44 |
+
column for each.
|
| 45 |
+
Archive members whose archive the same source also lists whole have other hashes than the archive, so they
|
| 46 |
are not subtracted. Without `distinct_bytes` (an index commit before it) the page shows referenced alone and says that
|
| 47 |
distinct content is not recorded yet.
|
| 48 |
|
app.js
CHANGED
|
@@ -621,10 +621,11 @@ function sortValue(s, k) {
|
|
| 621 |
const sizeText = (s) => (s.bytes == null ? 'Unknown' : fmtBytes(s.bytes));
|
| 622 |
const distinctText = (s) => (s.distinct == null ? 'Unknown' : fmtBytes(s.distinct));
|
| 623 |
// #364: the same words as the index README, section Size (artpool/projection_templates/text.json, size_text)
|
| 624 |
-
const SIZE_METHOD =
|
| 625 |
-
+ '(the same
|
| 626 |
-
+ '
|
| 627 |
-
+ 'so they are not subtracted.'
|
|
|
|
| 628 |
/** Distinct, referenced and rows without a size of the sources S: the rule of index_projection.size_totals. */
|
| 629 |
function sizeTotals(S) {
|
| 630 |
const known = S.filter((s) => s.distinct != null);
|
|
@@ -635,7 +636,7 @@ function sizeNote(S) {
|
|
| 635 |
const t = sizeTotals(S);
|
| 636 |
const unsized = t.unsized ? ` ${fmtInt(t.unsized)} rows have no size and are ${t.distinct == null ? 'not in that number' : 'in neither number'}.` : '';
|
| 637 |
return t.distinct == null
|
| 638 |
-
? `Referenced: ${fmtBytes(t.referenced)}, the sum of size_bytes over the rows, so content that several rows point to counts once per row.${unsized} Distinct content (referenced minus the sizes of the rows
|
| 639 |
: `Distinct content: ${fmtBytes(t.distinct)}. Referenced: ${fmtBytes(t.referenced)}, the sum of size_bytes over the rows, so content that several rows point to counts once per row.${unsized} ${SIZE_METHOD}`;
|
| 640 |
}
|
| 641 |
|
|
|
|
| 621 |
const sizeText = (s) => (s.bytes == null ? 'Unknown' : fmtBytes(s.bytes));
|
| 622 |
const distinctText = (s) => (s.distinct == null ? 'Unknown' : fmtBytes(s.distinct));
|
| 623 |
// #364: the same words as the index README, section Size (artpool/projection_templates/text.json, size_text)
|
| 624 |
+
const SIZE_METHOD = "Distinct content is referenced minus the sizes of the rows that repeat another row's content "
|
| 625 |
+
+ '(the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the '
|
| 626 |
+
+ 'first source by registry order), counted at every release, per source in the catalog column distinct_bytes; archive '
|
| 627 |
+
+ 'members whose archive the same source also lists whole have other hashes, so they are not subtracted. tools/fetch.py '
|
| 628 |
+
+ 'removes duplicates over your own selection when it reads it.';
|
| 629 |
/** Distinct, referenced and rows without a size of the sources S: the rule of index_projection.size_totals. */
|
| 630 |
function sizeTotals(S) {
|
| 631 |
const known = S.filter((s) => s.distinct != null);
|
|
|
|
| 636 |
const t = sizeTotals(S);
|
| 637 |
const unsized = t.unsized ? ` ${fmtInt(t.unsized)} rows have no size and are ${t.distinct == null ? 'not in that number' : 'in neither number'}.` : '';
|
| 638 |
return t.distinct == null
|
| 639 |
+
? `Referenced: ${fmtBytes(t.referenced)}, the sum of size_bytes over the rows, so content that several rows point to counts once per row.${unsized} Distinct content (referenced minus the sizes of the rows that repeat another row's content) is not recorded for this release yet.`
|
| 640 |
: `Distinct content: ${fmtBytes(t.distinct)}. Referenced: ${fmtBytes(t.referenced)}, the sum of size_bytes over the rows, so content that several rows point to counts once per row.${unsized} ${SIZE_METHOD}`;
|
| 641 |
}
|
| 642 |
|
data.js
CHANGED
|
@@ -278,7 +278,7 @@ export async function loadIndex(params, onNote) {
|
|
| 278 |
pii: r.pii || {},
|
| 279 |
rows,
|
| 280 |
bytes: bytes && bytes > 0 ? bytes : null,
|
| 281 |
-
// distinct content (#364): bytes minus the sizes of the rows
|
| 282 |
distinct: has('distinct_bytes') ? num(r.distinct_bytes) : null,
|
| 283 |
rowsNoSize: num(r.files_without_size) || 0,
|
| 284 |
topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
|
|
|
|
| 278 |
pii: r.pii || {},
|
| 279 |
rows,
|
| 280 |
bytes: bytes && bytes > 0 ? bytes : null,
|
| 281 |
+
// distinct content (#364): bytes minus the sizes of the rows that repeat another row's content; null when not recorded
|
| 282 |
distinct: has('distinct_bytes') ? num(r.distinct_bytes) : null,
|
| 283 |
rowsNoSize: num(r.files_without_size) || 0,
|
| 284 |
topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
|
first.json
CHANGED
|
@@ -1,14 +1,14 @@
|
|
| 1 |
{
|
| 2 |
-
"answers":
|
| 3 |
-
"bytes":
|
| 4 |
-
"date": "2026-10-
|
| 5 |
-
"distinct_bytes":
|
| 6 |
"format": "portal-first/1",
|
| 7 |
-
"index": "
|
| 8 |
-
"raw_rows":
|
| 9 |
"raw_sources": 89,
|
| 10 |
-
"ready_sources":
|
| 11 |
-
"ready_units":
|
| 12 |
-
"sources":
|
| 13 |
-
"unsized_rows":
|
| 14 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"answers": 215,
|
| 3 |
+
"bytes": 4132040796019874,
|
| 4 |
+
"date": "2026-10-28",
|
| 5 |
+
"distinct_bytes": 3768358919653868,
|
| 6 |
"format": "portal-first/1",
|
| 7 |
+
"index": "43b0080e9bb57d44e40ff1d82d918ba3ae309975",
|
| 8 |
+
"raw_rows": 1106900884,
|
| 9 |
"raw_sources": 89,
|
| 10 |
+
"ready_sources": 373,
|
| 11 |
+
"ready_units": 1060,
|
| 12 |
+
"sources": 462,
|
| 13 |
+
"unsized_rows": 250161364
|
| 14 |
}
|
llms.txt
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
# Agentic Env Index: for agents
|
| 2 |
|
| 3 |
-
- Pick sources from `meta/catalog.parquet` (one row per source, about 50 KB; do not download `meta/` whole
|
| 4 |
-
- Size of this release: Referenced: 4.13 PB, the sum of `size_bytes` over the rows, so content that several rows point to counts once per row.
|
| 5 |
- Task-ready sources are collections whose rows carry the roles unit, companion and answer (`answers` above 0, or `units` below `files`): each unit comes with its expected results. Every other source is a raw corpus to mine, one unit per row.
|
| 6 |
- Fetch with one command: `uv run https://huggingface.co/datasets/Battam/agentic-env-index/resolve/main/tools/fetch.py --domain D3 --form spreadsheets-with-answers --sample 3 --out ./data`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--domain`, `--form`, `--tag`, `--ext`; `--sample N` takes N units per source, `--dry-run` only counts and writes `_meta/fetch_manifest.jsonl` with one line per file of the selection (status selected, removed with the criterion in removed_by, or not_taken), as many lines as the files the summary counts.
|
| 7 |
- It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
|
| 8 |
-
- The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, collections awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), files whose license does not allow shipping (license.ship yes and conditional stay), rows recorded in a language other than English, personal data (catalog pii.level present or suspected, rows marked sensitive), and files above 20 MB. Before downloading it prints how many units each criterion removed; `--all` turns the profile off, and each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-license`, `--any-language`, `--allow-personal-data`, `--any-size`; `--allow-benchmark` also keeps rows not compared yet).
|
| 9 |
- Output: `./data/<domain>/<source>/...` holds the units and their companion files (README, LICENSE); `./data/answer/<source>/...` holds the answers and expected results, apart from the material; `./data/_meta/manifest.jsonl` has one line per file (source, path, url, sha256, bytes, spdx, licence_url, publisher, retrieved, answer, origin_kind, leaf, role, unit_group, answer_for); `./data/_meta/sizes.tsv` one line per source. Every file is checked against the sha256 the index records. A unit stored inside a container (such as a disk image inside a zip) is written as the unit itself: fetch.py takes the member out and checks it against the unit's recorded sha256, so no second tool is needed; only units marked expand (archives meant to be used as folders) are unpacked by `tools/to_pool.py`.
|
| 10 |
- Never put `answer/` into the environment the agent works in: the answers are there to check its result, not to be read by it. Each kind of answer, and what else to keep out:
|
| 11 |
- Answer files (role `answer`; an answer in JSON names its kind in the field `kind` or `answer_kind`): the expected result of one unit, such as an output, a value or a verdict. Keep out of the environment also any copy of it, and any file that holds the same result.
|
|
|
|
| 1 |
# Agentic Env Index: for agents
|
| 2 |
|
| 3 |
+
- Pick sources from `meta/catalog.parquet` (one row per source, about 50 KB; do not download `meta/` whole). Columns: `source`; `domain`, a list of codes D1 to D10 or leaves such as D3.5, or `multi`, `unclassified`; `form`, such as `test-suites` or `spreadsheets-with-answers`; `units` (a unit is one test case or one workbook; a raw corpus has one unit per row), `answers` (units that have an answer, the expected results kept apart) and `files` (files of every role); `language` (the language codes of the rows' labels, what the default profile filters on; `not recorded` when a row has none); `contamination.level` (`not scanned` when no comparison is recorded for the source); `license.ship`; `pii.level`; `distinct_bytes` (distinct content) and `bytes` (referenced, the sum over rows). The vocabulary is `meta/vocabulary.json`: every domain code with its name, what it covers and its parent, and every form with what it means.
|
| 4 |
+
- Size of this release: Distinct content: 3.77 PB. Referenced: 4.13 PB, the sum of `size_bytes` over the rows, so content that several rows point to counts once per row. 250,161,364 rows have no size and are in neither number. Distinct content is referenced minus the sizes of the rows that repeat another row's content (the same sha256, else sha1, else md5 and size, within the same hash scope; one copy of each content is kept, in the first source by registry order), counted at every release, per source in the catalog column `distinct_bytes`; archive members whose archive the same source also lists whole have other hashes, so they are not subtracted. `tools/fetch.py` removes duplicates over your own selection when it reads it.
|
| 5 |
- Task-ready sources are collections whose rows carry the roles unit, companion and answer (`answers` above 0, or `units` below `files`): each unit comes with its expected results. Every other source is a raw corpus to mine, one unit per row.
|
| 6 |
- Fetch with one command: `uv run https://huggingface.co/datasets/Battam/agentic-env-index/resolve/main/tools/fetch.py --domain D3 --form spreadsheets-with-answers --sample 3 --out ./data`. Without uv: `pip install duckdb`, download `tools/fetch.py` and run it with `python`. Select with `--source`, `--domain`, `--form`, `--tag`, `--ext`; `--sample N` takes N units per source, `--dry-run` only counts and writes `_meta/fetch_manifest.jsonl` with one line per file of the selection (status selected, removed with the criterion in removed_by, or not_taken), as many lines as the files the summary counts.
|
| 7 |
- It pins the commit it reads and prints it; pass `--revision <commit>` to get the same files later. A name that does not exist stops with exit code 2 and the closest names.
|
| 8 |
+
- The default profile removes duplicate copies, rows that matched a benchmark (copies of benchmark data, collections awaiting a ruling), rows not compared with the benchmark list yet (counted apart, with their own reason), files whose license does not allow shipping (license.ship yes and conditional stay), rows recorded in a language other than English, personal data (catalog pii.level present or suspected, rows marked sensitive), and files above 20 MB. Before downloading it prints how many units each criterion removed; `--all` turns the profile off, and each criterion has its own option (`--keep-duplicates`, `--allow-benchmark`, `--allow-not-compared`, `--any-license`, `--any-language`, `--allow-personal-data`, `--any-size`; `--allow-benchmark` also keeps rows not compared yet). Compared with the benchmark list (keys of 2026-10-24): 462 of 462 sources, 373 of 373 task-ready sources. The profile also removes every row whose data file was not compared (`--allow-benchmark` keeps them, unchecked).
|
| 9 |
- Output: `./data/<domain>/<source>/...` holds the units and their companion files (README, LICENSE); `./data/answer/<source>/...` holds the answers and expected results, apart from the material; `./data/_meta/manifest.jsonl` has one line per file (source, path, url, sha256, bytes, spdx, licence_url, publisher, retrieved, answer, origin_kind, leaf, role, unit_group, answer_for); `./data/_meta/sizes.tsv` one line per source. Every file is checked against the sha256 the index records. A unit stored inside a container (such as a disk image inside a zip) is written as the unit itself: fetch.py takes the member out and checks it against the unit's recorded sha256, so no second tool is needed; only units marked expand (archives meant to be used as folders) are unpacked by `tools/to_pool.py`.
|
| 10 |
- Never put `answer/` into the environment the agent works in: the answers are there to check its result, not to be read by it. Each kind of answer, and what else to keep out:
|
| 11 |
- Answer files (role `answer`; an answer in JSON names its kind in the field `kind` or `answer_kind`): the expected result of one unit, such as an output, a value or a verdict. Keep out of the environment also any copy of it, and any file that holds the same result.
|
portal.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
{
|
| 2 |
-
"commit": "
|
| 3 |
-
"digest": "
|
| 4 |
"files": 9,
|
| 5 |
"format": "portal-deploy/1",
|
| 6 |
-
"index": "
|
| 7 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"commit": "a8061e9d6cb39c2500a43b32b3fe38816e75605f",
|
| 3 |
+
"digest": "01249fb856c54a35a7f899fd95cca8c5829f6ea4f5c922b12406319554fb274f",
|
| 4 |
"files": 9,
|
| 5 |
"format": "portal-deploy/1",
|
| 6 |
+
"index": "43b0080e9bb57d44e40ff1d82d918ba3ae309975"
|
| 7 |
}
|