Battam's picture
portal: data page of repository commit 0ed03f2d7e11
689ce1d verified
Raw History Blame Contribute Delete
24.3 kB
// Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees.
import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm';
import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm';
import { downloadFile, pathsInfo } from 'https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.10/+esm';
import { HF, loadRevision as askRevision, REPO } from './commit.js';
import { t } from './i18n.js';
export { HF, REPO };
export const DATASET_URL = `${HF}/datasets/${REPO}`;
export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`;
export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v));
// the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus
export const MATERIAL_SET = 'material_set';
export const CORPUS = 'corpus';
export const KINDS = [MATERIAL_SET, CORPUS];
// #2065, #2066: the one table of Chinese names the release writes: per English text the index shows (a collection's
// name, a source's title, description or summary, as the catalog holds them) its Chinese text (columns en and zh)
const TRANSLATIONS = 'meta/translations.parquet';
// #2066: the projection's counts per training stage (artpool.stages FIGURES), with (#2065) each stage's funnel
const STAGE_FIGURES = 'meta/stage_figures.json';
const HUB_REPO = { type: 'dataset', name: REPO };
/** One whole file of the commit sha, read with downloadFile of @huggingface/hub, which answers null when the commit
* has no such file. */
export async function wholeFile(sha, path) {
const blob = await downloadFile({ repo: HUB_REPO, path, revision: sha });
if (!blob) throw new Error(`${path}: no such file at commit ${sha.slice(0, 7)}`);
return blob;
}
/** A file named by its URL in the page address (catalog=, vocabulary=), which may lie outside the dataset. */
async function urlFile(url) {
const r = await fetch(url);
if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
return r.blob();
}
const jsonOf = async (blob) => JSON.parse(await (await blob).text());
/** A small file read whole (the meta tables): one request instead of a size probe and ranges. */
async function readParquet(blob) {
return parquetReadObjects({ file: await (await blob).arrayBuffer(), compressors, utf8: false });
}
/** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd} (#1792:
* hyparquet documents no row filter, so a reader of one source reads its row range, filesOf, and keeps its rows by
* the column source itself). */
export async function readColumns(url, cols, range = {}) {
const file = await asyncBufferFromUrl({ url });
const metadata = await parquetMetadataAsync(file);
const names = new Set(metadata.schema.slice(1).map((s) => s.name));
const columns = cols.filter((c) => names.has(c));
const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range });
return { rows, total: num(metadata.num_rows) };
}
/** The commit to read: ?rev=<sha or branch>, else main. Returns {sha, date, files}. */
export async function loadRevision(rev) {
const r = await askRevision(rev);
if (!r.sha) throw new Error(t('error.noCommit'));
return r;
}
/** #1769: the date of the last commit of each batch of a source whose data files are files ({path, batches},
* loadIndex's files): the latest last commit among the files that hold the batch's rows (pathsInfo), never a folder
* named by the batch. -> Map of batch -> ISO date. */
export async function loadBatchDates(sha, files) {
const out = new Map();
if (!files.length) return out;
const info = await pathsInfo({ repo: HUB_REPO, paths: files.map((f) => f.path), revision: sha, expand: true });
const iso = (d) => (d instanceof Date ? d.toISOString() : d ? String(d) : null);
const at = new Map(info.map((e) => [e.path, iso(e.lastCommit?.date)]));
for (const f of files) {
const d = at.get(f.path);
for (const b of f.batches) if (d && !(out.get(b) >= d)) out.set(b, d);
}
return out;
}
// ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or
// unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without
// them everything shows as unclassified.
const pick = (o, keys) => {
if (!o || typeof o !== 'object') return undefined;
for (const k of keys) if (o[k] != null) return o[k];
return undefined;
};
const NAME_KEYS = ['name', 'label', 'title', 'en'];
const CODE_KEYS = ['code', 'id', 'key', 'value'];
const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items'];
const ABOUT_KEYS = ['description', 'about', 'definition'];
const ZH_KEYS = ['zh'];
function vocabSection(node) {
const names = new Map();
const zh = new Map();
const about = new Map();
const order = [];
const add = (code, name, text, zhName) => {
if (code == null) return;
code = String(code);
if (!names.has(code)) order.push(code);
if (typeof name === 'string' && name.trim()) names.set(code, name.trim());
else if (!names.has(code)) names.set(code, null);
if (typeof text === 'string' && text.trim()) about.set(code, text.trim());
if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim());
};
const walk = (n) => {
if (!n) return;
if (Array.isArray(n)) {
for (const it of n) {
if (typeof it === 'string') add(it, null);
else if (it && typeof it === 'object') {
add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS));
walk(pick(it, KIDS_KEYS));
}
}
} else if (typeof n === 'object') {
for (const [k, v] of Object.entries(n)) {
if (typeof v === 'string') add(k, v);
else if (v && typeof v === 'object') {
add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS));
walk(pick(v, KIDS_KEYS));
} else add(k, null);
}
}
};
walk(node);
return { names, zh, about, order };
}
// meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows
// its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites).
// zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about,
// zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the
// lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags
// of its material sets). terms: Map of the term table by its English term. #2397: profile [{criterion, option}], the
// criteria tools/fetch.py's default profile applies, in its order, each with the option that takes back what it
// leaves out (what Get the data shows the profile leaves out).
export function parseVocabulary(j) {
const form = vocabSection(pick(j, ['form', 'forms']));
for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' '));
const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l]));
const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({
code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '',
campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [],
}));
// terms [{term, zh, about}] (the index's term table: the English term, its Chinese term and one sentence on what it
// means), by the English term
const terms = new Map((Array.isArray(j?.terms) ? j.terms : []).filter((x) => x?.term).map((x) => [String(x.term), x]));
const profile = (Array.isArray(j?.profile) ? j.profile : []).map((p) => ({ criterion: String(p.criterion), option: String(p.option) }));
return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections, terms, profile };
}
/** #2066: meta/stage_figures.json -> {rules, stages: Map id -> stage, noStage}: the rules every stage's material passes,
* in their order, and per stage, in the stage table's order, its names, status and measures (counted_by) and its counts
* by them (counts, each null before the release counted the stage fields), which (#2422) the first screen's figure says;
* #2629: noStage, what no stage counts of each layer by reason ({kind: {reason: {measure: value}}}, null for figures
* written before it); null for a commit without the file. */
function parseStageFigures(j) {
if (!j) return null;
return { rules: j.rules || [], stages: new Map((j.stages || []).map((s) => [s.id, s])), noStage: j.no_stage || null };
}
export const topOf = (code) => String(code).split('.')[0];
function domainOf(v) {
let list = v;
if (typeof list === 'string') list = [list];
if (list && typeof list.toArray === 'function') list = list.toArray();
list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : [];
if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] };
if (list.includes('multi')) return { kind: 'multi', codes: [] };
return { kind: 'codes', codes: [...new Set(list)] };
}
/** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or
* conditional for every row) or 'no' (some rows are no or unknown). */
function permission(lr) {
if (!lr) return 'no';
const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k);
if (!keys.length) return 'no';
if (keys.every((k) => k === 'yes')) return 'yes';
if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond';
return 'no';
}
/** #1266: a catalog cell of license_counts as [{kind, spdx, training_ok, redistribution_ok, files}], null when empty. */
function licenceCounts(v) {
const items = Array.isArray(v) ? v : v?.toArray?.();
return items?.length ? items.map((x) => ({ kind: x.kind, spdx: x.spdx || null, training_ok: x.training_ok,
redistribution_ok: x.redistribution_ok, files: num(x.files) || 0 })) : null;
}
/** #1524: a source's file types as [[ext, rows]], most rows first, from its by_ext of meta/sources.json ({ext: {rows,
* bytes}}); the keys in brackets ((none), (other)) name no extension that fetch.py's --ext or a row's ext matches, and
* are left out. */
function extsOf(byExt) {
return Object.entries(byExt || {}).filter(([k]) => !k.startsWith('(')).map(([k, v]) => [k, num(v?.rows) || 0])
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
}
/** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}};
* json: the file is JSON (a list of rows, or {sources} or {rows}), else Parquet. */
async function readCatalog(blob, json) {
if (json) {
const j = await jsonOf(blob);
const rows = Array.isArray(j) ? j : j.sources || j.rows || [];
return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } };
}
const buf = await (await blob).arrayBuffer();
// DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf
const bytes = new Uint8Array(buf.slice(0));
return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } };
}
/** #2065: a sum over values some of which are not known (null), as the figures sum them (index_projection._total):
* {sum: the sum of the known values, null when none is known (0 when there are no values), missing: how many are
* not counted yet}. */
export function summed(vals) {
const known = vals.filter((v) => v != null);
return { sum: known.length ? known.reduce((a, v) => a + v, 0) : vals.length ? null : 0, missing: vals.length - known.length };
}
const listOf = (v) => (Array.isArray(v) ? v : v?.toArray?.() || []);
/** #1769: the data files that hold rows of the catalog row r, each {path, batches, first, last} (the batches of its
* rows in that file; #1792: the source's first and last row in it, null when the catalog does not record them), sorted
* by path: from the catalog column batch_files, else (a catalog with data_files only) from data_files, with no batches;
* never from a file's path. A file may hold rows of other sources too, so every read of one keeps the source's rows by
* the column source, and reads only its row range when known (a packed file holds a source's rows together). */
function filesOf(r) {
const by = new Map();
for (const x of listOf(r.batch_files)) {
const f = String(x.file);
if (!by.has(f)) by.set(f, { batches: [], first: num(x.first), last: num(x.last) });
by.get(f).batches.push(String(x.batch));
}
if (!by.size) for (const f of listOf(r.data_files)) by.set(String(f), { batches: [], first: null, last: null });
return [...by].map(([path, x]) => ({ path, ...x, batches: x.batches.sort() })).sort((a, b) => a.path.localeCompare(b.path));
}
/** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values):
* a note for the reader, as a key of the locale files. */
export async function loadIndex(params, onNote) {
const rev = await loadRevision(params.get('rev'));
const at = (p) => wholeFile(rev.sha, p);
const catalogUrl = params.get('catalog');
const has = (p) => rev.files.includes(p);
const vocabUrl = params.get('vocabulary');
const vocabFile = vocabUrl ? () => urlFile(vocabUrl) : has('meta/vocabulary.json') ? () => at('meta/vocabulary.json') : null;
const [cat, sourcesJson, coverage, batches, vocabJson, figures, record, translations, stageFigures] =
await Promise.all([
catalogUrl ? readCatalog(urlFile(catalogUrl), /\.json(\?|$)/i.test(catalogUrl)) : readCatalog(at('meta/catalog.parquet'), false),
jsonOf(at('meta/sources.json')),
jsonOf(at('meta/coverage.json')),
readParquet(at('meta/batches.parquet')).catch(() => []),
vocabFile ? jsonOf(vocabFile()).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null,
// #1158: the size figures of every level, written by the projection; a commit before it has none
has('meta/size_figures.json') ? jsonOf(at('meta/size_figures.json')) : null,
// #1197: the record of the release's projection, whose tag (DL13h, artpool/release_tag.py) the commands name and
// (#2422) whose tag and date the release line of the first screen says
has('meta/projection.json') ? jsonOf(at('meta/projection.json')) : null,
// #2066: the Chinese names of collections and sources (#2065); #2422: what each training stage can use, the
// first screen's figure; #2518: and, in the same file, the stages of each source
has(TRANSLATIONS) ? readParquet(at(TRANSLATIONS)) : [],
has(STAGE_FIGURES) ? jsonOf(at(STAGE_FIGURES)) : null,
]);
// #2518: a source's stages come from the commit the page reads: meta/stage_figures.json carries, per source, the ids
// of the stages one of its units passes (sources), beside the stage table's version and each stage's counts, so the
// two never describe different commits. A commit without the file, one whose release has not counted its stage
// fields, or one written before the figures listed the sources has no stage for any source, and the filter by stage
// says which in one short line (stagesWhy)
const stagesOk = stageFigures?.counted && stageFigures.sources ? stageFigures : null;
const stagesWhy = stagesOk ? null : !stageFigures ? 'noFigures' : !stageFigures.counted ? 'notCounted' : 'noSources';
const vocab = parseVocabulary(vocabJson || {});
const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e]));
const cov = coverage.by_source || {};
const batchesBy = new Map();
for (const b of batches) {
if (!batchesBy.has(b.source)) batchesBy.set(b.source, []);
batchesBy.get(b.source).push(b);
}
// the catalog column kind (#952); an index commit before it has none, and the page computes the kind below
const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== '');
const sources = cat.rows.map((r) => {
const info = srcInfo.get(r.source) || {};
const lic = r.license || r.licence || {};
const dom = domainOf(r.domain);
const rows = num(r.files ?? info.rows) || 0;
const bytes = num(r.bytes);
const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form);
// The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets
// the rule the index writes it by (#1633): a source that holds rows of a collection (sources.json campaign or
// extended_by) or has fewer units than rows is a material set, every other one a corpus.
// A unit is one unit_id (a test case, one workbook, one package); files are counted apart. The catalog columns
// units (the distinct units of a material set) and files (its rows; a corpus is counted in rows) are read as they
// are; #2065: a material set whose catalog row has no count of its units has none here (readyUnits null, not
// counted yet), as the figures count it (meta/size_figures.json units_not_counted), never its rows.
const roles = info.unit || {};
const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k]));
const isTask = hasKind ? r.kind === MATERIAL_SET : !!(info.campaign || info.extended_by) || (has('units') && num(r.units) < rows);
const units = has('units') ? num(r.units) : null;
const lr = info.license_rows?.redistribution_ok || {};
const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0);
const shipShare = rows ? Math.min(1, shipRows / rows) : 0;
// #1577: the fetch check, catalog column fetch_check; #2006: named fetch up to release-2026-11-09.1, read until a
// release with fetch_check is published
const fc = r.fetch_check || r.fetch;
return {
kind: isTask ? MATERIAL_SET : CORPUS,
isTask,
// the collection of a material set (#1083): a line of admit_collections.tsv; of a corpus (#1633), the collection
// that lists it under corpora in catalog/domains.yaml, else none
collection: r.collection ? String(r.collection) : null,
readyUnits: isTask ? units : 0,
rawRows: isTask ? 0 : rows,
fromCatalog: has('units'),
// what one row of a corpus points at (sources.json unit: file, archive member, ...); a collection's entry counts
// its rows as files (#1633: no roles)
kinds: Object.keys(roles),
units,
shipShare,
id: r.source,
title: r.title || r.source,
description: r.description || info.description || '',
summary: r.summary || '',
unit: r.unit || '',
status: r.status || null,
reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [],
domainKind: dom.kind,
codes: dom.codes,
tops: [...new Set(dom.codes.map(topOf))],
form,
tier: lic.tier || 'unknown',
ship: lic.ship || null,
spdx: lic.spdx || null,
licenceNote: lic.note || null,
contamination: r.contamination || {},
pii: r.pii || {},
// #1256: the distinct publisher names over all the rows of a material set (catalog column publishers); null
// when its rows carry none or the catalog has no such column
publishers: has('publishers') ? num(r.publishers) : null,
// #1266: the files of a material set by kind of file (upstream, index), licence and the permits for training and
// redistribution, as the projection counted them (catalog column license_counts); null without the column
licenceCounts: licenceCounts(r.license_counts),
rows,
bytes: bytes && bytes > 0 ? bytes : null,
// #2422: the one size the page shows, the deduplicated one (catalog column distinct_bytes, the same as
// distinct_bytes of the source's figures); the size filter buckets it
distinct: num(r.distinct_bytes) > 0 ? num(r.distinct_bytes) : null,
// #1158: the source's size figures from meta/size_figures.json (#2422: its size, the deduplicated one, the check
// and the estimate); #2066: the list shows and sorts by them
size: figures?.sources?.[r.source] || null,
// #1577: whether its upstream still serves its files (the fetch check fc: level, rows, fetched, read_at); null
// for a catalog without the column
fetch: fc && fc.level ? { level: String(fc.level), rows: num(fc.rows) || 0, fetched: num(fc.fetched) || 0, readAt: fc.read_at || null } : null,
// #1928: the release tag of the release its rows come from (catalog column released_in), and why a newer batch of
// it is held back, as the release notes give it (catalog column held_back); null for a catalog without them
releasedIn: r.released_in || null,
heldBack: r.held_back || null,
// #2588: the source's declared upstream (catalog columns upstream_kind, upstream_agent and upstream_entry): a
// dataset or a catalog, its agent and its entry address, the values joined by ' | ' while the source reads
// several upstreams; null for a catalog without them
upstream: r.upstream_kind ? { kind: String(r.upstream_kind), agent: String(r.upstream_agent ?? ''),
entry: String(r.upstream_entry ?? '') } : null,
topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
needsAuth: r.needs_auth || null,
train: permission(info.license_rows?.training_ok),
redist: permission(info.license_rows?.redistribution_ok),
// #1524: its file types and their rows, for the File type facet, the --ext options and the SQL and CSV filters
exts: extsOf(info.by_ext),
// #2006: the ids of the training stages one of its units passes (#2518: sources of the commit's stage figures),
// [] without stages
stages: stagesOk?.sources?.[r.source] || [],
info,
cov: cov[r.source] || null,
batches: batchesBy.get(r.source) || [],
files: filesOf(r),
};
});
return {
sha: rev.sha,
// the release tag of the commit (a tag is never moved), the commit itself before meta/projection.json had one;
// #2422: the day of that release, the commit's own before meta/projection.json had one
pin: record?.tag || rev.sha,
date: rev.date,
releaseDate: record?.date || rev.date,
overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') },
hasVocabulary: Boolean(vocabJson),
// #1577: who published each file of a material set and under which license, read when the attribution list is
// made; a commit before it has none
hasAttribution: has('meta/attribution.parquet'),
hasKind,
// #2006: the stage table the stages were computed with (version, and each stage with its names, status, about and
// counts), null when the commit has no stage of its sources; #2066: stagesWhy, why it has none (noFigures,
// notCounted or noSources); stageList, the stages of the release's figures, whose status (#2422) the first
// screen's figure says; stageFigures, the release's counts per stage (meta/stage_figures.json), what that figure
// says each can use
stages: stagesOk ? { version: stagesOk.version, list: stagesOk.stages || [] } : null,
stagesWhy,
stageList: stageFigures?.stages || [],
stageFigures: parseStageFigures(stageFigures),
// #2066: the Chinese text of each English text the release shows (TRANSLATIONS, #2065): English -> Chinese, empty
// for a commit without the table
zh: new Map(translations.map((r) => [r.en, r.zh])),
vocab,
sources,
// #1158: the size figures of the whole index and of each layer (meta/size_figures.json, read as they are; #2422:
// the page says their deduplicated size), null for a commit without them
sizes: figures ? { index: figures.index, ...figures.layers } : null,
catalogFile: cat.file,
coverage,
};
}