// Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees. import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm'; import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm'; import { downloadFile, pathsInfo } from 'https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.10/+esm'; import { HF, loadRevision as askRevision, REPO } from './commit.js'; import { t } from './i18n.js'; export { HF, REPO }; export const DATASET_URL = `${HF}/datasets/${REPO}`; export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`; export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v)); // the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus export const MATERIAL_SET = 'material_set'; export const CORPUS = 'corpus'; export const KINDS = [MATERIAL_SET, CORPUS]; // #2065, #2066: the one table of Chinese names the release writes: per English text the index shows (a collection's // name, a source's title, description or summary, as the catalog holds them) its Chinese text (columns en and zh) const TRANSLATIONS = 'meta/translations.parquet'; // #2066: the projection's counts per training stage (artpool.stages FIGURES), with (#2065) each stage's funnel const STAGE_FIGURES = 'meta/stage_figures.json'; const HUB_REPO = { type: 'dataset', name: REPO }; /** One whole file of the commit sha, read with downloadFile of @huggingface/hub, which answers null when the commit * has no such file. */ export async function wholeFile(sha, path) { const blob = await downloadFile({ repo: HUB_REPO, path, revision: sha }); if (!blob) throw new Error(`${path}: no such file at commit ${sha.slice(0, 7)}`); return blob; } /** A file named by its URL in the page address (catalog=, vocabulary=), which may lie outside the dataset. */ async function urlFile(url) { const r = await fetch(url); if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`); return r.blob(); } const jsonOf = async (blob) => JSON.parse(await (await blob).text()); /** A small file read whole (the meta tables): one request instead of a size probe and ranges. */ async function readParquet(blob) { return parquetReadObjects({ file: await (await blob).arrayBuffer(), compressors, utf8: false }); } /** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd} (#1792: * hyparquet documents no row filter, so a reader of one source reads its row range, filesOf, and keeps its rows by * the column source itself). */ export async function readColumns(url, cols, range = {}) { const file = await asyncBufferFromUrl({ url }); const metadata = await parquetMetadataAsync(file); const names = new Set(metadata.schema.slice(1).map((s) => s.name)); const columns = cols.filter((c) => names.has(c)); const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range }); return { rows, total: num(metadata.num_rows) }; } /** The commit to read: ?rev=, else main. Returns {sha, date, files}. */ export async function loadRevision(rev) { const r = await askRevision(rev); if (!r.sha) throw new Error(t('error.noCommit')); return r; } /** #1769: the date of the last commit of each batch of a source whose data files are files ({path, batches}, * loadIndex's files): the latest last commit among the files that hold the batch's rows (pathsInfo), never a folder * named by the batch. -> Map of batch -> ISO date. */ export async function loadBatchDates(sha, files) { const out = new Map(); if (!files.length) return out; const info = await pathsInfo({ repo: HUB_REPO, paths: files.map((f) => f.path), revision: sha, expand: true }); const iso = (d) => (d instanceof Date ? d.toISOString() : d ? String(d) : null); const at = new Map(info.map((e) => [e.path, iso(e.lastCommit?.date)])); for (const f of files) { const d = at.get(f.path); for (const b of f.batches) if (d && !(out.get(b) >= d)) out.set(b, d); } return out; } // ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or // unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without // them everything shows as unclassified. const pick = (o, keys) => { if (!o || typeof o !== 'object') return undefined; for (const k of keys) if (o[k] != null) return o[k]; return undefined; }; const NAME_KEYS = ['name', 'label', 'title', 'en']; const CODE_KEYS = ['code', 'id', 'key', 'value']; const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items']; const ABOUT_KEYS = ['description', 'about', 'definition']; const ZH_KEYS = ['zh']; function vocabSection(node) { const names = new Map(); const zh = new Map(); const about = new Map(); const order = []; const add = (code, name, text, zhName) => { if (code == null) return; code = String(code); if (!names.has(code)) order.push(code); if (typeof name === 'string' && name.trim()) names.set(code, name.trim()); else if (!names.has(code)) names.set(code, null); if (typeof text === 'string' && text.trim()) about.set(code, text.trim()); if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim()); }; const walk = (n) => { if (!n) return; if (Array.isArray(n)) { for (const it of n) { if (typeof it === 'string') add(it, null); else if (it && typeof it === 'object') { add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS)); walk(pick(it, KIDS_KEYS)); } } } else if (typeof n === 'object') { for (const [k, v] of Object.entries(n)) { if (typeof v === 'string') add(k, v); else if (v && typeof v === 'object') { add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS)); walk(pick(v, KIDS_KEYS)); } else add(k, null); } } }; walk(node); return { names, zh, about, order }; } // meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows // its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites). // zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about, // zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the // lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags // of its material sets). terms: Map of the term table by its English term. #2397: profile [{criterion, option}], the // criteria tools/fetch.py's default profile applies, in its order, each with the option that takes back what it // leaves out (what Get the data shows the profile leaves out). export function parseVocabulary(j) { const form = vocabSection(pick(j, ['form', 'forms'])); for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' ')); const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l])); const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({ code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '', campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [], })); // terms [{term, zh, about}] (the index's term table: the English term, its Chinese term and one sentence on what it // means), by the English term const terms = new Map((Array.isArray(j?.terms) ? j.terms : []).filter((x) => x?.term).map((x) => [String(x.term), x])); const profile = (Array.isArray(j?.profile) ? j.profile : []).map((p) => ({ criterion: String(p.criterion), option: String(p.option) })); return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections, terms, profile }; } /** #2066: meta/stage_figures.json -> {rules, stages: Map id -> stage, noStage}: the rules every stage's material passes, * in their order, and per stage, in the stage table's order, its names, status and measures (counted_by) and its counts * by them (counts, each null before the release counted the stage fields), which (#2422) the first screen's figure says; * #2629: noStage, what no stage counts of each layer by reason ({kind: {reason: {measure: value}}}, null for figures * written before it); null for a commit without the file. */ function parseStageFigures(j) { if (!j) return null; return { rules: j.rules || [], stages: new Map((j.stages || []).map((s) => [s.id, s])), noStage: j.no_stage || null }; } export const topOf = (code) => String(code).split('.')[0]; function domainOf(v) { let list = v; if (typeof list === 'string') list = [list]; if (list && typeof list.toArray === 'function') list = list.toArray(); list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : []; if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] }; if (list.includes('multi')) return { kind: 'multi', codes: [] }; return { kind: 'codes', codes: [...new Set(list)] }; } /** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or * conditional for every row) or 'no' (some rows are no or unknown). */ function permission(lr) { if (!lr) return 'no'; const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k); if (!keys.length) return 'no'; if (keys.every((k) => k === 'yes')) return 'yes'; if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond'; return 'no'; } /** #1266: a catalog cell of license_counts as [{kind, spdx, training_ok, redistribution_ok, files}], null when empty. */ function licenceCounts(v) { const items = Array.isArray(v) ? v : v?.toArray?.(); return items?.length ? items.map((x) => ({ kind: x.kind, spdx: x.spdx || null, training_ok: x.training_ok, redistribution_ok: x.redistribution_ok, files: num(x.files) || 0 })) : null; } /** #1524: a source's file types as [[ext, rows]], most rows first, from its by_ext of meta/sources.json ({ext: {rows, * bytes}}); the keys in brackets ((none), (other)) name no extension that fetch.py's --ext or a row's ext matches, and * are left out. */ function extsOf(byExt) { return Object.entries(byExt || {}).filter(([k]) => !k.startsWith('(')).map(([k, v]) => [k, num(v?.rows) || 0]) .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); } /** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}}; * json: the file is JSON (a list of rows, or {sources} or {rows}), else Parquet. */ async function readCatalog(blob, json) { if (json) { const j = await jsonOf(blob); const rows = Array.isArray(j) ? j : j.sources || j.rows || []; return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } }; } const buf = await (await blob).arrayBuffer(); // DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf const bytes = new Uint8Array(buf.slice(0)); return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } }; } /** #2065: a sum over values some of which are not known (null), as the figures sum them (index_projection._total): * {sum: the sum of the known values, null when none is known (0 when there are no values), missing: how many are * not counted yet}. */ export function summed(vals) { const known = vals.filter((v) => v != null); return { sum: known.length ? known.reduce((a, v) => a + v, 0) : vals.length ? null : 0, missing: vals.length - known.length }; } const listOf = (v) => (Array.isArray(v) ? v : v?.toArray?.() || []); /** #1769: the data files that hold rows of the catalog row r, each {path, batches, first, last} (the batches of its * rows in that file; #1792: the source's first and last row in it, null when the catalog does not record them), sorted * by path: from the catalog column batch_files, else (a catalog with data_files only) from data_files, with no batches; * never from a file's path. A file may hold rows of other sources too, so every read of one keeps the source's rows by * the column source, and reads only its row range when known (a packed file holds a source's rows together). */ function filesOf(r) { const by = new Map(); for (const x of listOf(r.batch_files)) { const f = String(x.file); if (!by.has(f)) by.set(f, { batches: [], first: num(x.first), last: num(x.last) }); by.get(f).batches.push(String(x.batch)); } if (!by.size) for (const f of listOf(r.data_files)) by.set(String(f), { batches: [], first: null, last: null }); return [...by].map(([path, x]) => ({ path, ...x, batches: x.batches.sort() })).sort((a, b) => a.path.localeCompare(b.path)); } /** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values): * a note for the reader, as a key of the locale files. */ export async function loadIndex(params, onNote) { const rev = await loadRevision(params.get('rev')); const at = (p) => wholeFile(rev.sha, p); const catalogUrl = params.get('catalog'); const has = (p) => rev.files.includes(p); const vocabUrl = params.get('vocabulary'); const vocabFile = vocabUrl ? () => urlFile(vocabUrl) : has('meta/vocabulary.json') ? () => at('meta/vocabulary.json') : null; const [cat, sourcesJson, coverage, batches, vocabJson, figures, record, translations, stageFigures] = await Promise.all([ catalogUrl ? readCatalog(urlFile(catalogUrl), /\.json(\?|$)/i.test(catalogUrl)) : readCatalog(at('meta/catalog.parquet'), false), jsonOf(at('meta/sources.json')), jsonOf(at('meta/coverage.json')), readParquet(at('meta/batches.parquet')).catch(() => []), vocabFile ? jsonOf(vocabFile()).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null, // #1158: the size figures of every level, written by the projection; a commit before it has none has('meta/size_figures.json') ? jsonOf(at('meta/size_figures.json')) : null, // #1197: the record of the release's projection, whose tag (DL13h, artpool/release_tag.py) the commands name and // (#2422) whose tag and date the release line of the first screen says has('meta/projection.json') ? jsonOf(at('meta/projection.json')) : null, // #2066: the Chinese names of collections and sources (#2065); #2422: what each training stage can use, the // first screen's figure; #2518: and, in the same file, the stages of each source has(TRANSLATIONS) ? readParquet(at(TRANSLATIONS)) : [], has(STAGE_FIGURES) ? jsonOf(at(STAGE_FIGURES)) : null, ]); // #2518: a source's stages come from the commit the page reads: meta/stage_figures.json carries, per source, the ids // of the stages one of its units passes (sources), beside the stage table's version and each stage's counts, so the // two never describe different commits. A commit without the file, one whose release has not counted its stage // fields, or one written before the figures listed the sources has no stage for any source, and the filter by stage // says which in one short line (stagesWhy) const stagesOk = stageFigures?.counted && stageFigures.sources ? stageFigures : null; const stagesWhy = stagesOk ? null : !stageFigures ? 'noFigures' : !stageFigures.counted ? 'notCounted' : 'noSources'; const vocab = parseVocabulary(vocabJson || {}); const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e])); const cov = coverage.by_source || {}; const batchesBy = new Map(); for (const b of batches) { if (!batchesBy.has(b.source)) batchesBy.set(b.source, []); batchesBy.get(b.source).push(b); } // the catalog column kind (#952); an index commit before it has none, and the page computes the kind below const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== ''); const sources = cat.rows.map((r) => { const info = srcInfo.get(r.source) || {}; const lic = r.license || r.licence || {}; const dom = domainOf(r.domain); const rows = num(r.files ?? info.rows) || 0; const bytes = num(r.bytes); const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form); // The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets // the rule the index writes it by (#1633): a source that holds rows of a collection (sources.json campaign or // extended_by) or has fewer units than rows is a material set, every other one a corpus. // A unit is one unit_id (a test case, one workbook, one package); files are counted apart. The catalog columns // units (the distinct units of a material set) and files (its rows; a corpus is counted in rows) are read as they // are; #2065: a material set whose catalog row has no count of its units has none here (readyUnits null, not // counted yet), as the figures count it (meta/size_figures.json units_not_counted), never its rows. const roles = info.unit || {}; const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k])); const isTask = hasKind ? r.kind === MATERIAL_SET : !!(info.campaign || info.extended_by) || (has('units') && num(r.units) < rows); const units = has('units') ? num(r.units) : null; const lr = info.license_rows?.redistribution_ok || {}; const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0); const shipShare = rows ? Math.min(1, shipRows / rows) : 0; // #1577: the fetch check, catalog column fetch_check; #2006: named fetch up to release-2026-11-09.1, read until a // release with fetch_check is published const fc = r.fetch_check || r.fetch; return { kind: isTask ? MATERIAL_SET : CORPUS, isTask, // the collection of a material set (#1083): a line of admit_collections.tsv; of a corpus (#1633), the collection // that lists it under corpora in catalog/domains.yaml, else none collection: r.collection ? String(r.collection) : null, readyUnits: isTask ? units : 0, rawRows: isTask ? 0 : rows, fromCatalog: has('units'), // what one row of a corpus points at (sources.json unit: file, archive member, ...); a collection's entry counts // its rows as files (#1633: no roles) kinds: Object.keys(roles), units, shipShare, id: r.source, title: r.title || r.source, description: r.description || info.description || '', summary: r.summary || '', unit: r.unit || '', status: r.status || null, reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [], domainKind: dom.kind, codes: dom.codes, tops: [...new Set(dom.codes.map(topOf))], form, tier: lic.tier || 'unknown', ship: lic.ship || null, spdx: lic.spdx || null, licenceNote: lic.note || null, contamination: r.contamination || {}, pii: r.pii || {}, // #1256: the distinct publisher names over all the rows of a material set (catalog column publishers); null // when its rows carry none or the catalog has no such column publishers: has('publishers') ? num(r.publishers) : null, // #1266: the files of a material set by kind of file (upstream, index), licence and the permits for training and // redistribution, as the projection counted them (catalog column license_counts); null without the column licenceCounts: licenceCounts(r.license_counts), rows, bytes: bytes && bytes > 0 ? bytes : null, // #2422: the one size the page shows, the deduplicated one (catalog column distinct_bytes, the same as // distinct_bytes of the source's figures); the size filter buckets it distinct: num(r.distinct_bytes) > 0 ? num(r.distinct_bytes) : null, // #1158: the source's size figures from meta/size_figures.json (#2422: its size, the deduplicated one, the check // and the estimate); #2066: the list shows and sorts by them size: figures?.sources?.[r.source] || null, // #1577: whether its upstream still serves its files (the fetch check fc: level, rows, fetched, read_at); null // for a catalog without the column fetch: fc && fc.level ? { level: String(fc.level), rows: num(fc.rows) || 0, fetched: num(fc.fetched) || 0, readAt: fc.read_at || null } : null, // #1928: the release tag of the release its rows come from (catalog column released_in), and why a newer batch of // it is held back, as the release notes give it (catalog column held_back); null for a catalog without them releasedIn: r.released_in || null, heldBack: r.held_back || null, // #2588: the source's declared upstream (catalog columns upstream_kind, upstream_agent and upstream_entry): a // dataset or a catalog, its agent and its entry address, the values joined by ' | ' while the source reads // several upstreams; null for a catalog without them upstream: r.upstream_kind ? { kind: String(r.upstream_kind), agent: String(r.upstream_agent ?? ''), entry: String(r.upstream_entry ?? '') } : null, topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [], needsAuth: r.needs_auth || null, train: permission(info.license_rows?.training_ok), redist: permission(info.license_rows?.redistribution_ok), // #1524: its file types and their rows, for the File type facet, the --ext options and the SQL and CSV filters exts: extsOf(info.by_ext), // #2006: the ids of the training stages one of its units passes (#2518: sources of the commit's stage figures), // [] without stages stages: stagesOk?.sources?.[r.source] || [], info, cov: cov[r.source] || null, batches: batchesBy.get(r.source) || [], files: filesOf(r), }; }); return { sha: rev.sha, // the release tag of the commit (a tag is never moved), the commit itself before meta/projection.json had one; // #2422: the day of that release, the commit's own before meta/projection.json had one pin: record?.tag || rev.sha, date: rev.date, releaseDate: record?.date || rev.date, overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') }, hasVocabulary: Boolean(vocabJson), // #1577: who published each file of a material set and under which license, read when the attribution list is // made; a commit before it has none hasAttribution: has('meta/attribution.parquet'), hasKind, // #2006: the stage table the stages were computed with (version, and each stage with its names, status, about and // counts), null when the commit has no stage of its sources; #2066: stagesWhy, why it has none (noFigures, // notCounted or noSources); stageList, the stages of the release's figures, whose status (#2422) the first // screen's figure says; stageFigures, the release's counts per stage (meta/stage_figures.json), what that figure // says each can use stages: stagesOk ? { version: stagesOk.version, list: stagesOk.stages || [] } : null, stagesWhy, stageList: stageFigures?.stages || [], stageFigures: parseStageFigures(stageFigures), // #2066: the Chinese text of each English text the release shows (TRANSLATIONS, #2065): English -> Chinese, empty // for a commit without the table zh: new Map(translations.map((r) => [r.en, r.zh])), vocab, sources, // #1158: the size figures of the whole index and of each layer (meta/size_figures.json, read as they are; #2422: // the page says their deduplicated size), null for a commit without them sizes: figures ? { index: figures.index, ...figures.layers } : null, catalogFile: cat.file, coverage, }; }