Spaces:
Running
Running
Download app.js from Battam/training-data-foundation: direct link, hf CLI and curl.
- Browser
- Download file 137 kB
-
https://huggingface.co/spaces/Battam/training-data-foundation/resolve/main/app.js
- Command line
-
hf download hf://spaces/Battam/training-data-foundation/app.js
-
curl -L -o app.js https://huggingface.co/spaces/Battam/training-data-foundation/resolve/main/app.js
137 kB
// The data page of the Training Data Foundation (#2422): a working screen first (second round, "Strata": the field of
// every source as the ground, and on it the training stages as a column, the filters, the selection with its largest
// sources and the one command that takes it), then the sources (their list, each with its details, and the linked
// charts) and Get the data (the one command and the selection file). The dataset card is the overview; this page is
// where data is picked and taken.
// Figures first, few words; every chart keeps a short label and a table view. The charts of Pick, its filter counts and
// the list of selected sources come from one crossfilter selection (linked.js). The two layers stay apart everywhere
// and are never added together (#1083): material sets, grouped into collections and counted in sets and units, in the
// main color; corpora, counted in rows, in gray. Pick shows one layer at a time (F.kind, material sets by default; a
// training stage takes the layer it counts).
import {
CORPUS, DATASET_URL, fileUrl, KINDS, loadBatchDates, loadIndex, MATERIAL_SET, num, readColumns, REPO, summed, topOf,
wholeFile,
} from './data.js';
import {
clear, el, figureWithToggle, fmtAnd, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtOr, fmtPct, fmtShare, fmtShort,
segmented, table, tableShown, vt,
} from './charts.js';
import { decorate, drawRelease, drawStages, markFromIndex, RULE_KEY, sizeLine, sizeOf, sizeParts, stageColor, stageWhat } from './first.js';
import { lang, onLanguage, setUpSwitch, t, tEn, translatePage } from './i18n.js';
import { formIcon, icon } from './icons.js';
const CSV_MAX_ROWS = 200000;
const ROW_COLUMNS = ['source', 'batch', 'uri', 'range_offset', 'range_length', 'member_path', 'size_bytes', 'sha256'];
const FIRST_ROWS = 20;
const HEAT_FORMS = 8;
const FORM_BARS = 10;
// #1524: how many units Get the data takes from each source (--sample N; all: no --sample, #1891); #2629: when the
// reader has not chosen (sample in the address), every unit of material sets, whose whole is small, and 3 of each
// corpus, whose whole can be petabytes (DEFAULT_SAMPLE by the layer the command takes, sampleNow)
const SAMPLES = ['1', '3', '10', '30', 'all'];
const DEFAULT_SAMPLE = { [MATERIAL_SET]: 'all', [CORPUS]: '3' };
const MAX_FILE = 20e6;
const MAX_LISTED = 30;
// the phone layout of style.css
const PHONE = '(max-width: 720px)';
// #1617: the address of the user manual's view (#manual; showView); #2614: and of each of its parts, #manual/<name>
const MANUAL_VIEW = 'manual';
const inManual = (hash) => hash === `#${MANUAL_VIEW}` || hash.startsWith(`#${MANUAL_VIEW}/`);
const $ = (id) => document.getElementById(id);
// A link that leaves the page opens in a new tab. The Hub shows this page in a frame whose sandbox allows popups but
// not navigating the top window, and a site such as huggingface.co refuses to be shown in a frame, so a link opened in
// the frame showed the browser's error page (the Dataset link of the header, 2026-11-12). One listener in the capture
// phase sets the target before the browser follows the link, for the links of the markup and those the page writes.
document.addEventListener('click', (e) => {
const a = e.target.closest?.('a[href]');
if (a && /^https?:$/.test(a.protocol) && a.origin !== location.origin) {
a.target = '_blank';
a.rel = 'noopener';
}
}, true);
// ---- plain names (codes only inside the SQL and the commands); the words are in the locale files, by code:
// tier.<tier>, permit.<answer>, unitKind.<kind> and unitCount.<kind>
const tierName = (k, tt = t) => tt(`tier.${k}`, { defaultValue: human(k) });
const PERMIT_ORDER = ['yes', 'no', 'conditional', 'unknown'];
const PERMIT_COLOR = { yes: 'var(--ours)', no: 'var(--problem)', conditional: 'var(--fill-3)', unknown: 'var(--other)' };
// #1228: the condition in words of a conditional share under the source's licence, by its licence tier
const CONDITION = { 'share-alike': 'permit.conditional.sameLicence', 'nc-research': 'permit.conditional.nonCommercial' };
// a row's own licence; empty when the row takes its batch licence
const LICENCE_COLUMNS = ['license_spdx', 'license_training_ok', 'license_redistribution_ok'];
// #1266: the kinds of file of a material set (catalog column license_counts): its upstream files and the files the
// index wrote for it
const FILE_KINDS = ['upstream', 'index'];
// the two layers by their kind: kind.<kind> names them (Material sets, Corpora), kindOne.<kind> one of them
const kindName = (k, tt = t) => tt(`kind.${k}`);
const kindOne = (k) => t(`kindOne.${k}`);
/** The kinds as a legend: a swatch in each kind's color and its name. */
const kindKey = (kinds = KINDS) => el('span', { class: 'kind-key' },
kinds.map((k) => el('span', {}, el('span', { class: `sw kind-${k}`, 'aria-hidden': 'true' }), kindName(k))));
// the kinds of unit of a corpus (sources.json unit) -> their key in the locale files
const UNIT_KIND = { file: 'file', 'API response': 'apiResponse', 'WARC record': 'warcRecord', 'archive member': 'archiveMember', 'byte range': 'byteRange' };
// file type -> the tool that opens it (TOOL_WORDS: not a tool's name, a key of the locale files)
const TOOLS = {
xlsx: 'openpyxl', xlsm: 'openpyxl', xls: 'LibreOffice', ods: 'LibreOffice', csv: 'DuckDB', tsv: 'DuckDB', parquet: 'DuckDB',
json: 'jq', jsonl: 'jq', ndjson: 'jq', pdf: 'pdftotext', doc: 'LibreOffice', docx: 'python-docx', ppt: 'LibreOffice',
pptx: 'python-pptx', gz: 'gzip', tgz: 'tar', tar: 'tar', bz2: 'bzip2', xz: 'xz', zst: 'zstd', zip: 'unzip', whl: 'unzip',
jar: 'unzip', '7z': '7-Zip', crate: 'tar', deb: 'dpkg-deb', rpm: 'rpm2cpio', warc: 'warcio', arc: 'warcio', pgn: 'python-chess',
py: 'Python', '(none)': 'file(1)', xml: 'xmllint',
yaml: 'yq', yml: 'yq', evtx: 'python-evtx', log: 'grep', rego: 'OPA', sql: 'DuckDB', db: 'SQLite',
safetensors: 'safetensors', final: 'TensorBoard', ipynb: 'Jupyter', eml: 'Python email', mbox: 'Python mailbox',
ics: 'icalendar', e01: 'The Sleuth Kit', img: 'The Sleuth Kit', dd: 'The Sleuth Kit', raw: 'The Sleuth Kit',
ys: 'Yosys', v: 'Icarus Verilog', sv: 'Verilator', go: 'Go', rs: 'Cargo', zig: 'Zig', lua: 'Lua', rb: 'Ruby', c: 'gcc',
tcl: 'tclsh', smt2: 'Z3', mzn: 'MiniZinc', dzn: 'MiniZinc', lp: 'HiGHS', mps: 'HiGHS',
};
const TOOL_WORDS = { md: 'tool.editor', txt: 'tool.editor', toml: 'tool.editor', html: 'tool.browser', htm: 'tool.browser', test: 'tool.suiteRunner' };
const toolOf = (ext) => TOOLS[ext] || (TOOL_WORDS[ext] ? t(TOOL_WORDS[ext]) : null);
const NONE = '_none';
const MULTI = '_multi';
const OTHER = '_other';
const human = (code) => String(code).replace(/[_-]+/g, ' ').replace(/^./, (c) => c.toUpperCase());
function sentence(s) {
let x = String(s).trim();
if (!/\s/.test(x)) x = x.replace(/[-_]+/g, ' ');
x = x.replace(/\b(ci|ctf|etl|ml|qa|sql|api)\b/gi, (w) => (w.toLowerCase() === 'qa' ? 'Q&A' : w.toUpperCase()));
return x.charAt(0).toUpperCase() + x.slice(1);
}
/** A count with its noun: key_one and key_other in the locale files, {{n}} the formatted number. */
const plural = (key, n, tt = t) => tt(key, { count: n, n: fmtInt(n) });
const sum = (xs, f) => xs.reduce((a, x) => a + f(x), 0);
// #2065: units as the figures count them (meta/size_figures.json): a material set without a count of its units
// (data.js readyUnits null) is not counted yet, never counted by its rows; a sum over material sets is the sum of the
// counted ones, with how many are not counted yet (data.js summed), in the card's words
const uncounted = (n) => (n ? t('n.uncounted', { n: fmtInt(n) }) : '');
const unitsText = (n, missing = 0) => (n == null ? t('n.unitsNotCounted') : plural('n.unit', n) + uncounted(missing));
const unitsNum = (n) => (n == null ? t('n.notCounted') : fmtInt(n));
const unitsOfSets = (S) => { const u = summed(S.map((s) => s.readyUnits)); return unitsText(u.sum, u.missing); };
const notCountedOf = (S) => summed(S.filter((s) => s.isTask).map((s) => s.readyUnits)).missing;
/** #2066: a chart of plots.js, draw(); until the module is loaded a skeleton of class sk, and the reason in its place
* when it could not be loaded (each figure's Table view has the same numbers). */
function plotted(draw, sk = 'sk sk-block') {
if (plotsError) return el('p', { class: 'small problem-text' }, t('chart.failed', { error: plotsError.message }));
if (!P) return el('div', { class: sk });
return el('div', { class: 'plot-box' }, draw());
}
let IX = null;
// linked.js once loaded (after the first screen has painted)
let L = null;
// #2066: plots.js (the page's Plot charts) once loaded, which starts with the page, or the error that kept it from
// loading
let P = null;
let plotsError = null;
const F = {};
// ids: the selected sources (the selection's answer, null until it comes); facets: per filter {m: value -> sources,
// total}; tables: the rows behind each linked chart, with the ids each one covers; charts: the vgplot elements;
// preview: the ids whose "Preview rows" is open; previewRows: per id its rows of meta/preview.parquet, once read
// across: the ids the filters keep in either layer (linked.js startAcross)
const UI = { across: null, allForms: false, showAll: false, detail: new Map(), preview: new Set(), previewRows: new Map(), csv: null, tab: 'cmd', fitKeyOpen: false, ids: null, facets: {},
tables: { dom: [], form: [], heat: [] }, charts: null };
// kind: the layer the browse section shows (material_set, the default, or corpus); metric: what the charts of material
// sets count (sets or units; corpora are counted in rows); pick: the ids of the picked sources, joined by commas; ext:
// the file types selected, joined by commas; sample: how many units Get the data takes from each source (SAMPLES);
// stage (#2006): the id of a training stage of the stage table (the commit's stage figures); pii (#2422): the personal-data mark (none,
// marked or unknown, piiMark). The panel shows no control for collection, leaf, tier, redist and status (#2422): from
// the address they still filter, each a chip that removes it
// view (#2422, second round): the arrangement of the field (VIEWS), strata by default
const KEYS = ['domain', 'leaf', 'form', 'collection', 'kind', 'train', 'redist', 'tier', 'size', 'status', 'q', 'ext', 'stage', 'pii', 'metric', 'sort', 'dir', 'open', 'pick', 'sample', 'view'];
function readUrl() {
const p = new URLSearchParams(location.search);
for (const k of KEYS) F[k] = p.get(k) || '';
if (F.kind !== CORPUS) F.kind = MATERIAL_SET;
if (F.metric !== 'units') F.metric = 'sets';
if (!F.sort) { F.sort = 'fit'; F.dir = 'desc'; }
if (!SAMPLES.includes(F.sample)) F.sample = '';
if (!VIEWS.includes(F.view)) F.view = VIEWS[0];
}
function writeUrl() {
const p = new URLSearchParams(location.search);
for (const k of KEYS) {
const v = F[k];
const dflt = (k === 'kind' && v === MATERIAL_SET) || (k === 'metric' && v === 'sets') || (k === 'sort' && v === 'fit') || (k === 'dir' && v === 'desc')
|| (k === 'view' && v === VIEWS[0]);
if (v && !dflt) p.set(k, v); else p.delete(k);
}
const s = p.toString();
history.replaceState(null, '', s ? `?${s}` : location.pathname);
}
// ---- names: the vocabulary's own (meta/vocabulary.json), in the page's language. In Chinese a name is its zh, without
// the bracketed note a zh name may carry (as the Chinese card shows it, index_projection.domain_name); in English its
// name, in sentence case.
const ZH_NOTE = /\s*\uff08[^\uff09]*\uff09/g;
const inChinese = () => lang() === 'zh';
const vocabName = (sec, code) => {
const zh = inChinese() ? IX.vocab[sec].zh.get(code) : null;
if (zh) return zh.replace(ZH_NOTE, '').trim();
const n = IX.vocab[sec].names.get(code);
return n ? sentence(n) : null;
};
function domainName(key) {
if (key === NONE) return t('name.notClassified');
if (key === MULTI) return vocabName('domain', 'multi') || t('name.severalDomains');
return vocabName('domain', key) || t(`domain.${key}`, { defaultValue: key });
}
function formName(key) {
if (key === NONE || key == null) return t('name.notClassified');
if (key === OTHER) return t('name.otherForms');
return vocabName('form', key) || sentence(key);
}
/** The name of a layer (meta/vocabulary.json layers: Task material, Corpora) in the page's language; the locale's name
* of the kind when the vocabulary has none. */
function layerName(kind) {
const l = IX?.vocab.layers.get(kind);
return (l && (lang() === 'zh' ? l.zh : l.name)) || kindName(kind);
}
/** The name of a collection (meta/vocabulary.json collections) in the page's language; #2066: in Chinese the Chinese
* text of its name in the table of Chinese names (#2065, zhOf), else the vocabulary's zh. */
function collectionName(code) {
const c = IX.vocab.collections.find((x) => x.code === code);
if (!c) return code || t('name.noCollection');
return (inChinese() && (IX.zh.get(c.name) || c.zh)) || c.name;
}
/** #2066: an English text the index shows (a source's title or description, as the catalog holds it) in the page's
* language: in Chinese its Chinese text in the table of Chinese names the release writes (#2065, data.js zh), else
* as it is (a text the table does not hold yet stays English). */
const zhOf = (en) => (inChinese() && en && IX.zh.get(en)) || en;
const titleOf = (s) => zhOf(s.title);
const descriptionOf = (s) => zhOf(s.description);
/** #2006: the name of a training stage of the stage table (IX.stages, from the commit's stage figures) in the page's language, marked
* when the stage is provisional (stageTitle); its id when the page has no such stage. */
function stageName(id) {
const st = IX.stages?.list.find((x) => x.id === id);
return st ? stageTitle(st) : id;
}
/** #2066: a stage's name ({name, zh, status} of the release's figures) in the page's language,
* marked when it is provisional. */
function stageTitle(st) {
const name = (inChinese() && st.zh) || st.name;
return st.status === 'provisional' ? t('stage.provisional', { name }) : name;
}
const domainKeysOf = (s) => (s.domainKind === 'codes' ? s.tops : [s.domainKind === 'multi' ? MULTI : NONE]);
const formKeyOf = (s) => s.form ?? NONE;
// #2422: a source's personal-data mark, a label the page filters on and counts and never leaves a source out of a count
// for: its catalog pii.level present or suspected (marked), none_known (none), else not judged yet (unknown)
const PII_MARKED = ['present', 'suspected'];
const piiMark = (s) => (PII_MARKED.includes(s.pii?.level) ? 'marked' : s.pii?.level === 'none_known' ? 'none' : 'unknown');
function sourceDomainText(s) {
if (s.domainKind !== 'codes') return domainName(s.domainKind === 'multi' ? MULTI : NONE);
const leaves = s.codes.filter((c) => c.includes('.'));
// a leaf without a name shows its domain's name, never a bare code
const names = (leaves.length ? leaves : s.codes).map((c) => (domainName(c) === c ? domainName(topOf(c)) : domainName(c)));
return [...new Set(names)].join('; ');
}
const inCategory = (s, code) => (code.includes('.') ? s.codes.includes(code) : s.tops.includes(code));
// ---- fit: what decides whether a source can feed a task (yes, part, no, unknown); labels fit.<k>, states fitWord.<state>
const FIT = [
{ k: 'ship', icon: 'share' },
{ k: 'bench', icon: 'target' },
{ k: 'pii', icon: 'person' },
{ k: 'size', icon: 'size' },
{ k: 'alone', icon: 'doc' },
{ k: 'login', icon: 'link' },
// #1577: whether the upstream still serves the files (catalog column fetch_check)
{ k: 'fetch', icon: 'get' },
];
const fitLabel = (k) => t(`fit.${k}`);
const fitWord = (state) => t(`fitWord.${state}`);
// #1577: the fit of a fetch level (catalog column fetch_check, index_projection.FETCH_LEVELS); #2597: needs auth (the
// check, which reads anonymously, could not read the rows) says nothing of what the upstream serves, so its fit is unknown
const FETCH_FIT = { all: 'yes', some: 'part', none: 'no' };
// #2597: the sentence of each level read (locales fetch.<key>); a level without one is not read yet (fetch.notRead)
const FETCH_SENTENCE = { all: 'all', some: 'some', none: 'none', 'needs auth': 'needsAuth' };
/** #1577: a source's fetch check in words: how many of its sampled rows read fetched and matched their record, out of
* how many, and the day read; or that none is read yet; #2597: or that the check could not read them anonymously. */
function fetchText(f) {
if (!f || !FETCH_SENTENCE[f.level]) return t('fetch.notRead');
return t(`fetch.${FETCH_SENTENCE[f.level]}`, { fetched: fmtInt(f.fetched), rows: fmtInt(f.rows), share: fmtShare(f.fetched, f.rows), date: f.readAt || t('common.unknown') });
}
function fitOf(s) {
const q = s.info.size_quantiles_upper_bytes || {};
const c = s.contamination?.level;
const p = s.pii?.level;
const kinds = s.isTask ? ['unit'] : s.kinds;
return {
ship: s.shipShare >= 0.999 ? 'yes' : s.shipShare <= 0.001 ? 'no' : 'part',
bench: c == null || c === 'not scanned' ? 'unknown' : c === 'none' ? 'yes' : 'no',
pii: p == null ? 'unknown' : p === 'none_known' ? 'yes' : 'no',
size: s.bytes != null && s.bytes <= MAX_FILE ? 'yes' : q.p99 != null && num(q.p99) < MAX_FILE ? 'yes'
: q.p50 != null && num(q.p50) >= MAX_FILE ? 'no' : q.p50 != null ? 'part' : 'unknown',
alone: kinds.every((k) => ['unit', 'file', 'API response'].includes(k)) ? 'yes' : 'part',
login: s.needsAuth === 'no' ? 'yes' : s.needsAuth === 'mixed' ? 'part' : 'no',
fetch: FETCH_FIT[s.fetch?.level] || 'unknown',
};
}
function fitDot(state, label) {
const name = t('fit.dot', { label, state: fitWord(state) });
return el('span', { class: `fit fit-${state}`, role: 'img', title: name, 'aria-label': name });
}
function fitStrip(s) {
const f = fitOf(s);
return el('span', { class: 'fit-strip' }, FIT.map((x) => fitDot(f[x.k], fitLabel(x.k))));
}
/** #1302: the names of the fit dots over their column, each an icon and a short label (fitShort.<k>) above its dot. */
function fitHeads() {
return el('span', { class: 'fit-heads' }, FIT.map((x) => el('span', { class: 'fit-head', title: fitLabel(x.k) },
icon(x.icon, { size: 14 }), el('span', {}, t(`fitShort.${x.k}`)))));
}
function fitKey() {
return el('span', { class: 'fit-key' }, ['yes', 'part', 'no', 'unknown'].map((k) => el('span', {}, el('span', { class: `fit fit-${k}`, 'aria-hidden': 'true' }), fitWord(k))));
}
// ---- filters: the values are the page's state and its address (F); the selection of linked.js applies them, and its
// answer (the ids of the selected sources) is what every part below the filters shows
const sizeBucket = (b) => (b == null ? 'unknown' : b < 1e9 ? 'lt1g' : b < 1e12 ? '1g1t' : 'ge1t');
/** One row of the page's DuckDB table src (linked.js): what the filters and the charts read of a source; n_units are
* the units of a material set (#2065: null, which SQL's sum leaves out, while not counted yet) and n_rows the rows of a
* corpus, 0 for the other kind; collection is that of a material
* set or (#1633) of a corpus a collection lists, '' otherwise; exts its file types and ext_rows the rows of each
* (data.js exts); #2006: stages the ids of the training stages one of its units passes (data.js stages); #2422: sizeb
* the size class of its deduplicated size, the one size the page shows, and pii its personal-data mark (piiMark). */
function pageRow(s, cols) {
return {
source: s.id, dkeys: domainKeysOf(s), codes: s.codes, formkey: formKeyOf(s), hcol: heatColOf(s, cols), kind: s.kind,
collection: s.collection || '', status: s.status, n_src: 1, n_units: s.readyUnits, n_rows: s.rawRows, sizeb: sizeBucket(s.distinct),
tier: s.tier, train: s.train, redist: s.redist, pii: piiMark(s),
// #2066: the search reads the title and description in both languages
qtext: [s.title, s.description, IX.zh.get(s.title), IX.zh.get(s.description), s.id].filter(Boolean).join('\n').toLowerCase(),
exts: s.exts.map(([e]) => e), ext_rows: s.exts.map(([, n]) => n), stages: s.stages,
};
}
const selected = () => (UI.ids ? IX.sources.filter((s) => UI.ids.has(s.id)) : []);
const FILTER_KEYS = ['domain', 'leaf', 'form', 'collection', 'train', 'redist', 'tier', 'size', 'status', 'q', 'ext', 'stage', 'pii'];
// the clauses of the selection: the filters and the layer shown (kind, always one of the two)
const SEL_KEYS = [...FILTER_KEYS, 'kind'];
const activeFilters = () => FILTER_KEYS.filter((k) => F[k]);
/** The filters that are on, as [label, value, key] in the page's language. */
function filterChips() {
const out = [];
const word = (key) => t(key, { defaultValue: '' });
if (F.domain) out.push([t('chip.domain'), F.leaf ? domainName(F.leaf) : domainName(F.domain), 'domain']);
if (F.form) out.push([t('chip.form'), formName(F.form), 'form']);
if (F.collection) out.push([t('chip.collection'), collectionName(F.collection), 'collection']);
if (F.stage) out.push([t('chip.stage'), stageName(F.stage), 'stage']);
if (F.train) out.push([t('chip.training'), word(`filter.permit.${F.train}`), 'train']);
if (F.pii) out.push([t('chip.pii'), word(`filter.pii.${F.pii}`), 'pii']);
if (F.redist) out.push([t('chip.redistribution'), word(`filter.permit.${F.redist}`), 'redist']);
if (F.tier) out.push([t('chip.tier'), tierName(F.tier), 'tier']);
if (F.size) out.push([t('chip.size'), word(`filter.size.${F.size}`), 'size']);
if (F.status) out.push([t('chip.status'), t(`filter.status.${F.status}`, { defaultValue: human(F.status) }), 'status']);
if (F.q) out.push([t('chip.search'), F.q, 'q']);
if (F.ext) out.push([t('chip.ext'), extList().map((e) => `.${e}`).join(', '), 'ext']);
return out;
}
/** #1524: the file types selected (F.ext), in the order of the address. */
const extList = () => (F.ext ? F.ext.split(',').filter(Boolean) : []);
/** What the charts count, in the layer shown: material sets in sets or units (F.metric), corpora in rows. */
const metricNow = () => (F.kind === CORPUS ? 'rows' : F.metric);
function metricOf(s, m = metricNow()) { return m === 'sets' ? 1 : m === 'units' ? s.readyUnits : s.rawRows; }
const metricFmt = () => (v) => fmtInt(v);
const metricWord = (m) => t(`metric.${m}`);
/** #2422: the layer a training stage counts: that of the sources one of whose units passes it (a stage's condition names
* one kind), else by its measures (a stage counted in units counts material sets). */
function stageKind(id) {
const s = IX.sources.find((x) => x.stages.includes(id));
if (s) return s.kind;
return IX.stageList.find((x) => x.id === id)?.counted_by?.includes('units') ? MATERIAL_SET : CORPUS;
}
/** Sets filters (patch of F); origin 'map' when a map cell set them. #2422: a training stage takes the layer it counts,
* and a layer leaves out a stage of the other one, as corpora leave out a collection (which belongs to material sets);
* a new layer draws the charts again in its measure. The selection answers with the new ids, and the parts below the
* filters are drawn again from them (renderScoped). */
function setFilter(patch, origin = '') {
const p = { ...patch };
if (p.stage) p.kind = stageKind(p.stage);
const kind = p.kind ?? F.kind;
if (kind === CORPUS && (p.collection ?? F.collection)) p.collection = '';
const stage = p.stage ?? F.stage;
if (stage && stageKind(stage) !== kind) p.stage = '';
const layer = kind !== F.kind;
Object.assign(F, p);
UI.showAll = false;
writeUrl();
const keys = Object.keys(p).filter((k) => SEL_KEYS.includes(k));
if (L && keys.length) L.apply(F, keys, origin);
syncFilterRow();
renderChips();
if ('stage' in p || layer) renderStageFig();
if (layer) { drawCharts(); renderMapHead(selected()); }
// the labels of the field say the stage and the domain the filters hold, and the last group's figure of the layer
if (FIELD && (layer || 'stage' in p || 'domain' in p)) { FIELD.setLabels(fieldLabels()); FIELD.pick(stageFigList().findIndex((s) => s.id === F.stage)); }
}
/** Shows one layer in Pick (material_set or corpus), with the charts in its measure (setFilter). */
function setLayer(kind) {
if (kind !== F.kind) setFilter({ kind });
}
// ---- the command: tools/fetch.py of the index (#268), run with uv, the script and the data pinned to the release tag
// of the commit the page reads (#1197: IX.pin, as the quick start of the dataset card; --revision: without it the
// script reads the main branch as it is when it starts)
const fetchUrl = () => fileUrl(IX.pin, 'tools/fetch.py');
/** #2629: the layer a command takes: the picked sources' while there are picks (corpora when one of them is a corpus),
* else the layer shown. */
function takenKind() {
const picks = IX ? picked() : [];
return picks.length ? (picks.some((s) => !s.isTask) ? CORPUS : MATERIAL_SET) : F.kind;
}
/** #2629: the units choice in force: the reader's (F.sample), else the default of the layer the command takes. */
const sampleNow = () => F.sample || DEFAULT_SAMPLE[takenKind()];
/** #1524: the options of a units choice (SAMPLES): --sample N, or none for every unit. #1891: never fetch.py's --all,
* which turns the default profile's other criteria off: "all" is a choice of how many units, and the profile stays on. */
const sampleArgs = (v = sampleNow()) => (v === 'all' ? [] : [`--sample ${v}`]);
/** The options of a command, one string each; exts: one --ext per file type (#1524); sample: the units choice it takes
* (#2629: the one in force unless given). */
function commandArgs({ domain, form, sources, tags, kind, exts, sample } = {}) {
const a = [`--revision ${IX.pin}`];
if (kind) a.push(`--kind ${kind}`);
if (sources) for (const s of sources) a.push(`--source ${s}`);
if (tags) for (const g of tags) a.push(`--tag campaign:${g}`);
if (domain) a.push(`--domain ${domain}`);
if (form) a.push(`--form ${form}`);
if (exts) for (const e of exts) a.push(`--ext ${e}`);
a.push(...sampleArgs(sample), '--out ./data');
return a;
}
// #1216: one option per line, continuation lines indented four spaces and every line but the last ending in " \", so
// the text shown is the text copied and runs as it is
const command = (args) => [`uv run ${fetchUrl()}`, ...args].join(' \\\n ');
/** The options that take exactly S, sources of one layer, within domain and form (#1083: one command per layer).
* whole: S is every source of its layer within domain and form, taken by fetch.py's --kind (#1216). Otherwise material
* sets are taken through the campaign tags of their collections (--tag campaign:<name>, any of them; a collection's
* material sets carry the campaign of the collection or of one of its earlier names, vocabulary
* collections[].campaigns), within --domain and --form; corpora, and material sets without a collection, one --source
* each. exts: the file types selected, one --ext each (#1524). */
function layerArgs(S, { domain, form, whole = false, exts } = {}) {
if (whole) return { args: commandArgs({ kind: S[0]?.isTask ? MATERIAL_SET : CORPUS, domain, form, exts }), listed: false };
const colls = new Set(S.map((s) => s.collection));
if (!S[0]?.isTask || colls.has(null)) return { args: commandArgs({ sources: S.map((s) => s.id), exts }), listed: true };
const tags = IX.vocab.collections.filter((c) => colls.has(c.code)).flatMap((c) => c.campaigns);
return { args: commandArgs({ tags, domain, form, exts }), listed: false };
}
/** The options that select exactly the page's selection (one layer). fetch.py's --domain matches a top-level code with
* its leaves, a leaf only itself, and multi and unclassified only themselves, as the page's domain filter does; --form
* matches one form id. So the layer's options within domain and form when only those filters and the collection are
* set (and the form is a classified one; without a collection the whole layer there, --kind), else one --source per
* selected source. #1524: the file types selected narrow the rows of either, one --ext each (fetch.py keeps the rows
* of any of them, so a source without such rows gives none). */
function selectionArgs(sel) {
const exts = extList();
const other = activeFilters().some((k) => !['domain', 'leaf', 'form', 'collection', 'ext'].includes(k)) || F.form === NONE;
if (other) return { args: commandArgs({ sources: sel.map((s) => s.id), exts }), listed: true };
const domain = F.domain === MULTI ? 'multi' : F.domain === NONE ? 'unclassified' : F.leaf || F.domain;
return layerArgs(sel, { domain, form: F.form, whole: !F.collection, exts });
}
async function copy(text, state) {
try { await navigator.clipboard.writeText(text); state.textContent = t('copy.done'); } catch { state.textContent = t('copy.select'); }
setTimeout(() => { state.textContent = ''; }, 2400);
}
function copyButton(label, text, { dark = false, iconName = 'copy' } = {}) {
const state = el('span', { class: 'copy-state', 'aria-live': 'polite' });
return el('span', { class: 'copy-wrap' },
el('button', { type: 'button', class: `btn${dark ? ' on-dark' : ''}`, onclick: (e) => { e.stopPropagation(); copy(text, state); } }, icon(iconName, { size: 16 }), label),
state);
}
// ---- #2422: the first screen: the three training stages (first.js drawStages), each with what it can use, and the
// release line. The stages are those of the release's figures (meta/stage_figures.json, in the stage table's order),
// each with its status; without the figures there is no band. A tile sets the filter by stage once the commit's figures
// list the stages of each source (#2518); what is missing is one short line under the tiles.
function stageFigList() {
const figs = IX.stageFigures?.stages;
// a band's hover text: the stage's condition (in Chinese its text in the table of Chinese names) and the version of
// the stage table the stages were computed with
const title = (s) => [(inChinese() && IX.zh.get(s.about)) || s.about, IX.stages ? t('stage.version', { v: IX.stages.version }) : null].filter(Boolean).join(' ');
return [...(figs?.values() || [])].map((s) => ({ ...s, title: title(s) }));
}
// ---- the words the page counts in and sorts by, under the stages: each a button that opens its meaning in place, the
// sentence of the index's term table (meta/vocabulary.json terms; in Chinese its text in the table of Chinese names,
// the English sentence while a release has not translated it), and for the domains their names, in the vocabulary's
// order. One popover for all of them (the Popover API: it closes on a click outside or Escape), placed under the
// button that opened it. A commit whose vocabulary has no term table shows no such line. #2614: the start of the
// manual has the same line ([data-terms]), drawn again with the manual's text (showView).
const TERMS = ['material set', 'corpus', 'unit', 'domain'];
function renderTerms() {
const terms = IX?.vocab.terms;
const have = TERMS.filter((k) => terms?.get(k)?.about);
for (const line of document.querySelectorAll('#terms, .manual-text [data-terms]')) {
line.hidden = !have.length;
if (!have.length) continue;
line.replaceChildren(el('span', { class: 'terms-lead' }, t('terms.lead')), ...have.map((k) => {
const x = terms.get(k);
const name = inChinese() ? (x.zh || k) : k;
return el('button', { type: 'button', class: 'term', 'aria-haspopup': 'dialog', onclick: (e) => showTerm(e.currentTarget, k, name, x.about) }, name);
}));
}
}
function showTerm(button, key, name, about) {
const pop = $('term-pop');
const kids = [el('strong', {}, name), el('p', {}, (inChinese() && IX.zh.get(about)) || about)];
if (key === 'domain') {
const tops = IX.vocab.domain.order.filter((c) => !c.includes('.') && c !== 'multi' && c !== 'unclassified');
kids.push(el('ul', {}, tops.map((c) => el('li', {}, domainName(c)))));
}
kids.push(el('a', { href: `#${MANUAL_VIEW}/glossary`, onclick: () => pop.hidePopover() }, t('terms.more')));
pop.replaceChildren(...kids);
if (typeof pop.showPopover !== 'function') return;
pop.showPopover();
// under the button, kept inside the window
const b = button.getBoundingClientRect();
const w = pop.offsetWidth, h = pop.offsetHeight;
pop.style.left = `${Math.max(8, Math.min(b.left, window.innerWidth - w - 8))}px`;
pop.style.top = `${b.bottom + 8 + h > window.innerHeight ? Math.max(8, b.top - h - 8) : b.bottom + 8}px`;
}
function renderStageFig() {
if (!IX) return;
const why = IX.stages ? [] : [stageWhy()];
drawStages(stageFigList(), { pick: F.stage, onPick: IX.stages ? pickStage : null, onPoint: (g) => FIELD?.front(g), why, rest: restWhat() });
}
/** A stage's band: the filter by stage set to it (with its layer), or cleared when it holds it already; the field and
* the panels of the screen show the result. */
function pickStage(id) {
setFilter({ stage: F.stage === id ? '' : id });
}
// ---- filter row: the options with their counts under the current selection (linked.js facets; #1216: an option that
// keeps no source is left out, unless it is the one selected, so it can be cleared), again in a new language and
// whenever the counts change; the listeners once
function option(value, text) { return el('option', { value }, text); }
/** Each filter's options as [value, name]; the first option, '' (all or any), is added by fillSelect. */
function filterOptions(key) {
const words = (prefix, values) => values.map((v) => [v, t(`${prefix}.${v}`)]);
switch (key) {
// #2006: the stages of the stage table, in its order
case 'stage': return (IX.stages?.list || []).map((x) => [x.id, stageName(x.id)]);
case 'domain': return domainRows().map((k) => [k, domainName(k)]);
case 'form': return formsByCount().map((k) => [k, formName(k)]);
case 'size': return words('filter.size', ['lt1g', '1g1t', 'ge1t', 'unknown']);
case 'train': return words('filter.permit', ['yes', 'cond']);
case 'pii': return words('filter.pii', ['none', 'marked', 'unknown']);
default: return [];
}
}
// #2422: the filters of the panel, each a select (the file types are checkboxes, fillExt), in the panel's order
const SELECTS = { stage: 'f-stage', domain: 'f-domain', form: 'f-form', size: 'f-size', train: 'f-train', pii: 'f-pii' };
const ANY = new Set(['size', 'train', 'pii']);
function fillSelect(key) {
if (key === 'ext') { fillExt(); return; }
const s = $(SELECTS[key]);
const f = UI.facets[key];
const text = (name, n) => (n == null ? name : t('filter.optCount', { name, n: fmtInt(n) }));
const opts = filterOptions(key).filter(([v]) => !f || f.m.get(v) > 0 || F[key] === v);
// #2422: a stage's count is of the sources of its own layer (linked.js), so All, over both layers, has none
clear(s).append(option('', text(t(ANY.has(key) ? 'filter.any' : 'filter.all'), key === 'stage' ? null : f?.total)),
...opts.map(([v, name]) => option(v, text(name, f ? f.m.get(v) || 0 : null))));
s.value = F[key] || '';
if (key === 'stage') {
// #2006: the selected stage's condition (#2066: in Chinese its text in the table of Chinese names, once the release
// has it) and the version of the stage table on hover; without stages of the commit the page reads, the filter is
// off and (#2422) one short line under it says why (stageWhy)
const st = IX.stages?.list.find((x) => x.id === F.stage);
s.disabled = !IX.stages;
s.title = IX.stages ? [inChinese() ? IX.zh.get(st?.about) : st?.about, t('stage.version', { v: IX.stages.version })]
.filter(Boolean).join('\n') : '';
$('f-stage-note').hidden = !!IX.stages;
$('f-stage-note').textContent = IX.stages ? '' : stageWhy();
}
}
/** #2422: why the commit the page reads has no stage for any source, in one short line (data.js stagesWhy): no stage
* figures in it, a release that has not counted its stage fields, or figures written before they listed the sources
* (#2518); the filter by stage and the first screen say it. */
const stageWhy = () => t(`stage.${IX.stagesWhy}`);
/** #1524: the File type facet, as CKAN's Formats facet: one checkbox per file type of the layer shown with its rows
* under the other filters (linked.js facet ext, from each source's by_ext), most rows first; a type that keeps no row
* is left out unless it is checked, so it can be cleared. Any number can be checked (F.ext). */
function fillExt() {
const box = $('f-ext');
const f = UI.facets.ext;
const on = extList();
const types = f ? [...f.m].filter(([k, n]) => k !== '*' && n > 0) : [];
for (const k of on) if (!types.some(([x]) => x === k)) types.push([k, f?.m.get(k) || 0]);
types.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
const toggle = (k, checked) => setFilter({ ext: [...on.filter((x) => x !== k), ...(checked ? [k] : [])].join(',') });
clear(box).append(...types.map(([k, n]) => el('label', {},
el('input', { type: 'checkbox', value: k, checked: on.includes(k), onchange: (e) => toggle(k, e.target.checked) }),
el('span', {}, t('filter.optCount', { name: `.${k}`, n: fmtInt(n) })))));
if (f && !types.length) box.append(el('span', { class: 'small muted' }, t('filter.extNone')));
// #2422: the fold's summary says how many types are checked
$('ext-on').textContent = on.length ? t('filter.on', { count: on.length, n: fmtInt(on.length) }) : '';
}
const fillFilterOptions = () => { Object.keys(SELECTS).forEach(fillSelect); fillExt(); };
function fillFilterRow() {
fillFilterOptions();
for (const [key, id] of Object.entries(SELECTS)) {
$(id).addEventListener('change', (e) => {
const patch = { [key]: e.target.value };
if (key === 'domain') patch.leaf = '';
setFilter(patch);
});
}
let timer = null;
$('f-q').addEventListener('input', (e) => { clearTimeout(timer); timer = setTimeout(() => setFilter({ q: e.target.value.trim() }), 200); });
$('f-clear').addEventListener('click', () => setFilter(Object.fromEntries(FILTER_KEYS.map((k) => [k, '']))));
$('filter-toggle').addEventListener('click', () => {
const open = !$('filters').classList.contains('open');
$('filters').classList.toggle('open', open);
$('filter-toggle').setAttribute('aria-expanded', String(open));
});
}
/** The filter row after a change: the values, the file types (their fold open while one is checked), and the count of
* the selection. */
function syncFilterRow() {
for (const [key, id] of Object.entries(SELECTS)) if ($(id).value !== F[key]) $(id).value = F[key];
fillExt();
if (document.activeElement !== $('f-q')) $('f-q').value = F.q;
if (F.ext) $('ext-fold').open = true;
const n = activeFilters().length;
$('f-clear').disabled = !n;
const count = UI.ids ? plural(F.kind === CORPUS ? 'n.corpus' : 'n.materialSet', UI.ids.size) : '';
$('filter-state').textContent = n ? [t('filter.on', { count: n, n: fmtInt(n) }), count].filter(Boolean).join(' · ') : t('filter.noneOn');
$('f-count').textContent = count;
}
/** The filters that are on, next to the charts: one removable chip each, and Clear when there are two or more. */
function renderChips() {
const box = $('sel-chips');
if (!box || !IX) return;
const chips = filterChips().map(([k, v, key]) => {
const off = key === 'domain' ? { domain: '', leaf: '' } : { [key]: '' };
return el('span', { class: 'chip sel-chip' }, el('span', { class: 'chip-k' }, k), el('b', {}, v),
el('button', { type: 'button', class: 'chip-x', 'aria-label': t('chip.remove', { k, v }), title: t('chip.remove', { k, v }),
onclick: () => setFilter(off, '') }, '×'));
});
// append writes an array or a null as text, so the chips go in one by one and Clear only when it is there
clear(box).append(...chips,
...(chips.length > 1 ? [el('button', { type: 'button', class: 'link', onclick: () => setFilter(Object.fromEntries(FILTER_KEYS.map((x) => [x, '']))) }, t('filter.clear'))] : []));
}
// ---- axes of the map
function numericCodeSort(a, b) {
const pa = a.replace(/^D/, '').split('.').map(Number);
const pb = b.replace(/^D/, '').split('.').map(Number);
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
const d = (pa[i] ?? -1) - (pb[i] ?? -1);
if (d) return d;
}
return a.localeCompare(b);
}
function domainRows() {
const tops = new Set();
for (const c of IX.vocab.domain.order) if (!c.includes('.') && c !== 'multi' && c !== 'unclassified') tops.add(c);
for (const s of IX.sources) for (const t of s.tops) tops.add(t);
const out = [...tops].sort(numericCodeSort);
if (IX.sources.some((s) => s.domainKind === 'multi') || IX.vocab.domain.names.has('multi')) out.push(MULTI);
// the row of sources not classified yet only while there are some
if (IX.sources.some((s) => s.domainKind === 'none') || F.domain === NONE) out.push(NONE);
return out;
}
let FORM_ORDER = null;
function formsByCount() {
if (FORM_ORDER) return FORM_ORDER;
const n = new Map();
for (const c of IX.vocab.form.order) if (c !== 'unclassified') n.set(c, 0);
for (const s of IX.sources) if (s.form) n.set(s.form, (n.get(s.form) || 0) + 1);
FORM_ORDER = [...n].sort((a, b) => b[1] - a[1] || formName(a[0]).localeCompare(formName(b[0]))).map(([k]) => k);
FORM_ORDER.push(NONE);
return FORM_ORDER;
}
function heatCols() {
const all = formsByCount().filter((k) => k !== NONE && IX.sources.some((s) => s.form === k));
const cols = all.length > HEAT_FORMS + 1 ? [...all.slice(0, HEAT_FORMS), OTHER] : all;
if (IX.sources.some((s) => !s.form)) cols.push(NONE);
return cols;
}
const heatColOf = (s, cols) => (cols.includes(formKeyOf(s)) ? formKeyOf(s) : OTHER);
/** #1216: the rows and columns of the map that hold a source of the layer shown (and the ones the filters name). */
function heatKeys() {
const cols = heatCols();
const S = IX.sources.filter((s) => s.kind === F.kind);
const rowKeys = new Set(S.flatMap(domainKeysOf));
const colKeys = new Set(S.map((s) => heatColOf(s, cols)));
return { rows: domainRows().filter((k) => rowKeys.has(k) || k === F.domain), cols: cols.filter((k) => colKeys.has(k) || k === F.form) };
}
// ---- 3. the map and the bars: the vgplot charts of linked.js. Each has its table (the rows behind it, under the same
// selection) and, in its Table view, the query behind that table for the dataset's SQL Console.
const FIGS = { heat: 'heat-fig', dom: 'dom-fig', form: 'form-fig' };
function contentWidth(id) {
const n = $(id);
const cs = getComputedStyle(n);
return Math.max(280, Math.floor(n.clientWidth - parseFloat(cs.paddingLeft) - parseFloat(cs.paddingRight)));
}
let drawnWidths = '';
function drawCharts() {
if (!L) return;
const m = metricNow();
const fmt = metricFmt();
const forms = formsByCount();
let formKeys = UI.allForms ? forms : forms.slice(0, FORM_BARS);
if (F.form && !formKeys.includes(F.form)) formKeys = [...formKeys, F.form];
const tableOf = (k) => (rows) => { UI.tables[k] = rows; if (tableShown(k)) renderChartFig(k); };
drawnWidths = Object.values(FIGS).map(contentWidth).join(' ');
// a release that changes no filter (a click on the empty part of a chart with nothing picked) does nothing
const pickDomain = (k) => { if (k !== F.domain) setFilter({ domain: k, leaf: '' }); };
const pickForm = (k) => { if (k !== F.form) setFilter({ form: k }); };
const pickCell = (p) => {
const next = p ? { domain: p.row, leaf: '', form: p.col === OTHER ? '' : p.col } : { domain: '', leaf: '', form: '' };
if (next.domain !== F.domain || next.form !== F.form) setFilter(next, 'map');
};
// the words the tooltips name their lines by; the layer shown, whose color the marks take
const words = { metricWord: metricWord(m), kind: F.kind };
UI.charts = L.draw({
dom: { field: 'dkey', keys: domainRows(), name: (k) => domainName(k), metric: m, fmt,
...words, keyWord: t('th.domain'), width: contentWidth('dom-fig'), onPick: pickDomain, onTable: tableOf('dom'), onError: linkError },
form: { field: 'formkey', keys: formKeys, name: (k) => formName(k), metric: m, fmt,
...words, keyWord: t('th.form'), width: contentWidth('form-fig'), onPick: pickForm, onTable: tableOf('form'), onError: linkError },
heat: { ...heatKeys(), rowName: (k) => domainName(k), colName: (k) => formName(k),
...words, rowWord: t('th.domain'), colWord: t('th.form'), metric: m, fmt,
width: contentWidth('heat-fig'), onTable: tableOf('heat'), onError: linkError, onPick: pickCell },
}, F);
for (const k of Object.keys(FIGS)) renderChartFig(k);
}
/** Draws the linked charts again when the width of their cards has changed (a rotated phone, a resized window); the
* charts of a source's detail have a width of their own (DETAIL_CHART_PX). */
function redrawOnResize() {
if (L && UI.charts && Object.values(FIGS).map(contentWidth).join(' ') !== drawnWidths) drawCharts();
}
function renderChartFig(k) {
const m = metricNow();
const title = k === 'heat' ? t('map.title', { metric: metricWord(m) }) : t(k === 'dom' ? 'chart.byDomain' : 'chart.byForm', { metric: metricWord(m) });
const keyCells = (r) => (k === 'heat' ? [domainName(r.k), formName(r.f)] : [k === 'dom' ? domainName(r.k) : formName(r.k)]);
const keyHeads = k === 'heat' ? [t('th.domain'), t('th.form')] : [k === 'dom' ? t('th.domain') : t('th.form')];
const forms = formsByCount();
const more = k === 'form' && forms.length > FORM_BARS
? () => el('button', { type: 'button', class: 'link', 'aria-expanded': String(UI.allForms),
onclick: () => { UI.allForms = !UI.allForms; drawCharts(); } }, UI.allForms ? t('common.fewer') : t('common.all', { n: fmtInt(forms.length) }))
: null;
figureWithToggle($(FIGS[k]), k, title, {
// the layer the chart draws, in its color, above the chart; the table holds that layer's columns only
chart: () => el('div', {}, kindKey([F.kind]), el('div', { class: 'plot-wrap' }, UI.charts[k])),
table: () => (F.kind === CORPUS
? table([...keyHeads, { label: t('th.corpora'), r: true }, { label: t('th.corpusRows'), r: true }],
UI.tables[k].map((r) => [...keyCells(r), fmtInt(num(r.n)), fmtInt(num(r.rows))]))
: table([...keyHeads, { label: t('th.materialSets'), r: true }, { label: t('th.readyUnits'), r: true }],
UI.tables[k].map((r) => [...keyCells(r), fmtInt(num(r.n)), fmtInt(num(r.units))]))),
sql: () => consoleSql(k),
}, more);
}
/** The heading of the map: what the selection holds, by the map's count, and the domains it covers. */
function renderMapHead(sel) {
const m = metricNow();
for (const b of $('layer-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.kind === F.kind));
for (const b of $('metric-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.metric === m));
$('metric-wrap').hidden = F.kind === CORPUS;
if (!UI.ids) return;
const total = sum(sel, (s) => metricOf(s, m));
const covered = new Set(sel.flatMap((s) => s.tops)).size;
const amount = m === 'sets' ? plural('n.materialSet', total)
: m === 'units' ? t('map.units', { count: total, n: fmtCompact(total) }) + uncounted(notCountedOf(sel))
: t('map.rows', { count: total, n: fmtCompact(total) });
$('map-h').textContent = !sel.length ? t('map.nothing') : covered ? t('map.head', { amount, domains: plural('n.domain', covered) }) : amount;
}
// ---- the queries behind the tables, for the dataset's SQL Console: over the catalog of the commit the page reads,
// filtered by the filters that filter that table (linked.js filtersOf), written over the catalog's own columns; the
// domain keys and the kind are those of data.js, in the catalog's words (unclassified, multi; the column kind, or the
// rule it is written by when the commit's catalog has none). A query counts the layer shown only, in its own measure:
// material sets and their units, or corpora and their rows. The licence filters read meta/sources.json, which is no
// table, so they name their sources by id (or the ids they leave out, whichever list is shorter).
const READY_SQL = 'coalesce(units < files, false)';
const kindSql = () => (IX.hasKind ? 'kind' : `CASE WHEN ${READY_SQL} THEN '${MATERIAL_SET}' ELSE '${CORPUS}' END`);
const DOMAIN_KEYS_SQL = "CASE WHEN domain IS NULL OR len(domain) = 0 OR list_contains(domain, 'unclassified') THEN ['unclassified'] "
+ "WHEN list_contains(domain, 'multi') THEN ['multi'] ELSE list_distinct([split_part(code, '.', 1) FOR code IN domain]) END";
const FORM_SQL = "coalesce(nullif(form, ''), 'unclassified')";
const sumsSql = () => (F.kind === CORPUS ? 'count(*) AS corpora, sum(files) AS corpus_rows' : 'count(*) AS material_sets, sum(units) AS units');
// #2422: the size classes of the deduplicated size, the one size the page shows (catalog column distinct_bytes)
const SIZE_SQL = { lt1g: 'distinct_bytes > 0 AND distinct_bytes < 1e9', '1g1t': 'distinct_bytes >= 1e9 AND distinct_bytes < 1e12',
ge1t: 'distinct_bytes >= 1e12', unknown: 'coalesce(distinct_bytes, 0) <= 0' };
// #2422: the personal-data mark in the catalog's words (pii.level)
const PII_SQL = { none: "pii.level = 'none_known'", marked: "pii.level IN ('present', 'suspected')",
unknown: "coalesce(pii.level, '') NOT IN ('none_known', 'present', 'suspected')" };
const catalogWord = (key) => (key === NONE ? 'unclassified' : key === MULTI ? 'multi' : key);
function idsSql(keep) {
const all = IX.sources.map((s) => s.id).sort();
const inside = all.filter((id) => keep.has(id));
const outside = all.filter((id) => !keep.has(id));
if (!inside.length) return 'false';
if (!outside.length) return 'true';
const list = (xs) => xs.map(sqlStr).join(', ');
return inside.length <= outside.length ? `source IN (${list(inside)})` : `source NOT IN (${list(outside)})`;
}
function filterSql(keys) {
const permitted = (col, v) => idsSql(new Set(IX.sources.filter((s) => (v === 'yes' ? s[col] === 'yes' : s[col] !== 'no')).map((s) => s.id)));
const sqlOf = {
domain: (v) => `list_contains(${DOMAIN_KEYS_SQL}, ${sqlStr(catalogWord(v))})`,
leaf: (v) => `list_contains(domain, ${sqlStr(v)})`,
form: (v) => `${FORM_SQL} = ${sqlStr(catalogWord(v))}`,
kind: (v) => `${kindSql()} = ${sqlStr(v)}`,
collection: (v) => `collection = ${sqlStr(v)}`,
status: (v) => `status = ${sqlStr(v)}`,
tier: (v) => `coalesce(license.tier, 'unknown') = ${sqlStr(v)}`,
size: (v) => SIZE_SQL[v],
pii: (v) => PII_SQL[v],
q: (v) => `contains(lower(concat_ws(' ', title, description, source)), ${sqlStr(v.toLowerCase())})`,
train: (v) => permitted('train', v),
redist: (v) => permitted('redist', v),
// #1524: the file types are in meta/sources.json (by_ext), no column of the catalog, so the sources are named by id
ext: (v) => idsSql(new Set(IX.sources.filter((s) => s.exts.some(([e]) => v.split(',').includes(e))).map((s) => s.id))),
// #2006: a source's stages are computed from the stage table (the commit's stage figures), no column of the catalog, so the
// sources are named by id
stage: (v) => idsSql(new Set(IX.sources.filter((s) => s.stages.includes(v)).map((s) => s.id))),
};
const parts = keys.filter((k) => F[k]).map((k) => `(${sqlOf[k](F[k])})`);
return parts.length ? `WHERE ${parts.join('\n AND ')}` : '';
}
function consoleSql(k) {
const cat = `read_parquet(${sqlStr(fileUrl(IX.sha, 'meta/catalog.parquet'))})`;
const lines = {
dom: () => [`SELECT domain_key, ${sumsSql()}`, `FROM ${cat} AS c, unnest(${DOMAIN_KEYS_SQL}) AS t(domain_key)`, filterSql(L ? L.filtersOf(k) : ['kind']),
'GROUP BY domain_key ORDER BY domain_key'],
form: () => [`SELECT ${FORM_SQL} AS form, ${sumsSql()}`, `FROM ${cat}`, filterSql(L ? L.filtersOf(k) : ['kind']), 'GROUP BY 1 ORDER BY count(*) DESC'],
heat: () => [`SELECT domain_key, ${FORM_SQL} AS form, ${sumsSql()}`, `FROM ${cat} AS c, unnest(${DOMAIN_KEYS_SQL}) AS t(domain_key)`,
filterSql(L ? L.filtersOf(k) : ['kind']), 'GROUP BY 1, 2 ORDER BY 1, 2'],
}[k]();
// #1633: these queries read the catalog as published and say so, as the rows query does (sql.noProfile)
return [`-- ${REPO} @ ${IX.sha}`, `-- ${t('sql.catalogNoProfile')}`, ...lines.filter(Boolean)].join('\n') + ';';
}
// ---- 4. sources; the column labels are col.<key> in the locale files
const COLS = [
{ key: 'title' },
{ key: 'domain' },
{ key: 'form' },
{ key: 'units', r: true },
{ key: 'size', r: true },
{ key: 'tier' },
{ key: 'fit' },
];
function fitScore(s) { const f = fitOf(s); return sum(FIT, (x) => ({ yes: 2, part: 1, unknown: 0.5, no: 0 })[f[x.k]]); }
function sortValue(s, k) {
switch (k) {
case 'title': return titleOf(s).toLowerCase();
case 'domain': return sourceDomainText(s).toLowerCase();
case 'form': return formName(formKeyOf(s)).toLowerCase();
// units of material sets above rows of corpora (never compared as one count): a corpus has fewer than 1e12 rows
case 'units': return s.isTask ? 1e12 + s.readyUnits : s.rawRows;
case 'size': return s.distinct ?? -1;
case 'tier': return tierName(s.tier).toLowerCase();
case 'fit': return fitScore(s);
default: return 0;
}
}
/** #2422: a source's size in the list (meta/size_figures.json sources), its deduplicated size, the one size the page
* shows, and, for its rows without a recorded size, their estimate (or their count when too few were read for one),
* labelled; the size check and the estimate's range on hover (the cell's title). */
function sizeCell(s) {
const g = s.size;
if (!g) return t('common.unknown');
return [el('b', {}, sizeOf(g)), g.unsized_rows ? el('span', { class: 'size-sub' }, g.estimate
? t('size.barEstimate', { size: fmtBytes(g.estimate.bytes) }) : t('size.barUnsized', { rows: fmtCompact(g.unsized_rows) })) : null];
}
// #364, #1158: the same words as docs/sources.md of the index (artpool/projection_templates/text.json, size_text), in
// the locale files as size.*
/** The size note under the list: the size of the layer shown (meta/size_figures.json, IX.sizes; #2422: its deduplicated
* size), with the estimate for its rows without a recorded size. */
function sizeNote() {
const g = IX.sizes?.[F.kind];
if (!g) return t('size.none');
const unsized = !g.unsized_rows ? null : g.estimate ? t('size.unsizedEst', { size: fmtBytes(g.estimate.bytes) })
: t('size.unsizedNone', { rows: fmtCompact(g.unsized_rows) });
return t('size.note', { layer: layerName(F.kind), line: [t('size.noteSize', { size: sizeOf(g) }), unsized].filter(Boolean).join(' \u00b7 ') });
}
/** #2422: how sizes are measured, folded under the list: which files count as one (by which hash, across what scope)
* and where the size before deduplication can still be read, with the index docs' explanation behind its link. */
function sizeHow() {
return el('details', { class: 'size-how' }, el('summary', {}, t('size.how')),
el('p', {}, t('size.howText'), ' ', el('a', { href: t('url.docsSources') }, t('size.howMore'))));
}
/** The place of a material set's collection in the vocabulary's order (one without a collection last). */
function collectionRank(s) {
const i = IX.vocab.collections.findIndex((c) => c.code === s.collection);
return i < 0 ? IX.vocab.collections.length : i;
}
/** The list of the selection, one layer: material sets grouped by collection (a header row per collection, in the
* vocabulary's order), corpora as one list, or (#1633) grouped the same way when a collection lists some of them;
* the heading names the layer shown only. */
function renderSources(sel) {
if (!UI.ids) return;
const dir = F.dir === 'asc' ? 1 : -1;
const sets = F.kind !== CORPUS;
const grouped = sets || sel.some((s) => s.collection);
const byColl = (a, b) => (grouped ? collectionRank(a) - collectionRank(b) : 0);
const list = [...sel].sort((a, b) => {
const x = sortValue(a, F.sort);
const y = sortValue(b, F.sort);
return byColl(a, b) || (x < y ? -1 : x > y ? 1 : 0) * dir || titleOf(a).localeCompare(titleOf(b));
});
$('sources-h').textContent = !sel.length ? t('sources.none')
: sets ? t('sources.headSets', { sets: plural('n.materialSet', sel.length) }) : t('sources.headCorpora', { corpora: plural('n.corpus', sel.length) });
const body = clear($('sources-body'));
if (!sel.length) return;
const shown = UI.showAll ? list : list.slice(0, FIRST_ROWS);
const opened = list.find((s) => s.id === F.open);
if (opened && !shown.includes(opened)) shown.push(opened);
shown.sort(byColl);
const inColl = new Map();
for (const s of list) inColl.set(s.collection, (inColl.get(s.collection) || 0) + 1);
// #1216: the forms of each group (a collection's material sets); a group of one form leaves its form cells out, its
// name is the group's
const formsIn = new Map();
for (const s of list) formsIn.set(s.collection, (formsIn.get(s.collection) || new Set()).add(formKeyOf(s)));
const oneForm = (s) => sets && formsIn.get(s.collection).size === 1;
// #1216: the kind mark only when the list holds both kinds
const bothKinds = new Set(shown.map((s) => s.kind)).size > 1;
// a label of a cell on a phone, with its colon from the locale files
const label = (text) => t('sources.label', { label: text });
const head = el('tr', {}, COLS.map((c) => {
const on = F.sort === c.key;
return el('th', { scope: 'col', class: `${c.r ? 'r' : ''} col-${c.key}`, 'aria-sort': on ? (F.dir === 'asc' ? 'ascending' : 'descending') : null },
el('button', { type: 'button', onclick: () => {
const d = on ? (F.dir === 'asc' ? 'desc' : 'asc') : (c.r || c.key === 'fit' ? 'desc' : 'asc');
F.sort = c.key; F.dir = d; writeUrl(); renderSources(selected());
} }, t(c.key === 'units' && !sets ? 'col.rows' : `col.${c.key}`), on ? (F.dir === 'asc' ? ' ↑' : ' ↓') : ''),
c.key === 'fit' ? fitHeads() : null);
}));
const tbody = el('tbody');
let group;
for (const s of shown) {
if (grouped && s.collection !== group) {
group = s.collection;
// #2422: a collection's heading among material sets is a toggle of the filter by collection, which the panel
// shows no control for
const name = group ? collectionName(group) : t('name.noCollection');
const cur = F.collection === group;
tbody.append(el('tr', { class: 'group-row' }, el('th', { colspan: String(COLS.length), scope: 'colgroup' },
sets && group ? el('button', { type: 'button', class: 'group-btn', 'aria-pressed': String(cur), title: t(cur ? 'coll.showing' : 'coll.open'),
onclick: () => setFilter({ collection: cur ? '' : group }) }, cur ? icon('check', { size: 16 }) : null, name) : name,
el('span', { class: 'small' }, plural(sets ? 'n.materialSet' : 'n.corpus', inColl.get(group))))));
}
const open = F.open === s.id;
const toggle = () => openSource(open ? '' : s.id);
const previewing = UI.preview.has(s.id);
const preview = () => { if (previewing) UI.preview.delete(s.id); else UI.preview.add(s.id); renderSources(selected()); };
const isPicked = pickIds().includes(s.id);
tbody.append(el('tr', { class: `row${open ? ' open' : ''}${isPicked ? ' picked' : ''}` },
el('td', { class: 'title' },
el('button', { type: 'button', 'aria-expanded': String(open), 'aria-controls': `d-${s.id}`, onclick: toggle },
el('span', { class: 'caret', 'aria-hidden': 'true' }, open ? '▾' : '▸'), titleOf(s)),
bothKinds ? el('span', { class: `kind-tag kind-tag-${s.kind}` }, el('span', { class: `sw kind-${s.kind}`, 'aria-hidden': 'true' }), kindOne(s.kind)) : null,
// #1633: a flagged source (catalog column status; an upstream that serves none of its sampled files is one) is
// marked as not usable as it is, its reasons on hover and in its detail
s.status === 'flagged' ? el('span', { class: 'small problem-text', title: s.reasons.map(zhOf).join('; ') }, ` ${t('status.flagged')}`) : null,
el('span', { class: 'title-foot' }, el('code', { class: 'src-id' }, s.id),
el('button', { type: 'button', class: 'link preview-btn', 'aria-expanded': String(previewing), 'aria-controls': `p-${s.id}`, onclick: preview },
previewing ? t('preview.hide') : t('preview.show')),
el('button', { type: 'button', class: 'link pick-btn', 'aria-pressed': String(isPicked), onclick: () => togglePick(s.id) },
isPicked ? [icon('check', { size: 14 }), t('pick.picked')] : t('pick.add')))),
el('td', { 'data-label': label(t('col.domain')), class: 'cat-cell' }, sourceDomainText(s)),
oneForm(s) ? el('td', { class: 'form-cell form-same' })
: el('td', { 'data-label': label(t('col.form')), class: 'form-cell' }, formIcon(formKeyOf(s), { size: 16 }), formName(formKeyOf(s))),
el('td', { class: 'r num', 'data-label': label(s.isTask ? t('sources.unitsLabel') : t('sources.rowsLabel')) }, s.isTask
? (s.readyUnits == null ? t('n.notCounted') : [fmtInt(s.readyUnits), el('span', { class: 'unit-word' }, t('sources.unitWord', { count: s.readyUnits }))])
: el('span', { class: 'muted' }, fmtCompact(s.rawRows), el('span', { class: 'unit-word' }, t('sources.corpusWord', { count: s.rawRows })))),
el('td', { class: 'r num size-cell', 'data-label': label(t('col.size')), title: s.size ? sizeLine(s.size, false) : null }, sizeCell(s)),
el('td', { 'data-label': label(t('col.tier')) }, tierName(s.tier)),
el('td', { class: 'fit-cell' }, fitStrip(s))));
// the source's first rows from meta/preview.parquet of the page's commit, read once opened
if (previewing) {
tbody.append(el('tr', { class: 'preview-row' }, el('td', { colspan: String(COLS.length), id: `p-${s.id}` }, previewTable(s))));
}
if (open) tbody.append(el('tr', { class: 'detail-row' }, el('td', { colspan: String(COLS.length), id: `d-${s.id}` }, detail(s))));
}
// #1216: the fit legend, folded into one line on a phone (its summary shows there only), open on a wider screen
const wide = !matchMedia(PHONE).matches;
body.append(el('details', { class: 'list-key-fold', open: wide || UI.fitKeyOpen, ontoggle: (e) => { if (!wide) UI.fitKeyOpen = e.currentTarget.open; } },
el('summary', {}, t('sources.aboutFit')),
el('div', { class: 'list-key' }, kindKey([F.kind]), el('span', { class: 'small' }, t('sources.fitKey')), FIT.map((x) => el('span', { class: 'fit-name' }, icon(x.icon, { size: 14 }), fitLabel(x.k))), fitKey())));
body.append(el('table', { class: 'src-table' }, el('caption', { class: 'visually-hidden' }, kindName(F.kind)), el('thead', {}, head), tbody));
body.append(el('div', { class: 'small size-note' }, el('p', {}, sizeNote()), sizeHow()));
if (list.length > FIRST_ROWS) {
body.append(el('p', { class: 'more' }, el('button', { type: 'button', class: 'link', 'aria-expanded': String(UI.showAll),
onclick: () => { UI.showAll = !UI.showAll; renderSources(selected()); } }, UI.showAll ? t('common.first', { n: fmtInt(FIRST_ROWS) }) : t('common.all', { n: fmtInt(list.length) }))));
}
}
// ---- the first rows of one source (the index's meta/preview.parquet, read in the page's DuckDB-WASM by linked.js):
// every column that has a value in one of them, what a row records first (PREVIEW_FIRST, #1256), then the others in
// the file's order, one cell each, cut to one line (the whole value on hover; #1302: an address in its middle)
const PREVIEW = 'meta/preview.parquet';
const PREVIEW_FIRST = ['uri', 'size_bytes', 'sha256', 'license_spdx'];
const cellText = (v) => (v == null ? '' : v instanceof Uint8Array ? hex(v) : String(v));
/** The preview rows of source s ('loading' until read, {error} when they could not be); once read, its preview table
* and its detail are drawn again (#1197: the detail's example is the first unit file among them). */
function previewOf(s) {
if (!UI.previewRows.has(s.id)) {
UI.previewRows.set(s.id, 'loading');
const redraw = (v) => {
UI.previewRows.set(s.id, v);
const cell = document.getElementById(`p-${s.id}`);
if (cell) clear(cell).append(previewTable(s));
const box = document.getElementById(`d-${s.id}`);
if (box) clear(box).append(detail(s));
};
L.previewRows(() => wholeFile(IX.sha, PREVIEW), s.id).then(redraw, (e) => redraw({ error: e.message }));
}
return UI.previewRows.get(s.id);
}
/** #1302: an address cut in the middle, its host at the start and its file name at the end (a long name cut in its
* own middle); the address itself when that is not shorter. */
const NAME_MAX = 40;
function shortUri(v) {
const m = v.match(/^(?:[a-z][a-z0-9+.-]*:\/\/)?([^/]+)\/.*\/([^/]*)$/i);
if (!m) return v;
const name = m[2].length > NAME_MAX ? `${m[2].slice(0, 14)}\u2026${m[2].slice(-(NAME_MAX - 15))}` : m[2];
const out = `${m[1]}/\u2026/${name}`;
return out.length < v.length ? out : v;
}
function previewTable(s) {
const rows = previewOf(s);
if (rows === 'loading') return el('p', { class: 'small' }, t('preview.loading'));
if (rows.error) return el('p', { class: 'small problem-text' }, t('preview.failed', { error: rows.error }));
if (!rows.length) return el('p', { class: 'small' }, t('preview.none'));
const keys = Object.keys(rows[0]);
const cols = [...PREVIEW_FIRST.filter((c) => keys.includes(c)), ...keys.filter((c) => !PREVIEW_FIRST.includes(c))]
.filter((c) => rows.some((r) => cellText(r[c]) !== ''));
const tbl = table(cols, rows.map((r) => cols.map((c) => {
const v = cellText(r[c]);
return c === 'uri' ? el('span', { class: 'uri', title: v }, shortUri(v)) : el('span', { title: v }, v);
})), t('preview.title', { n: fmtInt(rows.length) }));
tbl.classList.add('preview-table');
return el('div', { class: 'table-wrap sample' }, tbl);
}
// ---- the detail of one source, in the group's order; the first rows are read when it opens
const hex = (v) => (v instanceof Uint8Array ? [...v].map((b) => b.toString(16).padStart(2, '0')).join('') : v == null ? '' : String(v));
const firstSentence = (t) => {
const m = String(t).match(/^.*?[.!?](\s|$)/);
const s = m ? m[0].trim() : String(t);
return s.length > 120 ? `${s.slice(0, 117).replace(/\s+\S*$/, '')} …` : s;
};
function metaOf(r) { try { return JSON.parse(r.extra || '{}').source_meta || {}; } catch { return {}; } }
/** A row of a unit: one that carries a unit_group, or a row of no collection (every row of a corpus); the README.md and
* LICENSE of a collection source belong to the source (#1633: no roles). */
const isUnit = (m) => !m.campaign || !!m.unit_group;
/** An http(s) address as a URL, else null. */
function webUrl(u) { try { const x = new URL(u); return /^https?:$/.test(x.protocol) ? x : null; } catch { return null; } }
// pool_source.origin of a material set's rows: its one upstream, or several in the form
// multi-publisher:<source name>(<upstream 1>,<upstream 2>,...) (artpool/pool/selection.py Source.origin)
const MULTI_ORIGIN = /^multi-publisher:[^(]*\((.*)\)$/;
const upstreamsOf = (origin) => { const m = origin.match(MULTI_ORIGIN); return m ? m[1].split(',') : [origin]; };
/** The upstreams of the rows (addresses, which are no publishers), distinct, in order of first appearance. */
const upstreamsOfRows = (metas) => [...new Set(metas.flatMap((m) => (m.pool_source?.origin ? upstreamsOf(String(m.pool_source.origin)) : [])))];
/** The publisher names of the rows (the publisher of their pool_fields, the unit's line of the pool manifest),
* distinct, in order of first appearance. */
const publisherNames = (metas) => [...new Set(metas.flatMap((m) => (Array.isArray(m.pool_fields)
? m.pool_fields.filter(([k, v]) => k === 'publisher' && v).map(([, v]) => String(v)) : [])))];
// the summary sentences artpool admit.py writes for a material set (readme_forms, what): "<n> files published by ...",
// "<n> units (<m> files) from <k> publishers." and the like; #2065: without counts since, "Files published by ...",
// "Units from several publishers.", "Files from the upstream named in the provenance line.", and "Files from one
// publisher." where admit.README_COUNTS rewrote an older one. Not the upstream's words, so the page builds its own
// sentence in its language instead (#1216); the same pattern as the projection's (index_projection.ADMIT_WHAT)
const ADMIT_WHAT = /^(?:\d+ (?:units? \(\d+ files?\)|files?) (?:published by |from )|(?:Files|Units) (?:published by |from (?:one|several) publishers?\.|from the upstream named in ))/;
/** #1197: where a source comes from, in words: how many publishers when there are several (#1256: publishers, the
* catalog's count over all its rows; never its addresses), then its main site, the host most of the addresses are on
* (upstream: the upstreams of its rows, else the addresses of its rows), as one link with the host as its text; an
* upstream that is no address is written as it is. -> nodes, [] when none. */
function fromWords(addresses, upstream, publishers) {
const sites = new Map();
for (const u of addresses.map(webUrl).filter(Boolean)) {
const x = sites.get(u.host) || { origin: u.origin, n: 0 };
x.n++;
sites.set(u.host, x);
}
const [host, main] = [...sites].sort((a, b) => b[1].n - a[1].n)[0] || [];
const site = host ? el('a', { href: `${main.origin}/`, rel: 'noopener' }, host) : upstream ? addresses[0] : null;
if (!site) return [];
return publishers > 1 ? [`${t('detail.publishers', { count: publishers, n: fmtInt(publishers) })} `, site] : [site];
}
function upstream(uri) {
if (!uri) return null;
if (/^https?:\/\//.test(uri)) return uri;
if (uri.startsWith('campaign_files/')) return fileUrl(IX.sha, uri);
const s3 = uri.match(/^s3:\/\/([^/]+)\/(.*)$/);
if (s3) return `https://${s3[1]}.s3.amazonaws.com/${s3[2]}`;
return null;
}
// #2588: the declared upstream's values of one field, joined by ' | ' while the source reads several upstreams
// (catalog_rules.SEVERAL)
const SEVERAL = ' | ';
/** #2588: a source's declared upstream (catalog columns upstream_kind, upstream_agent, upstream_entry), one line per
* upstream: its kind in the page's language, its agent, and its entry address, a link when it is an http(s) one. */
function upstreamLines(u) {
const kinds = u.kind.split(SEVERAL), agents = u.agent.split(SEVERAL), entries = u.entry.split(SEVERAL);
return el('span', { class: 'upstreams' }, entries.map((e, i) => {
const kind = kinds[i] ?? kinds[0];
const href = webUrl(e);
return el('span', { class: 'upstream' }, el('b', {}, t(`upstreamKind.${kind}`, { defaultValue: kind })),
el('span', {}, agents[i] ?? agents[0]),
href ? el('a', { href: href.href, rel: 'noopener', class: 'mono' }, e) : el('span', { class: 'mono' }, e));
}));
}
// #2066: the sizes of a source's files and the shares of its permits, drawn by Observable Plot (plots.js) at the width
// of the detail's column (CSS scales them down where it is narrower)
const DETAIL_CHART_PX = 360;
/** #1197: the sizes of a source's files (sources.json size_quantiles_upper_bytes, each plus one byte) on a log axis:
* the band from p10 to p90, the median and the fetch tool's 20 MB, labelled (plots.js strip). */
function sizeStrip(q) {
const v = (k) => num(q[k]) + 1;
const qs = { p10: v('p10'), p50: v('p50'), p90: v('p90'), p99: v('p99') };
return plotted(() => P.strip(qs, { width: DETAIL_CHART_PX, limit: MAX_FILE, median: t('detail.median', { size: fmtBytes(qs.p50) }),
limitText: '20 MB', label: t('detail.stripAria', { median: fmtBytes(qs.p50), p99: fmtBytes(qs.p99) }) }), 'sk sk-strip');
}
/** #1228: the shares of total that parts [{value, color, label, note}] hold as one bar (plots.js shares), each part's
* count, label, share and note on hover, and its legend: each part's label, its share and its note (what the share
* counts and the licenses behind it). */
function shareBar(parts, total) {
const shown = parts.filter((p) => p.value > 0);
const said = (p) => `${p.label} ${fmtPct(p.value, total)}${p.note ? ` ${p.note}` : ''}`;
return el('div', {},
plotted(() => P.shares(shown.map((p) => ({ value: p.value, color: p.color, title: `${fmtInt(p.value)}\n${said(p)}` })),
{ width: DETAIL_CHART_PX, label: shown.map(said).join(', ') }), 'sk sk-strip'),
el('div', { class: 'stack-labels' }, shown.map((p) => el('span', {},
el('span', { class: 'sw', style: { background: p.color } }), `${p.label} `, el('b', {}, fmtPct(p.value, total)),
p.note ? ` ${p.note}` : null))));
}
function item(iconName, label, ...value) {
return el('div', { class: 'd-item' }, el('dt', {}, icon(iconName, { size: 18 }), label), el('dd', {}, ...value));
}
function detail(s) {
const st = UI.detail.get(s.id) || {};
UI.detail.set(s.id, st);
const info = s.info || {};
const lic = info.license || {};
const redraw = () => { const cell = document.getElementById(`d-${s.id}`); if (cell) clear(cell).append(detail(s)); };
// #1769: a data file may hold rows of other sources too, so every read keeps this source's rows by the column source;
// #1792: and reads only the source's row range in the file where the catalog records it (rowsOf)
const ownRows = (rows) => rows.filter((r) => r.source === s.id);
if (!st.sample && s.files.length) {
st.sample = 'loading';
const cols = s.isTask ? [...ROW_COLUMNS, 'extra'] : ROW_COLUMNS;
const unitFile = s.files[0];
readColumns(fileUrl(IX.sha, unitFile.path), cols, rowsOf(unitFile, FIRST_ROWS))
.then((r) => { st.sample = ownRows(r.rows); st.file = unitFile.path; redraw(); }, (e) => { st.sample = { error: e.message }; redraw(); });
}
// #1228: the licences behind the shares of a material set, from the licence columns of all its files; #1266: only
// for a catalog without the column license_counts
if (s.isTask && !s.licenceCounts && !st.licences && s.files.length) {
st.licences = 'loading';
Promise.all(s.files.map((f) => readColumns(fileUrl(IX.sha, f.path), [...LICENCE_COLUMNS, 'source'], rowsOf(f))))
.then((parts) => { st.licences = parts.flatMap((p) => ownRows(p.rows)); redraw(); }, (e) => { st.licences = { error: e.message }; redraw(); });
}
if (!st.batches) {
st.batches = 'loading';
loadBatchInfo(s).then((b) => { st.batches = b; redraw(); }, (e) => { st.batches = { error: e.message }; redraw(); });
}
const rows = Array.isArray(st.sample) ? st.sample : [];
const metas = rows.map(metaOf);
const fit = fitOf(s);
const box = el('div', { class: 'detail', 'data-done': st.sample !== 'loading' && st.batches !== 'loading' && st.licences !== 'loading' ? 'true' : null });
// fit at a glance
box.append(el('ul', { class: 'fit-row' }, FIT.map((x) => el('li', { class: `fit-item fit-${fit[x.k]}-item` },
icon(x.icon, { size: 18 }), el('span', {}, fitLabel(x.k)), fitDot(fit[x.k], fitLabel(x.k))))));
const dl = el('dl', { class: 'd-list' });
// 1 what it is: the description its upstream wrote; #1197, #1216: else its counts in the page's language (#1633: a
// material set's units and files, the catalog's counts the index's card and docs/sources.md give in the same words),
// with the publisher names of its rows when the rows read are all its rows, else (#1256) the catalog's count of its
// publishers over all its rows
const ups = upstreamsOfRows(metas);
const names = publisherNames(metas);
const allRead = rows.length >= s.rows;
const counted = s.isTask ? t('n.unitsFiles', { units: unitsText(s.readyUnits), files: plural('n.file', s.rows) }) : plural('n.corpusRow', s.rawRows);
// #2066: in the page's language (descriptionOf); the sentence admission writes (ADMIT_WHAT) is recognized in English
const own = s.description && !ADMIT_WHAT.test(s.description) ? descriptionOf(s) : '';
const text = own
|| (names.length && allRead ? t('detail.countsBy', { what: counted, publishers: fmtAnd(names) })
: s.publishers ? t('detail.countsFrom', { what: counted, publishers: plural('n.publisher', s.publishers) })
: t('detail.counts', { what: counted }));
dl.append(item('doc', t('detail.what'), text.length > 160 ? el('details', {}, el('summary', {}, firstSentence(text)), el('p', {}, text)) : text));
// 2 a material set's units and files, what each row of a corpus points to (#2065: a corpus is counted in rows)
const kindCount = (k) => {
const n = num(s.info.unit[k]);
return t(UNIT_KIND[k] ? `unitCount.${UNIT_KIND[k]}` : 'unitCount.code', { count: n, n: fmtCompact(n), kind: k });
};
const unitWhat = s.isTask
? [el('span', { class: 'chips' }, el('span', { class: 'chip' }, unitsText(s.readyUnits)), el('span', { class: 'chip' }, plural('n.file', s.rows)))]
: [el('span', {}, sentence(fmtOr(s.kinds.map((k) => (UNIT_KIND[k] ? t(`unitKind.${UNIT_KIND[k]}`) : k))))),
el('span', { class: 'chips' }, s.kinds.map((k) => el('span', { class: 'chip' }, kindCount(k))))];
dl.append(item('rows', t(s.isTask ? 'detail.files' : 'detail.oneUnit'), ...unitWhat));
// 3 where it comes from (#1197: in words): the number of its publishers (#1256: the catalog's) and the main site of
// its upstreams, else of the addresses of its rows
const from = ups.length ? fromWords(ups, true, s.publishers) : fromWords(rows.map((r) => r.uri), false, s.publishers);
dl.append(item('pin', t('detail.from'), st.sample === 'loading' ? '…' : from.length ? el('span', {}, ...from) : t('detail.notRecorded')));
// #2588: the upstream it declares, one dataset or one catalog with its agent and entry address (each of them while
// it still reads several)
if (s.upstream) dl.append(item('pin', t('detail.upstream'), upstreamLines(s.upstream)));
// 4 self-contained; #1216: the fit chip's answer (fit.alone), with the files of the source a material set of a
// collection comes with (a license and a readme leave a unit usable alone): its rows read without a unit
const comps = [...new Set(rows.filter((r, i) => !isUnit(metas[i])).map((r) => r.uri.split('/').pop()))];
const compChips = () => el('span', { class: 'chips' }, (comps.length ? comps : ['LICENSE', 'README.md']).map((c) => el('span', { class: 'chip mono-chip' }, c)));
const ofCollection = !!(info.campaign || info.extended_by);
dl.append(item('layers', t('detail.alone'), s.isTask
? (fit.alone !== 'yes' ? [el('span', {}, t('detail.withCompanions')), compChips()]
: ofCollection ? [el('span', {}, t('detail.alsoWith')), compChips()] : t('detail.yes'))
: fit.alone === 'yes' ? t('detail.eachFile') : t('detail.container')));
// 5 tools to open it
const exts = Object.entries(info.by_ext || {}).map(([k, v]) => [k, num(v.rows) || 0]).sort((a, b) => b[1] - a[1]).map(([k]) => k);
const tools = [];
for (const e of exts) { const tool = toolOf(e); if (tool && !tools.some((x) => x[1] === tool)) tools.push([e, tool]); if (tools.length >= 3) break; }
dl.append(item('tool', t('detail.opensWith'), tools.length ? el('span', { class: 'chips' }, tools.map(([e, tool]) => el('span', { class: 'chip' }, el('span', { class: 'mono' }, e === '(none)' ? t('detail.noExt') : `.${e}`), ` ${tool}`))) : t('detail.anyEditor')));
// 6 licence
const lr = info.license_rows || {};
// #1228: the licences behind a share: a material set's from its licence columns (an empty column: the batch
// licence), a corpus's the source's licence when the share is its tier, else each row's own
const behind = (field, k) => {
const read = s.isTask && Array.isArray(st.licences) && st.licences.length ? st.licences : null;
if (!read) {
if (s.isTask && st.licences === 'loading') return ['\u2026'];
return lic[field] === k && lic.name ? [lic.name] : [t('detail.ownLicences')];
}
const names = new Set();
for (const r of read) {
if ((r[`license_${field}`] ?? lic[field]) !== k) continue;
const name = r.license_spdx ?? (lic[field] === k ? lic.name : null);
if (name) names.add(String(name));
}
return [...names].sort();
};
const permitLabel = (k, names) => (k === 'conditional' && CONDITION[s.tier] && names.length === 1 && names[0] === lic.name
? t(CONDITION[s.tier]) : t(`permit.${k}`));
const permit = (field) => PERMIT_ORDER.map((k) => {
const names = behind(field, k);
return { k, value: num(lr[field]?.[k]?.rows) || 0, color: PERMIT_COLOR[k], label: permitLabel(k, names),
note: t(`detail.shareOf${s.isTask ? 'Set' : 'Corpus'}${names.length ? '' : 'Only'}`, { licences: names.join(', ') }) };
});
// #1197: one line when every row allows both, the two bars when its rows differ
const allAllowed = (field) => permit(field).every((x) => (x.k === 'yes' ? x.value > 0 : !x.value));
// #1266: a material set's licence by kind of file, as the projection counted it (license_counts): per kind its
// files, the permit classes for training and for redistribution as "x / y files" with the licences behind each, or
// one phrase when all its files allow both
const spdxOf = (xs) => [...new Set(xs.map((x) => x.spdx).filter(Boolean))].sort();
const kindLine = (kind) => {
const items = s.licenceCounts.filter((x) => x.kind === kind);
const all = sum(items, (x) => x.files);
if (!all) return null;
const head = el('b', {}, t(`detail.kindFiles.${kind}`, { count: all, n: fmtInt(all) }));
if (items.every((x) => x.training_ok === 'yes' && x.redistribution_ok === 'yes')) {
const names = spdxOf(items);
return el('span', { class: 'lic-kind' }, head, el('span', { class: 'small' }, names.length ? t('detail.kindAllowed', { licences: names.join(', ') }) : t('detail.allAllowed')));
}
const classes = (field, label) => el('span', { class: 'small' }, t('sources.label', { label }), PERMIT_ORDER.map((k) => {
const xs = items.filter((x) => x[field] === k);
const n = sum(xs, (x) => x.files);
const names = spdxOf(xs);
return n ? t(names.length ? 'detail.permitFiles' : 'detail.permitFilesOnly', { permit: permitLabel(k, names), n: fmtInt(n), of: fmtInt(all), licences: names.join(', ') }) : null;
}).filter(Boolean).join('; '));
return el('span', { class: 'lic-kind' }, head, classes('training_ok', t('detail.training')), classes('redistribution_ok', t('detail.shipping')));
};
dl.append(item('training', t('detail.licence'), el('span', { class: 'lic' }, el('b', {}, tierName(s.tier)), el('span', { class: 'small' }, lic.name || s.spdx || '')),
s.isTask && s.licenceCounts ? el('span', { class: 'lic-kinds' }, FILE_KINDS.map(kindLine))
: allAllowed('training_ok') && allAllowed('redistribution_ok') ? el('span', { class: 'small' }, t('detail.allAllowed'))
: el('span', { class: 'lic-bars' },
el('span', { class: 'mini-title' }, t('detail.training')), shareBar(permit('training_ok'), s.rows),
el('span', { class: 'mini-title' }, t('detail.shipping')), shareBar(permit('redistribution_ok'), s.rows))));
// #2006: the training stages one of its units passes, from the stage table (the commit's stage figures), when the page has the
// stages of its commit, with the version of the table they were computed with
if (IX.stages) {
dl.append(item('layers', t('detail.stages'), s.stages.length
? el('span', { class: 'chips' }, s.stages.map((id) => el('span', { class: 'chip' }, stageName(id))))
: t('detail.stagesNone'), el('span', { class: 'small' }, t('stage.version', { v: IX.stages.version }))));
}
// 7 size and units
const q = info.size_quantiles_upper_bytes || {};
const amounts = s.isTask ? `${unitsText(s.readyUnits)} · ${plural('n.file', s.rows)}` : plural('n.corpusRow', s.rawRows);
// #1158: the source's figures of meta/size_figures.json; #2422: its size first, the deduplicated one
dl.append(item('size', t('detail.size'), el('span', { class: 'lic' }, el('b', {}, s.size ? sizeOf(s.size) : t('size.none')),
el('span', { class: 'small' }, amounts), s.size ? el('span', { class: 'small size-parts' }, sizeParts(s.size, false).join(' \u00b7 ')) : null),
q.p50 ? sizeStrip(q) : null));
// #1577: whether its upstream still serves the files, from the sampled rows read in meta/sizes.parquet (catalog
// column fetch_check)
dl.append(item('get', t('detail.fetch'), fetchText(s.fetch)));
// #1928: the release its rows come from, and why a newer batch of it is held back (the release notes' reason)
if (s.releasedIn) dl.append(item('get', t('detail.releasedIn'), el('span', { class: 'mono' }, s.releasedIn)));
// #2674: a hold reads as one sentence in the page's language, the release notes' words (up to 600 characters of
// English) in a closed block; #2066: those words in the page's language where the table of Chinese names holds them
if (s.heldBack) dl.append(item('get', t('detail.heldBack'), el('span', {}, t('detail.heldBackText')),
el('details', {}, el('summary', {}, t('detail.heldBackNotes')), el('p', { class: 'problem-text' }, zhOf(s.heldBack)))));
// #1633: why a flagged source is not usable as it is (catalog column status_reasons)
if (s.status === 'flagged') dl.append(item('get', t('status.flagged'), el('span', { class: 'problem-text' }, s.reasons.map(zhOf).join('; ') || t('common.unknown'))));
// 8 one example file (#1197): the first file of a unit among its preview rows (meta/preview.parquet; isUnit); a
// source with none there has no example
const pv = previewOf(s);
const r = Array.isArray(pv) ? pv.find((x) => isUnit(metaOf(x))) : null;
if (pv === 'loading') dl.append(item('link', t('detail.example'), '…'));
else if (pv.error) dl.append(item('link', t('detail.example'), el('span', { class: 'problem-text' }, pv.error)));
else if (r) {
const href = upstream(r.uri);
const name = r.member_path || r.uri.split('/').pop() || r.uri;
dl.append(item('link', t('detail.example'), el('span', { class: 'example' },
href ? el('a', { href, rel: 'noopener', class: 'mono' }, name) : el('span', { class: 'mono' }, name),
r.size_bytes != null ? el('span', { class: 'small' }, ` · ${fmtBytes(num(r.size_bytes))}`) : null)));
}
// 9 the command that takes this source: #2629: a material set whole, with its units and its size beside it; a corpus,
// whose whole can be petabytes, by the units choice (the reader's, else 3 per source)
const cmd = command(commandArgs({ sources: [s.id], sample: s.isTask ? 'all' : F.sample || DEFAULT_SAMPLE[CORPUS] }));
dl.append(item('run', t('detail.take'), el('code', { class: 'cmd-inline' }, cmd), copyButton(t('copy.button'), cmd),
s.isTask ? el('span', { class: 'small' }, s.size ? t('detail.takeWhole', { units: unitsText(s.readyUnits), size: sizeOf(s.size) })
: t('detail.takeWholeNoSize', { units: unitsText(s.readyUnits) })) : null));
box.append(dl);
// batches and the rows read, on request
const b = st.batches;
const more = el('details', { class: 'd-more' }, el('summary', {}, t('detail.more')));
if (Array.isArray(b)) {
more.append(table([t('th.batch'), { label: t('th.rows'), r: true }, t('th.changed'), t('th.state')],
b.map((x) => [x.batch, x.rows == null ? t('detail.notCounted') : fmtInt(x.rows), fmtDate(x.date),
x.status ? t(`batch.${x.status}`, { defaultValue: human(x.status) }) : ''])));
}
if (rows.length) {
more.append(el('div', { class: 'table-wrap sample' }, table([t('th.address'), { label: t('th.size'), r: true }, 'sha256'],
rows.map((r) => [r.member_path ? `${r.uri} ! ${r.member_path}` : r.range_offset != null ? `${r.uri} @ ${num(r.range_offset)}+${num(r.range_length)}` : r.uri,
r.size_bytes == null ? '' : fmtInt(num(r.size_bytes)), hex(r.sha256).slice(0, 16) + (r.sha256 ? '…' : '')]), st.file)));
}
box.append(more);
return box;
}
// #1792: what a reader of one source reads of the data file f (filesOf): its row range there when the catalog records it
// (a packed file holds a source's rows together, sorted by source), the first n rows of it when n is given; else the
// file from its first row. hyparquet documents no row filter, so the reader keeps the source's rows itself.
function rowsOf(f, n) {
if (f.first == null || f.last == null) return n == null ? {} : { rowStart: 0, rowEnd: n };
return { rowStart: f.first, rowEnd: n == null ? f.last + 1 : Math.min(f.last + 1, f.first + n) };
}
async function loadBatchInfo(s) {
const dates = await loadBatchDates(IX.sha, s.files).catch(() => new Map());
const listed = new Map(s.batches.map((b) => [b.batch, b]));
const names = [...new Set([...s.files.flatMap((f) => f.batches), ...listed.keys(), ...dates.keys()])].sort();
// #1769: a file may hold other batches and sources too, so the rows are counted by the columns source and batch;
// #1792: each file read once, its row range of the source only
const counted = new Map();
const countsOf = (f) => {
if (!counted.has(f.path)) {
counted.set(f.path, readColumns(fileUrl(IX.sha, f.path), ['source', 'batch'], rowsOf(f)).then((p) => {
const by = new Map();
for (const r of p.rows) if (r.source === s.id) by.set(String(r.batch), (by.get(String(r.batch)) || 0) + 1);
return by;
}));
}
return counted.get(f.path);
};
const out = [];
for (const name of names) {
const b = listed.get(name);
let rows = b ? num(b.rows) : null;
const files = s.files.filter((f) => f.batches.includes(name));
if (rows == null && files.length && files.length <= 40) {
const parts = await Promise.all(files.map(countsOf));
rows = parts.reduce((a, by) => a + (by.get(name) || 0), 0);
}
out.push({ batch: name, rows, date: dates.get(name) || null, status: b?.status || null });
}
return out;
}
// ---- 5. get the data
const sqlStr = (v) => `'${String(v).replace(/'/g, "''")}'`;
function selectionSql(sel) {
const root = `hf://datasets/${REPO}@${IX.sha}`;
// #1746: the data files come from the catalog's column data_files, so the query lists no folder of data/
const ids = sel.map((s) => sqlStr(s.id)).join(', ');
const chips = filterChips();
const exts = extList();
const where = [`source IN (${ids})`, ...(exts.length ? [`ext IN (${exts.map(sqlStr).join(', ')})`] : [])];
// the selection is one layer: its sources in that layer's measure
const head = F.kind === CORPUS
? t('sql.headCorpora', { sha: IX.sha.slice(0, 7), corpora: plural('n.corpus', sel.length), rows: plural('n.corpusRow', sum(sel, (s) => s.rawRows)) })
: t('sql.headSets', { sha: IX.sha.slice(0, 7), sets: plural('n.materialSet', sel.length), units: unitsOfSets(sel) });
return [
`-- ${head}`,
`-- ${chips.length ? chips.map(([k, v]) => t('sql.chip', { k, v })).join('; ') : t('sql.all')}`,
// #1577: the query reads the rows as published; the default profile of the command is not applied, and it says so
`-- ${t('sql.noProfile')}`,
`SET VARIABLE files = (SELECT list(${sqlStr(`${root}/`)} || f) FROM (SELECT unnest(data_files) AS f FROM ${sqlStr(`${root}/meta/catalog.parquet`)} WHERE source IN (${ids})));`,
`SELECT ${ROW_COLUMNS.map((c) => (c === 'sha256' ? 'lower(hex(sha256)) AS sha256' : c)).join(', ')}`,
"FROM read_parquet(getvariable('files'), union_by_name = true)",
// #1524: the file types selected, as fetch.py's --ext
`WHERE ${where.join(' AND ')};`,
].join('\n');
}
const csvCell = (v) => {
if (v == null) return '';
const s = typeof v === 'bigint' ? v.toString() : String(v);
return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
};
/** The pointer rows of sel as CSV; #1524: with file types selected, only the rows of those types. #1769: each data
* file is read once, and only the rows of the sources of sel are kept (a file may hold rows of other sources). */
async function buildCsv(sel, onStep) {
const files = [...new Set(sel.flatMap((s) => s.files.map((f) => f.path)))];
const ids = new Set(sel.map((s) => s.id));
const exts = new Set(extList());
const cols = exts.size ? [...ROW_COLUMNS, 'ext'] : ROW_COLUMNS;
const lines = [ROW_COLUMNS.join(',')];
let done = 0;
const queue = [...files];
const parts = new Map();
const worker = async () => {
while (queue.length) {
const path = queue.shift();
const { rows } = await readColumns(fileUrl(IX.sha, path), cols);
parts.set(path, rows.filter((r) => ids.has(r.source) && (!exts.size || exts.has(r.ext))));
done++;
onStep(done, files.length);
}
};
await Promise.all([worker(), worker(), worker()]);
for (const path of files) for (const r of parts.get(path)) lines.push(ROW_COLUMNS.map((c) => csvCell(c === 'sha256' ? hex(r[c]) : r[c])).join(','));
return lines.join('\n') + '\n';
}
/** What ./data holds after the command: the layout of tools/fetch.py (#268; #1633: one folder per unit). <domain> is a
* row's pool_domain when it has one (such as D3_data), else the top-level code of the source's first domain (such as
* D3). */
function folderTree(sel) {
const named = sel.length === 1 || (sel.length <= MAX_LISTED && sel.length < IX.sources.length);
const first = named ? sel.find((s) => s.isTask) || sel[0] : null;
const src = first ? first.id : '<source>';
const line = (depth, iconName, name, note) => el('li', { style: { paddingLeft: `${depth * 20}px` } }, icon(iconName, { size: 16 }),
el('span', { class: 'tree-name' }, el('span', { class: 'mono' }, name), note ? el('span', { class: 'small' }, note) : null));
return el('figure', { class: 'card tree-fig' }, el('p', { class: 'chart-title' }, t('tree.title')),
el('ul', { class: 'tree' },
line(0, 'folder', 'data/'),
line(1, 'folder', `<domain>/${src}/`, t('tree.units')),
line(2, 'folder', '<unit folder>/', t('tree.unitFolder')),
line(1, 'folder', '_meta/'),
line(2, 'rows', 'manifest.jsonl', t('tree.manifest')),
line(2, 'rows', 'sizes.tsv', t('tree.sizes'))),
el('p', { class: 'small tree-note' }, el('span', { class: 'mono' }, '<domain>'), t('tree.domainNote')),
// #1891: what the default profile leaves out, under every units choice ("all" too: the profile stays on); #2397:
// the criteria tools/fetch.py applies by default, in its order, as the release lists them (meta/vocabulary.json
// profile), #2422 each with the flag that takes it back (its option); #2441: every license and personal data come
// by default, labels the filters choose by, not rules of the profile, and the two flags that leave them out are
// named under the list (tree.labelNote)
...(IX.vocab.profile.length ? [el('p', { class: 'chart-title profile-title' }, t('tree.leftOut')),
el('ul', { class: 'profile' }, IX.vocab.profile.map(({ criterion, option }) => el('li', {}, el('span', {}, t(`profile.${criterion}`)), el('code', {}, option)))),
el('p', { class: 'small tree-note' }, t('tree.flagNote'))] : []),
el('p', { class: 'small tree-note' }, t('tree.labelNote'), ' ', el('code', {}, '--training-only'), ' ', el('code', {}, '--no-personal-data')),
el('p', { class: 'small agent-links' }, icon('robot', { size: 16 }),
el('a', { href: t('url.forAgents') }, t('agents.readme')), ' · ', el('a', { href: 'llms.txt' }, 'llms.txt')));
}
function download(name, text, type) {
const a = el('a', { href: URL.createObjectURL(new Blob([text], { type })), download: name });
document.body.append(a); a.click(); a.remove();
setTimeout(() => URL.revokeObjectURL(a.href), 10000);
}
// ---- the selection kept as files: the selection file (the ids, the filters, the release tag of the commit, which the
// command beside it names too; tools/fetch.py --selection reads it, so the same file gives the same files later) and
// the attribution file (per source its title, licence and attribution statement from meta/sources.json; #1577: and
// its files by publisher, licence and licence URL from meta/attribution.parquet, each file's own, else its source's;
// in English, the language of the dataset)
const selectionName = () => `training-data-foundation-selection-${IX.sha.slice(0, 7)}.json`;
const ATTRIBUTION = 'meta/attribution.parquet';
const ATTRIBUTION_COLUMNS = ['source', 'publisher', 'license', 'license_url', 'files'];
function selectionFile(sel, picks) {
return `${JSON.stringify({
format: 'training-data-foundation-selection/1', dataset: REPO, revision: IX.pin,
filters: picks ? {} : Object.fromEntries([...activeFilters(), 'kind'].map((k) => [k, F[k]])), sources: sel.map((s) => s.id).sort(),
}, null, 1)}\n`;
}
async function attributionFile(sel) {
const ids = new Set(sel.map((s) => s.id));
const files = new Map();
if (IX.hasAttribution) {
for (const r of (await readColumns(fileUrl(IX.sha, ATTRIBUTION), ATTRIBUTION_COLUMNS)).rows) {
if (!ids.has(r.source)) continue;
if (!files.has(r.source)) files.set(r.source, []);
files.get(r.source).push(r);
}
}
const lines = [`# ${tEn('attr.title', { count: sel.length, n: fmtInt(sel.length), dataset: REPO })}`, '',
tEn('attr.commit', { sha: IX.sha, url: `${DATASET_URL}/tree/${IX.sha}` }), ''];
for (const s of [...sel].sort((a, b) => a.title.localeCompare(b.title))) {
const lic = s.info.license || {};
const own = files.get(s.id) || [];
lines.push(`## ${s.title}`, '', `- ${tEn('attr.source')}: \`${s.id}\``,
`- ${tEn('attr.licence')}: ${lic.name || s.spdx || tEn('attr.notStated')}`);
if (own.length) {
lines.push(`- ${tEn('attr.files')}:`, ...own.map((r) => {
const n = num(r.files);
return ` - ${tEn('attr.fileLine', { count: n, n: fmtInt(n), publisher: r.publisher || tEn('attr.noPublisher'),
licence: r.license || tEn('attr.notStated'), url: r.license_url || tEn('attr.notStated') })}`;
}));
} else lines.push(`- ${tEn('attr.licenceUrl')}: ${lic.url || tEn('attr.notStated')}`);
// #1633: without an attribution statement of its own, a source is attributed by the same per-file license URL the
// fetch manifest records (its files' own, else its source's), as CC BY 4.0 section 3(a)(2) allows
const urls = [...new Set(own.map((r) => r.license_url).filter(Boolean))];
lines.push(`- ${tEn('attr.attribution')}: ${lic.attribution || lic.statement || urls.join(', ') || lic.url || tEn('attr.notStated')}`, '');
}
return lines.join('\n');
}
/** The command that reads the selection file; picks: the file lists the picked sources. #1524: the units choice, and
* outside pick mode one --ext per file type selected (fetch.py --selection reads only the file's sources). */
const selectionCommand = (picks) => command([`--selection ${selectionName()}`, `--revision ${IX.pin}`,
...(picks ? [] : extList().map((e) => `--ext ${e}`)), ...sampleArgs(), '--out ./data']);
/** The two downloads of sel; picks: sel is the picked sources (the selection file then records no filters). */
function keepCard(sel, picks = false) {
return el('div', { class: 'keep-grid' },
el('div', { class: 'card keep-card' },
el('p', { class: 'chart-title' }, t('keep.title')),
el('div', { class: 'keep-actions' },
el('button', { type: 'button', class: 'btn solid', onclick: () => download(selectionName(), selectionFile(sel, picks), 'application/json') },
icon('get', { size: 16 }), t('keep.selection')),
el('button', { type: 'button', class: 'btn', onclick: async (ev) => {
const btn = ev.currentTarget;
btn.disabled = true;
try { download(`training-data-foundation-attribution-${IX.sha.slice(0, 7)}.md`, await attributionFile(sel), 'text/markdown'); } finally { btn.disabled = false; }
} },
icon('doc', { size: 16 }), t('keep.attribution'))),
el('p', { class: 'small' }, el('code', {}, selectionName()), ' ', t('keep.selectionNote', { sources: plural(picks ? 'n.picked' : layerCount(), sel.length), sha: IX.sha.slice(0, 7) }))),
codeCard('run', t('keep.again'), selectionCommand(picks), 2));
}
/** The key of the count word of the layer shown: n.materialSet or n.corpus. */
const layerCount = () => (F.kind === CORPUS ? 'n.corpus' : 'n.materialSet');
function codeCard(iconName, title, code, n) {
return el('div', { class: 'code-card' },
el('div', { class: 'code-head' }, el('span', { class: 'code-ico' }, icon(iconName, { size: 20 })),
n ? el('span', { class: 'step-n' }, String(n)) : null, el('span', { class: 'code-title' }, title),
copyButton(t('copy.button'), code, { dark: true })),
el('pre', { class: 'code' }, el('code', {}, code)));
}
// ---- picks (#1286): sources picked row by row in either layer, kept in the address as pick=<id>,<id> like the
// filters; while any is picked, "Get the data" takes exactly them, one --source each
const pickIds = () => (F.pick ? F.pick.split(',').filter(Boolean) : []);
function picked() {
const by = new Map(IX.sources.map((s) => [s.id, s]));
return pickIds().map((id) => by.get(id)).filter(Boolean);
}
function setPicks(ids) {
F.pick = ids.join(',');
writeUrl();
renderSources(selected());
renderGet(selected());
renderCard();
}
function togglePick(id) {
const ids = pickIds();
setPicks(ids.includes(id) ? ids.filter((x) => x !== id) : [...ids, id]);
}
/** #1524: the heading of Get the data for each of sources (their count in words), or for one source (sources null),
* by the units choice in force (#2629: sampleNow). */
function takeHeading(sources) {
const v = sampleNow();
const all = v === 'all';
if (sources == null) return all ? t('get.takeOneAll') : t('get.takeOne', { count: Number(v) });
return all ? t('get.takeEachAll', { sources }) : t('get.takeEach', { count: Number(v), sources });
}
/** #1524: a choice of how many units to take from each source (F.sample, kept in the address); every command and
* heading follows it, the command of a corpus's detail too. #2629: it shows the choice in force (sampleNow), and a
* click on the default of the layer the command takes clears the reader's choice, so the other layer takes its own
* default again. */
function chooseSample(v) {
F.sample = v === DEFAULT_SAMPLE[takenKind()] ? '' : v;
writeUrl();
vt(() => { renderGet(selected()); renderSources(selected()); });
}
function unitsChoice() {
return el('div', { class: 'units-row' }, segmented(t('get.units'), SAMPLES.map((v) => [v, v === 'all' ? t('get.unitsAll') : v]), sampleNow(), chooseSample));
}
/** #1524: the units choice, "Take" (the command) and the folder tree of the command args for the sources S. */
function commandSteps(args, S) {
return [unitsChoice(), el('div', { class: 'get-grid' },
el('div', { class: 'cmd-steps' }, codeCard('run', t('get.take'), command(args), 1)),
folderTree(S))];
}
/** Get the picked sources P: one --source per pick, with the same steps as a selection (#1524), and the two downloads
* of them. */
function renderPicks(picks) {
$('get-h').textContent = takeHeading(plural('n.picked', picks.length));
const body = clear($('get-body'));
body.append(el('div', { class: 'pick-row' },
el('span', { class: 'fact-line' }, icon('sources', { size: 18 }), el('b', {}, fmtInt(picks.length)), t('pick.word', { count: picks.length })),
el('span', { class: 'chips' }, picks.map((s) => el('span', { class: 'chip', title: s.id }, titleOf(s)))),
el('button', { type: 'button', class: 'btn', onclick: () => setPicks([]) }, t('pick.clear'))));
body.append(el('p', { class: 'small' }, t('pick.note')));
body.append(...commandSteps(commandArgs({ sources: picks.map((s) => s.id) }), picks));
body.append(keepCard(picks, true));
}
/** Get the data: the selection is one layer, so one command takes it (#1083); the other layer is one switch away.
* Picked sources, when there are any, take its place (renderPicks). */
function renderGet(sel) {
if (!UI.ids) return;
const picks = picked();
renderDock(sel, picks);
if (picks.length) { renderPicks(picks); return; }
const ready = sum(sel, (s) => s.readyUnits);
const raw = sum(sel, (s) => s.rawRows);
const other = F.kind === CORPUS ? MATERIAL_SET : CORPUS;
$('get-h').textContent = !sel.length ? t('get.nothing') : takeHeading(sel.length === 1 ? null : plural(layerCount(), sel.length));
const body = clear($('get-body'));
const switchOther = el('button', { type: 'button', class: 'link', onclick: () => setLayer(other) }, t('get.otherLayer', { name: kindName(other) }));
if (!sel.length) { body.append(el('p', { class: 'small' }, switchOther)); return; }
const chips = filterChips();
const tabs = segmented('', [['cmd', t('get.command')], ['sql', 'SQL'], ['csv', 'CSV']], UI.tab, (v) => { UI.tab = v; vt(() => renderGet(selected())); });
const pickRow = el('div', { class: 'pick-row' },
el('span', { class: 'fact-line' }, icon('sources', { size: 18 }), el('b', {}, fmtInt(sel.length)), t(F.kind === CORPUS ? 'get.corpusWord' : 'get.setWord', { count: sel.length })),
F.kind === CORPUS
? el('span', { class: 'fact-line muted' }, icon('rows', { size: 18 }), el('b', {}, fmtCompact(raw)), t('cat.lead.raw', { count: raw }))
: el('span', { class: 'fact-line' }, icon('check', { size: 18 }), el('b', {}, fmtInt(ready)), t('cat.lead.ready', { count: ready }) + uncounted(notCountedOf(sel))),
el('span', { class: 'chips' }, el('span', { class: 'chip' }, kindName(F.kind)), ...chips.map(([k, v]) => el('span', { class: 'chip', title: k }, v))),
el('a', { class: 'small', href: '#filters' }, t('get.change')), switchOther, tabs);
body.append(pickRow);
if (UI.tab === 'cmd') {
body.append(...commandSteps(selectionArgs(sel).args, sel));
body.append(keepCard(sel));
} else if (UI.tab === 'sql') {
body.append(codeCard('database', t('get.sql'), selectionSql(sel)));
} else {
// #1524: with file types selected, the rows of those types (by_ext)
const exts = extList();
const rows = exts.length ? sum(sel, (s) => sum(s.exts.filter(([e]) => exts.includes(e)), ([, n]) => n)) : sum(sel, (s) => s.rows);
const key = `${sel.map((s) => s.id).join(',')} ${F.ext}`;
const csvState = el('span', { class: 'copy-state', 'aria-live': 'polite' }, UI.csv?.key === key ? UI.csv.text : '');
const tooMany = rows > CSV_MAX_ROWS;
const btn = el('button', { type: 'button', class: 'btn solid', disabled: tooMany, onclick: async () => {
btn.disabled = true;
const say = (text) => { csvState.textContent = text; UI.csv = { key, text }; };
try {
const csv = await buildCsv(sel, (d, n) => say(t('csv.progress', { count: n, done: fmtInt(d), n: fmtInt(n) })));
download(`${REPO.split('/')[1]}-${IX.sha.slice(0, 7)}-${sel.length === 1 ? sel[0].id : `${sel.length}-${F.kind === CORPUS ? 'corpora' : 'material-sets'}`}.csv`, csv, 'text/csv');
say(plural('n.pointerRow', csv.split('\n').length - 2));
} catch (e) { say(e.message); }
btn.disabled = false;
} }, icon('csv', { size: 16 }), t('csv.button', { rows: plural('n.pointerRow', rows) }));
body.append(el('div', { class: 'card csv-card copy-row' }, icon('csv', { size: 28 }), btn, tooMany ? el('span', { class: 'small' }, t('csv.tooMany', { n: fmtShort(CSV_MAX_ROWS) })) : csvState));
}
}
// ---- #2422, second round ("Strata"): the working screen. The field (field.js, loaded after the first paint) draws
// every source of the index as a point in its stage group's color; this part gives it the points, their light, its
// labels and the open source, and draws the panels that float on it: the count of the selection with its list (the
// largest first), the open source in short, and the dock with the one command that takes the selection.
const VIEWS = ['strata', 'domains', 'scale'];
// how many sources the panel's list shows; the list below the screen has them all
const LIST_ROWS = 60;
// the light of a source of the other layer that the filters keep
const HALF = 0.45;
let FIELD = null;
// the points of the field ([{s, g, c, lg, dup}]: s the source's place in IX.sources) and, per source id, its points
let fieldPts = [];
const fieldIndex = new Map();
const wideWork = () => matchMedia('(min-width: 1100px)').matches;
const stillMotion = () => matchMedia('(prefers-reduced-motion: reduce)').matches;
/** The stage group of a source among the stages ids (the stage table's order): its first stage, else the last group,
* in no stage. */
const groupOf = (s, ids) => { const g = ids.findIndex((id) => s.stages.includes(id)); return g < 0 ? ids.length : g; };
// #2629: why something is in no stage (artpool.stages.NO_STAGE, the keys of no_stage of the stage figures), each by the
// key of its words; a reason the page does not know is said by its name
const WHY_KEY = { condition: 'condition', ...RULE_KEY, 'not counted': 'notCounted' };
// the color of a source's dot in the list: the picked stage's while the source feeds it, else its first stage's
const groupColor = (s, ids) => stageColor(F.stage && s.stages.includes(F.stage) ? F.stage : ids[groupOf(s, ids)]);
/** The points of the field: one per source and stage it feeds (a corpus that feeds two stages stands on both planes
* and in both lanes; the further ones are dup), a source of no stage one in the last group. Each has its domain cell
* and the logarithm of its deduplicated size. */
function fieldPoints(ids, doms) {
const out = [];
IX.sources.forEach((s, i) => {
const gs = ids.map((id, g) => (s.stages.includes(id) ? g : -1)).filter((g) => g >= 0);
(gs.length ? gs : [ids.length]).forEach((g, k) => out.push({ s: i, g, c: doms.indexOf(domainKeysOf(s)[0]), lg: Math.log10((s.distinct || 0) + 1), dup: k > 0 }));
});
return out;
}
/** The point that stands for a source in the ring: the one in the picked stage's group when it has one, else its
* first; -1 for a source with no point. */
function pointOf(id) {
const at = fieldIndex.get(id);
if (!at) return -1;
const g = stageFigList().findIndex((s) => s.id === F.stage);
return at.find((i) => fieldPts[i].g === g) ?? at[0];
}
/** The domain cells of the field: the top-level domains that have a source (a source of several is in its first), then
* several domains and not classified, while there are some. */
function fieldDomains() {
const keys = new Set(IX.sources.map((s) => domainKeysOf(s)[0]));
return [...[...keys].filter((k) => k !== MULTI && k !== NONE).sort(numericCodeSort), ...[MULTI, NONE].filter((k) => keys.has(k))];
}
/** The light of each point: 1 for a source of the selection, HALF for one the filters keep in the other layer, 0 for
* one they leave out; until the selection answers, by the layer alone. While a stage is picked, a selected source's
* point in another stage's group is at HALF, so the picked stage's plane is the lit one. */
function fieldLevels() {
const picked = stageFigList().findIndex((s) => s.id === F.stage);
return Float32Array.from(fieldPts, (p) => {
const s = IX.sources[p.s];
if (!UI.ids) return s.kind === F.kind ? 1 : HALF;
if (!UI.ids.has(s.id)) return UI.across?.has(s.id) ? HALF : 0;
return picked >= 0 && p.g !== picked ? HALF : 1;
});
}
/** #1083: what no stage counts, said per layer and never as one number: that of the layer shown. #2629: from the
* release's figures (no_stage of meta/stage_figures.json, data.js noStage): its sources as the figure, its units or
* corpus rows in the line under it, and its sources by reason as the steps (the condition, each rule, not counted yet;
* a reason of none left out), its units or rows by reason on hover; so post-training's figure plus these is what the
* selection of every material set counts. Figures written before #2629 give the sources of the layer shown in no
* stage, without reasons. */
function restWhat() {
const sets = F.kind !== CORPUS;
const head = { name: t('strata.rest'), title: t('strata.restTitle') };
const sourcesWord = (n) => t(sets ? 'get.setWord' : 'get.corpusWord', { count: n });
const reasons = Object.entries(IX.stageFigures?.noStage?.[F.kind] || {});
if (!reasons.length || reasons.some(([, c]) => Object.values(c).some((v) => v == null))) {
const here = IX.sources.filter((s) => !s.stages.length && s.kind === F.kind).length;
return { ...head, num: fmtInt(here), word: sourcesWord(here), line: '', steps: [] };
}
const [many, amount] = sets ? ['material_sets', 'units'] : ['corpora', 'rows'];
const fmt = sets ? fmtInt : fmtCompact;
const said = (m, f) => reasons.filter(([, c]) => c[m]).map(([why, c]) => t(`strata.why.${WHY_KEY[why] || 'other'}`, { count: c[m], n: f(c[m]), why }));
const n = sum(reasons, ([, c]) => c[many]);
const left = sum(reasons, ([, c]) => c[amount]);
return { ...head, title: `${head.title}\n${t(sets ? 'strata.whyUnits' : 'strata.whyRows', { list: said(amount, fmt).join(t('size.listSep')) })}`,
num: fmtInt(n), word: sourcesWord(n), line: t(sets ? 'strata.restUnits' : 'strata.restRows', { count: left, n: fmt(left) }), steps: said(many, fmtInt) };
}
/** The labels of the field in the page's language: per stage group its figure and name at its plane and its name at
* its lane, the domains' names, and the marks of the size axis. */
function fieldLabels() {
const groups = [...stageFigList().map((st) => ({ id: st.id, color: stageColor(st.id), name: (inChinese() && st.zh) || st.name, what: stageWhat(st) })),
{ color: '--none', name: t('strata.rest'), what: restWhat() }];
// a label of a stage or of a domain is a button for the pointer (out of the tab order: the column's bands and the
// filters are the way by keyboard): it sets that filter, or clears it when it holds it
const press = (on, act, kids, style) => el('button', { type: 'button', tabindex: '-1', 'aria-pressed': String(on), style, onclick: act }, kids);
const node = (g, full) => {
const kids = [el('i'), full ? [el('b', {}, g.what.num), [g.what.word, g.name].filter(Boolean).join(' · ')] : g.name];
const style = `--c: var(${g.color})`;
return g.id && IX.stages ? press(F.stage === g.id, () => pickStage(g.id), kids, style) : el('span', { style }, kids);
};
const domain = (k) => press(F.domain === k, () => setFilter({ domain: F.domain === k ? '' : k, leaf: '' }), domainName(k));
return { strata: groups.map((g) => node(g, true)), lanes: groups.map((g) => node(g, false)), domains: fieldDomains().map(domain),
ticks: [[3, '1 KB'], [6, '1 MB'], [9, '1 GB'], [12, '1 TB'], [15, '1 PB']] };
}
/** The arrangement of the field (F.view, kept in the address). */
function setView(v) {
F.view = v;
writeUrl();
for (const b of $('view-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.view === v));
FIELD?.setView(v);
}
/** The open source (F.open, kept in the address; '' closes it): ringed in the field, in short in the panel, its whole
* detail in the list below the screen. */
function openSource(id) {
F.open = id;
writeUrl();
renderSources(selected());
renderCard();
}
/** A click on a point or on a row of the panel's list: the source opens (a second click closes it), its layer first
* when it is of the one not shown. */
function focusSource(s) {
if (s.kind !== F.kind) setFilter({ kind: s.kind });
openSource(F.open === s.id ? '' : s.id);
}
/** What the panels leave to the scene of the field: its width and height in px, and how far above the field's middle
* its middle is. From 1100 px the panels float on the field: the room is between the left and the right panel, under
* the arrangements and above the dock; below that the field is alone, with the arrangements at its top. */
function fieldRoom() {
const host = $('field');
// the room starts under the arrangements (and, from 1100 px, the hint under them)
const top = wideWork() ? 88 : 52;
if (!wideWork()) return { w: host.clientWidth - 24, h: host.clientHeight - top - 12, lift: -top / 2 };
const side = (cls) => document.querySelector(cls).offsetWidth + 40;
const bottom = document.querySelector('.work-dock').offsetTop;
return { w: host.clientWidth - side('.work-left') - side('.work-right') - 40, h: bottom - top - 24, lift: host.clientHeight / 2 - (top + bottom) / 2 };
}
/** The field after the first paint: the scene with every source, or, without WebGL, one line in its place. */
async function startField() {
const host = $('field');
const ids = stageFigList().map((s) => s.id);
const doms = fieldDomains();
fieldPts = fieldPoints(ids, doms);
fieldPts.forEach((p, i) => { const id = IX.sources[p.s].id; fieldIndex.set(id, [...(fieldIndex.get(id) || []), i]); });
const at = (i) => IX.sources[fieldPts[i].s];
try {
const mod = await import('./field.js');
FIELD = await mod.createField(host, {
points: fieldPts, colors: [...ids.map(stageColor), '--none'], cells: doms.length, view: F.view,
room: fieldRoom,
tip: (i) => { const s = at(i); return [titleOf(s), [sourceDomainText(s), kindOne(s.kind), s.size ? sizeOf(s.size) : null].filter(Boolean).join(' · ')]; },
onHover: (i) => { const id = i < 0 ? null : at(i).id; for (const b of $('sel-list').querySelectorAll('.src')) b.classList.toggle('hot', b.dataset.id === id); },
onPick: (i) => focusSource(at(i)),
onTouch: () => $('top').classList.add('touched'),
});
} catch (e) {
console.error(e);
}
// the manual's shots wait for this mark (manual/shots.yml): the field has faded in, or the page goes without it
if (!FIELD) { document.body.dataset.field = 'off'; host.hidden = true; $('field-off').hidden = false; return; }
setTimeout(() => { document.body.dataset.field = 'on'; }, stillMotion() ? 0 : 2400);
$('top').classList.add('field-up');
for (const id of ['views', 'field-hint', 'field-legend']) $(id).hidden = false;
placePills();
FIELD.setLabels(fieldLabels());
FIELD.setLevels(fieldLevels());
FIELD.pick(stageFigList().findIndex((s) => s.id === F.stage));
renderCard();
}
// a figure counts up to its new value, at once with reduced motion
const figures = new WeakMap();
function countTo(node, to) {
const from = figures.get(node) ?? 0;
figures.set(node, to);
if (stillMotion() || from === to) { node.textContent = fmtInt(to); return; }
const t0 = performance.now();
const step = (now) => {
const k = Math.min(1, (now - t0) / 450);
node.textContent = fmtInt(from + (to - from) * (1 - (1 - k) ** 3));
if (k < 1 && figures.get(node) === to) requestAnimationFrame(step);
};
requestAnimationFrame(step);
}
/** The right panel: how many sources of the layer shown the selection holds, with their units or rows and the domains
* they cover; its largest sources, each with its stage group's dot, its domain, its units and its size (a bar on a
* logarithmic scale); and the light of the field's points. */
function renderWork(sel) {
if (!UI.ids) return;
const sets = F.kind !== CORPUS;
countTo($('sel-n'), sel.length);
$('sel-word').textContent = t(sets ? 'get.setWord' : 'get.corpusWord', { count: sel.length });
const ready = sum(sel, (s) => s.readyUnits);
const raw = sum(sel, (s) => s.rawRows);
const covered = new Set(sel.flatMap((s) => s.tops)).size;
const meta = [sets ? [fmtInt(ready), t('cat.lead.ready', { count: ready }) + uncounted(notCountedOf(sel))] : [fmtCompact(raw), t('cat.lead.raw', { count: raw })],
[fmtInt(covered), t('strata.domains', { count: covered })]];
clear($('sel-meta')).append(...meta.map(([n, word]) => el('span', {}, el('b', {}, n), ' ', word)));
const ids = stageFigList().map((s) => s.id);
const lg = (s) => Math.log10((s.distinct || 0) + 1);
const top = [...sel].sort((a, b) => (b.distinct ?? -1) - (a.distinct ?? -1) || titleOf(a).localeCompare(titleOf(b))).slice(0, LIST_ROWS);
const most = Math.max(1, ...top.map(lg));
const list = clear($('sel-list'));
list.append(...top.map((s, i) => el('li', { style: `--i: ${i}` }, el('button', { type: 'button', class: 'src', 'data-id': s.id, style: `--c: var(${groupColor(s, ids)})`,
onclick: () => focusSource(s), onpointerenter: () => FIELD?.hover(pointOf(s.id)), onpointerleave: () => FIELD?.hover(-1) },
el('span', { class: 'dot', 'aria-hidden': 'true' }),
el('span', { style: 'min-width: 0' }, el('span', { class: 't' }, titleOf(s)),
el('span', { class: 's' }, [sourceDomainText(s), s.isTask ? unitsText(s.readyUnits) : plural('n.corpusRow', s.rawRows)].join(' · '))),
el('span', { class: 'sz' }, s.size ? sizeOf(s.size) : ''),
el('span', { class: 'barw', 'aria-hidden': 'true' }, el('u', { style: `width: ${Math.max(2, (lg(s) / most) * 100).toFixed(0)}%` }))))));
if (!sel.length) list.append(el('li', {}, el('p', { class: 'src-none' }, t('sources.none'))));
$('sel-all').textContent = !sel.length ? '' : sel.length > top.length ? t('strata.all', { n: fmtInt(sel.length) }) : t('strata.table');
// no source matches: the scene says so in its middle, with the way back
$('field-empty').hidden = sel.length > 0;
renderCard();
FIELD?.setLevels(fieldLevels());
}
/** The open source in short, above the panel's list, while the selection holds it: what it is in seven lines, Pick, and
* a link to its whole detail in the list below the screen; its point is ringed in the field. */
function renderCard() {
const box = $('sel-card');
const s = F.open && UI.ids?.has(F.open) ? IX.sources.find((x) => x.id === F.open) : null;
FIELD?.open(s ? pointOf(s.id) : -1, s ? titleOf(s) : '');
for (const b of $('sel-list').querySelectorAll('.src')) b.classList.toggle('on', b.dataset.id === s?.id);
box.hidden = !s;
clear(box);
if (!s) return;
const stages = s.stages.length ? s.stages.map(stageName).join(t('size.listSep')) : t('strata.rest');
const rows = [[t('col.domain'), sourceDomainText(s)], [t('strata.layer'), kindOne(s.kind)], [t('col.form'), formName(formKeyOf(s))],
s.isTask ? [t('col.units'), unitsNum(s.readyUnits)] : [t('col.rows'), fmtCompact(s.rawRows)],
[t('col.size'), s.size ? sizeOf(s.size) : t('common.unknown')], [t('col.tier'), tierName(s.tier)], [t('filter.stage'), stages]];
const isPicked = pickIds().includes(s.id);
box.append(el('div', { class: 'card-s' },
el('button', { type: 'button', class: 'x', 'aria-label': t('strata.close'), title: t('strata.close'), onclick: () => openSource('') }, '×'),
el('h3', {}, titleOf(s)), el('code', { class: 'src-id' }, s.id),
el('dl', {}, rows.map(([k, v]) => [el('dt', {}, k), el('dd', {}, v)])),
el('div', { class: 'acts' },
el('button', { type: 'button', class: 'btn small-btn', 'aria-pressed': String(isPicked), onclick: () => togglePick(s.id) },
isPicked ? [icon('check', { size: 14 }), t('pick.picked')] : t('pick.add')),
el('a', { href: `#d-${s.id}` }, t('strata.details')))));
}
/** The dock: the command that takes the picked sources, else the selection (the same command as Get the data shows),
* with the choice of how many units per source (chooseSample; #2629: pressed, the choice in force). */
function drawDockSample() {
clear($('dock-sample')).append(...SAMPLES.map((v) => el('button', { type: 'button', 'data-s': v, 'aria-pressed': String(v === sampleNow()),
onclick: () => chooseSample(v) }, v === 'all' ? t('get.unitsAll') : v)));
}
function renderDock(sel, picks) {
const S = picks.length ? picks : sel;
$('dock-h').textContent = [t('get.take'), S.length ? plural(picks.length ? 'n.picked' : layerCount(), S.length) : null].filter(Boolean).join(' · ');
const text = !S.length ? t('get.nothing') : command(picks.length ? commandArgs({ sources: picks.map((s) => s.id) }) : selectionArgs(sel).args);
// the command as it is copied, its options' names and its line ends each in a color of their own
clear($('dock-cmd')).append(...text.split(/(--[a-z-]+| \\\n)/).map((part) => (part.startsWith('--') ? el('span', { class: 'flag' }, part)
: part.endsWith('\\\n') ? el('span', { class: 'cont' }, part) : part)));
$('dock-copy').disabled = !S.length;
const v = sampleNow();
for (const b of $('dock-sample').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.s === v));
}
/** Every segmented control of the page gets a pill that slides to its pressed part: placed under that part, and placed
* again when another part is pressed, the control is drawn again or its words change. placePills places them all (after
* a control that was hidden is shown, or the window's width changes). */
function placePill(seg) {
const on = seg.querySelector(':scope > button[aria-pressed="true"]');
let pill = seg.querySelector(':scope > .seg-pill');
if (!pill) { pill = el('span', { class: 'seg-pill', 'aria-hidden': 'true' }); seg.prepend(pill); }
seg.classList.toggle('pilled', !!on && on.offsetWidth > 0);
pill.style.opacity = on ? 1 : 0;
if (!on) return;
pill.style.width = `${on.offsetWidth}px`;
pill.style.height = `${on.offsetHeight}px`;
pill.style.transform = `translate(${on.offsetLeft}px, ${on.offsetTop}px)`;
}
const placePills = () => document.querySelectorAll('.seg').forEach(placePill);
function setUpPills() {
new MutationObserver((changes) => {
const segs = new Set();
for (const c of changes) {
const seg = (c.target.nodeType === 1 ? c.target : c.target.parentElement)?.closest('.seg');
if (seg) segs.add(seg);
for (const n of c.addedNodes) if (n.nodeType === 1) for (const s of [n, ...n.querySelectorAll('.seg')]) if (s.matches('.seg')) segs.add(s);
}
segs.forEach(placePill);
}).observe(document.body, { subtree: true, childList: true, characterData: true, attributes: true, attributeFilter: ['aria-pressed'] });
addEventListener('resize', placePills);
document.fonts?.ready.then(placePills);
placePills();
}
/** The controls of the working screen, once: the arrangements (the keys 1, 2 and 3 too), the zoom, the copy button of
* the dock, and the key / that goes to the search. */
function setUpWork() {
drawDockSample();
for (const b of $('view-seg').querySelectorAll('button')) {
b.setAttribute('aria-pressed', String(b.dataset.view === F.view));
b.addEventListener('click', () => setView(b.dataset.view));
}
$('zoom-in').addEventListener('click', () => FIELD?.zoomBy(1.2));
$('zoom-out').addEventListener('click', () => FIELD?.zoomBy(1 / 1.2));
$('zoom-reset').addEventListener('click', () => FIELD?.reset());
$('field-clear').addEventListener('click', () => $('f-clear').click());
setUpPills();
$('dock-copy').addEventListener('click', async () => {
const b = $('dock-copy');
try { await navigator.clipboard.writeText($('dock-cmd').textContent); b.textContent = t('copy.done'); } catch { b.textContent = t('copy.select'); }
setTimeout(() => { b.textContent = t('dock.copy'); }, 1600);
});
addEventListener('keydown', (e) => {
if (e.ctrlKey || e.metaKey || e.altKey || e.target.closest?.('input, select, textarea')) return;
if (e.key === '/') { e.preventDefault(); $('f-q').focus(); }
if (e.key === 'Escape' && F.open) openSource('');
const v = VIEWS[Number(e.key) - 1];
if (v && FIELD) setView(v);
});
}
// ---- the first screen (#2422: the training stages and the release line), the progress bar, reveal
function renderHeader() {
markFromIndex();
renderStageFig();
renderTerms();
drawRelease(IX.pin, IX.releaseDate);
$('foot-pin').textContent = t('foot.index', { sha: IX.sha.slice(0, 7), date: fmtDate(IX.date) });
if (IX.overrides.catalog || IX.overrides.vocabulary) {
const chip = $('preview-chip');
clear(chip).append(el('span', { class: 'dot other', 'aria-hidden': 'true' }), t('hero.preview'));
chip.title = [IX.overrides.catalog, IX.overrides.vocabulary].filter(Boolean).join(' ');
chip.hidden = false;
}
}
// #1524, #2066: the usage videos (the video element's id -> its files' name): each plays the video of the page's
// language with that language's subtitles, from media/ of the dataset (catalog/media.tsv: <name>.<en or zh>.mp4 and
// .vtt); the language switch changes both of each. Their pictures carry no words, so the track is the default one
// (HTML track default).
const VIDEOS = { 'video-player': 'usage-example', 'people-player': 'usage-people' };
function renderVideo() {
const l = lang();
for (const [id, name] of Object.entries(VIDEOS)) {
const v = $(id);
if (v.dataset.lang === l) continue;
v.dataset.lang = l;
v.querySelector('track')?.remove();
v.src = fileUrl('main', `media/${name}.${l}.mp4`);
v.append(el('track', { kind: 'subtitles', srclang: l, default: true, label: t('video.subtitles'),
src: fileUrl('main', `media/${name}.${l}.vtt`) }));
}
}
function progress() {
const parts = ['stages', 'pick', 'get'];
const line = 64 + 80;
let active = 0;
const frac = [];
parts.forEach((p, i) => {
const nodes = [...document.querySelectorAll(`main [data-part="${p}"], footer[data-part="${p}"]`)];
const top = Math.min(...nodes.map((n) => n.getBoundingClientRect().top));
const bottom = Math.max(...nodes.map((n) => n.getBoundingClientRect().bottom));
if (top <= line) active = i;
frac.push(Math.max(0, Math.min(1, (line - top) / Math.max(1, bottom - top))));
});
if (window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - 4) { active = parts.length - 1; frac[active] = 1; }
// #1216: the header is opaque once the page has scrolled
document.querySelector('.topnav')?.classList.toggle('scrolled', window.scrollY > 0);
document.querySelectorAll('.progress a').forEach((a, i) => {
a.setAttribute('aria-current', i === active ? 'true' : 'false');
a.querySelector('.fill').style.width = `${Math.round((i < active ? 1 : i === active ? Math.max(0.04, frac[i]) : 0) * 100)}%`;
});
}
/** Section heads fade in once as they scroll into view (CSS keeps them visible without motion or without JS). */
function reveal() {
const nodes = document.querySelectorAll('.reveal');
if (!('IntersectionObserver' in window)) { nodes.forEach((n) => n.classList.add('in')); return; }
const io = new IntersectionObserver((es) => es.forEach((e) => { if (e.isIntersecting) { e.target.classList.add('in'); io.unobserve(e.target); } }),
{ rootMargin: '0px 0px -8% 0px' });
document.documentElement.classList.add('js-reveal');
nodes.forEach((n) => io.observe(n));
}
/** #1617: the user manual is a view of its own: the address #manual shows it in place of the page (manual.js, loaded
* the first time it is shown, draws it in the page's language); #2614: #manual/<name> shows it at one of its parts,
* and a move between its parts keeps it as it is drawn; any other address shows the page, at the block it names. */
function showView() {
const on = inManual(location.hash);
const back = !on && $('page').hidden;
const was = !$('manual').hidden;
$('manual').hidden = !on;
$('page').hidden = on;
document.body.classList.toggle('manual-on', on);
if (on) {
if (!was) scrollTo(0, 0);
import('./manual.js').then((m) => m.showManual($('manual-body'), $('manual-toc'), renderTerms), (e) => {
clear($('manual-body')).append(el('p', { class: 'small problem-text' }, t('manual.missing', { error: e.message })));
console.error(e);
});
} else if (back) {
// the charts were drawn, if at all, while the page was hidden
redrawOnResize();
progress();
if (location.hash) document.getElementById(decodeURIComponent(location.hash.slice(1)))?.scrollIntoView();
}
}
/** Everything that shows the selection, drawn again when the selection answers with its ids. */
function renderScoped() {
const sel = selected();
syncFilterRow();
renderChips();
renderWork(sel);
renderMapHead(sel);
renderSources(sel);
renderGet(sel);
}
// the note above the page (a part of the index not read, or the index not read at all), kept so a new language
// writes it again: {key, values, problem}
let notice = null;
function drawNotice() {
if (!notice) return;
const n = $('notice');
clear(n).append(...(notice.problem ? [el('span', { class: 'dot problem', 'aria-hidden': 'true' })] : []), t(notice.key, notice.values));
n.hidden = false;
}
/** The page in the language just chosen: the words of index.html, then every part drawn from the index. */
function renderLanguage() {
translatePage();
renderVideo();
drawNotice();
if (!IX) return;
renderHeader();
fillFilterOptions();
drawDockSample();
FIELD?.setLabels(fieldLabels());
drawCharts();
renderScoped();
}
// #2066: the block the address names (a jump from the nav, such as #get, or a link such as #usage-video, which the
// dataset card links to) moves down or up as the parts above it are drawn, one after another; the page goes to it
// again after each, until the reader scrolls (moved), and from the start again on the next jump
let moved = false;
function toHash() {
if (moved || !location.hash || inManual(location.hash)) return;
document.getElementById(decodeURIComponent(location.hash.slice(1)))?.scrollIntoView();
}
function linkError(e) {
notice = { key: 'notice.linked', values: { error: e?.message || String(e) }, problem: true };
drawNotice();
console.error(e);
}
/** The linked views (linked.js), after the first screen has painted: DuckDB-WASM with the catalog of the page's commit,
* the filters from the address as clauses of the selection, the charts, the counts of the filter options, and the list
* of selected sources, which draws the rest of the browse section and Get the data. */
async function startLinked() {
try {
const mod = await import('./linked.js');
const cols = heatCols();
await mod.open(IX.catalogFile, IX.sources.map((s) => pageRow(s, cols)));
L = mod;
L.startFacets((key, m, total) => { UI.facets[key] = { m, total }; fillSelect(key); }, linkError);
// the sources the filters keep in either layer: the field draws those of the layer not shown at half light
L.startAcross((ids) => { UI.across = ids; FIELD?.setLevels(fieldLevels()); }, linkError);
L.apply(F, SEL_KEYS);
drawCharts();
L.connectFacets();
let first = true;
L.startSelection((ids) => {
UI.ids = ids;
renderScoped();
if (first && F.open) document.getElementById(`d-${F.open}`)?.scrollIntoView({ block: 'center' });
else if (first) toHash();
first = false;
}, linkError);
} catch (e) {
linkError(e);
}
}
async function main() {
readUrl();
// #2066: the Plot charts of a source's detail (plots.js) load as the page starts, beside the index
const plots = import('./plots.js').then((m) => { P = m; }, (e) => { plotsError = e; console.error(e); });
translatePage();
renderVideo();
setUpSwitch();
setUpWork();
onLanguage(renderLanguage);
decorate();
reveal();
showView();
addEventListener('hashchange', () => { moved = false; showView(); });
for (const ev of ['wheel', 'touchmove', 'keydown']) addEventListener(ev, () => { moved = true; }, { passive: true });
let ticking = false;
const onScroll = () => { if (!ticking) { ticking = true; requestAnimationFrame(() => { ticking = false; progress(); }); } };
addEventListener('scroll', onScroll, { passive: true });
let resizing = null;
addEventListener('resize', () => { onScroll(); clearTimeout(resizing); resizing = setTimeout(redrawOnResize, 300); });
progress();
try {
IX = await loadIndex(new URLSearchParams(location.search), (key, values) => { notice = { key, values }; drawNotice(); });
} catch (e) {
notice = { key: 'notice.failed', values: { error: e.message }, problem: true };
drawNotice();
console.error(e);
return;
}
// #2006: without stages of its commit the page has no stage to filter by, so a stage in the address would keep no
// source; #2422: with them, a stage in the address takes the layer it counts, as a click on its tile does
if (!IX.stages && F.stage) { F.stage = ''; writeUrl(); }
// without stages every source is in one stage group, so the field starts by domain unless the address names an
// arrangement
if (!IX.stages && !new URLSearchParams(location.search).get('view')) F.view = 'domains';
if (F.stage && stageKind(F.stage) !== F.kind) {
F.kind = stageKind(F.stage);
if (F.kind === CORPUS) F.collection = '';
writeUrl();
}
document.body.classList.add('ready');
renderHeader();
fillFilterRow();
syncFilterRow();
renderChips();
renderMapHead([]);
for (const b of $('layer-seg').querySelectorAll('button')) b.addEventListener('click', () => setLayer(b.dataset.kind));
for (const b of $('metric-seg').querySelectorAll('button')) {
b.addEventListener('click', () => { F.metric = b.dataset.metric; writeUrl(); drawCharts(); renderMapHead(selected()); });
}
progress();
toHash();
// the Plot charts of the open source's detail once their module is in (each a skeleton until then)
plots.then(() => {
if (UI.ids) renderSources(selected());
toHash();
});
// #2422: the linked charts take their colors from the page's theme when drawn (linked.js), so a new theme of the
// reader's draws them again; the rest of the page follows its colors by itself
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', drawCharts);
// the field and the linked views load after this screen has painted
requestAnimationFrame(() => setTimeout(() => { startField(); startLinked(); }, 0));
}
main();
// The data page of the Training Data Foundation (#2422): a working screen first (second round, "Strata": the field of
// every source as the ground, and on it the training stages as a column, the filters, the selection with its largest
// sources and the one command that takes it), then the sources (their list, each with its details, and the linked
// charts) and Get the data (the one command and the selection file). The dataset card is the overview; this page is
// where data is picked and taken.
// Figures first, few words; every chart keeps a short label and a table view. The charts of Pick, its filter counts and
// the list of selected sources come from one crossfilter selection (linked.js). The two layers stay apart everywhere
// and are never added together (#1083): material sets, grouped into collections and counted in sets and units, in the
// main color; corpora, counted in rows, in gray. Pick shows one layer at a time (F.kind, material sets by default; a
// training stage takes the layer it counts).
import {
CORPUS, DATASET_URL, fileUrl, KINDS, loadBatchDates, loadIndex, MATERIAL_SET, num, readColumns, REPO, summed, topOf,
wholeFile,
} from './data.js';
import {
clear, el, figureWithToggle, fmtAnd, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtOr, fmtPct, fmtShare, fmtShort,
segmented, table, tableShown, vt,
} from './charts.js';
import { decorate, drawRelease, drawStages, markFromIndex, RULE_KEY, sizeLine, sizeOf, sizeParts, stageColor, stageWhat } from './first.js';
import { lang, onLanguage, setUpSwitch, t, tEn, translatePage } from './i18n.js';
import { formIcon, icon } from './icons.js';
const CSV_MAX_ROWS = 200000;
const ROW_COLUMNS = ['source', 'batch', 'uri', 'range_offset', 'range_length', 'member_path', 'size_bytes', 'sha256'];
const FIRST_ROWS = 20;
const HEAT_FORMS = 8;
const FORM_BARS = 10;
// #1524: how many units Get the data takes from each source (--sample N; all: no --sample, #1891); #2629: when the
// reader has not chosen (sample in the address), every unit of material sets, whose whole is small, and 3 of each
// corpus, whose whole can be petabytes (DEFAULT_SAMPLE by the layer the command takes, sampleNow)
const SAMPLES = ['1', '3', '10', '30', 'all'];
const DEFAULT_SAMPLE = { [MATERIAL_SET]: 'all', [CORPUS]: '3' };
const MAX_FILE = 20e6;
const MAX_LISTED = 30;
// the phone layout of style.css
const PHONE = '(max-width: 720px)';
// #1617: the address of the user manual's view (#manual; showView); #2614: and of each of its parts, #manual/<name>
const MANUAL_VIEW = 'manual';
const inManual = (hash) => hash === `#${MANUAL_VIEW}` || hash.startsWith(`#${MANUAL_VIEW}/`);
const $ = (id) => document.getElementById(id);
// A link that leaves the page opens in a new tab. The Hub shows this page in a frame whose sandbox allows popups but
// not navigating the top window, and a site such as huggingface.co refuses to be shown in a frame, so a link opened in
// the frame showed the browser's error page (the Dataset link of the header, 2026-11-12). One listener in the capture
// phase sets the target before the browser follows the link, for the links of the markup and those the page writes.
document.addEventListener('click', (e) => {
const a = e.target.closest?.('a[href]');
if (a && /^https?:$/.test(a.protocol) && a.origin !== location.origin) {
a.target = '_blank';
a.rel = 'noopener';
}
}, true);
// ---- plain names (codes only inside the SQL and the commands); the words are in the locale files, by code:
// tier.<tier>, permit.<answer>, unitKind.<kind> and unitCount.<kind>
const tierName = (k, tt = t) => tt(`tier.${k}`, { defaultValue: human(k) });
const PERMIT_ORDER = ['yes', 'no', 'conditional', 'unknown'];
const PERMIT_COLOR = { yes: 'var(--ours)', no: 'var(--problem)', conditional: 'var(--fill-3)', unknown: 'var(--other)' };
// #1228: the condition in words of a conditional share under the source's licence, by its licence tier
const CONDITION = { 'share-alike': 'permit.conditional.sameLicence', 'nc-research': 'permit.conditional.nonCommercial' };
// a row's own licence; empty when the row takes its batch licence
const LICENCE_COLUMNS = ['license_spdx', 'license_training_ok', 'license_redistribution_ok'];
// #1266: the kinds of file of a material set (catalog column license_counts): its upstream files and the files the
// index wrote for it
const FILE_KINDS = ['upstream', 'index'];
// the two layers by their kind: kind.<kind> names them (Material sets, Corpora), kindOne.<kind> one of them
const kindName = (k, tt = t) => tt(`kind.${k}`);
const kindOne = (k) => t(`kindOne.${k}`);
/** The kinds as a legend: a swatch in each kind's color and its name. */
const kindKey = (kinds = KINDS) => el('span', { class: 'kind-key' },
kinds.map((k) => el('span', {}, el('span', { class: `sw kind-${k}`, 'aria-hidden': 'true' }), kindName(k))));
// the kinds of unit of a corpus (sources.json unit) -> their key in the locale files
const UNIT_KIND = { file: 'file', 'API response': 'apiResponse', 'WARC record': 'warcRecord', 'archive member': 'archiveMember', 'byte range': 'byteRange' };
// file type -> the tool that opens it (TOOL_WORDS: not a tool's name, a key of the locale files)
const TOOLS = {
xlsx: 'openpyxl', xlsm: 'openpyxl', xls: 'LibreOffice', ods: 'LibreOffice', csv: 'DuckDB', tsv: 'DuckDB', parquet: 'DuckDB',
json: 'jq', jsonl: 'jq', ndjson: 'jq', pdf: 'pdftotext', doc: 'LibreOffice', docx: 'python-docx', ppt: 'LibreOffice',
pptx: 'python-pptx', gz: 'gzip', tgz: 'tar', tar: 'tar', bz2: 'bzip2', xz: 'xz', zst: 'zstd', zip: 'unzip', whl: 'unzip',
jar: 'unzip', '7z': '7-Zip', crate: 'tar', deb: 'dpkg-deb', rpm: 'rpm2cpio', warc: 'warcio', arc: 'warcio', pgn: 'python-chess',
py: 'Python', '(none)': 'file(1)', xml: 'xmllint',
yaml: 'yq', yml: 'yq', evtx: 'python-evtx', log: 'grep', rego: 'OPA', sql: 'DuckDB', db: 'SQLite',
safetensors: 'safetensors', final: 'TensorBoard', ipynb: 'Jupyter', eml: 'Python email', mbox: 'Python mailbox',
ics: 'icalendar', e01: 'The Sleuth Kit', img: 'The Sleuth Kit', dd: 'The Sleuth Kit', raw: 'The Sleuth Kit',
ys: 'Yosys', v: 'Icarus Verilog', sv: 'Verilator', go: 'Go', rs: 'Cargo', zig: 'Zig', lua: 'Lua', rb: 'Ruby', c: 'gcc',
tcl: 'tclsh', smt2: 'Z3', mzn: 'MiniZinc', dzn: 'MiniZinc', lp: 'HiGHS', mps: 'HiGHS',
};
const TOOL_WORDS = { md: 'tool.editor', txt: 'tool.editor', toml: 'tool.editor', html: 'tool.browser', htm: 'tool.browser', test: 'tool.suiteRunner' };
const toolOf = (ext) => TOOLS[ext] || (TOOL_WORDS[ext] ? t(TOOL_WORDS[ext]) : null);
const NONE = '_none';
const MULTI = '_multi';
const OTHER = '_other';
const human = (code) => String(code).replace(/[_-]+/g, ' ').replace(/^./, (c) => c.toUpperCase());
function sentence(s) {
let x = String(s).trim();
if (!/\s/.test(x)) x = x.replace(/[-_]+/g, ' ');
x = x.replace(/\b(ci|ctf|etl|ml|qa|sql|api)\b/gi, (w) => (w.toLowerCase() === 'qa' ? 'Q&A' : w.toUpperCase()));
return x.charAt(0).toUpperCase() + x.slice(1);
}
/** A count with its noun: key_one and key_other in the locale files, {{n}} the formatted number. */
const plural = (key, n, tt = t) => tt(key, { count: n, n: fmtInt(n) });
const sum = (xs, f) => xs.reduce((a, x) => a + f(x), 0);