Battam's picture
portal: data page of repository commit 544a0b41b806
8c4f803 verified
Raw History Blame Contribute Delete
137 kB
// The data page of the Training Data Foundation (#2422): a working screen first (second round, "Strata": the field of // every source as the ground, and on it the training stages as a column, the filters, the selection with its largest // sources and the one command that takes it), then the sources (their list, each with its details, and the linked // charts) and Get the data (the one command and the selection file). The dataset card is the overview; this page is // where data is picked and taken. // Figures first, few words; every chart keeps a short label and a table view. The charts of Pick, its filter counts and // the list of selected sources come from one crossfilter selection (linked.js). The two layers stay apart everywhere // and are never added together (#1083): material sets, grouped into collections and counted in sets and units, in the // main color; corpora, counted in rows, in gray. Pick shows one layer at a time (F.kind, material sets by default; a // training stage takes the layer it counts). import { CORPUS, DATASET_URL, fileUrl, KINDS, loadBatchDates, loadIndex, MATERIAL_SET, num, readColumns, REPO, summed, topOf, wholeFile, } from './data.js'; import { clear, el, figureWithToggle, fmtAnd, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtOr, fmtPct, fmtShare, fmtShort, segmented, table, tableShown, vt, } from './charts.js'; import { decorate, drawRelease, drawStages, markFromIndex, RULE_KEY, sizeLine, sizeOf, sizeParts, stageColor, stageWhat } from './first.js'; import { lang, onLanguage, setUpSwitch, t, tEn, translatePage } from './i18n.js'; import { formIcon, icon } from './icons.js'; const CSV_MAX_ROWS = 200000; const ROW_COLUMNS = ['source', 'batch', 'uri', 'range_offset', 'range_length', 'member_path', 'size_bytes', 'sha256']; const FIRST_ROWS = 20; const HEAT_FORMS = 8; const FORM_BARS = 10; // #1524: how many units Get the data takes from each source (--sample N; all: no --sample, #1891); #2629: when the // reader has not chosen (sample in the address), every unit of material sets, whose whole is small, and 3 of each // corpus, whose whole can be petabytes (DEFAULT_SAMPLE by the layer the command takes, sampleNow) const SAMPLES = ['1', '3', '10', '30', 'all']; const DEFAULT_SAMPLE = { [MATERIAL_SET]: 'all', [CORPUS]: '3' }; const MAX_FILE = 20e6; const MAX_LISTED = 30; // the phone layout of style.css const PHONE = '(max-width: 720px)'; // #1617: the address of the user manual's view (#manual; showView); #2614: and of each of its parts, #manual/<name> const MANUAL_VIEW = 'manual'; const inManual = (hash) => hash === `#${MANUAL_VIEW}` || hash.startsWith(`#${MANUAL_VIEW}/`); const $ = (id) => document.getElementById(id); // A link that leaves the page opens in a new tab. The Hub shows this page in a frame whose sandbox allows popups but // not navigating the top window, and a site such as huggingface.co refuses to be shown in a frame, so a link opened in // the frame showed the browser's error page (the Dataset link of the header, 2026-11-12). One listener in the capture // phase sets the target before the browser follows the link, for the links of the markup and those the page writes. document.addEventListener('click', (e) => { const a = e.target.closest?.('a[href]'); if (a && /^https?:$/.test(a.protocol) && a.origin !== location.origin) { a.target = '_blank'; a.rel = 'noopener'; } }, true); // ---- plain names (codes only inside the SQL and the commands); the words are in the locale files, by code: // tier.<tier>, permit.<answer>, unitKind.<kind> and unitCount.<kind> const tierName = (k, tt = t) => tt(`tier.${k}`, { defaultValue: human(k) }); const PERMIT_ORDER = ['yes', 'no', 'conditional', 'unknown']; const PERMIT_COLOR = { yes: 'var(--ours)', no: 'var(--problem)', conditional: 'var(--fill-3)', unknown: 'var(--other)' }; // #1228: the condition in words of a conditional share under the source's licence, by its licence tier const CONDITION = { 'share-alike': 'permit.conditional.sameLicence', 'nc-research': 'permit.conditional.nonCommercial' }; // a row's own licence; empty when the row takes its batch licence const LICENCE_COLUMNS = ['license_spdx', 'license_training_ok', 'license_redistribution_ok']; // #1266: the kinds of file of a material set (catalog column license_counts): its upstream files and the files the // index wrote for it const FILE_KINDS = ['upstream', 'index']; // the two layers by their kind: kind.<kind> names them (Material sets, Corpora), kindOne.<kind> one of them const kindName = (k, tt = t) => tt(`kind.${k}`); const kindOne = (k) => t(`kindOne.${k}`); /** The kinds as a legend: a swatch in each kind's color and its name. */ const kindKey = (kinds = KINDS) => el('span', { class: 'kind-key' }, kinds.map((k) => el('span', {}, el('span', { class: `sw kind-${k}`, 'aria-hidden': 'true' }), kindName(k)))); // the kinds of unit of a corpus (sources.json unit) -> their key in the locale files const UNIT_KIND = { file: 'file', 'API response': 'apiResponse', 'WARC record': 'warcRecord', 'archive member': 'archiveMember', 'byte range': 'byteRange' }; // file type -> the tool that opens it (TOOL_WORDS: not a tool's name, a key of the locale files) const TOOLS = { xlsx: 'openpyxl', xlsm: 'openpyxl', xls: 'LibreOffice', ods: 'LibreOffice', csv: 'DuckDB', tsv: 'DuckDB', parquet: 'DuckDB', json: 'jq', jsonl: 'jq', ndjson: 'jq', pdf: 'pdftotext', doc: 'LibreOffice', docx: 'python-docx', ppt: 'LibreOffice', pptx: 'python-pptx', gz: 'gzip', tgz: 'tar', tar: 'tar', bz2: 'bzip2', xz: 'xz', zst: 'zstd', zip: 'unzip', whl: 'unzip', jar: 'unzip', '7z': '7-Zip', crate: 'tar', deb: 'dpkg-deb', rpm: 'rpm2cpio', warc: 'warcio', arc: 'warcio', pgn: 'python-chess', py: 'Python', '(none)': 'file(1)', xml: 'xmllint', yaml: 'yq', yml: 'yq', evtx: 'python-evtx', log: 'grep', rego: 'OPA', sql: 'DuckDB', db: 'SQLite', safetensors: 'safetensors', final: 'TensorBoard', ipynb: 'Jupyter', eml: 'Python email', mbox: 'Python mailbox', ics: 'icalendar', e01: 'The Sleuth Kit', img: 'The Sleuth Kit', dd: 'The Sleuth Kit', raw: 'The Sleuth Kit', ys: 'Yosys', v: 'Icarus Verilog', sv: 'Verilator', go: 'Go', rs: 'Cargo', zig: 'Zig', lua: 'Lua', rb: 'Ruby', c: 'gcc', tcl: 'tclsh', smt2: 'Z3', mzn: 'MiniZinc', dzn: 'MiniZinc', lp: 'HiGHS', mps: 'HiGHS', }; const TOOL_WORDS = { md: 'tool.editor', txt: 'tool.editor', toml: 'tool.editor', html: 'tool.browser', htm: 'tool.browser', test: 'tool.suiteRunner' }; const toolOf = (ext) => TOOLS[ext] || (TOOL_WORDS[ext] ? t(TOOL_WORDS[ext]) : null); const NONE = '_none'; const MULTI = '_multi'; const OTHER = '_other'; const human = (code) => String(code).replace(/[_-]+/g, ' ').replace(/^./, (c) => c.toUpperCase()); function sentence(s) { let x = String(s).trim(); if (!/\s/.test(x)) x = x.replace(/[-_]+/g, ' '); x = x.replace(/\b(ci|ctf|etl|ml|qa|sql|api)\b/gi, (w) => (w.toLowerCase() === 'qa' ? 'Q&A' : w.toUpperCase())); return x.charAt(0).toUpperCase() + x.slice(1); } /** A count with its noun: key_one and key_other in the locale files, {{n}} the formatted number. */ const plural = (key, n, tt = t) => tt(key, { count: n, n: fmtInt(n) }); const sum = (xs, f) => xs.reduce((a, x) => a + f(x), 0); // #2065: units as the figures count them (meta/size_figures.json): a material set without a count of its units // (data.js readyUnits null) is not counted yet, never counted by its rows; a sum over material sets is the sum of the // counted ones, with how many are not counted yet (data.js summed), in the card's words const uncounted = (n) => (n ? t('n.uncounted', { n: fmtInt(n) }) : ''); const unitsText = (n, missing = 0) => (n == null ? t('n.unitsNotCounted') : plural('n.unit', n) + uncounted(missing)); const unitsNum = (n) => (n == null ? t('n.notCounted') : fmtInt(n)); const unitsOfSets = (S) => { const u = summed(S.map((s) => s.readyUnits)); return unitsText(u.sum, u.missing); }; const notCountedOf = (S) => summed(S.filter((s) => s.isTask).map((s) => s.readyUnits)).missing; /** #2066: a chart of plots.js, draw(); until the module is loaded a skeleton of class sk, and the reason in its place * when it could not be loaded (each figure's Table view has the same numbers). */ function plotted(draw, sk = 'sk sk-block') { if (plotsError) return el('p', { class: 'small problem-text' }, t('chart.failed', { error: plotsError.message })); if (!P) return el('div', { class: sk }); return el('div', { class: 'plot-box' }, draw()); } let IX = null; // linked.js once loaded (after the first screen has painted) let L = null; // #2066: plots.js (the page's Plot charts) once loaded, which starts with the page, or the error that kept it from // loading let P = null; let plotsError = null; const F = {}; // ids: the selected sources (the selection's answer, null until it comes); facets: per filter {m: value -> sources, // total}; tables: the rows behind each linked chart, with the ids each one covers; charts: the vgplot elements; // preview: the ids whose "Preview rows" is open; previewRows: per id its rows of meta/preview.parquet, once read // across: the ids the filters keep in either layer (linked.js startAcross) const UI = { across: null, allForms: false, showAll: false, detail: new Map(), preview: new Set(), previewRows: new Map(), csv: null, tab: 'cmd', fitKeyOpen: false, ids: null, facets: {}, tables: { dom: [], form: [], heat: [] }, charts: null }; // kind: the layer the browse section shows (material_set, the default, or corpus); metric: what the charts of material // sets count (sets or units; corpora are counted in rows); pick: the ids of the picked sources, joined by commas; ext: // the file types selected, joined by commas; sample: how many units Get the data takes from each source (SAMPLES); // stage (#2006): the id of a training stage of the stage table (the commit's stage figures); pii (#2422): the personal-data mark (none, // marked or unknown, piiMark). The panel shows no control for collection, leaf, tier, redist and status (#2422): from // the address they still filter, each a chip that removes it // view (#2422, second round): the arrangement of the field (VIEWS), strata by default const KEYS = ['domain', 'leaf', 'form', 'collection', 'kind', 'train', 'redist', 'tier', 'size', 'status', 'q', 'ext', 'stage', 'pii', 'metric', 'sort', 'dir', 'open', 'pick', 'sample', 'view']; function readUrl() { const p = new URLSearchParams(location.search); for (const k of KEYS) F[k] = p.get(k) || ''; if (F.kind !== CORPUS) F.kind = MATERIAL_SET; if (F.metric !== 'units') F.metric = 'sets'; if (!F.sort) { F.sort = 'fit'; F.dir = 'desc'; } if (!SAMPLES.includes(F.sample)) F.sample = ''; if (!VIEWS.includes(F.view)) F.view = VIEWS[0]; } function writeUrl() { const p = new URLSearchParams(location.search); for (const k of KEYS) { const v = F[k]; const dflt = (k === 'kind' && v === MATERIAL_SET) || (k === 'metric' && v === 'sets') || (k === 'sort' && v === 'fit') || (k === 'dir' && v === 'desc') || (k === 'view' && v === VIEWS[0]); if (v && !dflt) p.set(k, v); else p.delete(k); } const s = p.toString(); history.replaceState(null, '', s ? `?${s}` : location.pathname); } // ---- names: the vocabulary's own (meta/vocabulary.json), in the page's language. In Chinese a name is its zh, without // the bracketed note a zh name may carry (as the Chinese card shows it, index_projection.domain_name); in English its // name, in sentence case. const ZH_NOTE = /\s*\uff08[^\uff09]*\uff09/g; const inChinese = () => lang() === 'zh'; const vocabName = (sec, code) => { const zh = inChinese() ? IX.vocab[sec].zh.get(code) : null; if (zh) return zh.replace(ZH_NOTE, '').trim(); const n = IX.vocab[sec].names.get(code); return n ? sentence(n) : null; }; function domainName(key) { if (key === NONE) return t('name.notClassified'); if (key === MULTI) return vocabName('domain', 'multi') || t('name.severalDomains'); return vocabName('domain', key) || t(`domain.${key}`, { defaultValue: key }); } function formName(key) { if (key === NONE || key == null) return t('name.notClassified'); if (key === OTHER) return t('name.otherForms'); return vocabName('form', key) || sentence(key); } /** The name of a layer (meta/vocabulary.json layers: Task material, Corpora) in the page's language; the locale's name * of the kind when the vocabulary has none. */ function layerName(kind) { const l = IX?.vocab.layers.get(kind); return (l && (lang() === 'zh' ? l.zh : l.name)) || kindName(kind); } /** The name of a collection (meta/vocabulary.json collections) in the page's language; #2066: in Chinese the Chinese * text of its name in the table of Chinese names (#2065, zhOf), else the vocabulary's zh. */ function collectionName(code) { const c = IX.vocab.collections.find((x) => x.code === code); if (!c) return code || t('name.noCollection'); return (inChinese() && (IX.zh.get(c.name) || c.zh)) || c.name; } /** #2066: an English text the index shows (a source's title or description, as the catalog holds it) in the page's * language: in Chinese its Chinese text in the table of Chinese names the release writes (#2065, data.js zh), else * as it is (a text the table does not hold yet stays English). */ const zhOf = (en) => (inChinese() && en && IX.zh.get(en)) || en; const titleOf = (s) => zhOf(s.title); const descriptionOf = (s) => zhOf(s.description); /** #2006: the name of a training stage of the stage table (IX.stages, from the commit's stage figures) in the page's language, marked * when the stage is provisional (stageTitle); its id when the page has no such stage. */ function stageName(id) { const st = IX.stages?.list.find((x) => x.id === id); return st ? stageTitle(st) : id; } /** #2066: a stage's name ({name, zh, status} of the release's figures) in the page's language, * marked when it is provisional. */ function stageTitle(st) { const name = (inChinese() && st.zh) || st.name; return st.status === 'provisional' ? t('stage.provisional', { name }) : name; } const domainKeysOf = (s) => (s.domainKind === 'codes' ? s.tops : [s.domainKind === 'multi' ? MULTI : NONE]); const formKeyOf = (s) => s.form ?? NONE; // #2422: a source's personal-data mark, a label the page filters on and counts and never leaves a source out of a count // for: its catalog pii.level present or suspected (marked), none_known (none), else not judged yet (unknown) const PII_MARKED = ['present', 'suspected']; const piiMark = (s) => (PII_MARKED.includes(s.pii?.level) ? 'marked' : s.pii?.level === 'none_known' ? 'none' : 'unknown'); function sourceDomainText(s) { if (s.domainKind !== 'codes') return domainName(s.domainKind === 'multi' ? MULTI : NONE); const leaves = s.codes.filter((c) => c.includes('.')); // a leaf without a name shows its domain's name, never a bare code const names = (leaves.length ? leaves : s.codes).map((c) => (domainName(c) === c ? domainName(topOf(c)) : domainName(c))); return [...new Set(names)].join('; '); } const inCategory = (s, code) => (code.includes('.') ? s.codes.includes(code) : s.tops.includes(code)); // ---- fit: what decides whether a source can feed a task (yes, part, no, unknown); labels fit.<k>, states fitWord.<state> const FIT = [ { k: 'ship', icon: 'share' }, { k: 'bench', icon: 'target' }, { k: 'pii', icon: 'person' }, { k: 'size', icon: 'size' }, { k: 'alone', icon: 'doc' }, { k: 'login', icon: 'link' }, // #1577: whether the upstream still serves the files (catalog column fetch_check) { k: 'fetch', icon: 'get' }, ]; const fitLabel = (k) => t(`fit.${k}`); const fitWord = (state) => t(`fitWord.${state}`); // #1577: the fit of a fetch level (catalog column fetch_check, index_projection.FETCH_LEVELS); #2597: needs auth (the // check, which reads anonymously, could not read the rows) says nothing of what the upstream serves, so its fit is unknown const FETCH_FIT = { all: 'yes', some: 'part', none: 'no' }; // #2597: the sentence of each level read (locales fetch.<key>); a level without one is not read yet (fetch.notRead) const FETCH_SENTENCE = { all: 'all', some: 'some', none: 'none', 'needs auth': 'needsAuth' }; /** #1577: a source's fetch check in words: how many of its sampled rows read fetched and matched their record, out of * how many, and the day read; or that none is read yet; #2597: or that the check could not read them anonymously. */ function fetchText(f) { if (!f || !FETCH_SENTENCE[f.level]) return t('fetch.notRead'); return t(`fetch.${FETCH_SENTENCE[f.level]}`, { fetched: fmtInt(f.fetched), rows: fmtInt(f.rows), share: fmtShare(f.fetched, f.rows), date: f.readAt || t('common.unknown') }); } function fitOf(s) { const q = s.info.size_quantiles_upper_bytes || {}; const c = s.contamination?.level; const p = s.pii?.level; const kinds = s.isTask ? ['unit'] : s.kinds; return { ship: s.shipShare >= 0.999 ? 'yes' : s.shipShare <= 0.001 ? 'no' : 'part', bench: c == null || c === 'not scanned' ? 'unknown' : c === 'none' ? 'yes' : 'no', pii: p == null ? 'unknown' : p === 'none_known' ? 'yes' : 'no', size: s.bytes != null && s.bytes <= MAX_FILE ? 'yes' : q.p99 != null && num(q.p99) < MAX_FILE ? 'yes' : q.p50 != null && num(q.p50) >= MAX_FILE ? 'no' : q.p50 != null ? 'part' : 'unknown', alone: kinds.every((k) => ['unit', 'file', 'API response'].includes(k)) ? 'yes' : 'part', login: s.needsAuth === 'no' ? 'yes' : s.needsAuth === 'mixed' ? 'part' : 'no', fetch: FETCH_FIT[s.fetch?.level] || 'unknown', }; } function fitDot(state, label) { const name = t('fit.dot', { label, state: fitWord(state) }); return el('span', { class: `fit fit-${state}`, role: 'img', title: name, 'aria-label': name }); } function fitStrip(s) { const f = fitOf(s); return el('span', { class: 'fit-strip' }, FIT.map((x) => fitDot(f[x.k], fitLabel(x.k)))); } /** #1302: the names of the fit dots over their column, each an icon and a short label (fitShort.<k>) above its dot. */ function fitHeads() { return el('span', { class: 'fit-heads' }, FIT.map((x) => el('span', { class: 'fit-head', title: fitLabel(x.k) }, icon(x.icon, { size: 14 }), el('span', {}, t(`fitShort.${x.k}`))))); } function fitKey() { return el('span', { class: 'fit-key' }, ['yes', 'part', 'no', 'unknown'].map((k) => el('span', {}, el('span', { class: `fit fit-${k}`, 'aria-hidden': 'true' }), fitWord(k)))); } // ---- filters: the values are the page's state and its address (F); the selection of linked.js applies them, and its // answer (the ids of the selected sources) is what every part below the filters shows const sizeBucket = (b) => (b == null ? 'unknown' : b < 1e9 ? 'lt1g' : b < 1e12 ? '1g1t' : 'ge1t'); /** One row of the page's DuckDB table src (linked.js): what the filters and the charts read of a source; n_units are * the units of a material set (#2065: null, which SQL's sum leaves out, while not counted yet) and n_rows the rows of a * corpus, 0 for the other kind; collection is that of a material * set or (#1633) of a corpus a collection lists, '' otherwise; exts its file types and ext_rows the rows of each * (data.js exts); #2006: stages the ids of the training stages one of its units passes (data.js stages); #2422: sizeb * the size class of its deduplicated size, the one size the page shows, and pii its personal-data mark (piiMark). */ function pageRow(s, cols) { return { source: s.id, dkeys: domainKeysOf(s), codes: s.codes, formkey: formKeyOf(s), hcol: heatColOf(s, cols), kind: s.kind, collection: s.collection || '', status: s.status, n_src: 1, n_units: s.readyUnits, n_rows: s.rawRows, sizeb: sizeBucket(s.distinct), tier: s.tier, train: s.train, redist: s.redist, pii: piiMark(s), // #2066: the search reads the title and description in both languages qtext: [s.title, s.description, IX.zh.get(s.title), IX.zh.get(s.description), s.id].filter(Boolean).join('\n').toLowerCase(), exts: s.exts.map(([e]) => e), ext_rows: s.exts.map(([, n]) => n), stages: s.stages, }; } const selected = () => (UI.ids ? IX.sources.filter((s) => UI.ids.has(s.id)) : []); const FILTER_KEYS = ['domain', 'leaf', 'form', 'collection', 'train', 'redist', 'tier', 'size', 'status', 'q', 'ext', 'stage', 'pii']; // the clauses of the selection: the filters and the layer shown (kind, always one of the two) const SEL_KEYS = [...FILTER_KEYS, 'kind']; const activeFilters = () => FILTER_KEYS.filter((k) => F[k]); /** The filters that are on, as [label, value, key] in the page's language. */ function filterChips() { const out = []; const word = (key) => t(key, { defaultValue: '' }); if (F.domain) out.push([t('chip.domain'), F.leaf ? domainName(F.leaf) : domainName(F.domain), 'domain']); if (F.form) out.push([t('chip.form'), formName(F.form), 'form']); if (F.collection) out.push([t('chip.collection'), collectionName(F.collection), 'collection']); if (F.stage) out.push([t('chip.stage'), stageName(F.stage), 'stage']); if (F.train) out.push([t('chip.training'), word(`filter.permit.${F.train}`), 'train']); if (F.pii) out.push([t('chip.pii'), word(`filter.pii.${F.pii}`), 'pii']); if (F.redist) out.push([t('chip.redistribution'), word(`filter.permit.${F.redist}`), 'redist']); if (F.tier) out.push([t('chip.tier'), tierName(F.tier), 'tier']); if (F.size) out.push([t('chip.size'), word(`filter.size.${F.size}`), 'size']); if (F.status) out.push([t('chip.status'), t(`filter.status.${F.status}`, { defaultValue: human(F.status) }), 'status']); if (F.q) out.push([t('chip.search'), F.q, 'q']); if (F.ext) out.push([t('chip.ext'), extList().map((e) => `.${e}`).join(', '), 'ext']); return out; } /** #1524: the file types selected (F.ext), in the order of the address. */ const extList = () => (F.ext ? F.ext.split(',').filter(Boolean) : []); /** What the charts count, in the layer shown: material sets in sets or units (F.metric), corpora in rows. */ const metricNow = () => (F.kind === CORPUS ? 'rows' : F.metric); function metricOf(s, m = metricNow()) { return m === 'sets' ? 1 : m === 'units' ? s.readyUnits : s.rawRows; } const metricFmt = () => (v) => fmtInt(v); const metricWord = (m) => t(`metric.${m}`); /** #2422: the layer a training stage counts: that of the sources one of whose units passes it (a stage's condition names * one kind), else by its measures (a stage counted in units counts material sets). */ function stageKind(id) { const s = IX.sources.find((x) => x.stages.includes(id)); if (s) return s.kind; return IX.stageList.find((x) => x.id === id)?.counted_by?.includes('units') ? MATERIAL_SET : CORPUS; } /** Sets filters (patch of F); origin 'map' when a map cell set them. #2422: a training stage takes the layer it counts, * and a layer leaves out a stage of the other one, as corpora leave out a collection (which belongs to material sets); * a new layer draws the charts again in its measure. The selection answers with the new ids, and the parts below the * filters are drawn again from them (renderScoped). */ function setFilter(patch, origin = '') { const p = { ...patch }; if (p.stage) p.kind = stageKind(p.stage); const kind = p.kind ?? F.kind; if (kind === CORPUS && (p.collection ?? F.collection)) p.collection = ''; const stage = p.stage ?? F.stage; if (stage && stageKind(stage) !== kind) p.stage = ''; const layer = kind !== F.kind; Object.assign(F, p); UI.showAll = false; writeUrl(); const keys = Object.keys(p).filter((k) => SEL_KEYS.includes(k)); if (L && keys.length) L.apply(F, keys, origin); syncFilterRow(); renderChips(); if ('stage' in p || layer) renderStageFig(); if (layer) { drawCharts(); renderMapHead(selected()); } // the labels of the field say the stage and the domain the filters hold, and the last group's figure of the layer if (FIELD && (layer || 'stage' in p || 'domain' in p)) { FIELD.setLabels(fieldLabels()); FIELD.pick(stageFigList().findIndex((s) => s.id === F.stage)); } } /** Shows one layer in Pick (material_set or corpus), with the charts in its measure (setFilter). */ function setLayer(kind) { if (kind !== F.kind) setFilter({ kind }); } // ---- the command: tools/fetch.py of the index (#268), run with uv, the script and the data pinned to the release tag // of the commit the page reads (#1197: IX.pin, as the quick start of the dataset card; --revision: without it the // script reads the main branch as it is when it starts) const fetchUrl = () => fileUrl(IX.pin, 'tools/fetch.py'); /** #2629: the layer a command takes: the picked sources' while there are picks (corpora when one of them is a corpus), * else the layer shown. */ function takenKind() { const picks = IX ? picked() : []; return picks.length ? (picks.some((s) => !s.isTask) ? CORPUS : MATERIAL_SET) : F.kind; } /** #2629: the units choice in force: the reader's (F.sample), else the default of the layer the command takes. */ const sampleNow = () => F.sample || DEFAULT_SAMPLE[takenKind()]; /** #1524: the options of a units choice (SAMPLES): --sample N, or none for every unit. #1891: never fetch.py's --all, * which turns the default profile's other criteria off: "all" is a choice of how many units, and the profile stays on. */ const sampleArgs = (v = sampleNow()) => (v === 'all' ? [] : [`--sample ${v}`]); /** The options of a command, one string each; exts: one --ext per file type (#1524); sample: the units choice it takes * (#2629: the one in force unless given). */ function commandArgs({ domain, form, sources, tags, kind, exts, sample } = {}) { const a = [`--revision ${IX.pin}`]; if (kind) a.push(`--kind ${kind}`); if (sources) for (const s of sources) a.push(`--source ${s}`); if (tags) for (const g of tags) a.push(`--tag campaign:${g}`); if (domain) a.push(`--domain ${domain}`); if (form) a.push(`--form ${form}`); if (exts) for (const e of exts) a.push(`--ext ${e}`); a.push(...sampleArgs(sample), '--out ./data'); return a; } // #1216: one option per line, continuation lines indented four spaces and every line but the last ending in " \", so // the text shown is the text copied and runs as it is const command = (args) => [`uv run ${fetchUrl()}`, ...args].join(' \\\n '); /** The options that take exactly S, sources of one layer, within domain and form (#1083: one command per layer). * whole: S is every source of its layer within domain and form, taken by fetch.py's --kind (#1216). Otherwise material * sets are taken through the campaign tags of their collections (--tag campaign:<name>, any of them; a collection's * material sets carry the campaign of the collection or of one of its earlier names, vocabulary * collections[].campaigns), within --domain and --form; corpora, and material sets without a collection, one --source * each. exts: the file types selected, one --ext each (#1524). */ function layerArgs(S, { domain, form, whole = false, exts } = {}) { if (whole) return { args: commandArgs({ kind: S[0]?.isTask ? MATERIAL_SET : CORPUS, domain, form, exts }), listed: false }; const colls = new Set(S.map((s) => s.collection)); if (!S[0]?.isTask || colls.has(null)) return { args: commandArgs({ sources: S.map((s) => s.id), exts }), listed: true }; const tags = IX.vocab.collections.filter((c) => colls.has(c.code)).flatMap((c) => c.campaigns); return { args: commandArgs({ tags, domain, form, exts }), listed: false }; } /** The options that select exactly the page's selection (one layer). fetch.py's --domain matches a top-level code with * its leaves, a leaf only itself, and multi and unclassified only themselves, as the page's domain filter does; --form * matches one form id. So the layer's options within domain and form when only those filters and the collection are * set (and the form is a classified one; without a collection the whole layer there, --kind), else one --source per * selected source. #1524: the file types selected narrow the rows of either, one --ext each (fetch.py keeps the rows * of any of them, so a source without such rows gives none). */ function selectionArgs(sel) { const exts = extList(); const other = activeFilters().some((k) => !['domain', 'leaf', 'form', 'collection', 'ext'].includes(k)) || F.form === NONE; if (other) return { args: commandArgs({ sources: sel.map((s) => s.id), exts }), listed: true }; const domain = F.domain === MULTI ? 'multi' : F.domain === NONE ? 'unclassified' : F.leaf || F.domain; return layerArgs(sel, { domain, form: F.form, whole: !F.collection, exts }); } async function copy(text, state) { try { await navigator.clipboard.writeText(text); state.textContent = t('copy.done'); } catch { state.textContent = t('copy.select'); } setTimeout(() => { state.textContent = ''; }, 2400); } function copyButton(label, text, { dark = false, iconName = 'copy' } = {}) { const state = el('span', { class: 'copy-state', 'aria-live': 'polite' }); return el('span', { class: 'copy-wrap' }, el('button', { type: 'button', class: `btn${dark ? ' on-dark' : ''}`, onclick: (e) => { e.stopPropagation(); copy(text, state); } }, icon(iconName, { size: 16 }), label), state); } // ---- #2422: the first screen: the three training stages (first.js drawStages), each with what it can use, and the // release line. The stages are those of the release's figures (meta/stage_figures.json, in the stage table's order), // each with its status; without the figures there is no band. A tile sets the filter by stage once the commit's figures // list the stages of each source (#2518); what is missing is one short line under the tiles. function stageFigList() { const figs = IX.stageFigures?.stages; // a band's hover text: the stage's condition (in Chinese its text in the table of Chinese names) and the version of // the stage table the stages were computed with const title = (s) => [(inChinese() && IX.zh.get(s.about)) || s.about, IX.stages ? t('stage.version', { v: IX.stages.version }) : null].filter(Boolean).join(' '); return [...(figs?.values() || [])].map((s) => ({ ...s, title: title(s) })); } // ---- the words the page counts in and sorts by, under the stages: each a button that opens its meaning in place, the // sentence of the index's term table (meta/vocabulary.json terms; in Chinese its text in the table of Chinese names, // the English sentence while a release has not translated it), and for the domains their names, in the vocabulary's // order. One popover for all of them (the Popover API: it closes on a click outside or Escape), placed under the // button that opened it. A commit whose vocabulary has no term table shows no such line. #2614: the start of the // manual has the same line ([data-terms]), drawn again with the manual's text (showView). const TERMS = ['material set', 'corpus', 'unit', 'domain']; function renderTerms() { const terms = IX?.vocab.terms; const have = TERMS.filter((k) => terms?.get(k)?.about); for (const line of document.querySelectorAll('#terms, .manual-text [data-terms]')) { line.hidden = !have.length; if (!have.length) continue; line.replaceChildren(el('span', { class: 'terms-lead' }, t('terms.lead')), ...have.map((k) => { const x = terms.get(k); const name = inChinese() ? (x.zh || k) : k; return el('button', { type: 'button', class: 'term', 'aria-haspopup': 'dialog', onclick: (e) => showTerm(e.currentTarget, k, name, x.about) }, name); })); } } function showTerm(button, key, name, about) { const pop = $('term-pop'); const kids = [el('strong', {}, name), el('p', {}, (inChinese() && IX.zh.get(about)) || about)]; if (key === 'domain') { const tops = IX.vocab.domain.order.filter((c) => !c.includes('.') && c !== 'multi' && c !== 'unclassified'); kids.push(el('ul', {}, tops.map((c) => el('li', {}, domainName(c))))); } kids.push(el('a', { href: `#${MANUAL_VIEW}/glossary`, onclick: () => pop.hidePopover() }, t('terms.more'))); pop.replaceChildren(...kids); if (typeof pop.showPopover !== 'function') return; pop.showPopover(); // under the button, kept inside the window const b = button.getBoundingClientRect(); const w = pop.offsetWidth, h = pop.offsetHeight; pop.style.left = `${Math.max(8, Math.min(b.left, window.innerWidth - w - 8))}px`; pop.style.top = `${b.bottom + 8 + h > window.innerHeight ? Math.max(8, b.top - h - 8) : b.bottom + 8}px`; } function renderStageFig() { if (!IX) return; const why = IX.stages ? [] : [stageWhy()]; drawStages(stageFigList(), { pick: F.stage, onPick: IX.stages ? pickStage : null, onPoint: (g) => FIELD?.front(g), why, rest: restWhat() }); } /** A stage's band: the filter by stage set to it (with its layer), or cleared when it holds it already; the field and * the panels of the screen show the result. */ function pickStage(id) { setFilter({ stage: F.stage === id ? '' : id }); } // ---- filter row: the options with their counts under the current selection (linked.js facets; #1216: an option that // keeps no source is left out, unless it is the one selected, so it can be cleared), again in a new language and // whenever the counts change; the listeners once function option(value, text) { return el('option', { value }, text); } /** Each filter's options as [value, name]; the first option, '' (all or any), is added by fillSelect. */ function filterOptions(key) { const words = (prefix, values) => values.map((v) => [v, t(`${prefix}.${v}`)]); switch (key) { // #2006: the stages of the stage table, in its order case 'stage': return (IX.stages?.list || []).map((x) => [x.id, stageName(x.id)]); case 'domain': return domainRows().map((k) => [k, domainName(k)]); case 'form': return formsByCount().map((k) => [k, formName(k)]); case 'size': return words('filter.size', ['lt1g', '1g1t', 'ge1t', 'unknown']); case 'train': return words('filter.permit', ['yes', 'cond']); case 'pii': return words('filter.pii', ['none', 'marked', 'unknown']); default: return []; } } // #2422: the filters of the panel, each a select (the file types are checkboxes, fillExt), in the panel's order const SELECTS = { stage: 'f-stage', domain: 'f-domain', form: 'f-form', size: 'f-size', train: 'f-train', pii: 'f-pii' }; const ANY = new Set(['size', 'train', 'pii']); function fillSelect(key) { if (key === 'ext') { fillExt(); return; } const s = $(SELECTS[key]); const f = UI.facets[key]; const text = (name, n) => (n == null ? name : t('filter.optCount', { name, n: fmtInt(n) })); const opts = filterOptions(key).filter(([v]) => !f || f.m.get(v) > 0 || F[key] === v); // #2422: a stage's count is of the sources of its own layer (linked.js), so All, over both layers, has none clear(s).append(option('', text(t(ANY.has(key) ? 'filter.any' : 'filter.all'), key === 'stage' ? null : f?.total)), ...opts.map(([v, name]) => option(v, text(name, f ? f.m.get(v) || 0 : null)))); s.value = F[key] || ''; if (key === 'stage') { // #2006: the selected stage's condition (#2066: in Chinese its text in the table of Chinese names, once the release // has it) and the version of the stage table on hover; without stages of the commit the page reads, the filter is // off and (#2422) one short line under it says why (stageWhy) const st = IX.stages?.list.find((x) => x.id === F.stage); s.disabled = !IX.stages; s.title = IX.stages ? [inChinese() ? IX.zh.get(st?.about) : st?.about, t('stage.version', { v: IX.stages.version })] .filter(Boolean).join('\n') : ''; $('f-stage-note').hidden = !!IX.stages; $('f-stage-note').textContent = IX.stages ? '' : stageWhy(); } } /** #2422: why the commit the page reads has no stage for any source, in one short line (data.js stagesWhy): no stage * figures in it, a release that has not counted its stage fields, or figures written before they listed the sources * (#2518); the filter by stage and the first screen say it. */ const stageWhy = () => t(`stage.${IX.stagesWhy}`); /** #1524: the File type facet, as CKAN's Formats facet: one checkbox per file type of the layer shown with its rows * under the other filters (linked.js facet ext, from each source's by_ext), most rows first; a type that keeps no row * is left out unless it is checked, so it can be cleared. Any number can be checked (F.ext). */ function fillExt() { const box = $('f-ext'); const f = UI.facets.ext; const on = extList(); const types = f ? [...f.m].filter(([k, n]) => k !== '*' && n > 0) : []; for (const k of on) if (!types.some(([x]) => x === k)) types.push([k, f?.m.get(k) || 0]); types.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); const toggle = (k, checked) => setFilter({ ext: [...on.filter((x) => x !== k), ...(checked ? [k] : [])].join(',') }); clear(box).append(...types.map(([k, n]) => el('label', {}, el('input', { type: 'checkbox', value: k, checked: on.includes(k), onchange: (e) => toggle(k, e.target.checked) }), el('span', {}, t('filter.optCount', { name: `.${k}`, n: fmtInt(n) }))))); if (f && !types.length) box.append(el('span', { class: 'small muted' }, t('filter.extNone'))); // #2422: the fold's summary says how many types are checked $('ext-on').textContent = on.length ? t('filter.on', { count: on.length, n: fmtInt(on.length) }) : ''; } const fillFilterOptions = () => { Object.keys(SELECTS).forEach(fillSelect); fillExt(); }; function fillFilterRow() { fillFilterOptions(); for (const [key, id] of Object.entries(SELECTS)) { $(id).addEventListener('change', (e) => { const patch = { [key]: e.target.value }; if (key === 'domain') patch.leaf = ''; setFilter(patch); }); } let timer = null; $('f-q').addEventListener('input', (e) => { clearTimeout(timer); timer = setTimeout(() => setFilter({ q: e.target.value.trim() }), 200); }); $('f-clear').addEventListener('click', () => setFilter(Object.fromEntries(FILTER_KEYS.map((k) => [k, ''])))); $('filter-toggle').addEventListener('click', () => { const open = !$('filters').classList.contains('open'); $('filters').classList.toggle('open', open); $('filter-toggle').setAttribute('aria-expanded', String(open)); }); } /** The filter row after a change: the values, the file types (their fold open while one is checked), and the count of * the selection. */ function syncFilterRow() { for (const [key, id] of Object.entries(SELECTS)) if ($(id).value !== F[key]) $(id).value = F[key]; fillExt(); if (document.activeElement !== $('f-q')) $('f-q').value = F.q; if (F.ext) $('ext-fold').open = true; const n = activeFilters().length; $('f-clear').disabled = !n; const count = UI.ids ? plural(F.kind === CORPUS ? 'n.corpus' : 'n.materialSet', UI.ids.size) : ''; $('filter-state').textContent = n ? [t('filter.on', { count: n, n: fmtInt(n) }), count].filter(Boolean).join(' · ') : t('filter.noneOn'); $('f-count').textContent = count; } /** The filters that are on, next to the charts: one removable chip each, and Clear when there are two or more. */ function renderChips() { const box = $('sel-chips'); if (!box || !IX) return; const chips = filterChips().map(([k, v, key]) => { const off = key === 'domain' ? { domain: '', leaf: '' } : { [key]: '' }; return el('span', { class: 'chip sel-chip' }, el('span', { class: 'chip-k' }, k), el('b', {}, v), el('button', { type: 'button', class: 'chip-x', 'aria-label': t('chip.remove', { k, v }), title: t('chip.remove', { k, v }), onclick: () => setFilter(off, '') }, '×')); }); // append writes an array or a null as text, so the chips go in one by one and Clear only when it is there clear(box).append(...chips, ...(chips.length > 1 ? [el('button', { type: 'button', class: 'link', onclick: () => setFilter(Object.fromEntries(FILTER_KEYS.map((x) => [x, '']))) }, t('filter.clear'))] : [])); } // ---- axes of the map function numericCodeSort(a, b) { const pa = a.replace(/^D/, '').split('.').map(Number); const pb = b.replace(/^D/, '').split('.').map(Number); for (let i = 0; i < Math.max(pa.length, pb.length); i++) { const d = (pa[i] ?? -1) - (pb[i] ?? -1); if (d) return d; } return a.localeCompare(b); } function domainRows() { const tops = new Set(); for (const c of IX.vocab.domain.order) if (!c.includes('.') && c !== 'multi' && c !== 'unclassified') tops.add(c); for (const s of IX.sources) for (const t of s.tops) tops.add(t); const out = [...tops].sort(numericCodeSort); if (IX.sources.some((s) => s.domainKind === 'multi') || IX.vocab.domain.names.has('multi')) out.push(MULTI); // the row of sources not classified yet only while there are some if (IX.sources.some((s) => s.domainKind === 'none') || F.domain === NONE) out.push(NONE); return out; } let FORM_ORDER = null; function formsByCount() { if (FORM_ORDER) return FORM_ORDER; const n = new Map(); for (const c of IX.vocab.form.order) if (c !== 'unclassified') n.set(c, 0); for (const s of IX.sources) if (s.form) n.set(s.form, (n.get(s.form) || 0) + 1); FORM_ORDER = [...n].sort((a, b) => b[1] - a[1] || formName(a[0]).localeCompare(formName(b[0]))).map(([k]) => k); FORM_ORDER.push(NONE); return FORM_ORDER; } function heatCols() { const all = formsByCount().filter((k) => k !== NONE && IX.sources.some((s) => s.form === k)); const cols = all.length > HEAT_FORMS + 1 ? [...all.slice(0, HEAT_FORMS), OTHER] : all; if (IX.sources.some((s) => !s.form)) cols.push(NONE); return cols; } const heatColOf = (s, cols) => (cols.includes(formKeyOf(s)) ? formKeyOf(s) : OTHER); /** #1216: the rows and columns of the map that hold a source of the layer shown (and the ones the filters name). */ function heatKeys() { const cols = heatCols(); const S = IX.sources.filter((s) => s.kind === F.kind); const rowKeys = new Set(S.flatMap(domainKeysOf)); const colKeys = new Set(S.map((s) => heatColOf(s, cols))); return { rows: domainRows().filter((k) => rowKeys.has(k) || k === F.domain), cols: cols.filter((k) => colKeys.has(k) || k === F.form) }; } // ---- 3. the map and the bars: the vgplot charts of linked.js. Each has its table (the rows behind it, under the same // selection) and, in its Table view, the query behind that table for the dataset's SQL Console. const FIGS = { heat: 'heat-fig', dom: 'dom-fig', form: 'form-fig' }; function contentWidth(id) { const n = $(id); const cs = getComputedStyle(n); return Math.max(280, Math.floor(n.clientWidth - parseFloat(cs.paddingLeft) - parseFloat(cs.paddingRight))); } let drawnWidths = ''; function drawCharts() { if (!L) return; const m = metricNow(); const fmt = metricFmt(); const forms = formsByCount(); let formKeys = UI.allForms ? forms : forms.slice(0, FORM_BARS); if (F.form && !formKeys.includes(F.form)) formKeys = [...formKeys, F.form]; const tableOf = (k) => (rows) => { UI.tables[k] = rows; if (tableShown(k)) renderChartFig(k); }; drawnWidths = Object.values(FIGS).map(contentWidth).join(' '); // a release that changes no filter (a click on the empty part of a chart with nothing picked) does nothing const pickDomain = (k) => { if (k !== F.domain) setFilter({ domain: k, leaf: '' }); }; const pickForm = (k) => { if (k !== F.form) setFilter({ form: k }); }; const pickCell = (p) => { const next = p ? { domain: p.row, leaf: '', form: p.col === OTHER ? '' : p.col } : { domain: '', leaf: '', form: '' }; if (next.domain !== F.domain || next.form !== F.form) setFilter(next, 'map'); }; // the words the tooltips name their lines by; the layer shown, whose color the marks take const words = { metricWord: metricWord(m), kind: F.kind }; UI.charts = L.draw({ dom: { field: 'dkey', keys: domainRows(), name: (k) => domainName(k), metric: m, fmt, ...words, keyWord: t('th.domain'), width: contentWidth('dom-fig'), onPick: pickDomain, onTable: tableOf('dom'), onError: linkError }, form: { field: 'formkey', keys: formKeys, name: (k) => formName(k), metric: m, fmt, ...words, keyWord: t('th.form'), width: contentWidth('form-fig'), onPick: pickForm, onTable: tableOf('form'), onError: linkError }, heat: { ...heatKeys(), rowName: (k) => domainName(k), colName: (k) => formName(k), ...words, rowWord: t('th.domain'), colWord: t('th.form'), metric: m, fmt, width: contentWidth('heat-fig'), onTable: tableOf('heat'), onError: linkError, onPick: pickCell }, }, F); for (const k of Object.keys(FIGS)) renderChartFig(k); } /** Draws the linked charts again when the width of their cards has changed (a rotated phone, a resized window); the * charts of a source's detail have a width of their own (DETAIL_CHART_PX). */ function redrawOnResize() { if (L && UI.charts && Object.values(FIGS).map(contentWidth).join(' ') !== drawnWidths) drawCharts(); } function renderChartFig(k) { const m = metricNow(); const title = k === 'heat' ? t('map.title', { metric: metricWord(m) }) : t(k === 'dom' ? 'chart.byDomain' : 'chart.byForm', { metric: metricWord(m) }); const keyCells = (r) => (k === 'heat' ? [domainName(r.k), formName(r.f)] : [k === 'dom' ? domainName(r.k) : formName(r.k)]); const keyHeads = k === 'heat' ? [t('th.domain'), t('th.form')] : [k === 'dom' ? t('th.domain') : t('th.form')]; const forms = formsByCount(); const more = k === 'form' && forms.length > FORM_BARS ? () => el('button', { type: 'button', class: 'link', 'aria-expanded': String(UI.allForms), onclick: () => { UI.allForms = !UI.allForms; drawCharts(); } }, UI.allForms ? t('common.fewer') : t('common.all', { n: fmtInt(forms.length) })) : null; figureWithToggle($(FIGS[k]), k, title, { // the layer the chart draws, in its color, above the chart; the table holds that layer's columns only chart: () => el('div', {}, kindKey([F.kind]), el('div', { class: 'plot-wrap' }, UI.charts[k])), table: () => (F.kind === CORPUS ? table([...keyHeads, { label: t('th.corpora'), r: true }, { label: t('th.corpusRows'), r: true }], UI.tables[k].map((r) => [...keyCells(r), fmtInt(num(r.n)), fmtInt(num(r.rows))])) : table([...keyHeads, { label: t('th.materialSets'), r: true }, { label: t('th.readyUnits'), r: true }], UI.tables[k].map((r) => [...keyCells(r), fmtInt(num(r.n)), fmtInt(num(r.units))]))), sql: () => consoleSql(k), }, more); } /** The heading of the map: what the selection holds, by the map's count, and the domains it covers. */ function renderMapHead(sel) { const m = metricNow(); for (const b of $('layer-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.kind === F.kind)); for (const b of $('metric-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.metric === m)); $('metric-wrap').hidden = F.kind === CORPUS; if (!UI.ids) return; const total = sum(sel, (s) => metricOf(s, m)); const covered = new Set(sel.flatMap((s) => s.tops)).size; const amount = m === 'sets' ? plural('n.materialSet', total) : m === 'units' ? t('map.units', { count: total, n: fmtCompact(total) }) + uncounted(notCountedOf(sel)) : t('map.rows', { count: total, n: fmtCompact(total) }); $('map-h').textContent = !sel.length ? t('map.nothing') : covered ? t('map.head', { amount, domains: plural('n.domain', covered) }) : amount; } // ---- the queries behind the tables, for the dataset's SQL Console: over the catalog of the commit the page reads, // filtered by the filters that filter that table (linked.js filtersOf), written over the catalog's own columns; the // domain keys and the kind are those of data.js, in the catalog's words (unclassified, multi; the column kind, or the // rule it is written by when the commit's catalog has none). A query counts the layer shown only, in its own measure: // material sets and their units, or corpora and their rows. The licence filters read meta/sources.json, which is no // table, so they name their sources by id (or the ids they leave out, whichever list is shorter). const READY_SQL = 'coalesce(units < files, false)'; const kindSql = () => (IX.hasKind ? 'kind' : `CASE WHEN ${READY_SQL} THEN '${MATERIAL_SET}' ELSE '${CORPUS}' END`); const DOMAIN_KEYS_SQL = "CASE WHEN domain IS NULL OR len(domain) = 0 OR list_contains(domain, 'unclassified') THEN ['unclassified'] " + "WHEN list_contains(domain, 'multi') THEN ['multi'] ELSE list_distinct([split_part(code, '.', 1) FOR code IN domain]) END"; const FORM_SQL = "coalesce(nullif(form, ''), 'unclassified')"; const sumsSql = () => (F.kind === CORPUS ? 'count(*) AS corpora, sum(files) AS corpus_rows' : 'count(*) AS material_sets, sum(units) AS units'); // #2422: the size classes of the deduplicated size, the one size the page shows (catalog column distinct_bytes) const SIZE_SQL = { lt1g: 'distinct_bytes > 0 AND distinct_bytes < 1e9', '1g1t': 'distinct_bytes >= 1e9 AND distinct_bytes < 1e12', ge1t: 'distinct_bytes >= 1e12', unknown: 'coalesce(distinct_bytes, 0) <= 0' }; // #2422: the personal-data mark in the catalog's words (pii.level) const PII_SQL = { none: "pii.level = 'none_known'", marked: "pii.level IN ('present', 'suspected')", unknown: "coalesce(pii.level, '') NOT IN ('none_known', 'present', 'suspected')" }; const catalogWord = (key) => (key === NONE ? 'unclassified' : key === MULTI ? 'multi' : key); function idsSql(keep) { const all = IX.sources.map((s) => s.id).sort(); const inside = all.filter((id) => keep.has(id)); const outside = all.filter((id) => !keep.has(id)); if (!inside.length) return 'false'; if (!outside.length) return 'true'; const list = (xs) => xs.map(sqlStr).join(', '); return inside.length <= outside.length ? `source IN (${list(inside)})` : `source NOT IN (${list(outside)})`; } function filterSql(keys) { const permitted = (col, v) => idsSql(new Set(IX.sources.filter((s) => (v === 'yes' ? s[col] === 'yes' : s[col] !== 'no')).map((s) => s.id))); const sqlOf = { domain: (v) => `list_contains(${DOMAIN_KEYS_SQL}, ${sqlStr(catalogWord(v))})`, leaf: (v) => `list_contains(domain, ${sqlStr(v)})`, form: (v) => `${FORM_SQL} = ${sqlStr(catalogWord(v))}`, kind: (v) => `${kindSql()} = ${sqlStr(v)}`, collection: (v) => `collection = ${sqlStr(v)}`, status: (v) => `status = ${sqlStr(v)}`, tier: (v) => `coalesce(license.tier, 'unknown') = ${sqlStr(v)}`, size: (v) => SIZE_SQL[v], pii: (v) => PII_SQL[v], q: (v) => `contains(lower(concat_ws(' ', title, description, source)), ${sqlStr(v.toLowerCase())})`, train: (v) => permitted('train', v), redist: (v) => permitted('redist', v), // #1524: the file types are in meta/sources.json (by_ext), no column of the catalog, so the sources are named by id ext: (v) => idsSql(new Set(IX.sources.filter((s) => s.exts.some(([e]) => v.split(',').includes(e))).map((s) => s.id))), // #2006: a source's stages are computed from the stage table (the commit's stage figures), no column of the catalog, so the // sources are named by id stage: (v) => idsSql(new Set(IX.sources.filter((s) => s.stages.includes(v)).map((s) => s.id))), }; const parts = keys.filter((k) => F[k]).map((k) => `(${sqlOf[k](F[k])})`); return parts.length ? `WHERE ${parts.join('\n AND ')}` : ''; } function consoleSql(k) { const cat = `read_parquet(${sqlStr(fileUrl(IX.sha, 'meta/catalog.parquet'))})`; const lines = { dom: () => [`SELECT domain_key, ${sumsSql()}`, `FROM ${cat} AS c, unnest(${DOMAIN_KEYS_SQL}) AS t(domain_key)`, filterSql(L ? L.filtersOf(k) : ['kind']), 'GROUP BY domain_key ORDER BY domain_key'], form: () => [`SELECT ${FORM_SQL} AS form, ${sumsSql()}`, `FROM ${cat}`, filterSql(L ? L.filtersOf(k) : ['kind']), 'GROUP BY 1 ORDER BY count(*) DESC'], heat: () => [`SELECT domain_key, ${FORM_SQL} AS form, ${sumsSql()}`, `FROM ${cat} AS c, unnest(${DOMAIN_KEYS_SQL}) AS t(domain_key)`, filterSql(L ? L.filtersOf(k) : ['kind']), 'GROUP BY 1, 2 ORDER BY 1, 2'], }[k](); // #1633: these queries read the catalog as published and say so, as the rows query does (sql.noProfile) return [`-- ${REPO} @ ${IX.sha}`, `-- ${t('sql.catalogNoProfile')}`, ...lines.filter(Boolean)].join('\n') + ';'; } // ---- 4. sources; the column labels are col.<key> in the locale files const COLS = [ { key: 'title' }, { key: 'domain' }, { key: 'form' }, { key: 'units', r: true }, { key: 'size', r: true }, { key: 'tier' }, { key: 'fit' }, ]; function fitScore(s) { const f = fitOf(s); return sum(FIT, (x) => ({ yes: 2, part: 1, unknown: 0.5, no: 0 })[f[x.k]]); } function sortValue(s, k) { switch (k) { case 'title': return titleOf(s).toLowerCase(); case 'domain': return sourceDomainText(s).toLowerCase(); case 'form': return formName(formKeyOf(s)).toLowerCase(); // units of material sets above rows of corpora (never compared as one count): a corpus has fewer than 1e12 rows case 'units': return s.isTask ? 1e12 + s.readyUnits : s.rawRows; case 'size': return s.distinct ?? -1; case 'tier': return tierName(s.tier).toLowerCase(); case 'fit': return fitScore(s); default: return 0; } } /** #2422: a source's size in the list (meta/size_figures.json sources), its deduplicated size, the one size the page * shows, and, for its rows without a recorded size, their estimate (or their count when too few were read for one), * labelled; the size check and the estimate's range on hover (the cell's title). */ function sizeCell(s) { const g = s.size; if (!g) return t('common.unknown'); return [el('b', {}, sizeOf(g)), g.unsized_rows ? el('span', { class: 'size-sub' }, g.estimate ? t('size.barEstimate', { size: fmtBytes(g.estimate.bytes) }) : t('size.barUnsized', { rows: fmtCompact(g.unsized_rows) })) : null]; } // #364, #1158: the same words as docs/sources.md of the index (artpool/projection_templates/text.json, size_text), in // the locale files as size.* /** The size note under the list: the size of the layer shown (meta/size_figures.json, IX.sizes; #2422: its deduplicated * size), with the estimate for its rows without a recorded size. */ function sizeNote() { const g = IX.sizes?.[F.kind]; if (!g) return t('size.none'); const unsized = !g.unsized_rows ? null : g.estimate ? t('size.unsizedEst', { size: fmtBytes(g.estimate.bytes) }) : t('size.unsizedNone', { rows: fmtCompact(g.unsized_rows) }); return t('size.note', { layer: layerName(F.kind), line: [t('size.noteSize', { size: sizeOf(g) }), unsized].filter(Boolean).join(' \u00b7 ') }); } /** #2422: how sizes are measured, folded under the list: which files count as one (by which hash, across what scope) * and where the size before deduplication can still be read, with the index docs' explanation behind its link. */ function sizeHow() { return el('details', { class: 'size-how' }, el('summary', {}, t('size.how')), el('p', {}, t('size.howText'), ' ', el('a', { href: t('url.docsSources') }, t('size.howMore')))); } /** The place of a material set's collection in the vocabulary's order (one without a collection last). */ function collectionRank(s) { const i = IX.vocab.collections.findIndex((c) => c.code === s.collection); return i < 0 ? IX.vocab.collections.length : i; } /** The list of the selection, one layer: material sets grouped by collection (a header row per collection, in the * vocabulary's order), corpora as one list, or (#1633) grouped the same way when a collection lists some of them; * the heading names the layer shown only. */ function renderSources(sel) { if (!UI.ids) return; const dir = F.dir === 'asc' ? 1 : -1; const sets = F.kind !== CORPUS; const grouped = sets || sel.some((s) => s.collection); const byColl = (a, b) => (grouped ? collectionRank(a) - collectionRank(b) : 0); const list = [...sel].sort((a, b) => { const x = sortValue(a, F.sort); const y = sortValue(b, F.sort); return byColl(a, b) || (x < y ? -1 : x > y ? 1 : 0) * dir || titleOf(a).localeCompare(titleOf(b)); }); $('sources-h').textContent = !sel.length ? t('sources.none') : sets ? t('sources.headSets', { sets: plural('n.materialSet', sel.length) }) : t('sources.headCorpora', { corpora: plural('n.corpus', sel.length) }); const body = clear($('sources-body')); if (!sel.length) return; const shown = UI.showAll ? list : list.slice(0, FIRST_ROWS); const opened = list.find((s) => s.id === F.open); if (opened && !shown.includes(opened)) shown.push(opened); shown.sort(byColl); const inColl = new Map(); for (const s of list) inColl.set(s.collection, (inColl.get(s.collection) || 0) + 1); // #1216: the forms of each group (a collection's material sets); a group of one form leaves its form cells out, its // name is the group's const formsIn = new Map(); for (const s of list) formsIn.set(s.collection, (formsIn.get(s.collection) || new Set()).add(formKeyOf(s))); const oneForm = (s) => sets && formsIn.get(s.collection).size === 1; // #1216: the kind mark only when the list holds both kinds const bothKinds = new Set(shown.map((s) => s.kind)).size > 1; // a label of a cell on a phone, with its colon from the locale files const label = (text) => t('sources.label', { label: text }); const head = el('tr', {}, COLS.map((c) => { const on = F.sort === c.key; return el('th', { scope: 'col', class: `${c.r ? 'r' : ''} col-${c.key}`, 'aria-sort': on ? (F.dir === 'asc' ? 'ascending' : 'descending') : null }, el('button', { type: 'button', onclick: () => { const d = on ? (F.dir === 'asc' ? 'desc' : 'asc') : (c.r || c.key === 'fit' ? 'desc' : 'asc'); F.sort = c.key; F.dir = d; writeUrl(); renderSources(selected()); } }, t(c.key === 'units' && !sets ? 'col.rows' : `col.${c.key}`), on ? (F.dir === 'asc' ? ' ↑' : ' ↓') : ''), c.key === 'fit' ? fitHeads() : null); })); const tbody = el('tbody'); let group; for (const s of shown) { if (grouped && s.collection !== group) { group = s.collection; // #2422: a collection's heading among material sets is a toggle of the filter by collection, which the panel // shows no control for const name = group ? collectionName(group) : t('name.noCollection'); const cur = F.collection === group; tbody.append(el('tr', { class: 'group-row' }, el('th', { colspan: String(COLS.length), scope: 'colgroup' }, sets && group ? el('button', { type: 'button', class: 'group-btn', 'aria-pressed': String(cur), title: t(cur ? 'coll.showing' : 'coll.open'), onclick: () => setFilter({ collection: cur ? '' : group }) }, cur ? icon('check', { size: 16 }) : null, name) : name, el('span', { class: 'small' }, plural(sets ? 'n.materialSet' : 'n.corpus', inColl.get(group)))))); } const open = F.open === s.id; const toggle = () => openSource(open ? '' : s.id); const previewing = UI.preview.has(s.id); const preview = () => { if (previewing) UI.preview.delete(s.id); else UI.preview.add(s.id); renderSources(selected()); }; const isPicked = pickIds().includes(s.id); tbody.append(el('tr', { class: `row${open ? ' open' : ''}${isPicked ? ' picked' : ''}` }, el('td', { class: 'title' }, el('button', { type: 'button', 'aria-expanded': String(open), 'aria-controls': `d-${s.id}`, onclick: toggle }, el('span', { class: 'caret', 'aria-hidden': 'true' }, open ? '▾' : '▸'), titleOf(s)), bothKinds ? el('span', { class: `kind-tag kind-tag-${s.kind}` }, el('span', { class: `sw kind-${s.kind}`, 'aria-hidden': 'true' }), kindOne(s.kind)) : null, // #1633: a flagged source (catalog column status; an upstream that serves none of its sampled files is one) is // marked as not usable as it is, its reasons on hover and in its detail s.status === 'flagged' ? el('span', { class: 'small problem-text', title: s.reasons.map(zhOf).join('; ') }, ` ${t('status.flagged')}`) : null, el('span', { class: 'title-foot' }, el('code', { class: 'src-id' }, s.id), el('button', { type: 'button', class: 'link preview-btn', 'aria-expanded': String(previewing), 'aria-controls': `p-${s.id}`, onclick: preview }, previewing ? t('preview.hide') : t('preview.show')), el('button', { type: 'button', class: 'link pick-btn', 'aria-pressed': String(isPicked), onclick: () => togglePick(s.id) }, isPicked ? [icon('check', { size: 14 }), t('pick.picked')] : t('pick.add')))), el('td', { 'data-label': label(t('col.domain')), class: 'cat-cell' }, sourceDomainText(s)), oneForm(s) ? el('td', { class: 'form-cell form-same' }) : el('td', { 'data-label': label(t('col.form')), class: 'form-cell' }, formIcon(formKeyOf(s), { size: 16 }), formName(formKeyOf(s))), el('td', { class: 'r num', 'data-label': label(s.isTask ? t('sources.unitsLabel') : t('sources.rowsLabel')) }, s.isTask ? (s.readyUnits == null ? t('n.notCounted') : [fmtInt(s.readyUnits), el('span', { class: 'unit-word' }, t('sources.unitWord', { count: s.readyUnits }))]) : el('span', { class: 'muted' }, fmtCompact(s.rawRows), el('span', { class: 'unit-word' }, t('sources.corpusWord', { count: s.rawRows })))), el('td', { class: 'r num size-cell', 'data-label': label(t('col.size')), title: s.size ? sizeLine(s.size, false) : null }, sizeCell(s)), el('td', { 'data-label': label(t('col.tier')) }, tierName(s.tier)), el('td', { class: 'fit-cell' }, fitStrip(s)))); // the source's first rows from meta/preview.parquet of the page's commit, read once opened if (previewing) { tbody.append(el('tr', { class: 'preview-row' }, el('td', { colspan: String(COLS.length), id: `p-${s.id}` }, previewTable(s)))); } if (open) tbody.append(el('tr', { class: 'detail-row' }, el('td', { colspan: String(COLS.length), id: `d-${s.id}` }, detail(s)))); } // #1216: the fit legend, folded into one line on a phone (its summary shows there only), open on a wider screen const wide = !matchMedia(PHONE).matches; body.append(el('details', { class: 'list-key-fold', open: wide || UI.fitKeyOpen, ontoggle: (e) => { if (!wide) UI.fitKeyOpen = e.currentTarget.open; } }, el('summary', {}, t('sources.aboutFit')), el('div', { class: 'list-key' }, kindKey([F.kind]), el('span', { class: 'small' }, t('sources.fitKey')), FIT.map((x) => el('span', { class: 'fit-name' }, icon(x.icon, { size: 14 }), fitLabel(x.k))), fitKey()))); body.append(el('table', { class: 'src-table' }, el('caption', { class: 'visually-hidden' }, kindName(F.kind)), el('thead', {}, head), tbody)); body.append(el('div', { class: 'small size-note' }, el('p', {}, sizeNote()), sizeHow())); if (list.length > FIRST_ROWS) { body.append(el('p', { class: 'more' }, el('button', { type: 'button', class: 'link', 'aria-expanded': String(UI.showAll), onclick: () => { UI.showAll = !UI.showAll; renderSources(selected()); } }, UI.showAll ? t('common.first', { n: fmtInt(FIRST_ROWS) }) : t('common.all', { n: fmtInt(list.length) })))); } } // ---- the first rows of one source (the index's meta/preview.parquet, read in the page's DuckDB-WASM by linked.js): // every column that has a value in one of them, what a row records first (PREVIEW_FIRST, #1256), then the others in // the file's order, one cell each, cut to one line (the whole value on hover; #1302: an address in its middle) const PREVIEW = 'meta/preview.parquet'; const PREVIEW_FIRST = ['uri', 'size_bytes', 'sha256', 'license_spdx']; const cellText = (v) => (v == null ? '' : v instanceof Uint8Array ? hex(v) : String(v)); /** The preview rows of source s ('loading' until read, {error} when they could not be); once read, its preview table * and its detail are drawn again (#1197: the detail's example is the first unit file among them). */ function previewOf(s) { if (!UI.previewRows.has(s.id)) { UI.previewRows.set(s.id, 'loading'); const redraw = (v) => { UI.previewRows.set(s.id, v); const cell = document.getElementById(`p-${s.id}`); if (cell) clear(cell).append(previewTable(s)); const box = document.getElementById(`d-${s.id}`); if (box) clear(box).append(detail(s)); }; L.previewRows(() => wholeFile(IX.sha, PREVIEW), s.id).then(redraw, (e) => redraw({ error: e.message })); } return UI.previewRows.get(s.id); } /** #1302: an address cut in the middle, its host at the start and its file name at the end (a long name cut in its * own middle); the address itself when that is not shorter. */ const NAME_MAX = 40; function shortUri(v) { const m = v.match(/^(?:[a-z][a-z0-9+.-]*:\/\/)?([^/]+)\/.*\/([^/]*)$/i); if (!m) return v; const name = m[2].length > NAME_MAX ? `${m[2].slice(0, 14)}\u2026${m[2].slice(-(NAME_MAX - 15))}` : m[2]; const out = `${m[1]}/\u2026/${name}`; return out.length < v.length ? out : v; } function previewTable(s) { const rows = previewOf(s); if (rows === 'loading') return el('p', { class: 'small' }, t('preview.loading')); if (rows.error) return el('p', { class: 'small problem-text' }, t('preview.failed', { error: rows.error })); if (!rows.length) return el('p', { class: 'small' }, t('preview.none')); const keys = Object.keys(rows[0]); const cols = [...PREVIEW_FIRST.filter((c) => keys.includes(c)), ...keys.filter((c) => !PREVIEW_FIRST.includes(c))] .filter((c) => rows.some((r) => cellText(r[c]) !== '')); const tbl = table(cols, rows.map((r) => cols.map((c) => { const v = cellText(r[c]); return c === 'uri' ? el('span', { class: 'uri', title: v }, shortUri(v)) : el('span', { title: v }, v); })), t('preview.title', { n: fmtInt(rows.length) })); tbl.classList.add('preview-table'); return el('div', { class: 'table-wrap sample' }, tbl); } // ---- the detail of one source, in the group's order; the first rows are read when it opens const hex = (v) => (v instanceof Uint8Array ? [...v].map((b) => b.toString(16).padStart(2, '0')).join('') : v == null ? '' : String(v)); const firstSentence = (t) => { const m = String(t).match(/^.*?[.!?](\s|$)/); const s = m ? m[0].trim() : String(t); return s.length > 120 ? `${s.slice(0, 117).replace(/\s+\S*$/, '')} …` : s; }; function metaOf(r) { try { return JSON.parse(r.extra || '{}').source_meta || {}; } catch { return {}; } } /** A row of a unit: one that carries a unit_group, or a row of no collection (every row of a corpus); the README.md and * LICENSE of a collection source belong to the source (#1633: no roles). */ const isUnit = (m) => !m.campaign || !!m.unit_group; /** An http(s) address as a URL, else null. */ function webUrl(u) { try { const x = new URL(u); return /^https?:$/.test(x.protocol) ? x : null; } catch { return null; } } // pool_source.origin of a material set's rows: its one upstream, or several in the form // multi-publisher:<source name>(<upstream 1>,<upstream 2>,...) (artpool/pool/selection.py Source.origin) const MULTI_ORIGIN = /^multi-publisher:[^(]*\((.*)\)$/; const upstreamsOf = (origin) => { const m = origin.match(MULTI_ORIGIN); return m ? m[1].split(',') : [origin]; }; /** The upstreams of the rows (addresses, which are no publishers), distinct, in order of first appearance. */ const upstreamsOfRows = (metas) => [...new Set(metas.flatMap((m) => (m.pool_source?.origin ? upstreamsOf(String(m.pool_source.origin)) : [])))]; /** The publisher names of the rows (the publisher of their pool_fields, the unit's line of the pool manifest), * distinct, in order of first appearance. */ const publisherNames = (metas) => [...new Set(metas.flatMap((m) => (Array.isArray(m.pool_fields) ? m.pool_fields.filter(([k, v]) => k === 'publisher' && v).map(([, v]) => String(v)) : [])))]; // the summary sentences artpool admit.py writes for a material set (readme_forms, what): "<n> files published by ...", // "<n> units (<m> files) from <k> publishers." and the like; #2065: without counts since, "Files published by ...", // "Units from several publishers.", "Files from the upstream named in the provenance line.", and "Files from one // publisher." where admit.README_COUNTS rewrote an older one. Not the upstream's words, so the page builds its own // sentence in its language instead (#1216); the same pattern as the projection's (index_projection.ADMIT_WHAT) const ADMIT_WHAT = /^(?:\d+ (?:units? \(\d+ files?\)|files?) (?:published by |from )|(?:Files|Units) (?:published by |from (?:one|several) publishers?\.|from the upstream named in ))/; /** #1197: where a source comes from, in words: how many publishers when there are several (#1256: publishers, the * catalog's count over all its rows; never its addresses), then its main site, the host most of the addresses are on * (upstream: the upstreams of its rows, else the addresses of its rows), as one link with the host as its text; an * upstream that is no address is written as it is. -> nodes, [] when none. */ function fromWords(addresses, upstream, publishers) { const sites = new Map(); for (const u of addresses.map(webUrl).filter(Boolean)) { const x = sites.get(u.host) || { origin: u.origin, n: 0 }; x.n++; sites.set(u.host, x); } const [host, main] = [...sites].sort((a, b) => b[1].n - a[1].n)[0] || []; const site = host ? el('a', { href: `${main.origin}/`, rel: 'noopener' }, host) : upstream ? addresses[0] : null; if (!site) return []; return publishers > 1 ? [`${t('detail.publishers', { count: publishers, n: fmtInt(publishers) })} `, site] : [site]; } function upstream(uri) { if (!uri) return null; if (/^https?:\/\//.test(uri)) return uri; if (uri.startsWith('campaign_files/')) return fileUrl(IX.sha, uri); const s3 = uri.match(/^s3:\/\/([^/]+)\/(.*)$/); if (s3) return `https://${s3[1]}.s3.amazonaws.com/${s3[2]}`; return null; } // #2588: the declared upstream's values of one field, joined by ' | ' while the source reads several upstreams // (catalog_rules.SEVERAL) const SEVERAL = ' | '; /** #2588: a source's declared upstream (catalog columns upstream_kind, upstream_agent, upstream_entry), one line per * upstream: its kind in the page's language, its agent, and its entry address, a link when it is an http(s) one. */ function upstreamLines(u) { const kinds = u.kind.split(SEVERAL), agents = u.agent.split(SEVERAL), entries = u.entry.split(SEVERAL); return el('span', { class: 'upstreams' }, entries.map((e, i) => { const kind = kinds[i] ?? kinds[0]; const href = webUrl(e); return el('span', { class: 'upstream' }, el('b', {}, t(`upstreamKind.${kind}`, { defaultValue: kind })), el('span', {}, agents[i] ?? agents[0]), href ? el('a', { href: href.href, rel: 'noopener', class: 'mono' }, e) : el('span', { class: 'mono' }, e)); })); } // #2066: the sizes of a source's files and the shares of its permits, drawn by Observable Plot (plots.js) at the width // of the detail's column (CSS scales them down where it is narrower) const DETAIL_CHART_PX = 360; /** #1197: the sizes of a source's files (sources.json size_quantiles_upper_bytes, each plus one byte) on a log axis: * the band from p10 to p90, the median and the fetch tool's 20 MB, labelled (plots.js strip). */ function sizeStrip(q) { const v = (k) => num(q[k]) + 1; const qs = { p10: v('p10'), p50: v('p50'), p90: v('p90'), p99: v('p99') }; return plotted(() => P.strip(qs, { width: DETAIL_CHART_PX, limit: MAX_FILE, median: t('detail.median', { size: fmtBytes(qs.p50) }), limitText: '20 MB', label: t('detail.stripAria', { median: fmtBytes(qs.p50), p99: fmtBytes(qs.p99) }) }), 'sk sk-strip'); } /** #1228: the shares of total that parts [{value, color, label, note}] hold as one bar (plots.js shares), each part's * count, label, share and note on hover, and its legend: each part's label, its share and its note (what the share * counts and the licenses behind it). */ function shareBar(parts, total) { const shown = parts.filter((p) => p.value > 0); const said = (p) => `${p.label} ${fmtPct(p.value, total)}${p.note ? ` ${p.note}` : ''}`; return el('div', {}, plotted(() => P.shares(shown.map((p) => ({ value: p.value, color: p.color, title: `${fmtInt(p.value)}\n${said(p)}` })), { width: DETAIL_CHART_PX, label: shown.map(said).join(', ') }), 'sk sk-strip'), el('div', { class: 'stack-labels' }, shown.map((p) => el('span', {}, el('span', { class: 'sw', style: { background: p.color } }), `${p.label} `, el('b', {}, fmtPct(p.value, total)), p.note ? ` ${p.note}` : null)))); } function item(iconName, label, ...value) { return el('div', { class: 'd-item' }, el('dt', {}, icon(iconName, { size: 18 }), label), el('dd', {}, ...value)); } function detail(s) { const st = UI.detail.get(s.id) || {}; UI.detail.set(s.id, st); const info = s.info || {}; const lic = info.license || {}; const redraw = () => { const cell = document.getElementById(`d-${s.id}`); if (cell) clear(cell).append(detail(s)); }; // #1769: a data file may hold rows of other sources too, so every read keeps this source's rows by the column source; // #1792: and reads only the source's row range in the file where the catalog records it (rowsOf) const ownRows = (rows) => rows.filter((r) => r.source === s.id); if (!st.sample && s.files.length) { st.sample = 'loading'; const cols = s.isTask ? [...ROW_COLUMNS, 'extra'] : ROW_COLUMNS; const unitFile = s.files[0]; readColumns(fileUrl(IX.sha, unitFile.path), cols, rowsOf(unitFile, FIRST_ROWS)) .then((r) => { st.sample = ownRows(r.rows); st.file = unitFile.path; redraw(); }, (e) => { st.sample = { error: e.message }; redraw(); }); } // #1228: the licences behind the shares of a material set, from the licence columns of all its files; #1266: only // for a catalog without the column license_counts if (s.isTask && !s.licenceCounts && !st.licences && s.files.length) { st.licences = 'loading'; Promise.all(s.files.map((f) => readColumns(fileUrl(IX.sha, f.path), [...LICENCE_COLUMNS, 'source'], rowsOf(f)))) .then((parts) => { st.licences = parts.flatMap((p) => ownRows(p.rows)); redraw(); }, (e) => { st.licences = { error: e.message }; redraw(); }); } if (!st.batches) { st.batches = 'loading'; loadBatchInfo(s).then((b) => { st.batches = b; redraw(); }, (e) => { st.batches = { error: e.message }; redraw(); }); } const rows = Array.isArray(st.sample) ? st.sample : []; const metas = rows.map(metaOf); const fit = fitOf(s); const box = el('div', { class: 'detail', 'data-done': st.sample !== 'loading' && st.batches !== 'loading' && st.licences !== 'loading' ? 'true' : null }); // fit at a glance box.append(el('ul', { class: 'fit-row' }, FIT.map((x) => el('li', { class: `fit-item fit-${fit[x.k]}-item` }, icon(x.icon, { size: 18 }), el('span', {}, fitLabel(x.k)), fitDot(fit[x.k], fitLabel(x.k)))))); const dl = el('dl', { class: 'd-list' }); // 1 what it is: the description its upstream wrote; #1197, #1216: else its counts in the page's language (#1633: a // material set's units and files, the catalog's counts the index's card and docs/sources.md give in the same words), // with the publisher names of its rows when the rows read are all its rows, else (#1256) the catalog's count of its // publishers over all its rows const ups = upstreamsOfRows(metas); const names = publisherNames(metas); const allRead = rows.length >= s.rows; const counted = s.isTask ? t('n.unitsFiles', { units: unitsText(s.readyUnits), files: plural('n.file', s.rows) }) : plural('n.corpusRow', s.rawRows); // #2066: in the page's language (descriptionOf); the sentence admission writes (ADMIT_WHAT) is recognized in English const own = s.description && !ADMIT_WHAT.test(s.description) ? descriptionOf(s) : ''; const text = own || (names.length && allRead ? t('detail.countsBy', { what: counted, publishers: fmtAnd(names) }) : s.publishers ? t('detail.countsFrom', { what: counted, publishers: plural('n.publisher', s.publishers) }) : t('detail.counts', { what: counted })); dl.append(item('doc', t('detail.what'), text.length > 160 ? el('details', {}, el('summary', {}, firstSentence(text)), el('p', {}, text)) : text)); // 2 a material set's units and files, what each row of a corpus points to (#2065: a corpus is counted in rows) const kindCount = (k) => { const n = num(s.info.unit[k]); return t(UNIT_KIND[k] ? `unitCount.${UNIT_KIND[k]}` : 'unitCount.code', { count: n, n: fmtCompact(n), kind: k }); }; const unitWhat = s.isTask ? [el('span', { class: 'chips' }, el('span', { class: 'chip' }, unitsText(s.readyUnits)), el('span', { class: 'chip' }, plural('n.file', s.rows)))] : [el('span', {}, sentence(fmtOr(s.kinds.map((k) => (UNIT_KIND[k] ? t(`unitKind.${UNIT_KIND[k]}`) : k))))), el('span', { class: 'chips' }, s.kinds.map((k) => el('span', { class: 'chip' }, kindCount(k))))]; dl.append(item('rows', t(s.isTask ? 'detail.files' : 'detail.oneUnit'), ...unitWhat)); // 3 where it comes from (#1197: in words): the number of its publishers (#1256: the catalog's) and the main site of // its upstreams, else of the addresses of its rows const from = ups.length ? fromWords(ups, true, s.publishers) : fromWords(rows.map((r) => r.uri), false, s.publishers); dl.append(item('pin', t('detail.from'), st.sample === 'loading' ? '…' : from.length ? el('span', {}, ...from) : t('detail.notRecorded'))); // #2588: the upstream it declares, one dataset or one catalog with its agent and entry address (each of them while // it still reads several) if (s.upstream) dl.append(item('pin', t('detail.upstream'), upstreamLines(s.upstream))); // 4 self-contained; #1216: the fit chip's answer (fit.alone), with the files of the source a material set of a // collection comes with (a license and a readme leave a unit usable alone): its rows read without a unit const comps = [...new Set(rows.filter((r, i) => !isUnit(metas[i])).map((r) => r.uri.split('/').pop()))]; const compChips = () => el('span', { class: 'chips' }, (comps.length ? comps : ['LICENSE', 'README.md']).map((c) => el('span', { class: 'chip mono-chip' }, c))); const ofCollection = !!(info.campaign || info.extended_by); dl.append(item('layers', t('detail.alone'), s.isTask ? (fit.alone !== 'yes' ? [el('span', {}, t('detail.withCompanions')), compChips()] : ofCollection ? [el('span', {}, t('detail.alsoWith')), compChips()] : t('detail.yes')) : fit.alone === 'yes' ? t('detail.eachFile') : t('detail.container'))); // 5 tools to open it const exts = Object.entries(info.by_ext || {}).map(([k, v]) => [k, num(v.rows) || 0]).sort((a, b) => b[1] - a[1]).map(([k]) => k); const tools = []; for (const e of exts) { const tool = toolOf(e); if (tool && !tools.some((x) => x[1] === tool)) tools.push([e, tool]); if (tools.length >= 3) break; } dl.append(item('tool', t('detail.opensWith'), tools.length ? el('span', { class: 'chips' }, tools.map(([e, tool]) => el('span', { class: 'chip' }, el('span', { class: 'mono' }, e === '(none)' ? t('detail.noExt') : `.${e}`), ` ${tool}`))) : t('detail.anyEditor'))); // 6 licence const lr = info.license_rows || {}; // #1228: the licences behind a share: a material set's from its licence columns (an empty column: the batch // licence), a corpus's the source's licence when the share is its tier, else each row's own const behind = (field, k) => { const read = s.isTask && Array.isArray(st.licences) && st.licences.length ? st.licences : null; if (!read) { if (s.isTask && st.licences === 'loading') return ['\u2026']; return lic[field] === k && lic.name ? [lic.name] : [t('detail.ownLicences')]; } const names = new Set(); for (const r of read) { if ((r[`license_${field}`] ?? lic[field]) !== k) continue; const name = r.license_spdx ?? (lic[field] === k ? lic.name : null); if (name) names.add(String(name)); } return [...names].sort(); }; const permitLabel = (k, names) => (k === 'conditional' && CONDITION[s.tier] && names.length === 1 && names[0] === lic.name ? t(CONDITION[s.tier]) : t(`permit.${k}`)); const permit = (field) => PERMIT_ORDER.map((k) => { const names = behind(field, k); return { k, value: num(lr[field]?.[k]?.rows) || 0, color: PERMIT_COLOR[k], label: permitLabel(k, names), note: t(`detail.shareOf${s.isTask ? 'Set' : 'Corpus'}${names.length ? '' : 'Only'}`, { licences: names.join(', ') }) }; }); // #1197: one line when every row allows both, the two bars when its rows differ const allAllowed = (field) => permit(field).every((x) => (x.k === 'yes' ? x.value > 0 : !x.value)); // #1266: a material set's licence by kind of file, as the projection counted it (license_counts): per kind its // files, the permit classes for training and for redistribution as "x / y files" with the licences behind each, or // one phrase when all its files allow both const spdxOf = (xs) => [...new Set(xs.map((x) => x.spdx).filter(Boolean))].sort(); const kindLine = (kind) => { const items = s.licenceCounts.filter((x) => x.kind === kind); const all = sum(items, (x) => x.files); if (!all) return null; const head = el('b', {}, t(`detail.kindFiles.${kind}`, { count: all, n: fmtInt(all) })); if (items.every((x) => x.training_ok === 'yes' && x.redistribution_ok === 'yes')) { const names = spdxOf(items); return el('span', { class: 'lic-kind' }, head, el('span', { class: 'small' }, names.length ? t('detail.kindAllowed', { licences: names.join(', ') }) : t('detail.allAllowed'))); } const classes = (field, label) => el('span', { class: 'small' }, t('sources.label', { label }), PERMIT_ORDER.map((k) => { const xs = items.filter((x) => x[field] === k); const n = sum(xs, (x) => x.files); const names = spdxOf(xs); return n ? t(names.length ? 'detail.permitFiles' : 'detail.permitFilesOnly', { permit: permitLabel(k, names), n: fmtInt(n), of: fmtInt(all), licences: names.join(', ') }) : null; }).filter(Boolean).join('; ')); return el('span', { class: 'lic-kind' }, head, classes('training_ok', t('detail.training')), classes('redistribution_ok', t('detail.shipping'))); }; dl.append(item('training', t('detail.licence'), el('span', { class: 'lic' }, el('b', {}, tierName(s.tier)), el('span', { class: 'small' }, lic.name || s.spdx || '')), s.isTask && s.licenceCounts ? el('span', { class: 'lic-kinds' }, FILE_KINDS.map(kindLine)) : allAllowed('training_ok') && allAllowed('redistribution_ok') ? el('span', { class: 'small' }, t('detail.allAllowed')) : el('span', { class: 'lic-bars' }, el('span', { class: 'mini-title' }, t('detail.training')), shareBar(permit('training_ok'), s.rows), el('span', { class: 'mini-title' }, t('detail.shipping')), shareBar(permit('redistribution_ok'), s.rows)))); // #2006: the training stages one of its units passes, from the stage table (the commit's stage figures), when the page has the // stages of its commit, with the version of the table they were computed with if (IX.stages) { dl.append(item('layers', t('detail.stages'), s.stages.length ? el('span', { class: 'chips' }, s.stages.map((id) => el('span', { class: 'chip' }, stageName(id)))) : t('detail.stagesNone'), el('span', { class: 'small' }, t('stage.version', { v: IX.stages.version })))); } // 7 size and units const q = info.size_quantiles_upper_bytes || {}; const amounts = s.isTask ? `${unitsText(s.readyUnits)} · ${plural('n.file', s.rows)}` : plural('n.corpusRow', s.rawRows); // #1158: the source's figures of meta/size_figures.json; #2422: its size first, the deduplicated one dl.append(item('size', t('detail.size'), el('span', { class: 'lic' }, el('b', {}, s.size ? sizeOf(s.size) : t('size.none')), el('span', { class: 'small' }, amounts), s.size ? el('span', { class: 'small size-parts' }, sizeParts(s.size, false).join(' \u00b7 ')) : null), q.p50 ? sizeStrip(q) : null)); // #1577: whether its upstream still serves the files, from the sampled rows read in meta/sizes.parquet (catalog // column fetch_check) dl.append(item('get', t('detail.fetch'), fetchText(s.fetch))); // #1928: the release its rows come from, and why a newer batch of it is held back (the release notes' reason) if (s.releasedIn) dl.append(item('get', t('detail.releasedIn'), el('span', { class: 'mono' }, s.releasedIn))); // #2674: a hold reads as one sentence in the page's language, the release notes' words (up to 600 characters of // English) in a closed block; #2066: those words in the page's language where the table of Chinese names holds them if (s.heldBack) dl.append(item('get', t('detail.heldBack'), el('span', {}, t('detail.heldBackText')), el('details', {}, el('summary', {}, t('detail.heldBackNotes')), el('p', { class: 'problem-text' }, zhOf(s.heldBack))))); // #1633: why a flagged source is not usable as it is (catalog column status_reasons) if (s.status === 'flagged') dl.append(item('get', t('status.flagged'), el('span', { class: 'problem-text' }, s.reasons.map(zhOf).join('; ') || t('common.unknown')))); // 8 one example file (#1197): the first file of a unit among its preview rows (meta/preview.parquet; isUnit); a // source with none there has no example const pv = previewOf(s); const r = Array.isArray(pv) ? pv.find((x) => isUnit(metaOf(x))) : null; if (pv === 'loading') dl.append(item('link', t('detail.example'), '…')); else if (pv.error) dl.append(item('link', t('detail.example'), el('span', { class: 'problem-text' }, pv.error))); else if (r) { const href = upstream(r.uri); const name = r.member_path || r.uri.split('/').pop() || r.uri; dl.append(item('link', t('detail.example'), el('span', { class: 'example' }, href ? el('a', { href, rel: 'noopener', class: 'mono' }, name) : el('span', { class: 'mono' }, name), r.size_bytes != null ? el('span', { class: 'small' }, ` · ${fmtBytes(num(r.size_bytes))}`) : null))); } // 9 the command that takes this source: #2629: a material set whole, with its units and its size beside it; a corpus, // whose whole can be petabytes, by the units choice (the reader's, else 3 per source) const cmd = command(commandArgs({ sources: [s.id], sample: s.isTask ? 'all' : F.sample || DEFAULT_SAMPLE[CORPUS] })); dl.append(item('run', t('detail.take'), el('code', { class: 'cmd-inline' }, cmd), copyButton(t('copy.button'), cmd), s.isTask ? el('span', { class: 'small' }, s.size ? t('detail.takeWhole', { units: unitsText(s.readyUnits), size: sizeOf(s.size) }) : t('detail.takeWholeNoSize', { units: unitsText(s.readyUnits) })) : null)); box.append(dl); // batches and the rows read, on request const b = st.batches; const more = el('details', { class: 'd-more' }, el('summary', {}, t('detail.more'))); if (Array.isArray(b)) { more.append(table([t('th.batch'), { label: t('th.rows'), r: true }, t('th.changed'), t('th.state')], b.map((x) => [x.batch, x.rows == null ? t('detail.notCounted') : fmtInt(x.rows), fmtDate(x.date), x.status ? t(`batch.${x.status}`, { defaultValue: human(x.status) }) : '']))); } if (rows.length) { more.append(el('div', { class: 'table-wrap sample' }, table([t('th.address'), { label: t('th.size'), r: true }, 'sha256'], rows.map((r) => [r.member_path ? `${r.uri} ! ${r.member_path}` : r.range_offset != null ? `${r.uri} @ ${num(r.range_offset)}+${num(r.range_length)}` : r.uri, r.size_bytes == null ? '' : fmtInt(num(r.size_bytes)), hex(r.sha256).slice(0, 16) + (r.sha256 ? '…' : '')]), st.file))); } box.append(more); return box; } // #1792: what a reader of one source reads of the data file f (filesOf): its row range there when the catalog records it // (a packed file holds a source's rows together, sorted by source), the first n rows of it when n is given; else the // file from its first row. hyparquet documents no row filter, so the reader keeps the source's rows itself. function rowsOf(f, n) { if (f.first == null || f.last == null) return n == null ? {} : { rowStart: 0, rowEnd: n }; return { rowStart: f.first, rowEnd: n == null ? f.last + 1 : Math.min(f.last + 1, f.first + n) }; } async function loadBatchInfo(s) { const dates = await loadBatchDates(IX.sha, s.files).catch(() => new Map()); const listed = new Map(s.batches.map((b) => [b.batch, b])); const names = [...new Set([...s.files.flatMap((f) => f.batches), ...listed.keys(), ...dates.keys()])].sort(); // #1769: a file may hold other batches and sources too, so the rows are counted by the columns source and batch; // #1792: each file read once, its row range of the source only const counted = new Map(); const countsOf = (f) => { if (!counted.has(f.path)) { counted.set(f.path, readColumns(fileUrl(IX.sha, f.path), ['source', 'batch'], rowsOf(f)).then((p) => { const by = new Map(); for (const r of p.rows) if (r.source === s.id) by.set(String(r.batch), (by.get(String(r.batch)) || 0) + 1); return by; })); } return counted.get(f.path); }; const out = []; for (const name of names) { const b = listed.get(name); let rows = b ? num(b.rows) : null; const files = s.files.filter((f) => f.batches.includes(name)); if (rows == null && files.length && files.length <= 40) { const parts = await Promise.all(files.map(countsOf)); rows = parts.reduce((a, by) => a + (by.get(name) || 0), 0); } out.push({ batch: name, rows, date: dates.get(name) || null, status: b?.status || null }); } return out; } // ---- 5. get the data const sqlStr = (v) => `'${String(v).replace(/'/g, "''")}'`; function selectionSql(sel) { const root = `hf://datasets/${REPO}@${IX.sha}`; // #1746: the data files come from the catalog's column data_files, so the query lists no folder of data/ const ids = sel.map((s) => sqlStr(s.id)).join(', '); const chips = filterChips(); const exts = extList(); const where = [`source IN (${ids})`, ...(exts.length ? [`ext IN (${exts.map(sqlStr).join(', ')})`] : [])]; // the selection is one layer: its sources in that layer's measure const head = F.kind === CORPUS ? t('sql.headCorpora', { sha: IX.sha.slice(0, 7), corpora: plural('n.corpus', sel.length), rows: plural('n.corpusRow', sum(sel, (s) => s.rawRows)) }) : t('sql.headSets', { sha: IX.sha.slice(0, 7), sets: plural('n.materialSet', sel.length), units: unitsOfSets(sel) }); return [ `-- ${head}`, `-- ${chips.length ? chips.map(([k, v]) => t('sql.chip', { k, v })).join('; ') : t('sql.all')}`, // #1577: the query reads the rows as published; the default profile of the command is not applied, and it says so `-- ${t('sql.noProfile')}`, `SET VARIABLE files = (SELECT list(${sqlStr(`${root}/`)} || f) FROM (SELECT unnest(data_files) AS f FROM ${sqlStr(`${root}/meta/catalog.parquet`)} WHERE source IN (${ids})));`, `SELECT ${ROW_COLUMNS.map((c) => (c === 'sha256' ? 'lower(hex(sha256)) AS sha256' : c)).join(', ')}`, "FROM read_parquet(getvariable('files'), union_by_name = true)", // #1524: the file types selected, as fetch.py's --ext `WHERE ${where.join(' AND ')};`, ].join('\n'); } const csvCell = (v) => { if (v == null) return ''; const s = typeof v === 'bigint' ? v.toString() : String(v); return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; }; /** The pointer rows of sel as CSV; #1524: with file types selected, only the rows of those types. #1769: each data * file is read once, and only the rows of the sources of sel are kept (a file may hold rows of other sources). */ async function buildCsv(sel, onStep) { const files = [...new Set(sel.flatMap((s) => s.files.map((f) => f.path)))]; const ids = new Set(sel.map((s) => s.id)); const exts = new Set(extList()); const cols = exts.size ? [...ROW_COLUMNS, 'ext'] : ROW_COLUMNS; const lines = [ROW_COLUMNS.join(',')]; let done = 0; const queue = [...files]; const parts = new Map(); const worker = async () => { while (queue.length) { const path = queue.shift(); const { rows } = await readColumns(fileUrl(IX.sha, path), cols); parts.set(path, rows.filter((r) => ids.has(r.source) && (!exts.size || exts.has(r.ext)))); done++; onStep(done, files.length); } }; await Promise.all([worker(), worker(), worker()]); for (const path of files) for (const r of parts.get(path)) lines.push(ROW_COLUMNS.map((c) => csvCell(c === 'sha256' ? hex(r[c]) : r[c])).join(',')); return lines.join('\n') + '\n'; } /** What ./data holds after the command: the layout of tools/fetch.py (#268; #1633: one folder per unit). <domain> is a * row's pool_domain when it has one (such as D3_data), else the top-level code of the source's first domain (such as * D3). */ function folderTree(sel) { const named = sel.length === 1 || (sel.length <= MAX_LISTED && sel.length < IX.sources.length); const first = named ? sel.find((s) => s.isTask) || sel[0] : null; const src = first ? first.id : '<source>'; const line = (depth, iconName, name, note) => el('li', { style: { paddingLeft: `${depth * 20}px` } }, icon(iconName, { size: 16 }), el('span', { class: 'tree-name' }, el('span', { class: 'mono' }, name), note ? el('span', { class: 'small' }, note) : null)); return el('figure', { class: 'card tree-fig' }, el('p', { class: 'chart-title' }, t('tree.title')), el('ul', { class: 'tree' }, line(0, 'folder', 'data/'), line(1, 'folder', `<domain>/${src}/`, t('tree.units')), line(2, 'folder', '<unit folder>/', t('tree.unitFolder')), line(1, 'folder', '_meta/'), line(2, 'rows', 'manifest.jsonl', t('tree.manifest')), line(2, 'rows', 'sizes.tsv', t('tree.sizes'))), el('p', { class: 'small tree-note' }, el('span', { class: 'mono' }, '<domain>'), t('tree.domainNote')), // #1891: what the default profile leaves out, under every units choice ("all" too: the profile stays on); #2397: // the criteria tools/fetch.py applies by default, in its order, as the release lists them (meta/vocabulary.json // profile), #2422 each with the flag that takes it back (its option); #2441: every license and personal data come // by default, labels the filters choose by, not rules of the profile, and the two flags that leave them out are // named under the list (tree.labelNote) ...(IX.vocab.profile.length ? [el('p', { class: 'chart-title profile-title' }, t('tree.leftOut')), el('ul', { class: 'profile' }, IX.vocab.profile.map(({ criterion, option }) => el('li', {}, el('span', {}, t(`profile.${criterion}`)), el('code', {}, option)))), el('p', { class: 'small tree-note' }, t('tree.flagNote'))] : []), el('p', { class: 'small tree-note' }, t('tree.labelNote'), ' ', el('code', {}, '--training-only'), ' ', el('code', {}, '--no-personal-data')), el('p', { class: 'small agent-links' }, icon('robot', { size: 16 }), el('a', { href: t('url.forAgents') }, t('agents.readme')), ' · ', el('a', { href: 'llms.txt' }, 'llms.txt'))); } function download(name, text, type) { const a = el('a', { href: URL.createObjectURL(new Blob([text], { type })), download: name }); document.body.append(a); a.click(); a.remove(); setTimeout(() => URL.revokeObjectURL(a.href), 10000); } // ---- the selection kept as files: the selection file (the ids, the filters, the release tag of the commit, which the // command beside it names too; tools/fetch.py --selection reads it, so the same file gives the same files later) and // the attribution file (per source its title, licence and attribution statement from meta/sources.json; #1577: and // its files by publisher, licence and licence URL from meta/attribution.parquet, each file's own, else its source's; // in English, the language of the dataset) const selectionName = () => `training-data-foundation-selection-${IX.sha.slice(0, 7)}.json`; const ATTRIBUTION = 'meta/attribution.parquet'; const ATTRIBUTION_COLUMNS = ['source', 'publisher', 'license', 'license_url', 'files']; function selectionFile(sel, picks) { return `${JSON.stringify({ format: 'training-data-foundation-selection/1', dataset: REPO, revision: IX.pin, filters: picks ? {} : Object.fromEntries([...activeFilters(), 'kind'].map((k) => [k, F[k]])), sources: sel.map((s) => s.id).sort(), }, null, 1)}\n`; } async function attributionFile(sel) { const ids = new Set(sel.map((s) => s.id)); const files = new Map(); if (IX.hasAttribution) { for (const r of (await readColumns(fileUrl(IX.sha, ATTRIBUTION), ATTRIBUTION_COLUMNS)).rows) { if (!ids.has(r.source)) continue; if (!files.has(r.source)) files.set(r.source, []); files.get(r.source).push(r); } } const lines = [`# ${tEn('attr.title', { count: sel.length, n: fmtInt(sel.length), dataset: REPO })}`, '', tEn('attr.commit', { sha: IX.sha, url: `${DATASET_URL}/tree/${IX.sha}` }), '']; for (const s of [...sel].sort((a, b) => a.title.localeCompare(b.title))) { const lic = s.info.license || {}; const own = files.get(s.id) || []; lines.push(`## ${s.title}`, '', `- ${tEn('attr.source')}: \`${s.id}\``, `- ${tEn('attr.licence')}: ${lic.name || s.spdx || tEn('attr.notStated')}`); if (own.length) { lines.push(`- ${tEn('attr.files')}:`, ...own.map((r) => { const n = num(r.files); return ` - ${tEn('attr.fileLine', { count: n, n: fmtInt(n), publisher: r.publisher || tEn('attr.noPublisher'), licence: r.license || tEn('attr.notStated'), url: r.license_url || tEn('attr.notStated') })}`; })); } else lines.push(`- ${tEn('attr.licenceUrl')}: ${lic.url || tEn('attr.notStated')}`); // #1633: without an attribution statement of its own, a source is attributed by the same per-file license URL the // fetch manifest records (its files' own, else its source's), as CC BY 4.0 section 3(a)(2) allows const urls = [...new Set(own.map((r) => r.license_url).filter(Boolean))]; lines.push(`- ${tEn('attr.attribution')}: ${lic.attribution || lic.statement || urls.join(', ') || lic.url || tEn('attr.notStated')}`, ''); } return lines.join('\n'); } /** The command that reads the selection file; picks: the file lists the picked sources. #1524: the units choice, and * outside pick mode one --ext per file type selected (fetch.py --selection reads only the file's sources). */ const selectionCommand = (picks) => command([`--selection ${selectionName()}`, `--revision ${IX.pin}`, ...(picks ? [] : extList().map((e) => `--ext ${e}`)), ...sampleArgs(), '--out ./data']); /** The two downloads of sel; picks: sel is the picked sources (the selection file then records no filters). */ function keepCard(sel, picks = false) { return el('div', { class: 'keep-grid' }, el('div', { class: 'card keep-card' }, el('p', { class: 'chart-title' }, t('keep.title')), el('div', { class: 'keep-actions' }, el('button', { type: 'button', class: 'btn solid', onclick: () => download(selectionName(), selectionFile(sel, picks), 'application/json') }, icon('get', { size: 16 }), t('keep.selection')), el('button', { type: 'button', class: 'btn', onclick: async (ev) => { const btn = ev.currentTarget; btn.disabled = true; try { download(`training-data-foundation-attribution-${IX.sha.slice(0, 7)}.md`, await attributionFile(sel), 'text/markdown'); } finally { btn.disabled = false; } } }, icon('doc', { size: 16 }), t('keep.attribution'))), el('p', { class: 'small' }, el('code', {}, selectionName()), ' ', t('keep.selectionNote', { sources: plural(picks ? 'n.picked' : layerCount(), sel.length), sha: IX.sha.slice(0, 7) }))), codeCard('run', t('keep.again'), selectionCommand(picks), 2)); } /** The key of the count word of the layer shown: n.materialSet or n.corpus. */ const layerCount = () => (F.kind === CORPUS ? 'n.corpus' : 'n.materialSet'); function codeCard(iconName, title, code, n) { return el('div', { class: 'code-card' }, el('div', { class: 'code-head' }, el('span', { class: 'code-ico' }, icon(iconName, { size: 20 })), n ? el('span', { class: 'step-n' }, String(n)) : null, el('span', { class: 'code-title' }, title), copyButton(t('copy.button'), code, { dark: true })), el('pre', { class: 'code' }, el('code', {}, code))); } // ---- picks (#1286): sources picked row by row in either layer, kept in the address as pick=<id>,<id> like the // filters; while any is picked, "Get the data" takes exactly them, one --source each const pickIds = () => (F.pick ? F.pick.split(',').filter(Boolean) : []); function picked() { const by = new Map(IX.sources.map((s) => [s.id, s])); return pickIds().map((id) => by.get(id)).filter(Boolean); } function setPicks(ids) { F.pick = ids.join(','); writeUrl(); renderSources(selected()); renderGet(selected()); renderCard(); } function togglePick(id) { const ids = pickIds(); setPicks(ids.includes(id) ? ids.filter((x) => x !== id) : [...ids, id]); } /** #1524: the heading of Get the data for each of sources (their count in words), or for one source (sources null), * by the units choice in force (#2629: sampleNow). */ function takeHeading(sources) { const v = sampleNow(); const all = v === 'all'; if (sources == null) return all ? t('get.takeOneAll') : t('get.takeOne', { count: Number(v) }); return all ? t('get.takeEachAll', { sources }) : t('get.takeEach', { count: Number(v), sources }); } /** #1524: a choice of how many units to take from each source (F.sample, kept in the address); every command and * heading follows it, the command of a corpus's detail too. #2629: it shows the choice in force (sampleNow), and a * click on the default of the layer the command takes clears the reader's choice, so the other layer takes its own * default again. */ function chooseSample(v) { F.sample = v === DEFAULT_SAMPLE[takenKind()] ? '' : v; writeUrl(); vt(() => { renderGet(selected()); renderSources(selected()); }); } function unitsChoice() { return el('div', { class: 'units-row' }, segmented(t('get.units'), SAMPLES.map((v) => [v, v === 'all' ? t('get.unitsAll') : v]), sampleNow(), chooseSample)); } /** #1524: the units choice, "Take" (the command) and the folder tree of the command args for the sources S. */ function commandSteps(args, S) { return [unitsChoice(), el('div', { class: 'get-grid' }, el('div', { class: 'cmd-steps' }, codeCard('run', t('get.take'), command(args), 1)), folderTree(S))]; } /** Get the picked sources P: one --source per pick, with the same steps as a selection (#1524), and the two downloads * of them. */ function renderPicks(picks) { $('get-h').textContent = takeHeading(plural('n.picked', picks.length)); const body = clear($('get-body')); body.append(el('div', { class: 'pick-row' }, el('span', { class: 'fact-line' }, icon('sources', { size: 18 }), el('b', {}, fmtInt(picks.length)), t('pick.word', { count: picks.length })), el('span', { class: 'chips' }, picks.map((s) => el('span', { class: 'chip', title: s.id }, titleOf(s)))), el('button', { type: 'button', class: 'btn', onclick: () => setPicks([]) }, t('pick.clear')))); body.append(el('p', { class: 'small' }, t('pick.note'))); body.append(...commandSteps(commandArgs({ sources: picks.map((s) => s.id) }), picks)); body.append(keepCard(picks, true)); } /** Get the data: the selection is one layer, so one command takes it (#1083); the other layer is one switch away. * Picked sources, when there are any, take its place (renderPicks). */ function renderGet(sel) { if (!UI.ids) return; const picks = picked(); renderDock(sel, picks); if (picks.length) { renderPicks(picks); return; } const ready = sum(sel, (s) => s.readyUnits); const raw = sum(sel, (s) => s.rawRows); const other = F.kind === CORPUS ? MATERIAL_SET : CORPUS; $('get-h').textContent = !sel.length ? t('get.nothing') : takeHeading(sel.length === 1 ? null : plural(layerCount(), sel.length)); const body = clear($('get-body')); const switchOther = el('button', { type: 'button', class: 'link', onclick: () => setLayer(other) }, t('get.otherLayer', { name: kindName(other) })); if (!sel.length) { body.append(el('p', { class: 'small' }, switchOther)); return; } const chips = filterChips(); const tabs = segmented('', [['cmd', t('get.command')], ['sql', 'SQL'], ['csv', 'CSV']], UI.tab, (v) => { UI.tab = v; vt(() => renderGet(selected())); }); const pickRow = el('div', { class: 'pick-row' }, el('span', { class: 'fact-line' }, icon('sources', { size: 18 }), el('b', {}, fmtInt(sel.length)), t(F.kind === CORPUS ? 'get.corpusWord' : 'get.setWord', { count: sel.length })), F.kind === CORPUS ? el('span', { class: 'fact-line muted' }, icon('rows', { size: 18 }), el('b', {}, fmtCompact(raw)), t('cat.lead.raw', { count: raw })) : el('span', { class: 'fact-line' }, icon('check', { size: 18 }), el('b', {}, fmtInt(ready)), t('cat.lead.ready', { count: ready }) + uncounted(notCountedOf(sel))), el('span', { class: 'chips' }, el('span', { class: 'chip' }, kindName(F.kind)), ...chips.map(([k, v]) => el('span', { class: 'chip', title: k }, v))), el('a', { class: 'small', href: '#filters' }, t('get.change')), switchOther, tabs); body.append(pickRow); if (UI.tab === 'cmd') { body.append(...commandSteps(selectionArgs(sel).args, sel)); body.append(keepCard(sel)); } else if (UI.tab === 'sql') { body.append(codeCard('database', t('get.sql'), selectionSql(sel))); } else { // #1524: with file types selected, the rows of those types (by_ext) const exts = extList(); const rows = exts.length ? sum(sel, (s) => sum(s.exts.filter(([e]) => exts.includes(e)), ([, n]) => n)) : sum(sel, (s) => s.rows); const key = `${sel.map((s) => s.id).join(',')} ${F.ext}`; const csvState = el('span', { class: 'copy-state', 'aria-live': 'polite' }, UI.csv?.key === key ? UI.csv.text : ''); const tooMany = rows > CSV_MAX_ROWS; const btn = el('button', { type: 'button', class: 'btn solid', disabled: tooMany, onclick: async () => { btn.disabled = true; const say = (text) => { csvState.textContent = text; UI.csv = { key, text }; }; try { const csv = await buildCsv(sel, (d, n) => say(t('csv.progress', { count: n, done: fmtInt(d), n: fmtInt(n) }))); download(`${REPO.split('/')[1]}-${IX.sha.slice(0, 7)}-${sel.length === 1 ? sel[0].id : `${sel.length}-${F.kind === CORPUS ? 'corpora' : 'material-sets'}`}.csv`, csv, 'text/csv'); say(plural('n.pointerRow', csv.split('\n').length - 2)); } catch (e) { say(e.message); } btn.disabled = false; } }, icon('csv', { size: 16 }), t('csv.button', { rows: plural('n.pointerRow', rows) })); body.append(el('div', { class: 'card csv-card copy-row' }, icon('csv', { size: 28 }), btn, tooMany ? el('span', { class: 'small' }, t('csv.tooMany', { n: fmtShort(CSV_MAX_ROWS) })) : csvState)); } } // ---- #2422, second round ("Strata"): the working screen. The field (field.js, loaded after the first paint) draws // every source of the index as a point in its stage group's color; this part gives it the points, their light, its // labels and the open source, and draws the panels that float on it: the count of the selection with its list (the // largest first), the open source in short, and the dock with the one command that takes the selection. const VIEWS = ['strata', 'domains', 'scale']; // how many sources the panel's list shows; the list below the screen has them all const LIST_ROWS = 60; // the light of a source of the other layer that the filters keep const HALF = 0.45; let FIELD = null; // the points of the field ([{s, g, c, lg, dup}]: s the source's place in IX.sources) and, per source id, its points let fieldPts = []; const fieldIndex = new Map(); const wideWork = () => matchMedia('(min-width: 1100px)').matches; const stillMotion = () => matchMedia('(prefers-reduced-motion: reduce)').matches; /** The stage group of a source among the stages ids (the stage table's order): its first stage, else the last group, * in no stage. */ const groupOf = (s, ids) => { const g = ids.findIndex((id) => s.stages.includes(id)); return g < 0 ? ids.length : g; }; // #2629: why something is in no stage (artpool.stages.NO_STAGE, the keys of no_stage of the stage figures), each by the // key of its words; a reason the page does not know is said by its name const WHY_KEY = { condition: 'condition', ...RULE_KEY, 'not counted': 'notCounted' }; // the color of a source's dot in the list: the picked stage's while the source feeds it, else its first stage's const groupColor = (s, ids) => stageColor(F.stage && s.stages.includes(F.stage) ? F.stage : ids[groupOf(s, ids)]); /** The points of the field: one per source and stage it feeds (a corpus that feeds two stages stands on both planes * and in both lanes; the further ones are dup), a source of no stage one in the last group. Each has its domain cell * and the logarithm of its deduplicated size. */ function fieldPoints(ids, doms) { const out = []; IX.sources.forEach((s, i) => { const gs = ids.map((id, g) => (s.stages.includes(id) ? g : -1)).filter((g) => g >= 0); (gs.length ? gs : [ids.length]).forEach((g, k) => out.push({ s: i, g, c: doms.indexOf(domainKeysOf(s)[0]), lg: Math.log10((s.distinct || 0) + 1), dup: k > 0 })); }); return out; } /** The point that stands for a source in the ring: the one in the picked stage's group when it has one, else its * first; -1 for a source with no point. */ function pointOf(id) { const at = fieldIndex.get(id); if (!at) return -1; const g = stageFigList().findIndex((s) => s.id === F.stage); return at.find((i) => fieldPts[i].g === g) ?? at[0]; } /** The domain cells of the field: the top-level domains that have a source (a source of several is in its first), then * several domains and not classified, while there are some. */ function fieldDomains() { const keys = new Set(IX.sources.map((s) => domainKeysOf(s)[0])); return [...[...keys].filter((k) => k !== MULTI && k !== NONE).sort(numericCodeSort), ...[MULTI, NONE].filter((k) => keys.has(k))]; } /** The light of each point: 1 for a source of the selection, HALF for one the filters keep in the other layer, 0 for * one they leave out; until the selection answers, by the layer alone. While a stage is picked, a selected source's * point in another stage's group is at HALF, so the picked stage's plane is the lit one. */ function fieldLevels() { const picked = stageFigList().findIndex((s) => s.id === F.stage); return Float32Array.from(fieldPts, (p) => { const s = IX.sources[p.s]; if (!UI.ids) return s.kind === F.kind ? 1 : HALF; if (!UI.ids.has(s.id)) return UI.across?.has(s.id) ? HALF : 0; return picked >= 0 && p.g !== picked ? HALF : 1; }); } /** #1083: what no stage counts, said per layer and never as one number: that of the layer shown. #2629: from the * release's figures (no_stage of meta/stage_figures.json, data.js noStage): its sources as the figure, its units or * corpus rows in the line under it, and its sources by reason as the steps (the condition, each rule, not counted yet; * a reason of none left out), its units or rows by reason on hover; so post-training's figure plus these is what the * selection of every material set counts. Figures written before #2629 give the sources of the layer shown in no * stage, without reasons. */ function restWhat() { const sets = F.kind !== CORPUS; const head = { name: t('strata.rest'), title: t('strata.restTitle') }; const sourcesWord = (n) => t(sets ? 'get.setWord' : 'get.corpusWord', { count: n }); const reasons = Object.entries(IX.stageFigures?.noStage?.[F.kind] || {}); if (!reasons.length || reasons.some(([, c]) => Object.values(c).some((v) => v == null))) { const here = IX.sources.filter((s) => !s.stages.length && s.kind === F.kind).length; return { ...head, num: fmtInt(here), word: sourcesWord(here), line: '', steps: [] }; } const [many, amount] = sets ? ['material_sets', 'units'] : ['corpora', 'rows']; const fmt = sets ? fmtInt : fmtCompact; const said = (m, f) => reasons.filter(([, c]) => c[m]).map(([why, c]) => t(`strata.why.${WHY_KEY[why] || 'other'}`, { count: c[m], n: f(c[m]), why })); const n = sum(reasons, ([, c]) => c[many]); const left = sum(reasons, ([, c]) => c[amount]); return { ...head, title: `${head.title}\n${t(sets ? 'strata.whyUnits' : 'strata.whyRows', { list: said(amount, fmt).join(t('size.listSep')) })}`, num: fmtInt(n), word: sourcesWord(n), line: t(sets ? 'strata.restUnits' : 'strata.restRows', { count: left, n: fmt(left) }), steps: said(many, fmtInt) }; } /** The labels of the field in the page's language: per stage group its figure and name at its plane and its name at * its lane, the domains' names, and the marks of the size axis. */ function fieldLabels() { const groups = [...stageFigList().map((st) => ({ id: st.id, color: stageColor(st.id), name: (inChinese() && st.zh) || st.name, what: stageWhat(st) })), { color: '--none', name: t('strata.rest'), what: restWhat() }]; // a label of a stage or of a domain is a button for the pointer (out of the tab order: the column's bands and the // filters are the way by keyboard): it sets that filter, or clears it when it holds it const press = (on, act, kids, style) => el('button', { type: 'button', tabindex: '-1', 'aria-pressed': String(on), style, onclick: act }, kids); const node = (g, full) => { const kids = [el('i'), full ? [el('b', {}, g.what.num), [g.what.word, g.name].filter(Boolean).join(' · ')] : g.name]; const style = `--c: var(${g.color})`; return g.id && IX.stages ? press(F.stage === g.id, () => pickStage(g.id), kids, style) : el('span', { style }, kids); }; const domain = (k) => press(F.domain === k, () => setFilter({ domain: F.domain === k ? '' : k, leaf: '' }), domainName(k)); return { strata: groups.map((g) => node(g, true)), lanes: groups.map((g) => node(g, false)), domains: fieldDomains().map(domain), ticks: [[3, '1 KB'], [6, '1 MB'], [9, '1 GB'], [12, '1 TB'], [15, '1 PB']] }; } /** The arrangement of the field (F.view, kept in the address). */ function setView(v) { F.view = v; writeUrl(); for (const b of $('view-seg').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.view === v)); FIELD?.setView(v); } /** The open source (F.open, kept in the address; '' closes it): ringed in the field, in short in the panel, its whole * detail in the list below the screen. */ function openSource(id) { F.open = id; writeUrl(); renderSources(selected()); renderCard(); } /** A click on a point or on a row of the panel's list: the source opens (a second click closes it), its layer first * when it is of the one not shown. */ function focusSource(s) { if (s.kind !== F.kind) setFilter({ kind: s.kind }); openSource(F.open === s.id ? '' : s.id); } /** What the panels leave to the scene of the field: its width and height in px, and how far above the field's middle * its middle is. From 1100 px the panels float on the field: the room is between the left and the right panel, under * the arrangements and above the dock; below that the field is alone, with the arrangements at its top. */ function fieldRoom() { const host = $('field'); // the room starts under the arrangements (and, from 1100 px, the hint under them) const top = wideWork() ? 88 : 52; if (!wideWork()) return { w: host.clientWidth - 24, h: host.clientHeight - top - 12, lift: -top / 2 }; const side = (cls) => document.querySelector(cls).offsetWidth + 40; const bottom = document.querySelector('.work-dock').offsetTop; return { w: host.clientWidth - side('.work-left') - side('.work-right') - 40, h: bottom - top - 24, lift: host.clientHeight / 2 - (top + bottom) / 2 }; } /** The field after the first paint: the scene with every source, or, without WebGL, one line in its place. */ async function startField() { const host = $('field'); const ids = stageFigList().map((s) => s.id); const doms = fieldDomains(); fieldPts = fieldPoints(ids, doms); fieldPts.forEach((p, i) => { const id = IX.sources[p.s].id; fieldIndex.set(id, [...(fieldIndex.get(id) || []), i]); }); const at = (i) => IX.sources[fieldPts[i].s]; try { const mod = await import('./field.js'); FIELD = await mod.createField(host, { points: fieldPts, colors: [...ids.map(stageColor), '--none'], cells: doms.length, view: F.view, room: fieldRoom, tip: (i) => { const s = at(i); return [titleOf(s), [sourceDomainText(s), kindOne(s.kind), s.size ? sizeOf(s.size) : null].filter(Boolean).join(' · ')]; }, onHover: (i) => { const id = i < 0 ? null : at(i).id; for (const b of $('sel-list').querySelectorAll('.src')) b.classList.toggle('hot', b.dataset.id === id); }, onPick: (i) => focusSource(at(i)), onTouch: () => $('top').classList.add('touched'), }); } catch (e) { console.error(e); } // the manual's shots wait for this mark (manual/shots.yml): the field has faded in, or the page goes without it if (!FIELD) { document.body.dataset.field = 'off'; host.hidden = true; $('field-off').hidden = false; return; } setTimeout(() => { document.body.dataset.field = 'on'; }, stillMotion() ? 0 : 2400); $('top').classList.add('field-up'); for (const id of ['views', 'field-hint', 'field-legend']) $(id).hidden = false; placePills(); FIELD.setLabels(fieldLabels()); FIELD.setLevels(fieldLevels()); FIELD.pick(stageFigList().findIndex((s) => s.id === F.stage)); renderCard(); } // a figure counts up to its new value, at once with reduced motion const figures = new WeakMap(); function countTo(node, to) { const from = figures.get(node) ?? 0; figures.set(node, to); if (stillMotion() || from === to) { node.textContent = fmtInt(to); return; } const t0 = performance.now(); const step = (now) => { const k = Math.min(1, (now - t0) / 450); node.textContent = fmtInt(from + (to - from) * (1 - (1 - k) ** 3)); if (k < 1 && figures.get(node) === to) requestAnimationFrame(step); }; requestAnimationFrame(step); } /** The right panel: how many sources of the layer shown the selection holds, with their units or rows and the domains * they cover; its largest sources, each with its stage group's dot, its domain, its units and its size (a bar on a * logarithmic scale); and the light of the field's points. */ function renderWork(sel) { if (!UI.ids) return; const sets = F.kind !== CORPUS; countTo($('sel-n'), sel.length); $('sel-word').textContent = t(sets ? 'get.setWord' : 'get.corpusWord', { count: sel.length }); const ready = sum(sel, (s) => s.readyUnits); const raw = sum(sel, (s) => s.rawRows); const covered = new Set(sel.flatMap((s) => s.tops)).size; const meta = [sets ? [fmtInt(ready), t('cat.lead.ready', { count: ready }) + uncounted(notCountedOf(sel))] : [fmtCompact(raw), t('cat.lead.raw', { count: raw })], [fmtInt(covered), t('strata.domains', { count: covered })]]; clear($('sel-meta')).append(...meta.map(([n, word]) => el('span', {}, el('b', {}, n), ' ', word))); const ids = stageFigList().map((s) => s.id); const lg = (s) => Math.log10((s.distinct || 0) + 1); const top = [...sel].sort((a, b) => (b.distinct ?? -1) - (a.distinct ?? -1) || titleOf(a).localeCompare(titleOf(b))).slice(0, LIST_ROWS); const most = Math.max(1, ...top.map(lg)); const list = clear($('sel-list')); list.append(...top.map((s, i) => el('li', { style: `--i: ${i}` }, el('button', { type: 'button', class: 'src', 'data-id': s.id, style: `--c: var(${groupColor(s, ids)})`, onclick: () => focusSource(s), onpointerenter: () => FIELD?.hover(pointOf(s.id)), onpointerleave: () => FIELD?.hover(-1) }, el('span', { class: 'dot', 'aria-hidden': 'true' }), el('span', { style: 'min-width: 0' }, el('span', { class: 't' }, titleOf(s)), el('span', { class: 's' }, [sourceDomainText(s), s.isTask ? unitsText(s.readyUnits) : plural('n.corpusRow', s.rawRows)].join(' · '))), el('span', { class: 'sz' }, s.size ? sizeOf(s.size) : ''), el('span', { class: 'barw', 'aria-hidden': 'true' }, el('u', { style: `width: ${Math.max(2, (lg(s) / most) * 100).toFixed(0)}%` })))))); if (!sel.length) list.append(el('li', {}, el('p', { class: 'src-none' }, t('sources.none')))); $('sel-all').textContent = !sel.length ? '' : sel.length > top.length ? t('strata.all', { n: fmtInt(sel.length) }) : t('strata.table'); // no source matches: the scene says so in its middle, with the way back $('field-empty').hidden = sel.length > 0; renderCard(); FIELD?.setLevels(fieldLevels()); } /** The open source in short, above the panel's list, while the selection holds it: what it is in seven lines, Pick, and * a link to its whole detail in the list below the screen; its point is ringed in the field. */ function renderCard() { const box = $('sel-card'); const s = F.open && UI.ids?.has(F.open) ? IX.sources.find((x) => x.id === F.open) : null; FIELD?.open(s ? pointOf(s.id) : -1, s ? titleOf(s) : ''); for (const b of $('sel-list').querySelectorAll('.src')) b.classList.toggle('on', b.dataset.id === s?.id); box.hidden = !s; clear(box); if (!s) return; const stages = s.stages.length ? s.stages.map(stageName).join(t('size.listSep')) : t('strata.rest'); const rows = [[t('col.domain'), sourceDomainText(s)], [t('strata.layer'), kindOne(s.kind)], [t('col.form'), formName(formKeyOf(s))], s.isTask ? [t('col.units'), unitsNum(s.readyUnits)] : [t('col.rows'), fmtCompact(s.rawRows)], [t('col.size'), s.size ? sizeOf(s.size) : t('common.unknown')], [t('col.tier'), tierName(s.tier)], [t('filter.stage'), stages]]; const isPicked = pickIds().includes(s.id); box.append(el('div', { class: 'card-s' }, el('button', { type: 'button', class: 'x', 'aria-label': t('strata.close'), title: t('strata.close'), onclick: () => openSource('') }, '×'), el('h3', {}, titleOf(s)), el('code', { class: 'src-id' }, s.id), el('dl', {}, rows.map(([k, v]) => [el('dt', {}, k), el('dd', {}, v)])), el('div', { class: 'acts' }, el('button', { type: 'button', class: 'btn small-btn', 'aria-pressed': String(isPicked), onclick: () => togglePick(s.id) }, isPicked ? [icon('check', { size: 14 }), t('pick.picked')] : t('pick.add')), el('a', { href: `#d-${s.id}` }, t('strata.details'))))); } /** The dock: the command that takes the picked sources, else the selection (the same command as Get the data shows), * with the choice of how many units per source (chooseSample; #2629: pressed, the choice in force). */ function drawDockSample() { clear($('dock-sample')).append(...SAMPLES.map((v) => el('button', { type: 'button', 'data-s': v, 'aria-pressed': String(v === sampleNow()), onclick: () => chooseSample(v) }, v === 'all' ? t('get.unitsAll') : v))); } function renderDock(sel, picks) { const S = picks.length ? picks : sel; $('dock-h').textContent = [t('get.take'), S.length ? plural(picks.length ? 'n.picked' : layerCount(), S.length) : null].filter(Boolean).join(' · '); const text = !S.length ? t('get.nothing') : command(picks.length ? commandArgs({ sources: picks.map((s) => s.id) }) : selectionArgs(sel).args); // the command as it is copied, its options' names and its line ends each in a color of their own clear($('dock-cmd')).append(...text.split(/(--[a-z-]+| \\\n)/).map((part) => (part.startsWith('--') ? el('span', { class: 'flag' }, part) : part.endsWith('\\\n') ? el('span', { class: 'cont' }, part) : part))); $('dock-copy').disabled = !S.length; const v = sampleNow(); for (const b of $('dock-sample').querySelectorAll('button')) b.setAttribute('aria-pressed', String(b.dataset.s === v)); } /** Every segmented control of the page gets a pill that slides to its pressed part: placed under that part, and placed * again when another part is pressed, the control is drawn again or its words change. placePills places them all (after * a control that was hidden is shown, or the window's width changes). */ function placePill(seg) { const on = seg.querySelector(':scope > button[aria-pressed="true"]'); let pill = seg.querySelector(':scope > .seg-pill'); if (!pill) { pill = el('span', { class: 'seg-pill', 'aria-hidden': 'true' }); seg.prepend(pill); } seg.classList.toggle('pilled', !!on && on.offsetWidth > 0); pill.style.opacity = on ? 1 : 0; if (!on) return; pill.style.width = `${on.offsetWidth}px`; pill.style.height = `${on.offsetHeight}px`; pill.style.transform = `translate(${on.offsetLeft}px, ${on.offsetTop}px)`; } const placePills = () => document.querySelectorAll('.seg').forEach(placePill); function setUpPills() { new MutationObserver((changes) => { const segs = new Set(); for (const c of changes) { const seg = (c.target.nodeType === 1 ? c.target : c.target.parentElement)?.closest('.seg'); if (seg) segs.add(seg); for (const n of c.addedNodes) if (n.nodeType === 1) for (const s of [n, ...n.querySelectorAll('.seg')]) if (s.matches('.seg')) segs.add(s); } segs.forEach(placePill); }).observe(document.body, { subtree: true, childList: true, characterData: true, attributes: true, attributeFilter: ['aria-pressed'] }); addEventListener('resize', placePills); document.fonts?.ready.then(placePills); placePills(); } /** The controls of the working screen, once: the arrangements (the keys 1, 2 and 3 too), the zoom, the copy button of * the dock, and the key / that goes to the search. */ function setUpWork() { drawDockSample(); for (const b of $('view-seg').querySelectorAll('button')) { b.setAttribute('aria-pressed', String(b.dataset.view === F.view)); b.addEventListener('click', () => setView(b.dataset.view)); } $('zoom-in').addEventListener('click', () => FIELD?.zoomBy(1.2)); $('zoom-out').addEventListener('click', () => FIELD?.zoomBy(1 / 1.2)); $('zoom-reset').addEventListener('click', () => FIELD?.reset()); $('field-clear').addEventListener('click', () => $('f-clear').click()); setUpPills(); $('dock-copy').addEventListener('click', async () => { const b = $('dock-copy'); try { await navigator.clipboard.writeText($('dock-cmd').textContent); b.textContent = t('copy.done'); } catch { b.textContent = t('copy.select'); } setTimeout(() => { b.textContent = t('dock.copy'); }, 1600); }); addEventListener('keydown', (e) => { if (e.ctrlKey || e.metaKey || e.altKey || e.target.closest?.('input, select, textarea')) return; if (e.key === '/') { e.preventDefault(); $('f-q').focus(); } if (e.key === 'Escape' && F.open) openSource(''); const v = VIEWS[Number(e.key) - 1]; if (v && FIELD) setView(v); }); } // ---- the first screen (#2422: the training stages and the release line), the progress bar, reveal function renderHeader() { markFromIndex(); renderStageFig(); renderTerms(); drawRelease(IX.pin, IX.releaseDate); $('foot-pin').textContent = t('foot.index', { sha: IX.sha.slice(0, 7), date: fmtDate(IX.date) }); if (IX.overrides.catalog || IX.overrides.vocabulary) { const chip = $('preview-chip'); clear(chip).append(el('span', { class: 'dot other', 'aria-hidden': 'true' }), t('hero.preview')); chip.title = [IX.overrides.catalog, IX.overrides.vocabulary].filter(Boolean).join(' '); chip.hidden = false; } } // #1524, #2066: the usage videos (the video element's id -> its files' name): each plays the video of the page's // language with that language's subtitles, from media/ of the dataset (catalog/media.tsv: <name>.<en or zh>.mp4 and // .vtt); the language switch changes both of each. Their pictures carry no words, so the track is the default one // (HTML track default). const VIDEOS = { 'video-player': 'usage-example', 'people-player': 'usage-people' }; function renderVideo() { const l = lang(); for (const [id, name] of Object.entries(VIDEOS)) { const v = $(id); if (v.dataset.lang === l) continue; v.dataset.lang = l; v.querySelector('track')?.remove(); v.src = fileUrl('main', `media/${name}.${l}.mp4`); v.append(el('track', { kind: 'subtitles', srclang: l, default: true, label: t('video.subtitles'), src: fileUrl('main', `media/${name}.${l}.vtt`) })); } } function progress() { const parts = ['stages', 'pick', 'get']; const line = 64 + 80; let active = 0; const frac = []; parts.forEach((p, i) => { const nodes = [...document.querySelectorAll(`main [data-part="${p}"], footer[data-part="${p}"]`)]; const top = Math.min(...nodes.map((n) => n.getBoundingClientRect().top)); const bottom = Math.max(...nodes.map((n) => n.getBoundingClientRect().bottom)); if (top <= line) active = i; frac.push(Math.max(0, Math.min(1, (line - top) / Math.max(1, bottom - top)))); }); if (window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - 4) { active = parts.length - 1; frac[active] = 1; } // #1216: the header is opaque once the page has scrolled document.querySelector('.topnav')?.classList.toggle('scrolled', window.scrollY > 0); document.querySelectorAll('.progress a').forEach((a, i) => { a.setAttribute('aria-current', i === active ? 'true' : 'false'); a.querySelector('.fill').style.width = `${Math.round((i < active ? 1 : i === active ? Math.max(0.04, frac[i]) : 0) * 100)}%`; }); } /** Section heads fade in once as they scroll into view (CSS keeps them visible without motion or without JS). */ function reveal() { const nodes = document.querySelectorAll('.reveal'); if (!('IntersectionObserver' in window)) { nodes.forEach((n) => n.classList.add('in')); return; } const io = new IntersectionObserver((es) => es.forEach((e) => { if (e.isIntersecting) { e.target.classList.add('in'); io.unobserve(e.target); } }), { rootMargin: '0px 0px -8% 0px' }); document.documentElement.classList.add('js-reveal'); nodes.forEach((n) => io.observe(n)); } /** #1617: the user manual is a view of its own: the address #manual shows it in place of the page (manual.js, loaded * the first time it is shown, draws it in the page's language); #2614: #manual/<name> shows it at one of its parts, * and a move between its parts keeps it as it is drawn; any other address shows the page, at the block it names. */ function showView() { const on = inManual(location.hash); const back = !on && $('page').hidden; const was = !$('manual').hidden; $('manual').hidden = !on; $('page').hidden = on; document.body.classList.toggle('manual-on', on); if (on) { if (!was) scrollTo(0, 0); import('./manual.js').then((m) => m.showManual($('manual-body'), $('manual-toc'), renderTerms), (e) => { clear($('manual-body')).append(el('p', { class: 'small problem-text' }, t('manual.missing', { error: e.message }))); console.error(e); }); } else if (back) { // the charts were drawn, if at all, while the page was hidden redrawOnResize(); progress(); if (location.hash) document.getElementById(decodeURIComponent(location.hash.slice(1)))?.scrollIntoView(); } } /** Everything that shows the selection, drawn again when the selection answers with its ids. */ function renderScoped() { const sel = selected(); syncFilterRow(); renderChips(); renderWork(sel); renderMapHead(sel); renderSources(sel); renderGet(sel); } // the note above the page (a part of the index not read, or the index not read at all), kept so a new language // writes it again: {key, values, problem} let notice = null; function drawNotice() { if (!notice) return; const n = $('notice'); clear(n).append(...(notice.problem ? [el('span', { class: 'dot problem', 'aria-hidden': 'true' })] : []), t(notice.key, notice.values)); n.hidden = false; } /** The page in the language just chosen: the words of index.html, then every part drawn from the index. */ function renderLanguage() { translatePage(); renderVideo(); drawNotice(); if (!IX) return; renderHeader(); fillFilterOptions(); drawDockSample(); FIELD?.setLabels(fieldLabels()); drawCharts(); renderScoped(); } // #2066: the block the address names (a jump from the nav, such as #get, or a link such as #usage-video, which the // dataset card links to) moves down or up as the parts above it are drawn, one after another; the page goes to it // again after each, until the reader scrolls (moved), and from the start again on the next jump let moved = false; function toHash() { if (moved || !location.hash || inManual(location.hash)) return; document.getElementById(decodeURIComponent(location.hash.slice(1)))?.scrollIntoView(); } function linkError(e) { notice = { key: 'notice.linked', values: { error: e?.message || String(e) }, problem: true }; drawNotice(); console.error(e); } /** The linked views (linked.js), after the first screen has painted: DuckDB-WASM with the catalog of the page's commit, * the filters from the address as clauses of the selection, the charts, the counts of the filter options, and the list * of selected sources, which draws the rest of the browse section and Get the data. */ async function startLinked() { try { const mod = await import('./linked.js'); const cols = heatCols(); await mod.open(IX.catalogFile, IX.sources.map((s) => pageRow(s, cols))); L = mod; L.startFacets((key, m, total) => { UI.facets[key] = { m, total }; fillSelect(key); }, linkError); // the sources the filters keep in either layer: the field draws those of the layer not shown at half light L.startAcross((ids) => { UI.across = ids; FIELD?.setLevels(fieldLevels()); }, linkError); L.apply(F, SEL_KEYS); drawCharts(); L.connectFacets(); let first = true; L.startSelection((ids) => { UI.ids = ids; renderScoped(); if (first && F.open) document.getElementById(`d-${F.open}`)?.scrollIntoView({ block: 'center' }); else if (first) toHash(); first = false; }, linkError); } catch (e) { linkError(e); } } async function main() { readUrl(); // #2066: the Plot charts of a source's detail (plots.js) load as the page starts, beside the index const plots = import('./plots.js').then((m) => { P = m; }, (e) => { plotsError = e; console.error(e); }); translatePage(); renderVideo(); setUpSwitch(); setUpWork(); onLanguage(renderLanguage); decorate(); reveal(); showView(); addEventListener('hashchange', () => { moved = false; showView(); }); for (const ev of ['wheel', 'touchmove', 'keydown']) addEventListener(ev, () => { moved = true; }, { passive: true }); let ticking = false; const onScroll = () => { if (!ticking) { ticking = true; requestAnimationFrame(() => { ticking = false; progress(); }); } }; addEventListener('scroll', onScroll, { passive: true }); let resizing = null; addEventListener('resize', () => { onScroll(); clearTimeout(resizing); resizing = setTimeout(redrawOnResize, 300); }); progress(); try { IX = await loadIndex(new URLSearchParams(location.search), (key, values) => { notice = { key, values }; drawNotice(); }); } catch (e) { notice = { key: 'notice.failed', values: { error: e.message }, problem: true }; drawNotice(); console.error(e); return; } // #2006: without stages of its commit the page has no stage to filter by, so a stage in the address would keep no // source; #2422: with them, a stage in the address takes the layer it counts, as a click on its tile does if (!IX.stages && F.stage) { F.stage = ''; writeUrl(); } // without stages every source is in one stage group, so the field starts by domain unless the address names an // arrangement if (!IX.stages && !new URLSearchParams(location.search).get('view')) F.view = 'domains'; if (F.stage && stageKind(F.stage) !== F.kind) { F.kind = stageKind(F.stage); if (F.kind === CORPUS) F.collection = ''; writeUrl(); } document.body.classList.add('ready'); renderHeader(); fillFilterRow(); syncFilterRow(); renderChips(); renderMapHead([]); for (const b of $('layer-seg').querySelectorAll('button')) b.addEventListener('click', () => setLayer(b.dataset.kind)); for (const b of $('metric-seg').querySelectorAll('button')) { b.addEventListener('click', () => { F.metric = b.dataset.metric; writeUrl(); drawCharts(); renderMapHead(selected()); }); } progress(); toHash(); // the Plot charts of the open source's detail once their module is in (each a skeleton until then) plots.then(() => { if (UI.ids) renderSources(selected()); toHash(); }); // #2422: the linked charts take their colors from the page's theme when drawn (linked.js), so a new theme of the // reader's draws them again; the rest of the page follows its colors by itself matchMedia('(prefers-color-scheme: dark)').addEventListener('change', drawCharts); // the field and the linked views load after this screen has painted requestAnimationFrame(() => setTimeout(() => { startField(); startLinked(); }, 0)); } main();
// The data page of the Training Data Foundation (#2422): a working screen first (second round, "Strata": the field of
// every source as the ground, and on it the training stages as a column, the filters, the selection with its largest
// sources and the one command that takes it), then the sources (their list, each with its details, and the linked
// charts) and Get the data (the one command and the selection file). The dataset card is the overview; this page is
// where data is picked and taken.
// Figures first, few words; every chart keeps a short label and a table view. The charts of Pick, its filter counts and
// the list of selected sources come from one crossfilter selection (linked.js). The two layers stay apart everywhere
// and are never added together (#1083): material sets, grouped into collections and counted in sets and units, in the
// main color; corpora, counted in rows, in gray. Pick shows one layer at a time (F.kind, material sets by default; a
// training stage takes the layer it counts).
import {
CORPUS, DATASET_URL, fileUrl, KINDS, loadBatchDates, loadIndex, MATERIAL_SET, num, readColumns, REPO, summed, topOf,
wholeFile,
} from './data.js';
import {
clear, el, figureWithToggle, fmtAnd, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtOr, fmtPct, fmtShare, fmtShort,
segmented, table, tableShown, vt,
} from './charts.js';
import { decorate, drawRelease, drawStages, markFromIndex, RULE_KEY, sizeLine, sizeOf, sizeParts, stageColor, stageWhat } from './first.js';
import { lang, onLanguage, setUpSwitch, t, tEn, translatePage } from './i18n.js';
import { formIcon, icon } from './icons.js';
const CSV_MAX_ROWS = 200000;
const ROW_COLUMNS = ['source', 'batch', 'uri', 'range_offset', 'range_length', 'member_path', 'size_bytes', 'sha256'];
const FIRST_ROWS = 20;
const HEAT_FORMS = 8;
const FORM_BARS = 10;
// #1524: how many units Get the data takes from each source (--sample N; all: no --sample, #1891); #2629: when the
// reader has not chosen (sample in the address), every unit of material sets, whose whole is small, and 3 of each
// corpus, whose whole can be petabytes (DEFAULT_SAMPLE by the layer the command takes, sampleNow)
const SAMPLES = ['1', '3', '10', '30', 'all'];
const DEFAULT_SAMPLE = { [MATERIAL_SET]: 'all', [CORPUS]: '3' };
const MAX_FILE = 20e6;
const MAX_LISTED = 30;
// the phone layout of style.css
const PHONE = '(max-width: 720px)';
// #1617: the address of the user manual's view (#manual; showView); #2614: and of each of its parts, #manual/<name>
const MANUAL_VIEW = 'manual';
const inManual = (hash) => hash === `#${MANUAL_VIEW}` || hash.startsWith(`#${MANUAL_VIEW}/`);
const $ = (id) => document.getElementById(id);
// A link that leaves the page opens in a new tab. The Hub shows this page in a frame whose sandbox allows popups but
// not navigating the top window, and a site such as huggingface.co refuses to be shown in a frame, so a link opened in
// the frame showed the browser's error page (the Dataset link of the header, 2026-11-12). One listener in the capture
// phase sets the target before the browser follows the link, for the links of the markup and those the page writes.
document.addEventListener('click', (e) => {
const a = e.target.closest?.('a[href]');
if (a && /^https?:$/.test(a.protocol) && a.origin !== location.origin) {
a.target = '_blank';
a.rel = 'noopener';
}
}, true);
// ---- plain names (codes only inside the SQL and the commands); the words are in the locale files, by code:
// tier.<tier>, permit.<answer>, unitKind.<kind> and unitCount.<kind>
const tierName = (k, tt = t) => tt(`tier.${k}`, { defaultValue: human(k) });
const PERMIT_ORDER = ['yes', 'no', 'conditional', 'unknown'];
const PERMIT_COLOR = { yes: 'var(--ours)', no: 'var(--problem)', conditional: 'var(--fill-3)', unknown: 'var(--other)' };
// #1228: the condition in words of a conditional share under the source's licence, by its licence tier
const CONDITION = { 'share-alike': 'permit.conditional.sameLicence', 'nc-research': 'permit.conditional.nonCommercial' };
// a row's own licence; empty when the row takes its batch licence
const LICENCE_COLUMNS = ['license_spdx', 'license_training_ok', 'license_redistribution_ok'];
// #1266: the kinds of file of a material set (catalog column license_counts): its upstream files and the files the
// index wrote for it
const FILE_KINDS = ['upstream', 'index'];
// the two layers by their kind: kind.<kind> names them (Material sets, Corpora), kindOne.<kind> one of them
const kindName = (k, tt = t) => tt(`kind.${k}`);
const kindOne = (k) => t(`kindOne.${k}`);
/** The kinds as a legend: a swatch in each kind's color and its name. */
const kindKey = (kinds = KINDS) => el('span', { class: 'kind-key' },
kinds.map((k) => el('span', {}, el('span', { class: `sw kind-${k}`, 'aria-hidden': 'true' }), kindName(k))));
// the kinds of unit of a corpus (sources.json unit) -> their key in the locale files
const UNIT_KIND = { file: 'file', 'API response': 'apiResponse', 'WARC record': 'warcRecord', 'archive member': 'archiveMember', 'byte range': 'byteRange' };
// file type -> the tool that opens it (TOOL_WORDS: not a tool's name, a key of the locale files)
const TOOLS = {
xlsx: 'openpyxl', xlsm: 'openpyxl', xls: 'LibreOffice', ods: 'LibreOffice', csv: 'DuckDB', tsv: 'DuckDB', parquet: 'DuckDB',
json: 'jq', jsonl: 'jq', ndjson: 'jq', pdf: 'pdftotext', doc: 'LibreOffice', docx: 'python-docx', ppt: 'LibreOffice',
pptx: 'python-pptx', gz: 'gzip', tgz: 'tar', tar: 'tar', bz2: 'bzip2', xz: 'xz', zst: 'zstd', zip: 'unzip', whl: 'unzip',
jar: 'unzip', '7z': '7-Zip', crate: 'tar', deb: 'dpkg-deb', rpm: 'rpm2cpio', warc: 'warcio', arc: 'warcio', pgn: 'python-chess',
py: 'Python', '(none)': 'file(1)', xml: 'xmllint',
yaml: 'yq', yml: 'yq', evtx: 'python-evtx', log: 'grep', rego: 'OPA', sql: 'DuckDB', db: 'SQLite',
safetensors: 'safetensors', final: 'TensorBoard', ipynb: 'Jupyter', eml: 'Python email', mbox: 'Python mailbox',
ics: 'icalendar', e01: 'The Sleuth Kit', img: 'The Sleuth Kit', dd: 'The Sleuth Kit', raw: 'The Sleuth Kit',
ys: 'Yosys', v: 'Icarus Verilog', sv: 'Verilator', go: 'Go', rs: 'Cargo', zig: 'Zig', lua: 'Lua', rb: 'Ruby', c: 'gcc',
tcl: 'tclsh', smt2: 'Z3', mzn: 'MiniZinc', dzn: 'MiniZinc', lp: 'HiGHS', mps: 'HiGHS',
};
const TOOL_WORDS = { md: 'tool.editor', txt: 'tool.editor', toml: 'tool.editor', html: 'tool.browser', htm: 'tool.browser', test: 'tool.suiteRunner' };
const toolOf = (ext) => TOOLS[ext] || (TOOL_WORDS[ext] ? t(TOOL_WORDS[ext]) : null);
const NONE = '_none';
const MULTI = '_multi';
const OTHER = '_other';
const human = (code) => String(code).replace(/[_-]+/g, ' ').replace(/^./, (c) => c.toUpperCase());
function sentence(s) {
let x = String(s).trim();
if (!/\s/.test(x)) x = x.replace(/[-_]+/g, ' ');
x = x.replace(/\b(ci|ctf|etl|ml|qa|sql|api)\b/gi, (w) => (w.toLowerCase() === 'qa' ? 'Q&A' : w.toUpperCase()));
return x.charAt(0).toUpperCase() + x.slice(1);
}
/** A count with its noun: key_one and key_other in the locale files, {{n}} the formatted number. */
const plural = (key, n, tt = t) => tt(key, { count: n, n: fmtInt(n) });
const sum = (xs, f) => xs.reduce((a, x) => a + f(x), 0);