// The first screen, drawn before the index is read: the pictograms and (#2422) the training stages of the stage table as // a column of bands, each with what it can use and (#2629) its funnel, from the release's meta/stage_figures.json of // the commit the page reads (commit.js), read on its own as soon as that commit is known, with the release line. app.js // draws both again once it has read the index (drawStages, drawRelease): each stage then a button that sets the filter // by stage, with its status from the stage table the page was deployed with, and under them what no stage counts, by // reason. It also has the words of a level's size (#2422: one size, the deduplicated // one; sizeOf, sizeParts, sizeLine, checkText) that app.js writes. import { clear, el, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtShare } from './charts.js'; import { HF, loadRevision, REPO } from './commit.js'; import { lang, onLanguage, t } from './i18n.js'; import { icon } from './icons.js'; /** Pictograms for every [data-icon] element, once. */ export function decorate(root = document) { for (const n of root.querySelectorAll('[data-icon]:not([data-drawn])')) { n.dataset.drawn = '1'; const inBar = n.closest('.progress'); n.prepend(icon(n.dataset.icon, { size: Number(n.dataset.size || (inBar ? 14 : 20)) })); } } /** #1225: the size check of a level (check of meta/size_figures.json): the share of the rows compared that match, * with their number; #1577: a level with rows read but none compared says so with its own reading, the rows read and * what kept each from its file (the same counts as the md card's size line), never that nothing was compared yet. */ const NOT_COMPARED = ['unreachable', 'gone', 'needs_auth', 'no_length', 'error']; const whyNot = (c) => NOT_COMPARED.filter((k) => c[k]).map((k) => t(`size.status.${k}`, { n: fmtInt(c[k]) })).join(t('size.listSep')); export const checkText = (c) => (c.compared ? t('size.check', { share: fmtShare(c.match, c.compared), n: fmtInt(c.compared) }) : c.rows ? t('size.checkNotCompared', { read: fmtInt(c.rows), why: whyNot(c) }) : t('size.checkNone')); /** #2422: a level's size (a level of meta/size_figures.json, read as it is and never computed here): its deduplicated * size, the one size the page shows, which counts each file's content once (size.howText); the size before * deduplication stays in the data files. */ export const sizeOf = (g) => fmtBytes(g.distinct_bytes); /** #1158: one level's size figures in words after its size (sizeOf): the check (checkText) and, when the level has rows * without a recorded size, their estimate, marked as one, with its range (summed: a level above a source, whose range * is the sum of its sources' ranges, says so). -> [text]; g null (a commit without the figures) -> one text saying so. */ export function sizeParts(g, summed) { if (!g) return [t('size.none')]; const e = g.estimate; const out = [checkText(g.check)]; if (g.unsized_rows && !e) out.push(t('size.estimateNone', { rows: fmtInt(g.unsized_rows) })); else if (g.unsized_rows) { out.push(t(e.rows === g.unsized_rows ? 'size.estimate' : 'size.estimatePart', { rows: fmtInt(e.rows), all: fmtInt(g.unsized_rows), rest: fmtInt(g.unsized_rows - e.rows), size: fmtBytes(e.bytes), low: fmtBytes(e.low), high: fmtBytes(e.high), readings: fmtInt(e.readings), summed: summed ? t('size.summed') : '', })); } return out; } /** A level's size and its other figures on one line, its size first and labelled as the size. */ export const sizeLine = (g, summed) => (g ? [t('size.size', { size: sizeOf(g) }), ...sizeParts(g, summed)] : sizeParts(g)).join(' \u00b7 '); // ---- #2422: the first screen's figure, the training stages of the stage table (catalog/stages.yaml), drawn as a // stratigraphic column: one band per stage, top to bottom as the field stacks them const STAGE_FIGURES = 'meta/stage_figures.json'; // the color token of each stage of the stage table (style.css), which its band, its points of the field and its dots // share; a stage the page does not know takes the neutral one const STAGE_COLOR = { 'post-training': '--post', 'mid-training': '--mid', 'pre-training': '--pre' }; export const stageColor = (id) => STAGE_COLOR[id] || '--none'; // #2629: the rules every stage's material passes (artpool.stages.RULES, the steps of a funnel), each by the key of its // words; a rule the page does not know is said by its name export const RULE_KEY = { benchmark: 'benchmark', 'not compared': 'notCompared', duplicate: 'duplicate' }; /** #2629: a stage's funnel (funnel of the stage figures, #2065: its counts before the rules, then after each rule) in * words by the measure m of its figure, each value formatted by fmt: what meets its condition, then what each rule * removes, ending at the stage's count; a rule that removes nothing is left out. [] while it is not counted. */ function funnelSteps(funnel, m, fmt) { const at = funnel.map((x) => x.counts?.[m]); if (!at.length || at.some((v) => v == null)) return []; const gone = funnel.slice(1).map((x, k) => [x.after, at[k] - at[k + 1]]).filter(([, n]) => n > 0); return [t('stage.meets', { count: at[0], n: fmt(at[0]) }), ...gone.map(([rule, n]) => t(`stage.removed.${RULE_KEY[rule] || 'other'}`, { count: n, n: fmt(n), rule }))]; } /** What a stage can use, in the measures it is counted by (counted_by of the stage figures, #2065), as the dataset card's * stage figure says it: the headline is the stage's size where it is counted in bytes (after deduplication, so the * page's one size), else its units, else its rows; the line under it the rows of a stage counted in bytes, the material * sets of one counted in units. #2629: steps, its funnel in the headline's measure (funnelSteps), which ends at the * headline. -> {num, word, line, steps}: the figure, its unit, the line and the funnel; not counted yet while its counts * are null. */ export function stageWhat(st) { const c = st.counts || {}; const by = st.counted_by || []; const steps = (m, fmt) => funnelSteps(st.funnel || [], m, fmt); if (by.includes('bytes') && c.bytes != null) { const [num, word] = fmtBytes(c.bytes).split(' '); return { num, word, line: c.rows == null ? '' : t('stage.rows', { count: c.rows, n: fmtCompact(c.rows) }), steps: steps('bytes', fmtBytes) }; } if (by.includes('units') && c.units != null) { return { num: fmtInt(c.units), word: t('stage.units', { count: c.units }), line: c.material_sets == null ? '' : t('stage.sets', { count: c.material_sets, n: fmtInt(c.material_sets) }), steps: steps('units', fmtInt) }; } if (by.includes('rows') && c.rows != null) { return { num: fmtCompact(c.rows), word: t('stage.rowsWord', { count: c.rows }), line: '', steps: steps('rows', fmtCompact) }; } return { num: '–', word: t('n.notCounted'), line: '', steps: [] }; } let drawn = null; /** #2422: the column: one band per stage of stages ([{id, name, zh, status, counted_by, counts, funnel, title}], in the * stage table's order): the stage's color, its name in the page's language, the mark of a provisional stage, what it * can use (stageWhat) and (#2629) under them its funnel. With onPick (app.js, once the page can filter by stage) each * band is a button that calls onPick(id), pressed for pick, the stage the filter holds, and a band that had the focus * keeps it. rest ({num, word, line, steps, name, title}, from app.js once the index is read): the last band, what no * stage counts, with (#2629) its sources by reason as its steps, which is no button. why: the short lines that say which * input the page misses. */ export function drawStages(stages, { pick = '', onPick = null, onPoint = null, why = [], rest = null } = {}) { drawn = { stages, pick, onPick, onPoint, why, rest }; // onPoint(g): the band the pointer or the focus is on (its place in the column; -1 for none), which the field brings // forward const point = (g) => ({ onpointerenter: () => onPoint?.(g), onpointerleave: () => onPoint?.(-1), onfocus: () => onPoint?.(g), onblur: () => onPoint?.(-1) }); const box = document.getElementById('stage-tiles'); const focused = box.contains(document.activeElement) ? document.activeElement.dataset.stage : null; const band = (name, mark, what) => [ el('span', { class: 'core' }), el('span', {}, el('span', { class: 'nm' }, name, mark ? el('span', { class: 'prov' }, mark) : null), el('span', { class: 'sub' }, what.line)), el('span', { class: 'amount' }, el('b', { class: 'big' }, what.num), what.word ? el('span', { class: 'unit' }, what.word) : null), what.steps?.length ? el('span', { class: 'steps' }, what.steps.join(' \u00b7 ')) : null, ]; // #2629: a band with a line of steps under its name and its figure const cls = (what) => (what.steps?.length ? 'stratum has-steps' : 'stratum'); const items = stages.map((st, g) => { const on = pick === st.id; const what = stageWhat(st); const kids = band((lang() === 'zh' && st.zh) || st.name, st.status === 'provisional' ? t('stage.provisionalMark') : null, what); const style = `--c: var(${stageColor(st.id)})`; return el('li', {}, onPick ? el('button', { type: 'button', class: cls(what), style, 'aria-pressed': String(on), 'data-stage': st.id, title: st.title, onclick: () => onPick(st.id), ...point(g) }, kids) : el('div', { class: cls(what), style, ...point(g) }, kids)); }); if (rest) items.push(el('li', {}, el('div', { class: cls(rest), style: '--c: var(--none)', title: rest.title, ...point(stages.length) }, band(rest.name, null, rest)))); clear(box).append(...items); box.classList.toggle('picked', !!pick); if (focused) box.querySelector(`[data-stage="${focused}"]`)?.focus({ preventScroll: true }); const w = document.getElementById('stage-why'); clear(w).append(...why.map((x) => el('span', {}, x))); w.hidden = !why.length; } let release = null; /** #2422: the release the page reads, in one line: its tag (a link to the dataset at that tag, the tag naming it a * release), its day and a link to the list of every release (the dataset's CHANGELOG.md). */ export function drawRelease(tag, date) { release = { tag, date }; clear(document.getElementById('release-line')).append( el('a', { class: 'mono', href: `${HF}/datasets/${REPO}/tree/${tag}` }, tag), ' \u00b7 ', fmtDate(date), ' \u00b7 ', el('a', { href: `${HF}/datasets/${REPO}/blob/main/CHANGELOG.md` }, t('release.all'))); } decorate(); let drawnFromIndex = false; /** app.js has drawn the first screen from the index: what this module read early is not drawn again. */ export const markFromIndex = () => { drawnFromIndex = true; }; // the first screen in a new language until app.js draws it: the same stages and release, written again onLanguage(() => { if (drawnFromIndex) return; if (drawn) drawStages(drawn.stages, drawn); if (release) drawRelease(release.tag, release.date); }); // the release's stage figures, read as soon as the commit the page reads is known (?rev= or main as it is now), so the // first screen shows them before the whole index is read; a commit without the file waits for app.js loadRevision(new URLSearchParams(location.search).get('rev')) .then((rev) => (rev.sha && rev.files.includes(STAGE_FIGURES) ? fetch(`${HF}/datasets/${REPO}/resolve/${rev.sha}/${STAGE_FIGURES}`).then((r) => (r.ok ? r.json() : null)) : null)) .then((doc) => { if (!doc || drawnFromIndex) return; drawStages(doc.stages || []); drawRelease(doc.release, doc.date); }) .catch(() => {});