Battam's picture
portal: data page of repository commit d0907770eb11
f77cc19 verified
Raw History Blame Contribute Delete
11.6 kB
// The first screen, drawn before the index is read: the pictograms and (#2422) the training stages of the stage table as
// a column of bands, each with what it can use and (#2629) its funnel, from the release's meta/stage_figures.json of
// the commit the page reads (commit.js), read on its own as soon as that commit is known, with the release line. app.js
// draws both again once it has read the index (drawStages, drawRelease): each stage then a button that sets the filter
// by stage, with its status from the stage table the page was deployed with, and under them what no stage counts, by
// reason. It also has the words of a level's size (#2422: one size, the deduplicated
// one; sizeOf, sizeParts, sizeLine, checkText) that app.js writes.
import { clear, el, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtShare } from './charts.js';
import { HF, loadRevision, REPO } from './commit.js';
import { lang, onLanguage, t } from './i18n.js';
import { icon } from './icons.js';
/** Pictograms for every [data-icon] element, once. */
export function decorate(root = document) {
for (const n of root.querySelectorAll('[data-icon]:not([data-drawn])')) {
n.dataset.drawn = '1';
const inBar = n.closest('.progress');
n.prepend(icon(n.dataset.icon, { size: Number(n.dataset.size || (inBar ? 14 : 20)) }));
}
}
/** #1225: the size check of a level (check of meta/size_figures.json): the share of the rows compared that match,
* with their number; #1577: a level with rows read but none compared says so with its own reading, the rows read and
* what kept each from its file (the same counts as the md card's size line), never that nothing was compared yet. */
const NOT_COMPARED = ['unreachable', 'gone', 'needs_auth', 'no_length', 'error'];
const whyNot = (c) => NOT_COMPARED.filter((k) => c[k]).map((k) => t(`size.status.${k}`, { n: fmtInt(c[k]) })).join(t('size.listSep'));
export const checkText = (c) => (c.compared ? t('size.check', { share: fmtShare(c.match, c.compared), n: fmtInt(c.compared) })
: c.rows ? t('size.checkNotCompared', { read: fmtInt(c.rows), why: whyNot(c) }) : t('size.checkNone'));
/** #2422: a level's size (a level of meta/size_figures.json, read as it is and never computed here): its deduplicated
* size, the one size the page shows, which counts each file's content once (size.howText); the size before
* deduplication stays in the data files. */
export const sizeOf = (g) => fmtBytes(g.distinct_bytes);
/** #1158: one level's size figures in words after its size (sizeOf): the check (checkText) and, when the level has rows
* without a recorded size, their estimate, marked as one, with its range (summed: a level above a source, whose range
* is the sum of its sources' ranges, says so). -> [text]; g null (a commit without the figures) -> one text saying so. */
export function sizeParts(g, summed) {
if (!g) return [t('size.none')];
const e = g.estimate;
const out = [checkText(g.check)];
if (g.unsized_rows && !e) out.push(t('size.estimateNone', { rows: fmtInt(g.unsized_rows) }));
else if (g.unsized_rows) {
out.push(t(e.rows === g.unsized_rows ? 'size.estimate' : 'size.estimatePart', {
rows: fmtInt(e.rows), all: fmtInt(g.unsized_rows), rest: fmtInt(g.unsized_rows - e.rows), size: fmtBytes(e.bytes),
low: fmtBytes(e.low), high: fmtBytes(e.high), readings: fmtInt(e.readings), summed: summed ? t('size.summed') : '',
}));
}
return out;
}
/** A level's size and its other figures on one line, its size first and labelled as the size. */
export const sizeLine = (g, summed) => (g ? [t('size.size', { size: sizeOf(g) }), ...sizeParts(g, summed)] : sizeParts(g)).join(' \u00b7 ');
// ---- #2422: the first screen's figure, the training stages of the stage table (catalog/stages.yaml), drawn as a
// stratigraphic column: one band per stage, top to bottom as the field stacks them
const STAGE_FIGURES = 'meta/stage_figures.json';
// the color token of each stage of the stage table (style.css), which its band, its points of the field and its dots
// share; a stage the page does not know takes the neutral one
const STAGE_COLOR = { 'post-training': '--post', 'mid-training': '--mid', 'pre-training': '--pre' };
export const stageColor = (id) => STAGE_COLOR[id] || '--none';
// #2629: the rules every stage's material passes (artpool.stages.RULES, the steps of a funnel), each by the key of its
// words; a rule the page does not know is said by its name
export const RULE_KEY = { benchmark: 'benchmark', 'not compared': 'notCompared', duplicate: 'duplicate' };
/** #2629: a stage's funnel (funnel of the stage figures, #2065: its counts before the rules, then after each rule) in
* words by the measure m of its figure, each value formatted by fmt: what meets its condition, then what each rule
* removes, ending at the stage's count; a rule that removes nothing is left out. [] while it is not counted. */
function funnelSteps(funnel, m, fmt) {
const at = funnel.map((x) => x.counts?.[m]);
if (!at.length || at.some((v) => v == null)) return [];
const gone = funnel.slice(1).map((x, k) => [x.after, at[k] - at[k + 1]]).filter(([, n]) => n > 0);
return [t('stage.meets', { count: at[0], n: fmt(at[0]) }),
...gone.map(([rule, n]) => t(`stage.removed.${RULE_KEY[rule] || 'other'}`, { count: n, n: fmt(n), rule }))];
}
/** What a stage can use, in the measures it is counted by (counted_by of the stage figures, #2065), as the dataset card's
* stage figure says it: the headline is the stage's size where it is counted in bytes (after deduplication, so the
* page's one size), else its units, else its rows; the line under it the rows of a stage counted in bytes, the material
* sets of one counted in units. #2629: steps, its funnel in the headline's measure (funnelSteps), which ends at the
* headline. -> {num, word, line, steps}: the figure, its unit, the line and the funnel; not counted yet while its counts
* are null. */
export function stageWhat(st) {
const c = st.counts || {};
const by = st.counted_by || [];
const steps = (m, fmt) => funnelSteps(st.funnel || [], m, fmt);
if (by.includes('bytes') && c.bytes != null) {
const [num, word] = fmtBytes(c.bytes).split(' ');
return { num, word, line: c.rows == null ? '' : t('stage.rows', { count: c.rows, n: fmtCompact(c.rows) }), steps: steps('bytes', fmtBytes) };
}
if (by.includes('units') && c.units != null) {
return { num: fmtInt(c.units), word: t('stage.units', { count: c.units }),
line: c.material_sets == null ? '' : t('stage.sets', { count: c.material_sets, n: fmtInt(c.material_sets) }), steps: steps('units', fmtInt) };
}
if (by.includes('rows') && c.rows != null) {
return { num: fmtCompact(c.rows), word: t('stage.rowsWord', { count: c.rows }), line: '', steps: steps('rows', fmtCompact) };
}
return { num: '–', word: t('n.notCounted'), line: '', steps: [] };
}
let drawn = null;
/** #2422: the column: one band per stage of stages ([{id, name, zh, status, counted_by, counts, funnel, title}], in the
* stage table's order): the stage's color, its name in the page's language, the mark of a provisional stage, what it
* can use (stageWhat) and (#2629) under them its funnel. With onPick (app.js, once the page can filter by stage) each
* band is a button that calls onPick(id), pressed for pick, the stage the filter holds, and a band that had the focus
* keeps it. rest ({num, word, line, steps, name, title}, from app.js once the index is read): the last band, what no
* stage counts, with (#2629) its sources by reason as its steps, which is no button. why: the short lines that say which
* input the page misses. */
export function drawStages(stages, { pick = '', onPick = null, onPoint = null, why = [], rest = null } = {}) {
drawn = { stages, pick, onPick, onPoint, why, rest };
// onPoint(g): the band the pointer or the focus is on (its place in the column; -1 for none), which the field brings
// forward
const point = (g) => ({ onpointerenter: () => onPoint?.(g), onpointerleave: () => onPoint?.(-1), onfocus: () => onPoint?.(g), onblur: () => onPoint?.(-1) });
const box = document.getElementById('stage-tiles');
const focused = box.contains(document.activeElement) ? document.activeElement.dataset.stage : null;
const band = (name, mark, what) => [
el('span', { class: 'core' }),
el('span', {}, el('span', { class: 'nm' }, name, mark ? el('span', { class: 'prov' }, mark) : null), el('span', { class: 'sub' }, what.line)),
el('span', { class: 'amount' }, el('b', { class: 'big' }, what.num), what.word ? el('span', { class: 'unit' }, what.word) : null),
what.steps?.length ? el('span', { class: 'steps' }, what.steps.join(' \u00b7 ')) : null,
];
// #2629: a band with a line of steps under its name and its figure
const cls = (what) => (what.steps?.length ? 'stratum has-steps' : 'stratum');
const items = stages.map((st, g) => {
const on = pick === st.id;
const what = stageWhat(st);
const kids = band((lang() === 'zh' && st.zh) || st.name, st.status === 'provisional' ? t('stage.provisionalMark') : null, what);
const style = `--c: var(${stageColor(st.id)})`;
return el('li', {}, onPick
? el('button', { type: 'button', class: cls(what), style, 'aria-pressed': String(on), 'data-stage': st.id, title: st.title, onclick: () => onPick(st.id), ...point(g) }, kids)
: el('div', { class: cls(what), style, ...point(g) }, kids));
});
if (rest) items.push(el('li', {}, el('div', { class: cls(rest), style: '--c: var(--none)', title: rest.title, ...point(stages.length) }, band(rest.name, null, rest))));
clear(box).append(...items);
box.classList.toggle('picked', !!pick);
if (focused) box.querySelector(`[data-stage="${focused}"]`)?.focus({ preventScroll: true });
const w = document.getElementById('stage-why');
clear(w).append(...why.map((x) => el('span', {}, x)));
w.hidden = !why.length;
}
let release = null;
/** #2422: the release the page reads, in one line: its tag (a link to the dataset at that tag, the tag naming it a
* release), its day and a link to the list of every release (the dataset's CHANGELOG.md). */
export function drawRelease(tag, date) {
release = { tag, date };
clear(document.getElementById('release-line')).append(
el('a', { class: 'mono', href: `${HF}/datasets/${REPO}/tree/${tag}` }, tag), ' \u00b7 ', fmtDate(date), ' \u00b7 ',
el('a', { href: `${HF}/datasets/${REPO}/blob/main/CHANGELOG.md` }, t('release.all')));
}
decorate();
let drawnFromIndex = false;
/** app.js has drawn the first screen from the index: what this module read early is not drawn again. */
export const markFromIndex = () => { drawnFromIndex = true; };
// the first screen in a new language until app.js draws it: the same stages and release, written again
onLanguage(() => {
if (drawnFromIndex) return;
if (drawn) drawStages(drawn.stages, drawn);
if (release) drawRelease(release.tag, release.date);
});
// the release's stage figures, read as soon as the commit the page reads is known (?rev= or main as it is now), so the
// first screen shows them before the whole index is read; a commit without the file waits for app.js
loadRevision(new URLSearchParams(location.search).get('rev'))
.then((rev) => (rev.sha && rev.files.includes(STAGE_FIGURES)
? fetch(`${HF}/datasets/${REPO}/resolve/${rev.sha}/${STAGE_FIGURES}`).then((r) => (r.ok ? r.json() : null)) : null))
.then((doc) => {
if (!doc || drawnFromIndex) return;
drawStages(doc.stages || []);
drawRelease(doc.release, doc.date);
})
.catch(() => {});