Spaces:
Running
Running
File size: 11,585 Bytes
967f228 f77cc19 967f228 856d716 4950a02 975e4ee c7ba267 975e4ee 4950a02 967f228 86bca38 967f228 86bca38 967f228 86bca38 967f228 f77cc19 c29d7c6 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 86bca38 f77cc19 627a3d6 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 967f228 f77cc19 967f228 c7ba267 967f228 856d716 967f228 856d716 967f228 856d716 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 | // The first screen, drawn before the index is read: the pictograms and (#2422) the training stages of the stage table as
// a column of bands, each with what it can use and (#2629) its funnel, from the release's meta/stage_figures.json of
// the commit the page reads (commit.js), read on its own as soon as that commit is known, with the release line. app.js
// draws both again once it has read the index (drawStages, drawRelease): each stage then a button that sets the filter
// by stage, with its status from the stage table the page was deployed with, and under them what no stage counts, by
// reason. It also has the words of a level's size (#2422: one size, the deduplicated
// one; sizeOf, sizeParts, sizeLine, checkText) that app.js writes.
import { clear, el, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtShare } from './charts.js';
import { HF, loadRevision, REPO } from './commit.js';
import { lang, onLanguage, t } from './i18n.js';
import { icon } from './icons.js';
/** Pictograms for every [data-icon] element, once. */
export function decorate(root = document) {
for (const n of root.querySelectorAll('[data-icon]:not([data-drawn])')) {
n.dataset.drawn = '1';
const inBar = n.closest('.progress');
n.prepend(icon(n.dataset.icon, { size: Number(n.dataset.size || (inBar ? 14 : 20)) }));
}
}
/** #1225: the size check of a level (check of meta/size_figures.json): the share of the rows compared that match,
* with their number; #1577: a level with rows read but none compared says so with its own reading, the rows read and
* what kept each from its file (the same counts as the md card's size line), never that nothing was compared yet. */
const NOT_COMPARED = ['unreachable', 'gone', 'needs_auth', 'no_length', 'error'];
const whyNot = (c) => NOT_COMPARED.filter((k) => c[k]).map((k) => t(`size.status.${k}`, { n: fmtInt(c[k]) })).join(t('size.listSep'));
export const checkText = (c) => (c.compared ? t('size.check', { share: fmtShare(c.match, c.compared), n: fmtInt(c.compared) })
: c.rows ? t('size.checkNotCompared', { read: fmtInt(c.rows), why: whyNot(c) }) : t('size.checkNone'));
/** #2422: a level's size (a level of meta/size_figures.json, read as it is and never computed here): its deduplicated
* size, the one size the page shows, which counts each file's content once (size.howText); the size before
* deduplication stays in the data files. */
export const sizeOf = (g) => fmtBytes(g.distinct_bytes);
/** #1158: one level's size figures in words after its size (sizeOf): the check (checkText) and, when the level has rows
* without a recorded size, their estimate, marked as one, with its range (summed: a level above a source, whose range
* is the sum of its sources' ranges, says so). -> [text]; g null (a commit without the figures) -> one text saying so. */
export function sizeParts(g, summed) {
if (!g) return [t('size.none')];
const e = g.estimate;
const out = [checkText(g.check)];
if (g.unsized_rows && !e) out.push(t('size.estimateNone', { rows: fmtInt(g.unsized_rows) }));
else if (g.unsized_rows) {
out.push(t(e.rows === g.unsized_rows ? 'size.estimate' : 'size.estimatePart', {
rows: fmtInt(e.rows), all: fmtInt(g.unsized_rows), rest: fmtInt(g.unsized_rows - e.rows), size: fmtBytes(e.bytes),
low: fmtBytes(e.low), high: fmtBytes(e.high), readings: fmtInt(e.readings), summed: summed ? t('size.summed') : '',
}));
}
return out;
}
/** A level's size and its other figures on one line, its size first and labelled as the size. */
export const sizeLine = (g, summed) => (g ? [t('size.size', { size: sizeOf(g) }), ...sizeParts(g, summed)] : sizeParts(g)).join(' \u00b7 ');
// ---- #2422: the first screen's figure, the training stages of the stage table (catalog/stages.yaml), drawn as a
// stratigraphic column: one band per stage, top to bottom as the field stacks them
const STAGE_FIGURES = 'meta/stage_figures.json';
// the color token of each stage of the stage table (style.css), which its band, its points of the field and its dots
// share; a stage the page does not know takes the neutral one
const STAGE_COLOR = { 'post-training': '--post', 'mid-training': '--mid', 'pre-training': '--pre' };
export const stageColor = (id) => STAGE_COLOR[id] || '--none';
// #2629: the rules every stage's material passes (artpool.stages.RULES, the steps of a funnel), each by the key of its
// words; a rule the page does not know is said by its name
export const RULE_KEY = { benchmark: 'benchmark', 'not compared': 'notCompared', duplicate: 'duplicate' };
/** #2629: a stage's funnel (funnel of the stage figures, #2065: its counts before the rules, then after each rule) in
* words by the measure m of its figure, each value formatted by fmt: what meets its condition, then what each rule
* removes, ending at the stage's count; a rule that removes nothing is left out. [] while it is not counted. */
function funnelSteps(funnel, m, fmt) {
const at = funnel.map((x) => x.counts?.[m]);
if (!at.length || at.some((v) => v == null)) return [];
const gone = funnel.slice(1).map((x, k) => [x.after, at[k] - at[k + 1]]).filter(([, n]) => n > 0);
return [t('stage.meets', { count: at[0], n: fmt(at[0]) }),
...gone.map(([rule, n]) => t(`stage.removed.${RULE_KEY[rule] || 'other'}`, { count: n, n: fmt(n), rule }))];
}
/** What a stage can use, in the measures it is counted by (counted_by of the stage figures, #2065), as the dataset card's
* stage figure says it: the headline is the stage's size where it is counted in bytes (after deduplication, so the
* page's one size), else its units, else its rows; the line under it the rows of a stage counted in bytes, the material
* sets of one counted in units. #2629: steps, its funnel in the headline's measure (funnelSteps), which ends at the
* headline. -> {num, word, line, steps}: the figure, its unit, the line and the funnel; not counted yet while its counts
* are null. */
export function stageWhat(st) {
const c = st.counts || {};
const by = st.counted_by || [];
const steps = (m, fmt) => funnelSteps(st.funnel || [], m, fmt);
if (by.includes('bytes') && c.bytes != null) {
const [num, word] = fmtBytes(c.bytes).split(' ');
return { num, word, line: c.rows == null ? '' : t('stage.rows', { count: c.rows, n: fmtCompact(c.rows) }), steps: steps('bytes', fmtBytes) };
}
if (by.includes('units') && c.units != null) {
return { num: fmtInt(c.units), word: t('stage.units', { count: c.units }),
line: c.material_sets == null ? '' : t('stage.sets', { count: c.material_sets, n: fmtInt(c.material_sets) }), steps: steps('units', fmtInt) };
}
if (by.includes('rows') && c.rows != null) {
return { num: fmtCompact(c.rows), word: t('stage.rowsWord', { count: c.rows }), line: '', steps: steps('rows', fmtCompact) };
}
return { num: '–', word: t('n.notCounted'), line: '', steps: [] };
}
let drawn = null;
/** #2422: the column: one band per stage of stages ([{id, name, zh, status, counted_by, counts, funnel, title}], in the
* stage table's order): the stage's color, its name in the page's language, the mark of a provisional stage, what it
* can use (stageWhat) and (#2629) under them its funnel. With onPick (app.js, once the page can filter by stage) each
* band is a button that calls onPick(id), pressed for pick, the stage the filter holds, and a band that had the focus
* keeps it. rest ({num, word, line, steps, name, title}, from app.js once the index is read): the last band, what no
* stage counts, with (#2629) its sources by reason as its steps, which is no button. why: the short lines that say which
* input the page misses. */
export function drawStages(stages, { pick = '', onPick = null, onPoint = null, why = [], rest = null } = {}) {
drawn = { stages, pick, onPick, onPoint, why, rest };
// onPoint(g): the band the pointer or the focus is on (its place in the column; -1 for none), which the field brings
// forward
const point = (g) => ({ onpointerenter: () => onPoint?.(g), onpointerleave: () => onPoint?.(-1), onfocus: () => onPoint?.(g), onblur: () => onPoint?.(-1) });
const box = document.getElementById('stage-tiles');
const focused = box.contains(document.activeElement) ? document.activeElement.dataset.stage : null;
const band = (name, mark, what) => [
el('span', { class: 'core' }),
el('span', {}, el('span', { class: 'nm' }, name, mark ? el('span', { class: 'prov' }, mark) : null), el('span', { class: 'sub' }, what.line)),
el('span', { class: 'amount' }, el('b', { class: 'big' }, what.num), what.word ? el('span', { class: 'unit' }, what.word) : null),
what.steps?.length ? el('span', { class: 'steps' }, what.steps.join(' \u00b7 ')) : null,
];
// #2629: a band with a line of steps under its name and its figure
const cls = (what) => (what.steps?.length ? 'stratum has-steps' : 'stratum');
const items = stages.map((st, g) => {
const on = pick === st.id;
const what = stageWhat(st);
const kids = band((lang() === 'zh' && st.zh) || st.name, st.status === 'provisional' ? t('stage.provisionalMark') : null, what);
const style = `--c: var(${stageColor(st.id)})`;
return el('li', {}, onPick
? el('button', { type: 'button', class: cls(what), style, 'aria-pressed': String(on), 'data-stage': st.id, title: st.title, onclick: () => onPick(st.id), ...point(g) }, kids)
: el('div', { class: cls(what), style, ...point(g) }, kids));
});
if (rest) items.push(el('li', {}, el('div', { class: cls(rest), style: '--c: var(--none)', title: rest.title, ...point(stages.length) }, band(rest.name, null, rest))));
clear(box).append(...items);
box.classList.toggle('picked', !!pick);
if (focused) box.querySelector(`[data-stage="${focused}"]`)?.focus({ preventScroll: true });
const w = document.getElementById('stage-why');
clear(w).append(...why.map((x) => el('span', {}, x)));
w.hidden = !why.length;
}
let release = null;
/** #2422: the release the page reads, in one line: its tag (a link to the dataset at that tag, the tag naming it a
* release), its day and a link to the list of every release (the dataset's CHANGELOG.md). */
export function drawRelease(tag, date) {
release = { tag, date };
clear(document.getElementById('release-line')).append(
el('a', { class: 'mono', href: `${HF}/datasets/${REPO}/tree/${tag}` }, tag), ' \u00b7 ', fmtDate(date), ' \u00b7 ',
el('a', { href: `${HF}/datasets/${REPO}/blob/main/CHANGELOG.md` }, t('release.all')));
}
decorate();
let drawnFromIndex = false;
/** app.js has drawn the first screen from the index: what this module read early is not drawn again. */
export const markFromIndex = () => { drawnFromIndex = true; };
// the first screen in a new language until app.js draws it: the same stages and release, written again
onLanguage(() => {
if (drawnFromIndex) return;
if (drawn) drawStages(drawn.stages, drawn);
if (release) drawRelease(release.tag, release.date);
});
// the release's stage figures, read as soon as the commit the page reads is known (?rev= or main as it is now), so the
// first screen shows them before the whole index is read; a commit without the file waits for app.js
loadRevision(new URLSearchParams(location.search).get('rev'))
.then((rev) => (rev.sha && rev.files.includes(STAGE_FIGURES)
? fetch(`${HF}/datasets/${REPO}/resolve/${rev.sha}/${STAGE_FIGURES}`).then((r) => (r.ok ? r.json() : null)) : null))
.then((doc) => {
if (!doc || drawnFromIndex) return;
drawStages(doc.stages || []);
drawRelease(doc.release, doc.date);
})
.catch(() => {});
|