File size: 11,585 Bytes
967f228
f77cc19
 
 
 
 
967f228
 
 
 
856d716
 
 
 
 
 
 
 
 
 
 
4950a02
975e4ee
 
 
 
c7ba267
975e4ee
4950a02
967f228
 
 
 
 
 
 
 
86bca38
 
 
967f228
86bca38
 
 
 
 
 
 
 
 
967f228
 
86bca38
967f228
 
 
 
 
 
 
f77cc19
 
 
 
 
 
 
 
 
 
 
 
 
 
c29d7c6
967f228
 
 
f77cc19
 
 
967f228
 
 
f77cc19
967f228
 
f77cc19
967f228
 
 
f77cc19
 
 
 
86bca38
f77cc19
627a3d6
 
967f228
f77cc19
 
 
 
 
 
 
967f228
 
 
 
 
 
 
 
 
 
 
f77cc19
967f228
f77cc19
 
967f228
 
f77cc19
 
967f228
 
f77cc19
 
967f228
f77cc19
967f228
 
 
 
 
 
c7ba267
967f228
 
 
 
 
 
 
 
 
856d716
 
 
 
967f228
856d716
967f228
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
856d716
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
// The first screen, drawn before the index is read: the pictograms and (#2422) the training stages of the stage table as
// a column of bands, each with what it can use and (#2629) its funnel, from the release's meta/stage_figures.json of
// the commit the page reads (commit.js), read on its own as soon as that commit is known, with the release line. app.js
// draws both again once it has read the index (drawStages, drawRelease): each stage then a button that sets the filter
// by stage, with its status from the stage table the page was deployed with, and under them what no stage counts, by
// reason. It also has the words of a level's size (#2422: one size, the deduplicated
// one; sizeOf, sizeParts, sizeLine, checkText) that app.js writes.
import { clear, el, fmtBytes, fmtCompact, fmtDate, fmtInt, fmtShare } from './charts.js';
import { HF, loadRevision, REPO } from './commit.js';
import { lang, onLanguage, t } from './i18n.js';
import { icon } from './icons.js';

/** Pictograms for every [data-icon] element, once. */
export function decorate(root = document) {
  for (const n of root.querySelectorAll('[data-icon]:not([data-drawn])')) {
    n.dataset.drawn = '1';
    const inBar = n.closest('.progress');
    n.prepend(icon(n.dataset.icon, { size: Number(n.dataset.size || (inBar ? 14 : 20)) }));
  }
}

/** #1225: the size check of a level (check of meta/size_figures.json): the share of the rows compared that match,
 * with their number; #1577: a level with rows read but none compared says so with its own reading, the rows read and
 * what kept each from its file (the same counts as the md card's size line), never that nothing was compared yet. */
const NOT_COMPARED = ['unreachable', 'gone', 'needs_auth', 'no_length', 'error'];
const whyNot = (c) => NOT_COMPARED.filter((k) => c[k]).map((k) => t(`size.status.${k}`, { n: fmtInt(c[k]) })).join(t('size.listSep'));
export const checkText = (c) => (c.compared ? t('size.check', { share: fmtShare(c.match, c.compared), n: fmtInt(c.compared) })
  : c.rows ? t('size.checkNotCompared', { read: fmtInt(c.rows), why: whyNot(c) }) : t('size.checkNone'));

/** #2422: a level's size (a level of meta/size_figures.json, read as it is and never computed here): its deduplicated
 * size, the one size the page shows, which counts each file's content once (size.howText); the size before
 * deduplication stays in the data files. */
export const sizeOf = (g) => fmtBytes(g.distinct_bytes);

/** #1158: one level's size figures in words after its size (sizeOf): the check (checkText) and, when the level has rows
 * without a recorded size, their estimate, marked as one, with its range (summed: a level above a source, whose range
 * is the sum of its sources' ranges, says so). -> [text]; g null (a commit without the figures) -> one text saying so. */
export function sizeParts(g, summed) {
  if (!g) return [t('size.none')];
  const e = g.estimate;
  const out = [checkText(g.check)];
  if (g.unsized_rows && !e) out.push(t('size.estimateNone', { rows: fmtInt(g.unsized_rows) }));
  else if (g.unsized_rows) {
    out.push(t(e.rows === g.unsized_rows ? 'size.estimate' : 'size.estimatePart', {
      rows: fmtInt(e.rows), all: fmtInt(g.unsized_rows), rest: fmtInt(g.unsized_rows - e.rows), size: fmtBytes(e.bytes),
      low: fmtBytes(e.low), high: fmtBytes(e.high), readings: fmtInt(e.readings), summed: summed ? t('size.summed') : '',
    }));
  }
  return out;
}
/** A level's size and its other figures on one line, its size first and labelled as the size. */
export const sizeLine = (g, summed) => (g ? [t('size.size', { size: sizeOf(g) }), ...sizeParts(g, summed)] : sizeParts(g)).join(' \u00b7 ');

// ---- #2422: the first screen's figure, the training stages of the stage table (catalog/stages.yaml), drawn as a
// stratigraphic column: one band per stage, top to bottom as the field stacks them
const STAGE_FIGURES = 'meta/stage_figures.json';
// the color token of each stage of the stage table (style.css), which its band, its points of the field and its dots
// share; a stage the page does not know takes the neutral one
const STAGE_COLOR = { 'post-training': '--post', 'mid-training': '--mid', 'pre-training': '--pre' };
export const stageColor = (id) => STAGE_COLOR[id] || '--none';
// #2629: the rules every stage's material passes (artpool.stages.RULES, the steps of a funnel), each by the key of its
// words; a rule the page does not know is said by its name
export const RULE_KEY = { benchmark: 'benchmark', 'not compared': 'notCompared', duplicate: 'duplicate' };

/** #2629: a stage's funnel (funnel of the stage figures, #2065: its counts before the rules, then after each rule) in
 * words by the measure m of its figure, each value formatted by fmt: what meets its condition, then what each rule
 * removes, ending at the stage's count; a rule that removes nothing is left out. [] while it is not counted. */
function funnelSteps(funnel, m, fmt) {
  const at = funnel.map((x) => x.counts?.[m]);
  if (!at.length || at.some((v) => v == null)) return [];
  const gone = funnel.slice(1).map((x, k) => [x.after, at[k] - at[k + 1]]).filter(([, n]) => n > 0);
  return [t('stage.meets', { count: at[0], n: fmt(at[0]) }),
    ...gone.map(([rule, n]) => t(`stage.removed.${RULE_KEY[rule] || 'other'}`, { count: n, n: fmt(n), rule }))];
}

/** What a stage can use, in the measures it is counted by (counted_by of the stage figures, #2065), as the dataset card's
 * stage figure says it: the headline is the stage's size where it is counted in bytes (after deduplication, so the
 * page's one size), else its units, else its rows; the line under it the rows of a stage counted in bytes, the material
 * sets of one counted in units. #2629: steps, its funnel in the headline's measure (funnelSteps), which ends at the
 * headline. -> {num, word, line, steps}: the figure, its unit, the line and the funnel; not counted yet while its counts
 * are null. */
export function stageWhat(st) {
  const c = st.counts || {};
  const by = st.counted_by || [];
  const steps = (m, fmt) => funnelSteps(st.funnel || [], m, fmt);
  if (by.includes('bytes') && c.bytes != null) {
    const [num, word] = fmtBytes(c.bytes).split(' ');
    return { num, word, line: c.rows == null ? '' : t('stage.rows', { count: c.rows, n: fmtCompact(c.rows) }), steps: steps('bytes', fmtBytes) };
  }
  if (by.includes('units') && c.units != null) {
    return { num: fmtInt(c.units), word: t('stage.units', { count: c.units }),
      line: c.material_sets == null ? '' : t('stage.sets', { count: c.material_sets, n: fmtInt(c.material_sets) }), steps: steps('units', fmtInt) };
  }
  if (by.includes('rows') && c.rows != null) {
    return { num: fmtCompact(c.rows), word: t('stage.rowsWord', { count: c.rows }), line: '', steps: steps('rows', fmtCompact) };
  }
  return { num: '–', word: t('n.notCounted'), line: '', steps: [] };
}

let drawn = null;
/** #2422: the column: one band per stage of stages ([{id, name, zh, status, counted_by, counts, funnel, title}], in the
 * stage table's order): the stage's color, its name in the page's language, the mark of a provisional stage, what it
 * can use (stageWhat) and (#2629) under them its funnel. With onPick (app.js, once the page can filter by stage) each
 * band is a button that calls onPick(id), pressed for pick, the stage the filter holds, and a band that had the focus
 * keeps it. rest ({num, word, line, steps, name, title}, from app.js once the index is read): the last band, what no
 * stage counts, with (#2629) its sources by reason as its steps, which is no button. why: the short lines that say which
 * input the page misses. */
export function drawStages(stages, { pick = '', onPick = null, onPoint = null, why = [], rest = null } = {}) {
  drawn = { stages, pick, onPick, onPoint, why, rest };
  // onPoint(g): the band the pointer or the focus is on (its place in the column; -1 for none), which the field brings
  // forward
  const point = (g) => ({ onpointerenter: () => onPoint?.(g), onpointerleave: () => onPoint?.(-1), onfocus: () => onPoint?.(g), onblur: () => onPoint?.(-1) });
  const box = document.getElementById('stage-tiles');
  const focused = box.contains(document.activeElement) ? document.activeElement.dataset.stage : null;
  const band = (name, mark, what) => [
    el('span', { class: 'core' }),
    el('span', {}, el('span', { class: 'nm' }, name, mark ? el('span', { class: 'prov' }, mark) : null), el('span', { class: 'sub' }, what.line)),
    el('span', { class: 'amount' }, el('b', { class: 'big' }, what.num), what.word ? el('span', { class: 'unit' }, what.word) : null),
    what.steps?.length ? el('span', { class: 'steps' }, what.steps.join(' \u00b7 ')) : null,
  ];
  // #2629: a band with a line of steps under its name and its figure
  const cls = (what) => (what.steps?.length ? 'stratum has-steps' : 'stratum');
  const items = stages.map((st, g) => {
    const on = pick === st.id;
    const what = stageWhat(st);
    const kids = band((lang() === 'zh' && st.zh) || st.name, st.status === 'provisional' ? t('stage.provisionalMark') : null, what);
    const style = `--c: var(${stageColor(st.id)})`;
    return el('li', {}, onPick
      ? el('button', { type: 'button', class: cls(what), style, 'aria-pressed': String(on), 'data-stage': st.id, title: st.title, onclick: () => onPick(st.id), ...point(g) }, kids)
      : el('div', { class: cls(what), style, ...point(g) }, kids));
  });
  if (rest) items.push(el('li', {}, el('div', { class: cls(rest), style: '--c: var(--none)', title: rest.title, ...point(stages.length) }, band(rest.name, null, rest))));
  clear(box).append(...items);
  box.classList.toggle('picked', !!pick);
  if (focused) box.querySelector(`[data-stage="${focused}"]`)?.focus({ preventScroll: true });
  const w = document.getElementById('stage-why');
  clear(w).append(...why.map((x) => el('span', {}, x)));
  w.hidden = !why.length;
}

let release = null;
/** #2422: the release the page reads, in one line: its tag (a link to the dataset at that tag, the tag naming it a
 * release), its day and a link to the list of every release (the dataset's CHANGELOG.md). */
export function drawRelease(tag, date) {
  release = { tag, date };
  clear(document.getElementById('release-line')).append(
    el('a', { class: 'mono', href: `${HF}/datasets/${REPO}/tree/${tag}` }, tag), ' \u00b7 ', fmtDate(date), ' \u00b7 ',
    el('a', { href: `${HF}/datasets/${REPO}/blob/main/CHANGELOG.md` }, t('release.all')));
}

decorate();
let drawnFromIndex = false;
/** app.js has drawn the first screen from the index: what this module read early is not drawn again. */
export const markFromIndex = () => { drawnFromIndex = true; };
// the first screen in a new language until app.js draws it: the same stages and release, written again
onLanguage(() => {
  if (drawnFromIndex) return;
  if (drawn) drawStages(drawn.stages, drawn);
  if (release) drawRelease(release.tag, release.date);
});
// the release's stage figures, read as soon as the commit the page reads is known (?rev= or main as it is now), so the
// first screen shows them before the whole index is read; a commit without the file waits for app.js
loadRevision(new URLSearchParams(location.search).get('rev'))
  .then((rev) => (rev.sha && rev.files.includes(STAGE_FIGURES)
    ? fetch(`${HF}/datasets/${REPO}/resolve/${rev.sha}/${STAGE_FIGURES}`).then((r) => (r.ok ? r.json() : null)) : null))
  .then((doc) => {
    if (!doc || drawnFromIndex) return;
    drawStages(doc.stages || []);
    drawRelease(doc.release, doc.date);
  })
  .catch(() => {});