File size: 24,305 Bytes
a942796
 
 
2e7ce94
638568b
fcd1156
a942796
638568b
a942796
 
 
 
 
94bddc8
638568b
 
 
c7ba267
 
 
 
 
638568b
db3a530
a942796
db3a530
 
 
 
 
 
a942796
 
db3a530
 
a942796
 
db3a530
638568b
 
db3a530
 
638568b
db3a530
 
a942796
 
2e7ce94
 
 
a942796
 
 
 
 
 
 
 
 
 
 
638568b
 
 
a942796
 
2e7ce94
 
 
 
a942796
2e7ce94
 
 
 
 
 
 
a942796
 
 
 
 
0784d2a
 
a942796
 
 
 
 
 
 
 
 
0784d2a
a942796
 
 
0784d2a
a942796
 
0784d2a
a942796
 
 
 
 
 
0784d2a
a942796
 
 
 
 
 
 
0784d2a
a942796
 
 
 
 
 
 
0784d2a
a942796
 
 
 
 
 
0784d2a
a942796
 
0784d2a
 
94bddc8
 
 
689ce1d
 
 
a942796
856d716
 
94bddc8
 
 
 
 
894be04
 
 
689ce1d
 
a942796
 
f77cc19
 
 
 
 
c7ba267
 
f77cc19
c7ba267
 
a942796
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d0b101b
 
 
 
 
 
 
bec7df7
 
 
 
 
 
 
 
db3a530
 
 
 
 
638568b
 
a942796
db3a530
638568b
 
 
 
 
c7ba267
 
 
 
 
 
 
 
2e7ce94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fcd1156
 
a942796
 
db3a530
 
a942796
db3a530
 
a942796
631e2c6
967f228
 
 
 
 
 
 
 
 
 
 
 
631e2c6
967f228
 
 
631e2c6
 
 
 
 
 
 
a942796
 
 
 
 
 
 
 
 
638568b
 
a942796
638568b
a942796
 
 
 
 
 
638568b
2e7ce94
 
 
c7ba267
 
 
856d716
 
2e7ce94
c7ba267
856d716
 
 
7ade112
 
 
a942796
638568b
856d716
2e7ce94
 
 
856d716
 
 
2e7ce94
 
 
856d716
 
a942796
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
349bca8
 
 
d0b101b
 
 
a942796
 
967f228
 
 
 
 
86bca38
7ade112
975e4ee
7ade112
e18ef8b
 
 
 
a4caba4
 
 
 
 
a942796
 
 
 
bec7df7
 
631e2c6
 
7ade112
a942796
 
 
2e7ce94
a942796
 
 
 
 
967f228
 
2c4652f
a942796
967f228
a942796
 
975e4ee
 
 
638568b
7ade112
631e2c6
 
 
 
7ade112
c7ba267
631e2c6
c7ba267
 
 
 
a942796
 
967f228
 
 
638568b
a942796
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
// Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees.
import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm';
import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm';
import { downloadFile, pathsInfo } from 'https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.10/+esm';
import { HF, loadRevision as askRevision, REPO } from './commit.js';
import { t } from './i18n.js';

export { HF, REPO };
export const DATASET_URL = `${HF}/datasets/${REPO}`;

export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`;
export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v));

// the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus
export const MATERIAL_SET = 'material_set';
export const CORPUS = 'corpus';
export const KINDS = [MATERIAL_SET, CORPUS];
// #2065, #2066: the one table of Chinese names the release writes: per English text the index shows (a collection's
// name, a source's title, description or summary, as the catalog holds them) its Chinese text (columns en and zh)
const TRANSLATIONS = 'meta/translations.parquet';
// #2066: the projection's counts per training stage (artpool.stages FIGURES), with (#2065) each stage's funnel
const STAGE_FIGURES = 'meta/stage_figures.json';

const HUB_REPO = { type: 'dataset', name: REPO };

/** One whole file of the commit sha, read with downloadFile of @huggingface/hub, which answers null when the commit
 * has no such file. */
export async function wholeFile(sha, path) {
  const blob = await downloadFile({ repo: HUB_REPO, path, revision: sha });
  if (!blob) throw new Error(`${path}: no such file at commit ${sha.slice(0, 7)}`);
  return blob;
}

/** A file named by its URL in the page address (catalog=, vocabulary=), which may lie outside the dataset. */
async function urlFile(url) {
  const r = await fetch(url);
  if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
  return r.blob();
}

const jsonOf = async (blob) => JSON.parse(await (await blob).text());

/** A small file read whole (the meta tables): one request instead of a size probe and ranges. */
async function readParquet(blob) {
  return parquetReadObjects({ file: await (await blob).arrayBuffer(), compressors, utf8: false });
}

/** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd} (#1792:
 * hyparquet documents no row filter, so a reader of one source reads its row range, filesOf, and keeps its rows by
 * the column source itself). */
export async function readColumns(url, cols, range = {}) {
  const file = await asyncBufferFromUrl({ url });
  const metadata = await parquetMetadataAsync(file);
  const names = new Set(metadata.schema.slice(1).map((s) => s.name));
  const columns = cols.filter((c) => names.has(c));
  const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range });
  return { rows, total: num(metadata.num_rows) };
}

/** The commit to read: ?rev=<sha or branch>, else main. Returns {sha, date, files}. */
export async function loadRevision(rev) {
  const r = await askRevision(rev);
  if (!r.sha) throw new Error(t('error.noCommit'));
  return r;
}

/** #1769: the date of the last commit of each batch of a source whose data files are files ({path, batches},
 * loadIndex's files): the latest last commit among the files that hold the batch's rows (pathsInfo), never a folder
 * named by the batch. -> Map of batch -> ISO date. */
export async function loadBatchDates(sha, files) {
  const out = new Map();
  if (!files.length) return out;
  const info = await pathsInfo({ repo: HUB_REPO, paths: files.map((f) => f.path), revision: sha, expand: true });
  const iso = (d) => (d instanceof Date ? d.toISOString() : d ? String(d) : null);
  const at = new Map(info.map((e) => [e.path, iso(e.lastCommit?.date)]));
  for (const f of files) {
    const d = at.get(f.path);
    for (const b of f.batches) if (d && !(out.get(b) >= d)) out.set(b, d);
  }
  return out;
}

// ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or
// unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without
// them everything shows as unclassified.
const pick = (o, keys) => {
  if (!o || typeof o !== 'object') return undefined;
  for (const k of keys) if (o[k] != null) return o[k];
  return undefined;
};
const NAME_KEYS = ['name', 'label', 'title', 'en'];
const CODE_KEYS = ['code', 'id', 'key', 'value'];
const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items'];
const ABOUT_KEYS = ['description', 'about', 'definition'];
const ZH_KEYS = ['zh'];

function vocabSection(node) {
  const names = new Map();
  const zh = new Map();
  const about = new Map();
  const order = [];
  const add = (code, name, text, zhName) => {
    if (code == null) return;
    code = String(code);
    if (!names.has(code)) order.push(code);
    if (typeof name === 'string' && name.trim()) names.set(code, name.trim());
    else if (!names.has(code)) names.set(code, null);
    if (typeof text === 'string' && text.trim()) about.set(code, text.trim());
    if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim());
  };
  const walk = (n) => {
    if (!n) return;
    if (Array.isArray(n)) {
      for (const it of n) {
        if (typeof it === 'string') add(it, null);
        else if (it && typeof it === 'object') {
          add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS));
          walk(pick(it, KIDS_KEYS));
        }
      }
    } else if (typeof n === 'object') {
      for (const [k, v] of Object.entries(n)) {
        if (typeof v === 'string') add(k, v);
        else if (v && typeof v === 'object') {
          add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS));
          walk(pick(v, KIDS_KEYS));
        } else add(k, null);
      }
    }
  };
  walk(node);
  return { names, zh, about, order };
}

// meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows
// its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites).
// zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about,
// zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the
// lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags
// of its material sets). terms: Map of the term table by its English term. #2397: profile [{criterion, option}], the
// criteria tools/fetch.py's default profile applies, in its order, each with the option that takes back what it
// leaves out (what Get the data shows the profile leaves out).
export function parseVocabulary(j) {
  const form = vocabSection(pick(j, ['form', 'forms']));
  for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' '));
  const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l]));
  const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({
    code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '',
    campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [],
  }));
  // terms [{term, zh, about}] (the index's term table: the English term, its Chinese term and one sentence on what it
  // means), by the English term
  const terms = new Map((Array.isArray(j?.terms) ? j.terms : []).filter((x) => x?.term).map((x) => [String(x.term), x]));
  const profile = (Array.isArray(j?.profile) ? j.profile : []).map((p) => ({ criterion: String(p.criterion), option: String(p.option) }));
  return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections, terms, profile };
}

/** #2066: meta/stage_figures.json -> {rules, stages: Map id -> stage, noStage}: the rules every stage's material passes,
 * in their order, and per stage, in the stage table's order, its names, status and measures (counted_by) and its counts
 * by them (counts, each null before the release counted the stage fields), which (#2422) the first screen's figure says;
 * #2629: noStage, what no stage counts of each layer by reason ({kind: {reason: {measure: value}}}, null for figures
 * written before it); null for a commit without the file. */
function parseStageFigures(j) {
  if (!j) return null;
  return { rules: j.rules || [], stages: new Map((j.stages || []).map((s) => [s.id, s])), noStage: j.no_stage || null };
}

export const topOf = (code) => String(code).split('.')[0];

function domainOf(v) {
  let list = v;
  if (typeof list === 'string') list = [list];
  if (list && typeof list.toArray === 'function') list = list.toArray();
  list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : [];
  if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] };
  if (list.includes('multi')) return { kind: 'multi', codes: [] };
  return { kind: 'codes', codes: [...new Set(list)] };
}

/** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or
 * conditional for every row) or 'no' (some rows are no or unknown). */
function permission(lr) {
  if (!lr) return 'no';
  const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k);
  if (!keys.length) return 'no';
  if (keys.every((k) => k === 'yes')) return 'yes';
  if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond';
  return 'no';
}

/** #1266: a catalog cell of license_counts as [{kind, spdx, training_ok, redistribution_ok, files}], null when empty. */
function licenceCounts(v) {
  const items = Array.isArray(v) ? v : v?.toArray?.();
  return items?.length ? items.map((x) => ({ kind: x.kind, spdx: x.spdx || null, training_ok: x.training_ok,
    redistribution_ok: x.redistribution_ok, files: num(x.files) || 0 })) : null;
}

/** #1524: a source's file types as [[ext, rows]], most rows first, from its by_ext of meta/sources.json ({ext: {rows,
 * bytes}}); the keys in brackets ((none), (other)) name no extension that fetch.py's --ext or a row's ext matches, and
 * are left out. */
function extsOf(byExt) {
  return Object.entries(byExt || {}).filter(([k]) => !k.startsWith('(')).map(([k, v]) => [k, num(v?.rows) || 0])
    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
}

/** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}};
 * json: the file is JSON (a list of rows, or {sources} or {rows}), else Parquet. */
async function readCatalog(blob, json) {
  if (json) {
    const j = await jsonOf(blob);
    const rows = Array.isArray(j) ? j : j.sources || j.rows || [];
    return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } };
  }
  const buf = await (await blob).arrayBuffer();
  // DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf
  const bytes = new Uint8Array(buf.slice(0));
  return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } };
}

/** #2065: a sum over values some of which are not known (null), as the figures sum them (index_projection._total):
 * {sum: the sum of the known values, null when none is known (0 when there are no values), missing: how many are
 * not counted yet}. */
export function summed(vals) {
  const known = vals.filter((v) => v != null);
  return { sum: known.length ? known.reduce((a, v) => a + v, 0) : vals.length ? null : 0, missing: vals.length - known.length };
}

const listOf = (v) => (Array.isArray(v) ? v : v?.toArray?.() || []);

/** #1769: the data files that hold rows of the catalog row r, each {path, batches, first, last} (the batches of its
 * rows in that file; #1792: the source's first and last row in it, null when the catalog does not record them), sorted
 * by path: from the catalog column batch_files, else (a catalog with data_files only) from data_files, with no batches;
 * never from a file's path. A file may hold rows of other sources too, so every read of one keeps the source's rows by
 * the column source, and reads only its row range when known (a packed file holds a source's rows together). */
function filesOf(r) {
  const by = new Map();
  for (const x of listOf(r.batch_files)) {
    const f = String(x.file);
    if (!by.has(f)) by.set(f, { batches: [], first: num(x.first), last: num(x.last) });
    by.get(f).batches.push(String(x.batch));
  }
  if (!by.size) for (const f of listOf(r.data_files)) by.set(String(f), { batches: [], first: null, last: null });
  return [...by].map(([path, x]) => ({ path, ...x, batches: x.batches.sort() })).sort((a, b) => a.path.localeCompare(b.path));
}

/** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values):
 * a note for the reader, as a key of the locale files. */
export async function loadIndex(params, onNote) {
  const rev = await loadRevision(params.get('rev'));
  const at = (p) => wholeFile(rev.sha, p);
  const catalogUrl = params.get('catalog');
  const has = (p) => rev.files.includes(p);
  const vocabUrl = params.get('vocabulary');
  const vocabFile = vocabUrl ? () => urlFile(vocabUrl) : has('meta/vocabulary.json') ? () => at('meta/vocabulary.json') : null;

  const [cat, sourcesJson, coverage, batches, vocabJson, figures, record, translations, stageFigures] =
    await Promise.all([
      catalogUrl ? readCatalog(urlFile(catalogUrl), /\.json(\?|$)/i.test(catalogUrl)) : readCatalog(at('meta/catalog.parquet'), false),
      jsonOf(at('meta/sources.json')),
      jsonOf(at('meta/coverage.json')),
      readParquet(at('meta/batches.parquet')).catch(() => []),
      vocabFile ? jsonOf(vocabFile()).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null,
      // #1158: the size figures of every level, written by the projection; a commit before it has none
      has('meta/size_figures.json') ? jsonOf(at('meta/size_figures.json')) : null,
      // #1197: the record of the release's projection, whose tag (DL13h, artpool/release_tag.py) the commands name and
      // (#2422) whose tag and date the release line of the first screen says
      has('meta/projection.json') ? jsonOf(at('meta/projection.json')) : null,
      // #2066: the Chinese names of collections and sources (#2065); #2422: what each training stage can use, the
      // first screen's figure; #2518: and, in the same file, the stages of each source
      has(TRANSLATIONS) ? readParquet(at(TRANSLATIONS)) : [],
      has(STAGE_FIGURES) ? jsonOf(at(STAGE_FIGURES)) : null,
    ]);
  // #2518: a source's stages come from the commit the page reads: meta/stage_figures.json carries, per source, the ids
  // of the stages one of its units passes (sources), beside the stage table's version and each stage's counts, so the
  // two never describe different commits. A commit without the file, one whose release has not counted its stage
  // fields, or one written before the figures listed the sources has no stage for any source, and the filter by stage
  // says which in one short line (stagesWhy)
  const stagesOk = stageFigures?.counted && stageFigures.sources ? stageFigures : null;
  const stagesWhy = stagesOk ? null : !stageFigures ? 'noFigures' : !stageFigures.counted ? 'notCounted' : 'noSources';

  const vocab = parseVocabulary(vocabJson || {});
  const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e]));
  const cov = coverage.by_source || {};
  const batchesBy = new Map();
  for (const b of batches) {
    if (!batchesBy.has(b.source)) batchesBy.set(b.source, []);
    batchesBy.get(b.source).push(b);
  }
  // the catalog column kind (#952); an index commit before it has none, and the page computes the kind below
  const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== '');

  const sources = cat.rows.map((r) => {
    const info = srcInfo.get(r.source) || {};
    const lic = r.license || r.licence || {};
    const dom = domainOf(r.domain);
    const rows = num(r.files ?? info.rows) || 0;
    const bytes = num(r.bytes);
    const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form);
    // The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets
    // the rule the index writes it by (#1633): a source that holds rows of a collection (sources.json campaign or
    // extended_by) or has fewer units than rows is a material set, every other one a corpus.
    // A unit is one unit_id (a test case, one workbook, one package); files are counted apart. The catalog columns
    // units (the distinct units of a material set) and files (its rows; a corpus is counted in rows) are read as they
    // are; #2065: a material set whose catalog row has no count of its units has none here (readyUnits null, not
    // counted yet), as the figures count it (meta/size_figures.json units_not_counted), never its rows.
    const roles = info.unit || {};
    const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k]));
    const isTask = hasKind ? r.kind === MATERIAL_SET : !!(info.campaign || info.extended_by) || (has('units') && num(r.units) < rows);
    const units = has('units') ? num(r.units) : null;
    const lr = info.license_rows?.redistribution_ok || {};
    const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0);
    const shipShare = rows ? Math.min(1, shipRows / rows) : 0;
    // #1577: the fetch check, catalog column fetch_check; #2006: named fetch up to release-2026-11-09.1, read until a
    // release with fetch_check is published
    const fc = r.fetch_check || r.fetch;
    return {
      kind: isTask ? MATERIAL_SET : CORPUS,
      isTask,
      // the collection of a material set (#1083): a line of admit_collections.tsv; of a corpus (#1633), the collection
      // that lists it under corpora in catalog/domains.yaml, else none
      collection: r.collection ? String(r.collection) : null,
      readyUnits: isTask ? units : 0,
      rawRows: isTask ? 0 : rows,
      fromCatalog: has('units'),
      // what one row of a corpus points at (sources.json unit: file, archive member, ...); a collection's entry counts
      // its rows as files (#1633: no roles)
      kinds: Object.keys(roles),
      units,
      shipShare,
      id: r.source,
      title: r.title || r.source,
      description: r.description || info.description || '',
      summary: r.summary || '',
      unit: r.unit || '',
      status: r.status || null,
      reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [],
      domainKind: dom.kind,
      codes: dom.codes,
      tops: [...new Set(dom.codes.map(topOf))],
      form,
      tier: lic.tier || 'unknown',
      ship: lic.ship || null,
      spdx: lic.spdx || null,
      licenceNote: lic.note || null,
      contamination: r.contamination || {},
      pii: r.pii || {},
      // #1256: the distinct publisher names over all the rows of a material set (catalog column publishers); null
      // when its rows carry none or the catalog has no such column
      publishers: has('publishers') ? num(r.publishers) : null,
      // #1266: the files of a material set by kind of file (upstream, index), licence and the permits for training and
      // redistribution, as the projection counted them (catalog column license_counts); null without the column
      licenceCounts: licenceCounts(r.license_counts),
      rows,
      bytes: bytes && bytes > 0 ? bytes : null,
      // #2422: the one size the page shows, the deduplicated one (catalog column distinct_bytes, the same as
      // distinct_bytes of the source's figures); the size filter buckets it
      distinct: num(r.distinct_bytes) > 0 ? num(r.distinct_bytes) : null,
      // #1158: the source's size figures from meta/size_figures.json (#2422: its size, the deduplicated one, the check
      // and the estimate); #2066: the list shows and sorts by them
      size: figures?.sources?.[r.source] || null,
      // #1577: whether its upstream still serves its files (the fetch check fc: level, rows, fetched, read_at); null
      // for a catalog without the column
      fetch: fc && fc.level ? { level: String(fc.level), rows: num(fc.rows) || 0, fetched: num(fc.fetched) || 0, readAt: fc.read_at || null } : null,
      // #1928: the release tag of the release its rows come from (catalog column released_in), and why a newer batch of
      // it is held back, as the release notes give it (catalog column held_back); null for a catalog without them
      releasedIn: r.released_in || null,
      heldBack: r.held_back || null,
      // #2588: the source's declared upstream (catalog columns upstream_kind, upstream_agent and upstream_entry): a
      // dataset or a catalog, its agent and its entry address, the values joined by ' | ' while the source reads
      // several upstreams; null for a catalog without them
      upstream: r.upstream_kind ? { kind: String(r.upstream_kind), agent: String(r.upstream_agent ?? ''),
        entry: String(r.upstream_entry ?? '') } : null,
      topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
      needsAuth: r.needs_auth || null,
      train: permission(info.license_rows?.training_ok),
      redist: permission(info.license_rows?.redistribution_ok),
      // #1524: its file types and their rows, for the File type facet, the --ext options and the SQL and CSV filters
      exts: extsOf(info.by_ext),
      // #2006: the ids of the training stages one of its units passes (#2518: sources of the commit's stage figures),
      // [] without stages
      stages: stagesOk?.sources?.[r.source] || [],
      info,
      cov: cov[r.source] || null,
      batches: batchesBy.get(r.source) || [],
      files: filesOf(r),
    };
  });

  return {
    sha: rev.sha,
    // the release tag of the commit (a tag is never moved), the commit itself before meta/projection.json had one;
    // #2422: the day of that release, the commit's own before meta/projection.json had one
    pin: record?.tag || rev.sha,
    date: rev.date,
    releaseDate: record?.date || rev.date,
    overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') },
    hasVocabulary: Boolean(vocabJson),
    // #1577: who published each file of a material set and under which license, read when the attribution list is
    // made; a commit before it has none
    hasAttribution: has('meta/attribution.parquet'),
    hasKind,
    // #2006: the stage table the stages were computed with (version, and each stage with its names, status, about and
    // counts), null when the commit has no stage of its sources; #2066: stagesWhy, why it has none (noFigures,
    // notCounted or noSources); stageList, the stages of the release's figures, whose status (#2422) the first
    // screen's figure says; stageFigures, the release's counts per stage (meta/stage_figures.json), what that figure
    // says each can use
    stages: stagesOk ? { version: stagesOk.version, list: stagesOk.stages || [] } : null,
    stagesWhy,
    stageList: stageFigures?.stages || [],
    stageFigures: parseStageFigures(stageFigures),
    // #2066: the Chinese text of each English text the release shows (TRANSLATIONS, #2065): English -> Chinese, empty
    // for a commit without the table
    zh: new Map(translations.map((r) => [r.en, r.zh])),
    vocab,
    sources,
    // #1158: the size figures of the whole index and of each layer (meta/size_figures.json, read as they are; #2422:
    // the page says their deduplicated size), null for a commit without them
    sizes: figures ? { index: figures.index, ...figures.layers } : null,
    catalogFile: cat.file,
    coverage,
  };
}