Spaces:
Running
Running
File size: 24,305 Bytes
a942796 2e7ce94 638568b fcd1156 a942796 638568b a942796 94bddc8 638568b c7ba267 638568b db3a530 a942796 db3a530 a942796 db3a530 a942796 db3a530 638568b db3a530 638568b db3a530 a942796 2e7ce94 a942796 638568b a942796 2e7ce94 a942796 2e7ce94 a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a a942796 0784d2a 94bddc8 689ce1d a942796 856d716 94bddc8 894be04 689ce1d a942796 f77cc19 c7ba267 f77cc19 c7ba267 a942796 d0b101b bec7df7 db3a530 638568b a942796 db3a530 638568b c7ba267 2e7ce94 fcd1156 a942796 db3a530 a942796 db3a530 a942796 631e2c6 967f228 631e2c6 967f228 631e2c6 a942796 638568b a942796 638568b a942796 638568b 2e7ce94 c7ba267 856d716 2e7ce94 c7ba267 856d716 7ade112 a942796 638568b 856d716 2e7ce94 856d716 2e7ce94 856d716 a942796 349bca8 d0b101b a942796 967f228 86bca38 7ade112 975e4ee 7ade112 e18ef8b a4caba4 a942796 bec7df7 631e2c6 7ade112 a942796 2e7ce94 a942796 967f228 2c4652f a942796 967f228 a942796 975e4ee 638568b 7ade112 631e2c6 7ade112 c7ba267 631e2c6 c7ba267 a942796 967f228 638568b a942796 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 | // Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees.
import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm';
import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm';
import { downloadFile, pathsInfo } from 'https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.10/+esm';
import { HF, loadRevision as askRevision, REPO } from './commit.js';
import { t } from './i18n.js';
export { HF, REPO };
export const DATASET_URL = `${HF}/datasets/${REPO}`;
export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`;
export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v));
// the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus
export const MATERIAL_SET = 'material_set';
export const CORPUS = 'corpus';
export const KINDS = [MATERIAL_SET, CORPUS];
// #2065, #2066: the one table of Chinese names the release writes: per English text the index shows (a collection's
// name, a source's title, description or summary, as the catalog holds them) its Chinese text (columns en and zh)
const TRANSLATIONS = 'meta/translations.parquet';
// #2066: the projection's counts per training stage (artpool.stages FIGURES), with (#2065) each stage's funnel
const STAGE_FIGURES = 'meta/stage_figures.json';
const HUB_REPO = { type: 'dataset', name: REPO };
/** One whole file of the commit sha, read with downloadFile of @huggingface/hub, which answers null when the commit
* has no such file. */
export async function wholeFile(sha, path) {
const blob = await downloadFile({ repo: HUB_REPO, path, revision: sha });
if (!blob) throw new Error(`${path}: no such file at commit ${sha.slice(0, 7)}`);
return blob;
}
/** A file named by its URL in the page address (catalog=, vocabulary=), which may lie outside the dataset. */
async function urlFile(url) {
const r = await fetch(url);
if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`);
return r.blob();
}
const jsonOf = async (blob) => JSON.parse(await (await blob).text());
/** A small file read whole (the meta tables): one request instead of a size probe and ranges. */
async function readParquet(blob) {
return parquetReadObjects({ file: await (await blob).arrayBuffer(), compressors, utf8: false });
}
/** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd} (#1792:
* hyparquet documents no row filter, so a reader of one source reads its row range, filesOf, and keeps its rows by
* the column source itself). */
export async function readColumns(url, cols, range = {}) {
const file = await asyncBufferFromUrl({ url });
const metadata = await parquetMetadataAsync(file);
const names = new Set(metadata.schema.slice(1).map((s) => s.name));
const columns = cols.filter((c) => names.has(c));
const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range });
return { rows, total: num(metadata.num_rows) };
}
/** The commit to read: ?rev=<sha or branch>, else main. Returns {sha, date, files}. */
export async function loadRevision(rev) {
const r = await askRevision(rev);
if (!r.sha) throw new Error(t('error.noCommit'));
return r;
}
/** #1769: the date of the last commit of each batch of a source whose data files are files ({path, batches},
* loadIndex's files): the latest last commit among the files that hold the batch's rows (pathsInfo), never a folder
* named by the batch. -> Map of batch -> ISO date. */
export async function loadBatchDates(sha, files) {
const out = new Map();
if (!files.length) return out;
const info = await pathsInfo({ repo: HUB_REPO, paths: files.map((f) => f.path), revision: sha, expand: true });
const iso = (d) => (d instanceof Date ? d.toISOString() : d ? String(d) : null);
const at = new Map(info.map((e) => [e.path, iso(e.lastCommit?.date)]));
for (const f of files) {
const d = at.get(f.path);
for (const b of f.batches) if (d && !(out.get(b) >= d)) out.set(b, d);
}
return out;
}
// ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or
// unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without
// them everything shows as unclassified.
const pick = (o, keys) => {
if (!o || typeof o !== 'object') return undefined;
for (const k of keys) if (o[k] != null) return o[k];
return undefined;
};
const NAME_KEYS = ['name', 'label', 'title', 'en'];
const CODE_KEYS = ['code', 'id', 'key', 'value'];
const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items'];
const ABOUT_KEYS = ['description', 'about', 'definition'];
const ZH_KEYS = ['zh'];
function vocabSection(node) {
const names = new Map();
const zh = new Map();
const about = new Map();
const order = [];
const add = (code, name, text, zhName) => {
if (code == null) return;
code = String(code);
if (!names.has(code)) order.push(code);
if (typeof name === 'string' && name.trim()) names.set(code, name.trim());
else if (!names.has(code)) names.set(code, null);
if (typeof text === 'string' && text.trim()) about.set(code, text.trim());
if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim());
};
const walk = (n) => {
if (!n) return;
if (Array.isArray(n)) {
for (const it of n) {
if (typeof it === 'string') add(it, null);
else if (it && typeof it === 'object') {
add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS));
walk(pick(it, KIDS_KEYS));
}
}
} else if (typeof n === 'object') {
for (const [k, v] of Object.entries(n)) {
if (typeof v === 'string') add(k, v);
else if (v && typeof v === 'object') {
add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS));
walk(pick(v, KIDS_KEYS));
} else add(k, null);
}
}
};
walk(node);
return { names, zh, about, order };
}
// meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows
// its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites).
// zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about,
// zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the
// lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags
// of its material sets). terms: Map of the term table by its English term. #2397: profile [{criterion, option}], the
// criteria tools/fetch.py's default profile applies, in its order, each with the option that takes back what it
// leaves out (what Get the data shows the profile leaves out).
export function parseVocabulary(j) {
const form = vocabSection(pick(j, ['form', 'forms']));
for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' '));
const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l]));
const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({
code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '',
campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [],
}));
// terms [{term, zh, about}] (the index's term table: the English term, its Chinese term and one sentence on what it
// means), by the English term
const terms = new Map((Array.isArray(j?.terms) ? j.terms : []).filter((x) => x?.term).map((x) => [String(x.term), x]));
const profile = (Array.isArray(j?.profile) ? j.profile : []).map((p) => ({ criterion: String(p.criterion), option: String(p.option) }));
return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections, terms, profile };
}
/** #2066: meta/stage_figures.json -> {rules, stages: Map id -> stage, noStage}: the rules every stage's material passes,
* in their order, and per stage, in the stage table's order, its names, status and measures (counted_by) and its counts
* by them (counts, each null before the release counted the stage fields), which (#2422) the first screen's figure says;
* #2629: noStage, what no stage counts of each layer by reason ({kind: {reason: {measure: value}}}, null for figures
* written before it); null for a commit without the file. */
function parseStageFigures(j) {
if (!j) return null;
return { rules: j.rules || [], stages: new Map((j.stages || []).map((s) => [s.id, s])), noStage: j.no_stage || null };
}
export const topOf = (code) => String(code).split('.')[0];
function domainOf(v) {
let list = v;
if (typeof list === 'string') list = [list];
if (list && typeof list.toArray === 'function') list = list.toArray();
list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : [];
if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] };
if (list.includes('multi')) return { kind: 'multi', codes: [] };
return { kind: 'codes', codes: [...new Set(list)] };
}
/** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or
* conditional for every row) or 'no' (some rows are no or unknown). */
function permission(lr) {
if (!lr) return 'no';
const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k);
if (!keys.length) return 'no';
if (keys.every((k) => k === 'yes')) return 'yes';
if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond';
return 'no';
}
/** #1266: a catalog cell of license_counts as [{kind, spdx, training_ok, redistribution_ok, files}], null when empty. */
function licenceCounts(v) {
const items = Array.isArray(v) ? v : v?.toArray?.();
return items?.length ? items.map((x) => ({ kind: x.kind, spdx: x.spdx || null, training_ok: x.training_ok,
redistribution_ok: x.redistribution_ok, files: num(x.files) || 0 })) : null;
}
/** #1524: a source's file types as [[ext, rows]], most rows first, from its by_ext of meta/sources.json ({ext: {rows,
* bytes}}); the keys in brackets ((none), (other)) name no extension that fetch.py's --ext or a row's ext matches, and
* are left out. */
function extsOf(byExt) {
return Object.entries(byExt || {}).filter(([k]) => !k.startsWith('(')).map(([k, v]) => [k, num(v?.rows) || 0])
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
}
/** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}};
* json: the file is JSON (a list of rows, or {sources} or {rows}), else Parquet. */
async function readCatalog(blob, json) {
if (json) {
const j = await jsonOf(blob);
const rows = Array.isArray(j) ? j : j.sources || j.rows || [];
return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } };
}
const buf = await (await blob).arrayBuffer();
// DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf
const bytes = new Uint8Array(buf.slice(0));
return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } };
}
/** #2065: a sum over values some of which are not known (null), as the figures sum them (index_projection._total):
* {sum: the sum of the known values, null when none is known (0 when there are no values), missing: how many are
* not counted yet}. */
export function summed(vals) {
const known = vals.filter((v) => v != null);
return { sum: known.length ? known.reduce((a, v) => a + v, 0) : vals.length ? null : 0, missing: vals.length - known.length };
}
const listOf = (v) => (Array.isArray(v) ? v : v?.toArray?.() || []);
/** #1769: the data files that hold rows of the catalog row r, each {path, batches, first, last} (the batches of its
* rows in that file; #1792: the source's first and last row in it, null when the catalog does not record them), sorted
* by path: from the catalog column batch_files, else (a catalog with data_files only) from data_files, with no batches;
* never from a file's path. A file may hold rows of other sources too, so every read of one keeps the source's rows by
* the column source, and reads only its row range when known (a packed file holds a source's rows together). */
function filesOf(r) {
const by = new Map();
for (const x of listOf(r.batch_files)) {
const f = String(x.file);
if (!by.has(f)) by.set(f, { batches: [], first: num(x.first), last: num(x.last) });
by.get(f).batches.push(String(x.batch));
}
if (!by.size) for (const f of listOf(r.data_files)) by.set(String(f), { batches: [], first: null, last: null });
return [...by].map(([path, x]) => ({ path, ...x, batches: x.batches.sort() })).sort((a, b) => a.path.localeCompare(b.path));
}
/** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values):
* a note for the reader, as a key of the locale files. */
export async function loadIndex(params, onNote) {
const rev = await loadRevision(params.get('rev'));
const at = (p) => wholeFile(rev.sha, p);
const catalogUrl = params.get('catalog');
const has = (p) => rev.files.includes(p);
const vocabUrl = params.get('vocabulary');
const vocabFile = vocabUrl ? () => urlFile(vocabUrl) : has('meta/vocabulary.json') ? () => at('meta/vocabulary.json') : null;
const [cat, sourcesJson, coverage, batches, vocabJson, figures, record, translations, stageFigures] =
await Promise.all([
catalogUrl ? readCatalog(urlFile(catalogUrl), /\.json(\?|$)/i.test(catalogUrl)) : readCatalog(at('meta/catalog.parquet'), false),
jsonOf(at('meta/sources.json')),
jsonOf(at('meta/coverage.json')),
readParquet(at('meta/batches.parquet')).catch(() => []),
vocabFile ? jsonOf(vocabFile()).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null,
// #1158: the size figures of every level, written by the projection; a commit before it has none
has('meta/size_figures.json') ? jsonOf(at('meta/size_figures.json')) : null,
// #1197: the record of the release's projection, whose tag (DL13h, artpool/release_tag.py) the commands name and
// (#2422) whose tag and date the release line of the first screen says
has('meta/projection.json') ? jsonOf(at('meta/projection.json')) : null,
// #2066: the Chinese names of collections and sources (#2065); #2422: what each training stage can use, the
// first screen's figure; #2518: and, in the same file, the stages of each source
has(TRANSLATIONS) ? readParquet(at(TRANSLATIONS)) : [],
has(STAGE_FIGURES) ? jsonOf(at(STAGE_FIGURES)) : null,
]);
// #2518: a source's stages come from the commit the page reads: meta/stage_figures.json carries, per source, the ids
// of the stages one of its units passes (sources), beside the stage table's version and each stage's counts, so the
// two never describe different commits. A commit without the file, one whose release has not counted its stage
// fields, or one written before the figures listed the sources has no stage for any source, and the filter by stage
// says which in one short line (stagesWhy)
const stagesOk = stageFigures?.counted && stageFigures.sources ? stageFigures : null;
const stagesWhy = stagesOk ? null : !stageFigures ? 'noFigures' : !stageFigures.counted ? 'notCounted' : 'noSources';
const vocab = parseVocabulary(vocabJson || {});
const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e]));
const cov = coverage.by_source || {};
const batchesBy = new Map();
for (const b of batches) {
if (!batchesBy.has(b.source)) batchesBy.set(b.source, []);
batchesBy.get(b.source).push(b);
}
// the catalog column kind (#952); an index commit before it has none, and the page computes the kind below
const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== '');
const sources = cat.rows.map((r) => {
const info = srcInfo.get(r.source) || {};
const lic = r.license || r.licence || {};
const dom = domainOf(r.domain);
const rows = num(r.files ?? info.rows) || 0;
const bytes = num(r.bytes);
const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form);
// The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets
// the rule the index writes it by (#1633): a source that holds rows of a collection (sources.json campaign or
// extended_by) or has fewer units than rows is a material set, every other one a corpus.
// A unit is one unit_id (a test case, one workbook, one package); files are counted apart. The catalog columns
// units (the distinct units of a material set) and files (its rows; a corpus is counted in rows) are read as they
// are; #2065: a material set whose catalog row has no count of its units has none here (readyUnits null, not
// counted yet), as the figures count it (meta/size_figures.json units_not_counted), never its rows.
const roles = info.unit || {};
const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k]));
const isTask = hasKind ? r.kind === MATERIAL_SET : !!(info.campaign || info.extended_by) || (has('units') && num(r.units) < rows);
const units = has('units') ? num(r.units) : null;
const lr = info.license_rows?.redistribution_ok || {};
const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0);
const shipShare = rows ? Math.min(1, shipRows / rows) : 0;
// #1577: the fetch check, catalog column fetch_check; #2006: named fetch up to release-2026-11-09.1, read until a
// release with fetch_check is published
const fc = r.fetch_check || r.fetch;
return {
kind: isTask ? MATERIAL_SET : CORPUS,
isTask,
// the collection of a material set (#1083): a line of admit_collections.tsv; of a corpus (#1633), the collection
// that lists it under corpora in catalog/domains.yaml, else none
collection: r.collection ? String(r.collection) : null,
readyUnits: isTask ? units : 0,
rawRows: isTask ? 0 : rows,
fromCatalog: has('units'),
// what one row of a corpus points at (sources.json unit: file, archive member, ...); a collection's entry counts
// its rows as files (#1633: no roles)
kinds: Object.keys(roles),
units,
shipShare,
id: r.source,
title: r.title || r.source,
description: r.description || info.description || '',
summary: r.summary || '',
unit: r.unit || '',
status: r.status || null,
reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [],
domainKind: dom.kind,
codes: dom.codes,
tops: [...new Set(dom.codes.map(topOf))],
form,
tier: lic.tier || 'unknown',
ship: lic.ship || null,
spdx: lic.spdx || null,
licenceNote: lic.note || null,
contamination: r.contamination || {},
pii: r.pii || {},
// #1256: the distinct publisher names over all the rows of a material set (catalog column publishers); null
// when its rows carry none or the catalog has no such column
publishers: has('publishers') ? num(r.publishers) : null,
// #1266: the files of a material set by kind of file (upstream, index), licence and the permits for training and
// redistribution, as the projection counted them (catalog column license_counts); null without the column
licenceCounts: licenceCounts(r.license_counts),
rows,
bytes: bytes && bytes > 0 ? bytes : null,
// #2422: the one size the page shows, the deduplicated one (catalog column distinct_bytes, the same as
// distinct_bytes of the source's figures); the size filter buckets it
distinct: num(r.distinct_bytes) > 0 ? num(r.distinct_bytes) : null,
// #1158: the source's size figures from meta/size_figures.json (#2422: its size, the deduplicated one, the check
// and the estimate); #2066: the list shows and sorts by them
size: figures?.sources?.[r.source] || null,
// #1577: whether its upstream still serves its files (the fetch check fc: level, rows, fetched, read_at); null
// for a catalog without the column
fetch: fc && fc.level ? { level: String(fc.level), rows: num(fc.rows) || 0, fetched: num(fc.fetched) || 0, readAt: fc.read_at || null } : null,
// #1928: the release tag of the release its rows come from (catalog column released_in), and why a newer batch of
// it is held back, as the release notes give it (catalog column held_back); null for a catalog without them
releasedIn: r.released_in || null,
heldBack: r.held_back || null,
// #2588: the source's declared upstream (catalog columns upstream_kind, upstream_agent and upstream_entry): a
// dataset or a catalog, its agent and its entry address, the values joined by ' | ' while the source reads
// several upstreams; null for a catalog without them
upstream: r.upstream_kind ? { kind: String(r.upstream_kind), agent: String(r.upstream_agent ?? ''),
entry: String(r.upstream_entry ?? '') } : null,
topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [],
needsAuth: r.needs_auth || null,
train: permission(info.license_rows?.training_ok),
redist: permission(info.license_rows?.redistribution_ok),
// #1524: its file types and their rows, for the File type facet, the --ext options and the SQL and CSV filters
exts: extsOf(info.by_ext),
// #2006: the ids of the training stages one of its units passes (#2518: sources of the commit's stage figures),
// [] without stages
stages: stagesOk?.sources?.[r.source] || [],
info,
cov: cov[r.source] || null,
batches: batchesBy.get(r.source) || [],
files: filesOf(r),
};
});
return {
sha: rev.sha,
// the release tag of the commit (a tag is never moved), the commit itself before meta/projection.json had one;
// #2422: the day of that release, the commit's own before meta/projection.json had one
pin: record?.tag || rev.sha,
date: rev.date,
releaseDate: record?.date || rev.date,
overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') },
hasVocabulary: Boolean(vocabJson),
// #1577: who published each file of a material set and under which license, read when the attribution list is
// made; a commit before it has none
hasAttribution: has('meta/attribution.parquet'),
hasKind,
// #2006: the stage table the stages were computed with (version, and each stage with its names, status, about and
// counts), null when the commit has no stage of its sources; #2066: stagesWhy, why it has none (noFigures,
// notCounted or noSources); stageList, the stages of the release's figures, whose status (#2422) the first
// screen's figure says; stageFigures, the release's counts per stage (meta/stage_figures.json), what that figure
// says each can use
stages: stagesOk ? { version: stagesOk.version, list: stagesOk.stages || [] } : null,
stagesWhy,
stageList: stageFigures?.stages || [],
stageFigures: parseStageFigures(stageFigures),
// #2066: the Chinese text of each English text the release shows (TRANSLATIONS, #2065): English -> Chinese, empty
// for a commit without the table
zh: new Map(translations.map((r) => [r.en, r.zh])),
vocab,
sources,
// #1158: the size figures of the whole index and of each layer (meta/size_figures.json, read as they are; #2422:
// the page says their deduplicated size), null for a commit without them
sizes: figures ? { index: figures.index, ...figures.layers } : null,
catalogFile: cat.file,
coverage,
};
}
|