Spaces:
Running
Running
Download data.js from Battam/training-data-foundation: direct link, hf CLI and curl.
- Browser
- Download file 24.3 kB
-
https://huggingface.co/spaces/Battam/training-data-foundation/resolve/main/data.js
- Command line
-
hf download hf://spaces/Battam/training-data-foundation/data.js
-
curl -L -o data.js https://huggingface.co/spaces/Battam/training-data-foundation/resolve/main/data.js
24.3 kB
| // Reading the index in the browser. Every file comes from one commit of the dataset, so every number agrees. | |
| import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.5/+esm'; | |
| import { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm'; | |
| import { downloadFile, pathsInfo } from 'https://cdn.jsdelivr.net/npm/@huggingface/hub@2.17.10/+esm'; | |
| import { HF, loadRevision as askRevision, REPO } from './commit.js'; | |
| import { t } from './i18n.js'; | |
| export { HF, REPO }; | |
| export const DATASET_URL = `${HF}/datasets/${REPO}`; | |
| export const fileUrl = (sha, path) => `${DATASET_URL}/resolve/${sha}/${path}`; | |
| export const num = (v) => (v == null ? null : typeof v === 'bigint' ? Number(v) : Number(v)); | |
| // the two layers, the values of the catalog column kind (#952, #1083): a material set (task material) and a corpus | |
| export const MATERIAL_SET = 'material_set'; | |
| export const CORPUS = 'corpus'; | |
| export const KINDS = [MATERIAL_SET, CORPUS]; | |
| // #2065, #2066: the one table of Chinese names the release writes: per English text the index shows (a collection's | |
| // name, a source's title, description or summary, as the catalog holds them) its Chinese text (columns en and zh) | |
| const TRANSLATIONS = 'meta/translations.parquet'; | |
| // #2066: the projection's counts per training stage (artpool.stages FIGURES), with (#2065) each stage's funnel | |
| const STAGE_FIGURES = 'meta/stage_figures.json'; | |
| const HUB_REPO = { type: 'dataset', name: REPO }; | |
| /** One whole file of the commit sha, read with downloadFile of @huggingface/hub, which answers null when the commit | |
| * has no such file. */ | |
| export async function wholeFile(sha, path) { | |
| const blob = await downloadFile({ repo: HUB_REPO, path, revision: sha }); | |
| if (!blob) throw new Error(`${path}: no such file at commit ${sha.slice(0, 7)}`); | |
| return blob; | |
| } | |
| /** A file named by its URL in the page address (catalog=, vocabulary=), which may lie outside the dataset. */ | |
| async function urlFile(url) { | |
| const r = await fetch(url); | |
| if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`); | |
| return r.blob(); | |
| } | |
| const jsonOf = async (blob) => JSON.parse(await (await blob).text()); | |
| /** A small file read whole (the meta tables): one request instead of a size probe and ranges. */ | |
| async function readParquet(blob) { | |
| return parquetReadObjects({ file: await (await blob).arrayBuffer(), compressors, utf8: false }); | |
| } | |
| /** The columns of cols that the file has, read with hyparquet (byte ranges); range: {rowStart, rowEnd} (#1792: | |
| * hyparquet documents no row filter, so a reader of one source reads its row range, filesOf, and keeps its rows by | |
| * the column source itself). */ | |
| export async function readColumns(url, cols, range = {}) { | |
| const file = await asyncBufferFromUrl({ url }); | |
| const metadata = await parquetMetadataAsync(file); | |
| const names = new Set(metadata.schema.slice(1).map((s) => s.name)); | |
| const columns = cols.filter((c) => names.has(c)); | |
| const rows = await parquetReadObjects({ file, metadata, columns, compressors, utf8: false, ...range }); | |
| return { rows, total: num(metadata.num_rows) }; | |
| } | |
| /** The commit to read: ?rev=<sha or branch>, else main. Returns {sha, date, files}. */ | |
| export async function loadRevision(rev) { | |
| const r = await askRevision(rev); | |
| if (!r.sha) throw new Error(t('error.noCommit')); | |
| return r; | |
| } | |
| /** #1769: the date of the last commit of each batch of a source whose data files are files ({path, batches}, | |
| * loadIndex's files): the latest last commit among the files that hold the batch's rows (pathsInfo), never a folder | |
| * named by the batch. -> Map of batch -> ISO date. */ | |
| export async function loadBatchDates(sha, files) { | |
| const out = new Map(); | |
| if (!files.length) return out; | |
| const info = await pathsInfo({ repo: HUB_REPO, paths: files.map((f) => f.path), revision: sha, expand: true }); | |
| const iso = (d) => (d instanceof Date ? d.toISOString() : d ? String(d) : null); | |
| const at = new Map(info.map((e) => [e.path, iso(e.lastCommit?.date)])); | |
| for (const f of files) { | |
| const d = at.get(f.path); | |
| for (const b of f.batches) if (d && !(out.get(b) >= d)) out.set(b, d); | |
| } | |
| return out; | |
| } | |
| // ---- the classification contract (#252): domain is a list of codes (D1 to D10, with leaves), or multi, or | |
| // unclassified; form is one id; names come from meta/vocabulary.json, in English (name) and in Chinese (zh). Without | |
| // them everything shows as unclassified. | |
| const pick = (o, keys) => { | |
| if (!o || typeof o !== 'object') return undefined; | |
| for (const k of keys) if (o[k] != null) return o[k]; | |
| return undefined; | |
| }; | |
| const NAME_KEYS = ['name', 'label', 'title', 'en']; | |
| const CODE_KEYS = ['code', 'id', 'key', 'value']; | |
| const KIDS_KEYS = ['children', 'leaves', 'subdomains', 'items']; | |
| const ABOUT_KEYS = ['description', 'about', 'definition']; | |
| const ZH_KEYS = ['zh']; | |
| function vocabSection(node) { | |
| const names = new Map(); | |
| const zh = new Map(); | |
| const about = new Map(); | |
| const order = []; | |
| const add = (code, name, text, zhName) => { | |
| if (code == null) return; | |
| code = String(code); | |
| if (!names.has(code)) order.push(code); | |
| if (typeof name === 'string' && name.trim()) names.set(code, name.trim()); | |
| else if (!names.has(code)) names.set(code, null); | |
| if (typeof text === 'string' && text.trim()) about.set(code, text.trim()); | |
| if (typeof zhName === 'string' && zhName.trim()) zh.set(code, zhName.trim()); | |
| }; | |
| const walk = (n) => { | |
| if (!n) return; | |
| if (Array.isArray(n)) { | |
| for (const it of n) { | |
| if (typeof it === 'string') add(it, null); | |
| else if (it && typeof it === 'object') { | |
| add(pick(it, CODE_KEYS), pick(it, NAME_KEYS), pick(it, ABOUT_KEYS), pick(it, ZH_KEYS)); | |
| walk(pick(it, KIDS_KEYS)); | |
| } | |
| } | |
| } else if (typeof n === 'object') { | |
| for (const [k, v] of Object.entries(n)) { | |
| if (typeof v === 'string') add(k, v); | |
| else if (v && typeof v === 'object') { | |
| add(k, pick(v, NAME_KEYS), pick(v, ABOUT_KEYS), pick(v, ZH_KEYS)); | |
| walk(pick(v, KIDS_KEYS)); | |
| } else add(k, null); | |
| } | |
| } | |
| }; | |
| walk(node); | |
| return { names, zh, about, order }; | |
| } | |
| // meta/vocabulary.json (#304, #1052): domains [{code, name, zh, about, parent}], forms [{id, zh, about}]. A domain shows | |
| // its name; a form has no English name, so in English it shows its id written as words (test-suites -> test suites). | |
| // zh is the name in Chinese. #1083: layers [{kind, name, zh, about}] (task material, then corpora), relation {about, | |
| // zh} (the sentence that says how the two layers differ) and collections [{code, name, zh, about, campaigns}] (the | |
| // lines of admit_collections.tsv, in their order; campaigns: the collection and its earlier names, the campaign tags | |
| // of its material sets). terms: Map of the term table by its English term. #2397: profile [{criterion, option}], the | |
| // criteria tools/fetch.py's default profile applies, in its order, each with the option that takes back what it | |
| // leaves out (what Get the data shows the profile leaves out). | |
| export function parseVocabulary(j) { | |
| const form = vocabSection(pick(j, ['form', 'forms'])); | |
| for (const [id, name] of form.names) if (!name) form.names.set(id, id.replace(/[-_]+/g, ' ')); | |
| const layers = new Map((Array.isArray(j?.layers) ? j.layers : []).map((l) => [l.kind, l])); | |
| const collections = (Array.isArray(j?.collections) ? j.collections : []).map((c) => ({ | |
| code: String(c.code), name: c.name || String(c.code), zh: c.zh || '', about: c.about || '', | |
| campaigns: Array.isArray(c.campaigns) ? c.campaigns.map(String) : [], | |
| })); | |
| // terms [{term, zh, about}] (the index's term table: the English term, its Chinese term and one sentence on what it | |
| // means), by the English term | |
| const terms = new Map((Array.isArray(j?.terms) ? j.terms : []).filter((x) => x?.term).map((x) => [String(x.term), x])); | |
| const profile = (Array.isArray(j?.profile) ? j.profile : []).map((p) => ({ criterion: String(p.criterion), option: String(p.option) })); | |
| return { domain: vocabSection(pick(j, ['domain', 'domains'])), form, layers, relation: j?.relation || null, collections, terms, profile }; | |
| } | |
| /** #2066: meta/stage_figures.json -> {rules, stages: Map id -> stage, noStage}: the rules every stage's material passes, | |
| * in their order, and per stage, in the stage table's order, its names, status and measures (counted_by) and its counts | |
| * by them (counts, each null before the release counted the stage fields), which (#2422) the first screen's figure says; | |
| * #2629: noStage, what no stage counts of each layer by reason ({kind: {reason: {measure: value}}}, null for figures | |
| * written before it); null for a commit without the file. */ | |
| function parseStageFigures(j) { | |
| if (!j) return null; | |
| return { rules: j.rules || [], stages: new Map((j.stages || []).map((s) => [s.id, s])), noStage: j.no_stage || null }; | |
| } | |
| export const topOf = (code) => String(code).split('.')[0]; | |
| function domainOf(v) { | |
| let list = v; | |
| if (typeof list === 'string') list = [list]; | |
| if (list && typeof list.toArray === 'function') list = list.toArray(); | |
| list = Array.isArray(list) ? list.map(String).map((s) => s.trim()).filter(Boolean) : []; | |
| if (!list.length || list.includes('unclassified')) return { kind: 'none', codes: [] }; | |
| if (list.includes('multi')) return { kind: 'multi', codes: [] }; | |
| return { kind: 'codes', codes: [...new Set(list)] }; | |
| } | |
| /** What every row of a source allows, from sources.json license_rows: 'yes' (every row), 'cond' (yes or | |
| * conditional for every row) or 'no' (some rows are no or unknown). */ | |
| function permission(lr) { | |
| if (!lr) return 'no'; | |
| const keys = Object.entries(lr).filter(([, v]) => num(v?.rows) > 0).map(([k]) => k); | |
| if (!keys.length) return 'no'; | |
| if (keys.every((k) => k === 'yes')) return 'yes'; | |
| if (keys.every((k) => k === 'yes' || k === 'conditional')) return 'cond'; | |
| return 'no'; | |
| } | |
| /** #1266: a catalog cell of license_counts as [{kind, spdx, training_ok, redistribution_ok, files}], null when empty. */ | |
| function licenceCounts(v) { | |
| const items = Array.isArray(v) ? v : v?.toArray?.(); | |
| return items?.length ? items.map((x) => ({ kind: x.kind, spdx: x.spdx || null, training_ok: x.training_ok, | |
| redistribution_ok: x.redistribution_ok, files: num(x.files) || 0 })) : null; | |
| } | |
| /** #1524: a source's file types as [[ext, rows]], most rows first, from its by_ext of meta/sources.json ({ext: {rows, | |
| * bytes}}); the keys in brackets ((none), (other)) name no extension that fetch.py's --ext or a row's ext matches, and | |
| * are left out. */ | |
| function extsOf(byExt) { | |
| return Object.entries(byExt || {}).filter(([k]) => !k.startsWith('(')).map(([k, v]) => [k, num(v?.rows) || 0]) | |
| .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); | |
| } | |
| /** The catalog's rows and its file as read, for the page's DuckDB table (linked.js): {rows, file: {format, bytes}}; | |
| * json: the file is JSON (a list of rows, or {sources} or {rows}), else Parquet. */ | |
| async function readCatalog(blob, json) { | |
| if (json) { | |
| const j = await jsonOf(blob); | |
| const rows = Array.isArray(j) ? j : j.sources || j.rows || []; | |
| return { rows, file: { format: 'json', bytes: new TextEncoder().encode(JSON.stringify(rows)) } }; | |
| } | |
| const buf = await (await blob).arrayBuffer(); | |
| // DuckDB-WASM takes its own copy: the rows hyparquet returns may still point into buf | |
| const bytes = new Uint8Array(buf.slice(0)); | |
| return { rows: await parquetReadObjects({ file: buf, compressors, utf8: false }), file: { format: 'parquet', bytes } }; | |
| } | |
| /** #2065: a sum over values some of which are not known (null), as the figures sum them (index_projection._total): | |
| * {sum: the sum of the known values, null when none is known (0 when there are no values), missing: how many are | |
| * not counted yet}. */ | |
| export function summed(vals) { | |
| const known = vals.filter((v) => v != null); | |
| return { sum: known.length ? known.reduce((a, v) => a + v, 0) : vals.length ? null : 0, missing: vals.length - known.length }; | |
| } | |
| const listOf = (v) => (Array.isArray(v) ? v : v?.toArray?.() || []); | |
| /** #1769: the data files that hold rows of the catalog row r, each {path, batches, first, last} (the batches of its | |
| * rows in that file; #1792: the source's first and last row in it, null when the catalog does not record them), sorted | |
| * by path: from the catalog column batch_files, else (a catalog with data_files only) from data_files, with no batches; | |
| * never from a file's path. A file may hold rows of other sources too, so every read of one keeps the source's rows by | |
| * the column source, and reads only its row range when known (a packed file holds a source's rows together). */ | |
| function filesOf(r) { | |
| const by = new Map(); | |
| for (const x of listOf(r.batch_files)) { | |
| const f = String(x.file); | |
| if (!by.has(f)) by.set(f, { batches: [], first: num(x.first), last: num(x.last) }); | |
| by.get(f).batches.push(String(x.batch)); | |
| } | |
| if (!by.size) for (const f of listOf(r.data_files)) by.set(String(f), { batches: [], first: null, last: null }); | |
| return [...by].map(([path, x]) => ({ path, ...x, batches: x.batches.sort() })).sort((a, b) => a.path.localeCompare(b.path)); | |
| } | |
| /** Everything the page shows, from one commit. params: URLSearchParams (rev, catalog, vocabulary); onNote(key, values): | |
| * a note for the reader, as a key of the locale files. */ | |
| export async function loadIndex(params, onNote) { | |
| const rev = await loadRevision(params.get('rev')); | |
| const at = (p) => wholeFile(rev.sha, p); | |
| const catalogUrl = params.get('catalog'); | |
| const has = (p) => rev.files.includes(p); | |
| const vocabUrl = params.get('vocabulary'); | |
| const vocabFile = vocabUrl ? () => urlFile(vocabUrl) : has('meta/vocabulary.json') ? () => at('meta/vocabulary.json') : null; | |
| const [cat, sourcesJson, coverage, batches, vocabJson, figures, record, translations, stageFigures] = | |
| await Promise.all([ | |
| catalogUrl ? readCatalog(urlFile(catalogUrl), /\.json(\?|$)/i.test(catalogUrl)) : readCatalog(at('meta/catalog.parquet'), false), | |
| jsonOf(at('meta/sources.json')), | |
| jsonOf(at('meta/coverage.json')), | |
| readParquet(at('meta/batches.parquet')).catch(() => []), | |
| vocabFile ? jsonOf(vocabFile()).catch((e) => { onNote?.('notice.vocabulary', { error: e.message }); return null; }) : null, | |
| // #1158: the size figures of every level, written by the projection; a commit before it has none | |
| has('meta/size_figures.json') ? jsonOf(at('meta/size_figures.json')) : null, | |
| // #1197: the record of the release's projection, whose tag (DL13h, artpool/release_tag.py) the commands name and | |
| // (#2422) whose tag and date the release line of the first screen says | |
| has('meta/projection.json') ? jsonOf(at('meta/projection.json')) : null, | |
| // #2066: the Chinese names of collections and sources (#2065); #2422: what each training stage can use, the | |
| // first screen's figure; #2518: and, in the same file, the stages of each source | |
| has(TRANSLATIONS) ? readParquet(at(TRANSLATIONS)) : [], | |
| has(STAGE_FIGURES) ? jsonOf(at(STAGE_FIGURES)) : null, | |
| ]); | |
| // #2518: a source's stages come from the commit the page reads: meta/stage_figures.json carries, per source, the ids | |
| // of the stages one of its units passes (sources), beside the stage table's version and each stage's counts, so the | |
| // two never describe different commits. A commit without the file, one whose release has not counted its stage | |
| // fields, or one written before the figures listed the sources has no stage for any source, and the filter by stage | |
| // says which in one short line (stagesWhy) | |
| const stagesOk = stageFigures?.counted && stageFigures.sources ? stageFigures : null; | |
| const stagesWhy = stagesOk ? null : !stageFigures ? 'noFigures' : !stageFigures.counted ? 'notCounted' : 'noSources'; | |
| const vocab = parseVocabulary(vocabJson || {}); | |
| const srcInfo = new Map((sourcesJson.sources || []).map((e) => [e.source, e])); | |
| const cov = coverage.by_source || {}; | |
| const batchesBy = new Map(); | |
| for (const b of batches) { | |
| if (!batchesBy.has(b.source)) batchesBy.set(b.source, []); | |
| batchesBy.get(b.source).push(b); | |
| } | |
| // the catalog column kind (#952); an index commit before it has none, and the page computes the kind below | |
| const hasKind = cat.rows.some((r) => r.kind != null && r.kind !== ''); | |
| const sources = cat.rows.map((r) => { | |
| const info = srcInfo.get(r.source) || {}; | |
| const lic = r.license || r.licence || {}; | |
| const dom = domainOf(r.domain); | |
| const rows = num(r.files ?? info.rows) || 0; | |
| const bytes = num(r.bytes); | |
| const form = r.form == null || r.form === '' || r.form === 'unclassified' ? null : String(r.form); | |
| // The kind is the catalog column kind: material_set (task-ready) or corpus (raw). An index commit without it gets | |
| // the rule the index writes it by (#1633): a source that holds rows of a collection (sources.json campaign or | |
| // extended_by) or has fewer units than rows is a material set, every other one a corpus. | |
| // A unit is one unit_id (a test case, one workbook, one package); files are counted apart. The catalog columns | |
| // units (the distinct units of a material set) and files (its rows; a corpus is counted in rows) are read as they | |
| // are; #2065: a material set whose catalog row has no count of its units has none here (readyUnits null, not | |
| // counted yet), as the figures count it (meta/size_figures.json units_not_counted), never its rows. | |
| const roles = info.unit || {}; | |
| const has = (k) => r[k] != null && r[k] !== '' && Number.isFinite(num(r[k])); | |
| const isTask = hasKind ? r.kind === MATERIAL_SET : !!(info.campaign || info.extended_by) || (has('units') && num(r.units) < rows); | |
| const units = has('units') ? num(r.units) : null; | |
| const lr = info.license_rows?.redistribution_ok || {}; | |
| const shipRows = (num(lr.yes?.rows) || 0) + (num(lr.conditional?.rows) || 0); | |
| const shipShare = rows ? Math.min(1, shipRows / rows) : 0; | |
| // #1577: the fetch check, catalog column fetch_check; #2006: named fetch up to release-2026-11-09.1, read until a | |
| // release with fetch_check is published | |
| const fc = r.fetch_check || r.fetch; | |
| return { | |
| kind: isTask ? MATERIAL_SET : CORPUS, | |
| isTask, | |
| // the collection of a material set (#1083): a line of admit_collections.tsv; of a corpus (#1633), the collection | |
| // that lists it under corpora in catalog/domains.yaml, else none | |
| collection: r.collection ? String(r.collection) : null, | |
| readyUnits: isTask ? units : 0, | |
| rawRows: isTask ? 0 : rows, | |
| fromCatalog: has('units'), | |
| // what one row of a corpus points at (sources.json unit: file, archive member, ...); a collection's entry counts | |
| // its rows as files (#1633: no roles) | |
| kinds: Object.keys(roles), | |
| units, | |
| shipShare, | |
| id: r.source, | |
| title: r.title || r.source, | |
| description: r.description || info.description || '', | |
| summary: r.summary || '', | |
| unit: r.unit || '', | |
| status: r.status || null, | |
| reasons: Array.isArray(r.status_reasons) ? r.status_reasons : r.status_reasons?.toArray?.() || [], | |
| domainKind: dom.kind, | |
| codes: dom.codes, | |
| tops: [...new Set(dom.codes.map(topOf))], | |
| form, | |
| tier: lic.tier || 'unknown', | |
| ship: lic.ship || null, | |
| spdx: lic.spdx || null, | |
| licenceNote: lic.note || null, | |
| contamination: r.contamination || {}, | |
| pii: r.pii || {}, | |
| // #1256: the distinct publisher names over all the rows of a material set (catalog column publishers); null | |
| // when its rows carry none or the catalog has no such column | |
| publishers: has('publishers') ? num(r.publishers) : null, | |
| // #1266: the files of a material set by kind of file (upstream, index), licence and the permits for training and | |
| // redistribution, as the projection counted them (catalog column license_counts); null without the column | |
| licenceCounts: licenceCounts(r.license_counts), | |
| rows, | |
| bytes: bytes && bytes > 0 ? bytes : null, | |
| // #2422: the one size the page shows, the deduplicated one (catalog column distinct_bytes, the same as | |
| // distinct_bytes of the source's figures); the size filter buckets it | |
| distinct: num(r.distinct_bytes) > 0 ? num(r.distinct_bytes) : null, | |
| // #1158: the source's size figures from meta/size_figures.json (#2422: its size, the deduplicated one, the check | |
| // and the estimate); #2066: the list shows and sorts by them | |
| size: figures?.sources?.[r.source] || null, | |
| // #1577: whether its upstream still serves its files (the fetch check fc: level, rows, fetched, read_at); null | |
| // for a catalog without the column | |
| fetch: fc && fc.level ? { level: String(fc.level), rows: num(fc.rows) || 0, fetched: num(fc.fetched) || 0, readAt: fc.read_at || null } : null, | |
| // #1928: the release tag of the release its rows come from (catalog column released_in), and why a newer batch of | |
| // it is held back, as the release notes give it (catalog column held_back); null for a catalog without them | |
| releasedIn: r.released_in || null, | |
| heldBack: r.held_back || null, | |
| // #2588: the source's declared upstream (catalog columns upstream_kind, upstream_agent and upstream_entry): a | |
| // dataset or a catalog, its agent and its entry address, the values joined by ' | ' while the source reads | |
| // several upstreams; null for a catalog without them | |
| upstream: r.upstream_kind ? { kind: String(r.upstream_kind), agent: String(r.upstream_agent ?? ''), | |
| entry: String(r.upstream_entry ?? '') } : null, | |
| topTypes: Array.isArray(r.top_types) ? r.top_types : r.top_types?.toArray?.() || [], | |
| needsAuth: r.needs_auth || null, | |
| train: permission(info.license_rows?.training_ok), | |
| redist: permission(info.license_rows?.redistribution_ok), | |
| // #1524: its file types and their rows, for the File type facet, the --ext options and the SQL and CSV filters | |
| exts: extsOf(info.by_ext), | |
| // #2006: the ids of the training stages one of its units passes (#2518: sources of the commit's stage figures), | |
| // [] without stages | |
| stages: stagesOk?.sources?.[r.source] || [], | |
| info, | |
| cov: cov[r.source] || null, | |
| batches: batchesBy.get(r.source) || [], | |
| files: filesOf(r), | |
| }; | |
| }); | |
| return { | |
| sha: rev.sha, | |
| // the release tag of the commit (a tag is never moved), the commit itself before meta/projection.json had one; | |
| // #2422: the day of that release, the commit's own before meta/projection.json had one | |
| pin: record?.tag || rev.sha, | |
| date: rev.date, | |
| releaseDate: record?.date || rev.date, | |
| overrides: { catalog: params.get('catalog'), vocabulary: params.get('vocabulary') }, | |
| hasVocabulary: Boolean(vocabJson), | |
| // #1577: who published each file of a material set and under which license, read when the attribution list is | |
| // made; a commit before it has none | |
| hasAttribution: has('meta/attribution.parquet'), | |
| hasKind, | |
| // #2006: the stage table the stages were computed with (version, and each stage with its names, status, about and | |
| // counts), null when the commit has no stage of its sources; #2066: stagesWhy, why it has none (noFigures, | |
| // notCounted or noSources); stageList, the stages of the release's figures, whose status (#2422) the first | |
| // screen's figure says; stageFigures, the release's counts per stage (meta/stage_figures.json), what that figure | |
| // says each can use | |
| stages: stagesOk ? { version: stagesOk.version, list: stagesOk.stages || [] } : null, | |
| stagesWhy, | |
| stageList: stageFigures?.stages || [], | |
| stageFigures: parseStageFigures(stageFigures), | |
| // #2066: the Chinese text of each English text the release shows (TRANSLATIONS, #2065): English -> Chinese, empty | |
| // for a commit without the table | |
| zh: new Map(translations.map((r) => [r.en, r.zh])), | |
| vocab, | |
| sources, | |
| // #1158: the size figures of the whole index and of each layer (meta/size_figures.json, read as they are; #2422: | |
| // the page says their deduplicated size), null for a commit without them | |
| sizes: figures ? { index: figures.index, ...figures.layers } : null, | |
| catalogFile: cat.file, | |
| coverage, | |
| }; | |
| } | |