const DATASET_LABELS = { combined: "Combined", mcv26: "Common Voice 26", fleurs: "FLEURS", poems: "Poems", movies: "Movies", infocom_may_26: "News", }; const DEFAULT_METRICS = ["WER-NN", "CER-NN", "WER-norm-NN", "CER-norm-NN"]; const METRIC_PATTERN = /^(?:WER|CER)(?:-norm)?(?:-(?:NN|LLM))?$/; const METRIC_LABELS = { "WER-NN": "WER", "CER-NN": "CER", "WER-norm-NN": "WER-norm", "CER-norm-NN": "CER-norm", "WER-LLM": "WER-LLM", "CER-LLM": "CER-LLM", }; const DEFAULT_VERSION = "v1.0"; const state = { records: [], datasets: [], metrics: DEFAULT_METRICS, columns: new Map(), view: "matrix", sortKey: "combined|WER-NN", sortDirection: "asc", colorScores: false, availability: new Set(["Open", "Closed"]), availabilityMenuOpen: false, chartDataset: "combined", chartMetric: DEFAULT_METRICS[0], chartAvailability: "All", version: DEFAULT_VERSION, versionedResults: {}, }; function setTheme(theme, persist = true) { document.documentElement.dataset.theme = theme; const toggle = document.querySelector("#theme-toggle"); if (toggle) { const nextTheme = theme === "dark" ? "light" : "dark"; toggle.setAttribute("aria-label", `Switch to ${nextTheme} mode`); toggle.title = `Switch to ${nextTheme} mode`; } if (persist) { try { localStorage.setItem("arm-asr-theme", theme); } catch (_) {} } } function metricLabel(metric) { return METRIC_LABELS[metric] || metric; } function parseCsv(text) { const rows = []; let row = []; let field = ""; let quoted = false; for (let index = 0; index < text.length; index += 1) { const char = text[index]; if (quoted) { if (char === '"' && text[index + 1] === '"') { field += '"'; index += 1; } else if (char === '"') { quoted = false; } else { field += char; } } else if (char === '"') { quoted = true; } else if (char === ",") { row.push(field); field = ""; } else if (char === "\n") { row.push(field.replace(/\r$/, "")); if (row.some((value) => value !== "")) { rows.push(row); } row = []; field = ""; } else { field += char; } } if (field || row.length) { row.push(field.replace(/\r$/, "")); rows.push(row); } return rows; } function ingest(rows) { const [sections, metrics] = rows.slice(0, 2); const modelIndex = sections.indexOf("model"); const urlIndex = sections.indexOf("model_url"); const availabilityIndex = urlIndex + 1; const originIndex = sections.indexOf("origin"); let currentDataset = ""; state.records = []; state.datasets = []; state.columns = new Map(); state.metrics = []; for (let index = availabilityIndex + 1; index < sections.length; index += 1) { if (sections[index]) { currentDataset = sections[index]; } if (currentDataset && METRIC_PATTERN.test(metrics[index])) { state.columns.set( `${currentDataset}|${metrics[index]}`, index ); if (!state.datasets.includes(currentDataset)) { state.datasets.push(currentDataset); } if (!state.metrics.includes(metrics[index])) { state.metrics.push(metrics[index]); } } } state.records = rows.slice(2).map((row) => { const values = {}; for (const [key, index] of state.columns) { values[key] = row[index] === "" || row[index] === undefined ? null : Number(row[index]); } return { model: row[modelIndex], url: row[urlIndex], availability: row[availabilityIndex], origin: originIndex >= 0 ? row[originIndex] : "", values, }; }); } function currentDataset() { return ( document.querySelector('input[name="dataset"]:checked')?.value || state.datasets[0] ); } function currentMatrixMetric() { return ( document.querySelector('input[name="matrix-metric"]:checked')?.value || state.metrics[0] ); } function rankRecords(records, dataset, metric = state.metrics[0]) { const rankKey = `${dataset}|${metric}`; const ranked = [...records].sort((left, right) => { const a = left.values[rankKey]; const b = right.values[rankKey]; if (a === null) return 1; if (b === null) return -1; return a - b || left.model.localeCompare(right.model); }); return new Map( ranked.map((record, index) => [ record.model, index + 1, ]) ); } function makeCell(tag, text, className = "") { const cell = document.createElement(tag); cell.textContent = text; if (className) { cell.className = className; } return cell; } function matrixRanges(columns) { const familyValues = { wer: [], cer: [], }; columns.forEach((column) => { const metric = column.key.split("|")[1]; const family = metric.startsWith("CER") ? "cer" : "wer"; state.records.forEach((record) => { const value = record.values[column.key]; if (value !== null) { familyValues[family].push(value); } }); }); const familyRanges = Object.fromEntries( Object.entries(familyValues).map(([family, values]) => [ family, values.length ? { min: Math.min(...values), max: Math.max(...values), } : null, ]) ); return new Map( columns.map((column) => { const metric = column.key.split("|")[1]; const family = metric.startsWith("CER") ? "cer" : "wer"; return [ column.key, familyRanges[family], ]; }) ); } function heatClass(value, range) { if ( value === null || !range || range.max === range.min ) { return ""; } const ratio = (value - range.min) / (range.max - range.min); return ` heat-score-${Math.min( 5, Math.floor(ratio * 5) + 1 )}`; } function renderComparisonChart() { const container = document.querySelector("#comparison-bars"); const empty = document.querySelector("#comparison-chart-empty"); if (!container || !empty) { return; } const key = `${state.chartDataset}|${state.chartMetric}`; const records = state.records .filter((record) => { return ( state.chartAvailability === "All" || record.availability === state.chartAvailability ); }) .map((record) => ({ model: record.model, url: record.url, availability: record.availability, value: record.values[key], })) .filter((record) => { return ( record.value !== null && Number.isFinite(record.value) ); }) .sort((left, right) => { return ( left.value - right.value || left.model.localeCompare(right.model) ); }); empty.hidden = records.length !== 0; if (!records.length) { container.replaceChildren(); return; } const maxValue = Math.max( ...records.map((record) => record.value) ); const rows = records.map((record) => { const row = document.createElement("div"); row.className = "comparison-bar-row"; const modelWrap = document.createElement("div"); modelWrap.className = "comparison-model"; const availabilityDot = document.createElement("span"); availabilityDot.className = `comparison-availability-dot ${record.availability.toLowerCase()}`; availabilityDot.title = record.availability === "Open" ? "Open model" : "Closed model"; availabilityDot.setAttribute( "aria-label", record.availability === "Open" ? "Open model" : "Closed model" ); modelWrap.append( availabilityDot ); if (record.url) { const modelLink = document.createElement("a"); modelLink.className = "comparison-model-link"; modelLink.href = record.url; modelLink.target = "_blank"; modelLink.rel = "noreferrer"; modelLink.title = `Open ${record.model}`; modelLink.setAttribute( "aria-label", `Open ${record.model} model page` ); const modelName = document.createElement("span"); modelName.textContent = record.model; const icon = document.createElement("svg"); icon.setAttribute( "viewBox", "0 0 24 24" ); icon.setAttribute( "aria-hidden", "true" ); icon.innerHTML = ''; modelLink.append( modelName, icon ); modelWrap.append( modelLink ); } else { const modelName = document.createElement("span"); modelName.className = "comparison-model-name"; modelName.textContent = record.model; modelName.title = record.model; modelWrap.append( modelName ); } const track = document.createElement("div"); track.className = "comparison-track"; track.setAttribute( "aria-hidden", "true" ); const fill = document.createElement("div"); fill.className = "comparison-fill"; fill.style.width = `${ maxValue ? (record.value / maxValue) * 100 : 0 }%`; track.append(fill); const value = document.createElement("div"); value.className = "comparison-value"; value.textContent = `${record.value.toFixed(2)}%`; row.setAttribute( "aria-label", `${record.model}: ${record.value.toFixed( 2 )} percent ${metricLabel(state.chartMetric)}, ${record.availability}` ); row.append( modelWrap, track, value ); return row; }); container.replaceChildren( ...rows ); } function buildChartControls() { const datasetSelect = document.querySelector("#chart-dataset"); const metricSelect = document.querySelector("#chart-metric"); const availabilitySelect = document.querySelector("#chart-availability"); if ( !datasetSelect || !metricSelect || !availabilitySelect ) { return; } state.datasets.forEach((dataset) => { const option = document.createElement("option"); option.value = dataset; option.textContent = DATASET_LABELS[dataset] || dataset; datasetSelect.append( option ); }); state.metrics.forEach((metric) => { const option = document.createElement("option"); option.value = metric; option.textContent = metricLabel(metric); metricSelect.append( option ); }); state.chartDataset = state.datasets.includes("combined") ? "combined" : state.datasets[0]; state.chartMetric = state.metrics[0]; state.chartAvailability = "All"; datasetSelect.value = state.chartDataset; metricSelect.value = state.chartMetric; availabilitySelect.value = state.chartAvailability; datasetSelect.addEventListener( "change", () => { state.chartDataset = datasetSelect.value; renderComparisonChart(); } ); metricSelect.addEventListener( "change", () => { state.chartMetric = metricSelect.value; renderComparisonChart(); } ); availabilitySelect.addEventListener( "change", () => { state.chartAvailability = availabilitySelect.value; renderComparisonChart(); } ); } function render() { const dataset = currentDataset(); const matrixMetric = currentMatrixMetric(); const isMatrix = state.view === "matrix"; const columns = isMatrix ? state.datasets.map((item) => ({ label: DATASET_LABELS[item] || item, key: `${item}|${matrixMetric}`, })) : state.metrics.map((metric) => ({ label: metricLabel(metric), key: `${dataset}|${metric}`, })); const table = document.querySelector( "#leaderboard-table" ); const head = table.querySelector("thead"); const body = table.querySelector("tbody"); const rankDataset = isMatrix ? "combined" : dataset; const rankMetric = isMatrix ? matrixMetric : state.metrics[0]; const ranks = rankRecords( state.records, rankDataset, rankMetric ); const ranges = state.colorScores ? matrixRanges(columns) : new Map(); let records = state.records.filter( (record) => state.availability.has( record.availability ) ); if ( columns.some( (column) => column.key === state.sortKey ) ) { const direction = state.sortDirection === "asc" ? 1 : -1; records = [...records].sort( (left, right) => { const a = left.values[ state.sortKey ]; const b = right.values[ state.sortKey ]; if (a === null) return 1; if (b === null) return -1; return ( (a - b) * direction || left.model.localeCompare( right.model ) ); } ); } const headerRow = document.createElement("tr"); const rankHeader = makeCell("th", "Rank"); rankHeader.title = `Ranked by ${metricLabel(rankMetric)} for ${ DATASET_LABELS[rankDataset] || rankDataset }`; const availabilityHeader = document.createElement("th"); const availabilityControl = document.createElement("span"); availabilityControl.className = "availability-control"; const availabilityLabel = document.createElement("button"); availabilityLabel.type = "button"; availabilityLabel.className = "availability-label"; availabilityLabel.setAttribute( "aria-expanded", String( state.availabilityMenuOpen ) ); availabilityLabel.setAttribute( "aria-label", "Filter models by availability" ); const availabilitySummary = state.availability.size === 2 ? "All" : [...state.availability][0]; availabilityLabel.innerHTML = `Availability ${availabilitySummary}`; availabilityLabel.addEventListener( "click", (event) => { event.stopPropagation(); state.availabilityMenuOpen = !state.availabilityMenuOpen; render(); } ); const availabilityFilters = document.createElement("span"); availabilityFilters.className = "availability-filters"; availabilityFilters.hidden = !state.availabilityMenuOpen; availabilityFilters.addEventListener( "click", (event) => event.stopPropagation() ); ["Open", "Closed"].forEach( (value) => { const button = document.createElement( "button" ); button.type = "button"; button.className = `availability-filter ${value.toLowerCase()}`; button.classList.toggle( "is-active", state.availability.has( value ) ); button.setAttribute( "aria-pressed", String( state.availability.has( value ) ) ); button.setAttribute( "aria-label", `Toggle ${value.toLowerCase()} models` ); button.textContent = value; button.addEventListener( "click", () => { if ( state.availability.has( value ) && state.availability.size > 1 ) { state.availability.delete( value ); } else { state.availability.add( value ); } render(); } ); availabilityFilters.append( button ); } ); availabilityControl.append( availabilityLabel, availabilityFilters ); availabilityHeader.append( availabilityControl ); headerRow.append( rankHeader, makeCell("th", "Model"), availabilityHeader ); columns.forEach((column) => { const header = document.createElement("th"); if ( isMatrix && column.key.startsWith( "combined|" ) ) { header.classList.add( "combined-score" ); } const button = document.createElement( "button" ); button.type = "button"; button.className = "sort-button"; if ( state.sortKey === column.key ) { button.classList.add( `is-${state.sortDirection}` ); } button.textContent = column.label; button.setAttribute( "aria-label", `Sort by ${column.label}` ); button.addEventListener( "click", () => { if ( state.sortKey === column.key ) { state.sortDirection = state.sortDirection === "asc" ? "desc" : "asc"; } else { state.sortKey = column.key; state.sortDirection = "asc"; } render(); } ); header.append(button); headerRow.append(header); }); head.replaceChildren( headerRow ); const rows = records.map((record) => { const row = document.createElement("tr"); const modelCell = document.createElement( "td" ); const modelWrap = document.createElement( "div" ); modelWrap.className = "model-cell"; const model = document.createElement( "span" ); model.className = "model-name"; model.textContent = record.model; modelWrap.append( model ); if (record.origin) { const origin = document.createElement("span"); origin.className = "model-origin-flag"; origin.title = "Armenian-origin model"; origin.setAttribute("aria-label", "Armenian-origin model"); modelWrap.append(origin); } if (record.url) { const link = document.createElement( "a" ); link.className = "model-link"; link.href = record.url; link.target = "_blank"; link.rel = "noreferrer"; link.title = `Open ${record.model}`; link.setAttribute( "aria-label", `Open ${record.model} model page` ); link.innerHTML = ''; modelWrap.append( link ); } modelCell.append( modelWrap ); const rank = ranks.get( record.model ); const rankCell = makeCell( "td", rank ? `#${rank}` : "—", `rank${ rank <= 3 ? " rank-top" : "" }` ); const availabilityCell = document.createElement( "td" ); const badge = document.createElement( "span" ); badge.className = `availability-badge ${record.availability.toLowerCase()}`; badge.textContent = record.availability; availabilityCell.append( badge ); row.append( rankCell, modelCell, availabilityCell ); columns.forEach( (column) => { const value = record.values[ column.key ]; const heat = state.colorScores ? `heat-cell${heatClass( value, ranges.get( column.key ) )}` : ""; const cell = makeCell( "td", value === null ? "—" : value.toFixed(2), heat ); if ( isMatrix && column.key.startsWith( "combined|" ) ) { cell.classList.add( "combined-score" ); } row.append( cell ); } ); return row; }); body.replaceChildren( ...rows ); document.querySelector( "#table-title" ).textContent = isMatrix ? `${metricLabel(matrixMetric)} across all datasets` : `${ DATASET_LABELS[dataset] || dataset } leaderboard`; const colorNote = state.colorScores ? " Colors are scaled separately within the WER and CER metric families: green is better, red is worse." : ""; document.querySelector( "#table-description" ).textContent = ( isMatrix ? `Rank is based on Overall ${metricLabel(matrixMetric)}. Scores are percentages; lower is better.` : `Rank is based on ${metricLabel(state.metrics[0])}. Scores are percentages; lower is better.` ) + colorNote; document.querySelector( "#result-count" ).textContent = `${records.length} ${ records.length === 1 ? "model" : "models" }`; document.querySelector( "#empty-state" ).hidden = records.length !== 0; table.hidden = records.length === 0; } function setView(view) { state.view = view; const isMatrix = view === "matrix"; document .querySelector( "#leaderboards" ) .classList.toggle( "matrix-view", isMatrix ); document.querySelector( ".matrix-metric-field" ).hidden = !isMatrix; document .querySelectorAll( ".view-button" ) .forEach((button) => { const active = button.dataset.view === view; button.classList.toggle( "is-active", active ); button.setAttribute( "aria-pressed", String(active) ); }); state.sortKey = isMatrix ? `combined|${currentMatrixMetric()}` : `${currentDataset()}|${state.metrics[0]}`; state.sortDirection = "asc"; render(); } function syncMetricControls() { const matrixOptions = document.querySelector( "#matrix-metric-options" ); const chartMetric = document.querySelector("#chart-metric"); matrixOptions.replaceChildren(); chartMetric.replaceChildren(); state.metrics.forEach((metric, index) => { const label = document.createElement("label"); label.className = "choice-pill"; const input = document.createElement("input"); input.type = "radio"; input.name = "matrix-metric"; input.value = metric; input.checked = index === 0; input.addEventListener("change", () => { state.sortKey = `combined|${input.value}`; state.sortDirection = "asc"; render(); }); label.append(input, document.createTextNode(metricLabel(metric))); matrixOptions.append(label); const option = document.createElement("option"); option.value = metric; option.textContent = metricLabel(metric); chartMetric.append(option); }); } function setVersion(version) { const csv = state.versionedResults[version]; if (!csv || version === state.version) { return; } ingest(parseCsv(csv)); state.version = version; state.sortKey = `combined|${state.metrics[0]}`; state.sortDirection = "asc"; state.availability = new Set(["Open", "Closed"]); state.availabilityMenuOpen = false; state.chartDataset = state.datasets.includes("combined") ? "combined" : state.datasets[0]; state.chartMetric = state.metrics[0]; syncMetricControls(); const viewingOldVersion = version === "v0.1"; document.querySelector("#version-select").value = version; document.querySelector("#old-version-notice").hidden = !viewingOldVersion; document.querySelector("#chart-dataset").value = state.chartDataset; document.querySelector("#chart-metric").value = state.chartMetric; render(); renderComparisonChart(); } function activateTab( name, updateHash = true ) { const validName = [ "leaderboards", "datasets", "about", "changelog", ].includes(name) ? name : "leaderboards"; document .querySelectorAll( ".tab" ) .forEach((tab) => { const active = tab.dataset.tab === validName; tab.classList.toggle( "is-active", active ); tab.setAttribute( "aria-selected", String(active) ); }); document .querySelectorAll( ".panel" ) .forEach((panel) => { const active = panel.id === validName; panel.classList.toggle( "is-active", active ); panel.hidden = !active; }); if (updateHash) { history.replaceState( null, "", `#${validName}` ); } } function buildControls() { const datasetOptions = document.querySelector( "#dataset-options" ); state.datasets.forEach( (dataset, index) => { const label = document.createElement( "label" ); label.className = "choice-pill"; if ( dataset === "combined" ) { label.classList.add( "combined-choice" ); } const input = document.createElement( "input" ); input.type = "radio"; input.name = "dataset"; input.value = dataset; input.checked = index === 0; input.addEventListener( "change", () => { state.sortKey = `${dataset}|${state.metrics[0]}`; state.sortDirection = "asc"; render(); } ); label.append( input, document.createTextNode( DATASET_LABELS[ dataset ] || dataset ) ); datasetOptions.append( label ); } ); const matrixOptions = document.querySelector( "#matrix-metric-options" ); state.metrics.forEach( (metric, index) => { const matrixLabel = document.createElement( "label" ); matrixLabel.className = "choice-pill"; const matrixInput = document.createElement( "input" ); matrixInput.type = "radio"; matrixInput.name = "matrix-metric"; matrixInput.value = metric; matrixInput.checked = index === 0; matrixInput.addEventListener( "change", () => { state.sortKey = `combined|${matrixInput.value}`; state.sortDirection = "asc"; render(); } ); matrixLabel.append( matrixInput, document.createTextNode( metricLabel(metric) ) ); matrixOptions.append( matrixLabel ); } ); document .querySelector( "#color-toggle" ) .addEventListener( "click", (event) => { state.colorScores = !state.colorScores; event.currentTarget.classList.toggle( "is-active", state.colorScores ); event.currentTarget.setAttribute( "aria-pressed", String( state.colorScores ) ); render(); } ); document .querySelectorAll( ".view-button" ) .forEach((button) => { button.addEventListener( "click", () => { setView( button.dataset.view ); } ); }); document .querySelectorAll( ".tab" ) .forEach((tab) => { tab.addEventListener( "click", () => { activateTab( tab.dataset.tab ); } ); }); document .querySelector( "#theme-toggle" ) .addEventListener( "click", () => { setTheme( document.documentElement .dataset.theme === "dark" ? "light" : "dark" ); } ); document .querySelector("#version-select") .addEventListener("change", (event) => { setVersion(event.currentTarget.value); }); document .querySelector("#new-version-link") .addEventListener("click", () => { setVersion(DEFAULT_VERSION); }); document.addEventListener( "click", () => { if ( !state.availabilityMenuOpen ) { return; } state.availabilityMenuOpen = false; render(); } ); document.addEventListener( "keydown", (event) => { if ( event.key !== "Escape" || !state.availabilityMenuOpen ) { return; } state.availabilityMenuOpen = false; render(); } ); setTheme( document.documentElement .dataset.theme, false ); } async function start() { try { const embeddedResults = document.querySelector( "#results-data" ); if (!embeddedResults) { throw new Error( "Could not find the embedded results. Run build_site.py." ); } state.versionedResults = JSON.parse(embeddedResults.textContent); if (!state.versionedResults[DEFAULT_VERSION]) { throw new Error(`Could not find ${DEFAULT_VERSION} results.`); } ingest(parseCsv(state.versionedResults[DEFAULT_VERSION])); buildControls(); buildChartControls(); activateTab( location.hash.slice(1), false ); render(); renderComparisonChart(); } catch (error) { document.querySelector( "#table-title" ).textContent = "Leaderboard unavailable"; document.querySelector( "#leaderboard-table" ).hidden = true; const empty = document.querySelector( "#empty-state" ); empty.hidden = false; empty.querySelector( "h3" ).textContent = "Could not load results"; empty.querySelector( "p" ).textContent = error.message; } } if ( typeof module !== "undefined" ) { module.exports = { heatClass, ingest, matrixRanges, parseCsv, rankRecords, setTheme, state, }; } if ( typeof document !== "undefined" ) { start(); }