Add model-type colors and interactive reconstruction comparison plotter
Browse files- README.md +3 -1
- app.js +8 -2
- evaluation/README.md +20 -1
- evaluation/metadata.example.json +3 -1
- index.html +22 -7
- plot-data.mjs +104 -0
- plotter.css +46 -0
- plotter.mjs +152 -0
README.md
CHANGED
|
@@ -12,7 +12,7 @@ short_description: Scientific table reconstruction with RMS and exact mapping
|
|
| 12 |
|
| 13 |
# Scientific Table Reconstruction
|
| 14 |
|
| 15 |
-
A free static leaderboard
|
| 16 |
|
| 17 |
Dataset: https://huggingface.co/datasets/SciKnowOrg/Scientific-Table-Reconstruction
|
| 18 |
|
|
@@ -26,3 +26,5 @@ See evaluation/README.md for the scoring protocol and contribution workflow.
|
|
| 26 |
|
| 27 |
The board starts without model entries. Sanity-check fixtures are never ranked.
|
| 28 |
New results are reviewed and committed by maintainers after scoring saved predictions.
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
# Scientific Table Reconstruction
|
| 14 |
|
| 15 |
+
A free static leaderboard comparing OCR pipelines and vision-language models on scientific table reconstruction, derived from TabulusBench Step 2.
|
| 16 |
|
| 17 |
Dataset: https://huggingface.co/datasets/SciKnowOrg/Scientific-Table-Reconstruction
|
| 18 |
|
|
|
|
| 26 |
|
| 27 |
The board starts without model entries. Sanity-check fixtures are never ranked.
|
| 28 |
New results are reviewed and committed by maintainers after scoring saved predictions.
|
| 29 |
+
|
| 30 |
+
The interactive comparison plotter supports model-size and score axes, type colors, domain filters, local summary previews and SVG downloads. Local previews do not enter public rankings.
|
app.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
| 1 |
"use strict";
|
|
|
|
|
|
|
| 2 |
const state = {board: null, metric: "rms", domain: "all", search: "", sort: "macro.f1", descending: true};
|
| 3 |
const $ = id => document.getElementById(id);
|
| 4 |
const pretty = value => value.replaceAll("_", " ");
|
|
@@ -15,7 +17,7 @@ function render() {
|
|
| 15 |
});
|
| 16 |
const body = $("results"); body.replaceChildren();
|
| 17 |
if (!rows.length) {
|
| 18 |
-
const cell = body.insertRow().insertCell(); cell.colSpan =
|
| 19 |
const empty = document.createElement("div"); empty.className = "empty-state";
|
| 20 |
const icon = document.createElement("span"); icon.className = "empty-icon"; icon.textContent = "▥"; icon.setAttribute("aria-hidden", "true");
|
| 21 |
const heading = document.createElement("h3"); heading.textContent = state.board.results.length ? "No matching submissions." : "Ready for the first result.";
|
|
@@ -31,6 +33,10 @@ function render() {
|
|
| 31 |
link.textContent = result.submission.method_name;
|
| 32 |
const detail = document.createElement("small"); detail.textContent = result.submission.model_revision;
|
| 33 |
name.append(link, detail);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
for (const average of ["macro", "micro"])
|
| 35 |
for (const measure of ["precision", "recall", "f1"])
|
| 36 |
row.insertCell().textContent = stats[average][state.metric][measure].toFixed(2);
|
|
@@ -68,7 +74,7 @@ async function init() {
|
|
| 68 |
const select = $("example-select"); select.replaceChildren();
|
| 69 |
examples.forEach((example, index) => {const option = document.createElement("option"); option.value = index; option.textContent = pretty(example.domain); select.append(option);});
|
| 70 |
select.addEventListener("change", () => showExample(examples, Number(select.value)));
|
| 71 |
-
showExample(examples, 0); render();
|
| 72 |
} catch (error) { $("load-error").hidden = false; $("load-error").textContent = "Benchmark data could not be loaded. Please refresh or visit the dataset directly."; console.error(error); }
|
| 73 |
}
|
| 74 |
document.querySelectorAll("[data-metric]").forEach(button => button.addEventListener("click", () => {
|
|
|
|
| 1 |
"use strict";
|
| 2 |
+
import {MODEL_TYPES, modelType} from "./plot-data.mjs";
|
| 3 |
+
import {setupPlotter} from "./plotter.mjs";
|
| 4 |
const state = {board: null, metric: "rms", domain: "all", search: "", sort: "macro.f1", descending: true};
|
| 5 |
const $ = id => document.getElementById(id);
|
| 6 |
const pretty = value => value.replaceAll("_", " ");
|
|
|
|
| 17 |
});
|
| 18 |
const body = $("results"); body.replaceChildren();
|
| 19 |
if (!rows.length) {
|
| 20 |
+
const cell = body.insertRow().insertCell(); cell.colSpan = 11;
|
| 21 |
const empty = document.createElement("div"); empty.className = "empty-state";
|
| 22 |
const icon = document.createElement("span"); icon.className = "empty-icon"; icon.textContent = "▥"; icon.setAttribute("aria-hidden", "true");
|
| 23 |
const heading = document.createElement("h3"); heading.textContent = state.board.results.length ? "No matching submissions." : "Ready for the first result.";
|
|
|
|
| 33 |
link.textContent = result.submission.method_name;
|
| 34 |
const detail = document.createElement("small"); detail.textContent = result.submission.model_revision;
|
| 35 |
name.append(link, detail);
|
| 36 |
+
const type = modelType(result.submission), typeCell = row.insertCell();
|
| 37 |
+
const badge = document.createElement("span"); badge.className = "model-type-badge";
|
| 38 |
+
const dot = document.createElement("i"); dot.style.setProperty("--color", MODEL_TYPES[type].color);
|
| 39 |
+
badge.append(dot, document.createTextNode(MODEL_TYPES[type].label)); typeCell.append(badge);
|
| 40 |
for (const average of ["macro", "micro"])
|
| 41 |
for (const measure of ["precision", "recall", "f1"])
|
| 42 |
row.insertCell().textContent = stats[average][state.metric][measure].toFixed(2);
|
|
|
|
| 74 |
const select = $("example-select"); select.replaceChildren();
|
| 75 |
examples.forEach((example, index) => {const option = document.createElement("option"); option.value = index; option.textContent = pretty(example.domain); select.append(option);});
|
| 76 |
select.addEventListener("change", () => showExample(examples, Number(select.value)));
|
| 77 |
+
showExample(examples, 0); render(); setupPlotter(board);
|
| 78 |
} catch (error) { $("load-error").hidden = false; $("load-error").textContent = "Benchmark data could not be loaded. Please refresh or visit the dataset directly."; console.error(error); }
|
| 79 |
}
|
| 80 |
document.querySelectorAll("[data-metric]").forEach(button => button.addEventListener("click", () => {
|
evaluation/README.md
CHANGED
|
@@ -25,7 +25,7 @@ Create one JSON object per table, one line each, with exactly four fields:
|
|
| 25 |
|
| 26 |
Use the exact public IDs and preserve every prediction, including headers and row labels. Every one of the 610 IDs is required. On failure, submit status "error" or "empty" with rows []. Explicitly empty predictions are scored as failures. Duplicate, unknown, missing IDs and non-string cells are rejected. A table may contain at most 10,000 rows and 1,000,000 cells; oversized responses must be reported as errors. Do not submit just the subset that worked.
|
| 27 |
|
| 28 |
-
Copy metadata.example.json to metadata.json and fill
|
| 29 |
|
| 30 |
## Metrics and denominators
|
| 31 |
|
|
@@ -60,6 +60,25 @@ To add a reviewed result after independently running bench.py:
|
|
| 60 |
|
| 61 |
This updates only the local board file; maintainers review and commit that change to the Space.
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
## Reproducibility and licenses
|
| 64 |
|
| 65 |
bench.py verifies the content fingerprint and frozen scoring implementation against the release manifest. It records hashes of canonicalized predictions and the metric implementation, dependency versions, sufficient per-table statistics, aggregate denominators and domain breakdowns.
|
|
|
|
| 25 |
|
| 26 |
Use the exact public IDs and preserve every prediction, including headers and row labels. Every one of the 610 IDs is required. On failure, submit status "error" or "empty" with rows []. Explicitly empty predictions are scored as failures. Duplicate, unknown, missing IDs and non-string cells are rejected. A table may contain at most 10,000 rows and 1,000,000 cells; oversized responses must be reported as errors. Do not submit just the subset that worked.
|
| 27 |
|
| 28 |
+
Copy metadata.example.json to metadata.json and fill the required fields. Include immutable model/code revisions, full prompt, parser, generation settings and training-overlap disclosure. The required evaluation_mode is "physical-table"; input_mode is "image-only". The dataset fingerprint must match.
|
| 29 |
|
| 30 |
## Metrics and denominators
|
| 31 |
|
|
|
|
| 60 |
|
| 61 |
This updates only the local board file; maintainers review and commit that change to the Space.
|
| 62 |
|
| 63 |
+
|
| 64 |
+
## Model type and comparison plots
|
| 65 |
+
|
| 66 |
+
Set model_type in submission metadata to one of:
|
| 67 |
+
|
| 68 |
+
- VLM: a general-purpose vision-language model.
|
| 69 |
+
- specialist: a neural model specialized for OCR, documents or table reconstruction.
|
| 70 |
+
- classical: a traditional OCR engine/pipeline.
|
| 71 |
+
- hybrid: a pipeline combining distinct model types; describe its components.
|
| 72 |
+
- unspecified: type is unknown or not yet supplied.
|
| 73 |
+
|
| 74 |
+
This descriptive classification appears beside the model name and determines the same point color in the plot. It does not alter scoring. Existing summaries without this field display as unspecified.
|
| 75 |
+
|
| 76 |
+
Optionally provide parameter_count_b as a positive JSON number, in billions of total model parameters, or null when unknown. For a mixture-of-experts model, use total parameters, not active parameters, and explain the choice. For multi-model pipelines, disclose the components and counting convention in your evidence. Do not infer size from a model name. Proprietary or classical models with unknown counts remain comparable on score-versus-score plots.
|
| 77 |
+
|
| 78 |
+
The bottom-of-page plotter supports model size (logarithmic or linear) versus any of the twelve scores, or any pair of scores, with domain and model-type filters, keyboard/touch point selection, and SVG export. Parameter counts are declared metadata, not latency or inference-cost measurements.
|
| 79 |
+
|
| 80 |
+
Load saved summary.json files for private local comparison. They must match this dataset fingerprint, evaluation mode and complete population. Files are parsed only in the browser, never uploaded. Local points are outlined and marked unreviewed; they do not enter the public leaderboard. A new file with the same local submission ID replaces that local preview. Clearing previews removes them from the page.
|
| 81 |
+
|
| 82 |
## Reproducibility and licenses
|
| 83 |
|
| 84 |
bench.py verifies the content fingerprint and frozen scoring implementation against the release manifest. It records hashes of canonicalized predictions and the metric implementation, dependency versions, sufficient per-table statistics, aggregate denominators and domain breakdowns.
|
evaluation/metadata.example.json
CHANGED
|
@@ -10,5 +10,7 @@
|
|
| 10 |
"training_disclosure": "Disclose any training/tuning overlap with these papers or tables",
|
| 11 |
"dataset_fingerprint": "f026daa52fe17c24e66068f474cb7fef5f309d3e688ad72bc07e0c412040784e",
|
| 12 |
"evaluation_mode": "physical-table",
|
| 13 |
-
"input_mode": "image-only"
|
|
|
|
|
|
|
| 14 |
}
|
|
|
|
| 10 |
"training_disclosure": "Disclose any training/tuning overlap with these papers or tables",
|
| 11 |
"dataset_fingerprint": "f026daa52fe17c24e66068f474cb7fef5f309d3e688ad72bc07e0c412040784e",
|
| 12 |
"evaluation_mode": "physical-table",
|
| 13 |
+
"input_mode": "image-only",
|
| 14 |
+
"model_type": "unspecified",
|
| 15 |
+
"parameter_count_b": null
|
| 16 |
}
|
index.html
CHANGED
|
@@ -2,18 +2,18 @@
|
|
| 2 |
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
| 5 |
-
<meta name="description" content="
|
| 6 |
<title>Scientific Table Reconstruction</title>
|
| 7 |
-
<link rel="stylesheet" href="style.css"><script src="app.js"
|
| 8 |
</head>
|
| 9 |
<body>
|
| 10 |
<div class="page">
|
| 11 |
-
<header><a class="brand" href="https://huggingface.co/SciKnowOrg" target="_blank" rel="noopener"><span class="mark" aria-hidden="true">▦</span> SciKnowOrg
|
| 12 |
<main>
|
| 13 |
<section class="hero" aria-labelledby="title">
|
| 14 |
-
<div class="eyebrow"><span class="dot"></span>
|
| 15 |
<h1 id="title">From table images<br>to structured knowledge.</h1>
|
| 16 |
-
<p class="intro">
|
| 17 |
<div class="actions"><a class="button primary" href="https://huggingface.co/datasets/SciKnowOrg/Scientific-Table-Reconstruction" target="_blank" rel="noopener">Explore the dataset <span>↗</span></a><a class="button secondary" href="#submit">Submit a result <span>→</span></a></div>
|
| 18 |
<div class="stats" aria-label="Benchmark size"><div><strong>610</strong><span>table images</span></div><div><strong>253</strong><span>scientific papers</span></div><div><strong>5</strong><span>research domains</span></div><div><strong>12</strong><span>reported measures</span></div></div>
|
| 19 |
<div class="version-line"><span class="tiny-square"></span> PUBLIC TEST SET · <span id="version">v1.0.1</span><span class="version-dot">·</span> Image-only inputs <span class="version-dot">·</span> Open gold annotations</div>
|
|
@@ -23,7 +23,7 @@
|
|
| 23 |
<div class="board">
|
| 24 |
<div class="board-toolbar"><div class="metric-tabs" role="group" aria-label="Mapping metric"><button class="active" data-metric="rms" aria-pressed="true">RMS <span>approximate</span></button><button data-metric="exact-mapping" aria-pressed="false">Exact Mapping Match <span>strict</span></button></div><label class="domain-label">Domain<select id="domain"><option value="all">All five domains</option></select></label></div>
|
| 25 |
<div class="board-subtitle"><p id="metric-note">Relative Mapping Similarity · approximate matching with text and numeric tolerances.</p><label class="search-label"><span class="sr-only">Find a model</span><input id="search" type="search" placeholder="Find a model…" aria-label="Find a model"></label></div>
|
| 26 |
-
<div class="table-scroll"><table class="scores"><thead><tr class="group-head"><th rowspan="2" scope="col">#</th><th rowspan="2" scope="col" class="method">Model / method</th><th colspan="3" scope="colgroup">Macro <span>equal weight per table</span></th><th colspan="3" scope="colgroup">Micro <span>pooled mappings</span></th><th rowspan="2" scope="col">Evaluation mode</th><th rowspan="2" scope="col">Coverage</th></tr><tr><th><button data-sort="macro.precision">Precision</button></th><th><button data-sort="macro.recall">Recall</button></th><th><button data-sort="macro.f1" class="selected" aria-label="Sort by macro F1 descending">F1 ↓</button></th><th><button data-sort="micro.precision">Precision</button></th><th><button data-sort="micro.recall">Recall</button></th><th><button data-sort="micro.f1">F1</button></th></tr></thead><tbody id="results"><tr><td colspan="
|
| 27 |
<div class="board-footer"><span id="board-count">0 reviewed submissions</span><span>Scores 0–100 · higher is better · default rank: macro RMS F1</span></div>
|
| 28 |
</div>
|
| 29 |
<p class="caption">Every submission covers the full test set. Empty and failed extractions stay in the denominator. Domain views are diagnostic; the official ranking uses all 610 tables.</p>
|
|
@@ -41,7 +41,22 @@
|
|
| 41 |
<div class="details-list"><details open><summary>What counts as a mapping?</summary><p>The first row provides column headers and the first column provides row labels. Their combined key maps to each body value. Whitespace and case are normalized before comparing mappings. This measures associations, not raw CSV bytes.</p></details><details><summary>How do macro and micro differ?</summary><p>Macro averages each table’s precision, recall and F1 separately. Micro pools matched credit and gold/predicted mapping counts, then computes the three scores. Failed tables receive zero macro scores and keep their gold counts for micro recall and F1.</p></details><details><summary>What does “exact” mean here?</summary><p>Exact Mapping Match uses identical normalized keys and values. RMS uses approximate text matching (threshold 0.5) and relative numeric matching (threshold 0.1). Each selects the better of original and transposed predictions using per-table F1. Evaluation mode is physical-table; model input is image-only.</p></details><details><summary>What should I know about the gold?</summary><p>This is an open-gold public test set. Do not use gold in inference or tuning; disclose training overlap. Source gold is preserved, including merged-cell and repeated-label conventions. The inherited mapping parser collapses duplicate keys and has a few delimiter/zero-mapping edge cases. The manifest and protocol document these limitations and empty-set conventions.</p></details></div>
|
| 42 |
</section>
|
| 43 |
<section id="submit" class="section submit" aria-labelledby="submit-heading"><div class="submit-intro"><div class="eyebrow">MAKE YOUR WORK COMPARABLE</div><h2 id="submit-heading">Add your model<br>to the leaderboard.</h2><p>Inference runs on your own hardware. The dataset, evaluator and static leaderboard are open and free to access.</p><a class="button primary" href="https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/discussions" target="_blank" rel="noopener">Open a submission discussion ↗</a></div><ol class="steps"><li><span>01</span><div><h3>Load the images</h3><p>Import the v1.0.1 test split from SciKnowOrg/Scientific-Table-Reconstruction. Give the model only each table image.</p></div></li><li><span>02</span><div><h3>Reconstruct and score</h3><p>Save all 610 predictions as rows of strings, with explicit failures. Run the pinned CPU evaluator for all twelve measures.</p><a href="evaluation/README.md">Evaluator & prediction format ↗</a></div></li><li><span>03</span><div><h3>Share the evidence</h3><p>Publish predictions, prompt, model/code revisions, settings and training-overlap disclosure. Maintainers rescore before adding an entry.</p><a href="evaluation/metadata.example.json" download>Download metadata template ↓</a></div></li></ol></section>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
</main>
|
| 45 |
-
<footer><div class="brand"><span class="mark" aria-hidden="true">▦</span> SciKnowOrg
|
| 46 |
</div>
|
| 47 |
</body></html>
|
|
|
|
| 2 |
<html lang="en">
|
| 3 |
<head>
|
| 4 |
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
| 5 |
+
<meta name="description" content="Compare OCR pipelines and vision-language models on reconstructing scientific tables. RMS and exact mapping scores across 253 papers in five domains.">
|
| 6 |
<title>Scientific Table Reconstruction</title>
|
| 7 |
+
<link rel="stylesheet" href="style.css"><link rel="stylesheet" href="plotter.css"><script src="app.js" type="module"></script>
|
| 8 |
</head>
|
| 9 |
<body>
|
| 10 |
<div class="page">
|
| 11 |
+
<header><a class="brand" href="https://huggingface.co/SciKnowOrg" target="_blank" rel="noopener"><span class="mark" aria-hidden="true">▦</span> SciKnowOrg</a><nav aria-label="Main"><a href="#leaderboard">Leaderboard</a><a href="#dataset">Dataset</a><a href="#plotter">Plotter</a><a href="#submit">Contribute <span aria-hidden="true">↗</span></a></nav></header>
|
| 12 |
<main>
|
| 13 |
<section class="hero" aria-labelledby="title">
|
| 14 |
+
<div class="eyebrow"><span class="dot"></span> OCR & VISION-LANGUAGE MODELS <span class="pill">LEADERBOARD</span></div>
|
| 15 |
<h1 id="title">From table images<br>to structured knowledge.</h1>
|
| 16 |
+
<p class="intro">Compare OCR pipelines and vision-language models (VLMs) on reconstructing scientific tables. One table image in, structured rows and cells out—across five research domains.</p>
|
| 17 |
<div class="actions"><a class="button primary" href="https://huggingface.co/datasets/SciKnowOrg/Scientific-Table-Reconstruction" target="_blank" rel="noopener">Explore the dataset <span>↗</span></a><a class="button secondary" href="#submit">Submit a result <span>→</span></a></div>
|
| 18 |
<div class="stats" aria-label="Benchmark size"><div><strong>610</strong><span>table images</span></div><div><strong>253</strong><span>scientific papers</span></div><div><strong>5</strong><span>research domains</span></div><div><strong>12</strong><span>reported measures</span></div></div>
|
| 19 |
<div class="version-line"><span class="tiny-square"></span> PUBLIC TEST SET · <span id="version">v1.0.1</span><span class="version-dot">·</span> Image-only inputs <span class="version-dot">·</span> Open gold annotations</div>
|
|
|
|
| 23 |
<div class="board">
|
| 24 |
<div class="board-toolbar"><div class="metric-tabs" role="group" aria-label="Mapping metric"><button class="active" data-metric="rms" aria-pressed="true">RMS <span>approximate</span></button><button data-metric="exact-mapping" aria-pressed="false">Exact Mapping Match <span>strict</span></button></div><label class="domain-label">Domain<select id="domain"><option value="all">All five domains</option></select></label></div>
|
| 25 |
<div class="board-subtitle"><p id="metric-note">Relative Mapping Similarity · approximate matching with text and numeric tolerances.</p><label class="search-label"><span class="sr-only">Find a model</span><input id="search" type="search" placeholder="Find a model…" aria-label="Find a model"></label></div>
|
| 26 |
+
<div class="table-scroll"><table class="scores"><thead><tr class="group-head"><th rowspan="2" scope="col">#</th><th rowspan="2" scope="col" class="method">Model / method</th><th rowspan="2" scope="col" class="model-type-heading" title="VLM: general-purpose vision-language model; specialist: neural OCR/document/table model; classical: traditional OCR; hybrid: mixed pipeline.">Model type ⓘ</th><th colspan="3" scope="colgroup">Macro <span>equal weight per table</span></th><th colspan="3" scope="colgroup">Micro <span>pooled mappings</span></th><th rowspan="2" scope="col">Evaluation mode</th><th rowspan="2" scope="col">Coverage</th></tr><tr><th><button data-sort="macro.precision">Precision</button></th><th><button data-sort="macro.recall">Recall</button></th><th><button data-sort="macro.f1" class="selected" aria-label="Sort by macro F1 descending">F1 ↓</button></th><th><button data-sort="micro.precision">Precision</button></th><th><button data-sort="micro.recall">Recall</button></th><th><button data-sort="micro.f1">F1</button></th></tr></thead><tbody id="results"><tr><td colspan="11"><div class="empty-state"><span class="empty-icon" aria-hidden="true">▥</span><h3>Ready for the first result.</h3><p>No reviewed model submissions yet.<br>Run your model on the test images and help set the benchmark.</p><a href="#submit">See the submission guide <span>→</span></a></div></td></tr></tbody></table></div>
|
| 27 |
<div class="board-footer"><span id="board-count">0 reviewed submissions</span><span>Scores 0–100 · higher is better · default rank: macro RMS F1</span></div>
|
| 28 |
</div>
|
| 29 |
<p class="caption">Every submission covers the full test set. Empty and failed extractions stay in the denominator. Domain views are diagnostic; the official ranking uses all 610 tables.</p>
|
|
|
|
| 41 |
<div class="details-list"><details open><summary>What counts as a mapping?</summary><p>The first row provides column headers and the first column provides row labels. Their combined key maps to each body value. Whitespace and case are normalized before comparing mappings. This measures associations, not raw CSV bytes.</p></details><details><summary>How do macro and micro differ?</summary><p>Macro averages each table’s precision, recall and F1 separately. Micro pools matched credit and gold/predicted mapping counts, then computes the three scores. Failed tables receive zero macro scores and keep their gold counts for micro recall and F1.</p></details><details><summary>What does “exact” mean here?</summary><p>Exact Mapping Match uses identical normalized keys and values. RMS uses approximate text matching (threshold 0.5) and relative numeric matching (threshold 0.1). Each selects the better of original and transposed predictions using per-table F1. Evaluation mode is physical-table; model input is image-only.</p></details><details><summary>What should I know about the gold?</summary><p>This is an open-gold public test set. Do not use gold in inference or tuning; disclose training overlap. Source gold is preserved, including merged-cell and repeated-label conventions. The inherited mapping parser collapses duplicate keys and has a few delimiter/zero-mapping edge cases. The manifest and protocol document these limitations and empty-set conventions.</p></details></div>
|
| 42 |
</section>
|
| 43 |
<section id="submit" class="section submit" aria-labelledby="submit-heading"><div class="submit-intro"><div class="eyebrow">MAKE YOUR WORK COMPARABLE</div><h2 id="submit-heading">Add your model<br>to the leaderboard.</h2><p>Inference runs on your own hardware. The dataset, evaluator and static leaderboard are open and free to access.</p><a class="button primary" href="https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/discussions" target="_blank" rel="noopener">Open a submission discussion ↗</a></div><ol class="steps"><li><span>01</span><div><h3>Load the images</h3><p>Import the v1.0.1 test split from SciKnowOrg/Scientific-Table-Reconstruction. Give the model only each table image.</p></div></li><li><span>02</span><div><h3>Reconstruct and score</h3><p>Save all 610 predictions as rows of strings, with explicit failures. Run the pinned CPU evaluator for all twelve measures.</p><a href="evaluation/README.md">Evaluator & prediction format ↗</a></div></li><li><span>03</span><div><h3>Share the evidence</h3><p>Publish predictions, prompt, model/code revisions, settings and training-overlap disclosure. Maintainers rescore before adding an entry.</p><a href="evaluation/metadata.example.json" download>Download metadata template ↓</a></div></li></ol></section>
|
| 44 |
+
<section id="plotter" class="section" aria-labelledby="plot-heading">
|
| 45 |
+
<div class="section-top"><div><div class="eyebrow">EXPLORE THE RESULTS</div><h2 id="plot-heading">Size, accuracy, and trade-offs.</h2></div><button id="download-plot" class="button secondary" type="button" disabled>Download SVG ↓</button></div>
|
| 46 |
+
<p class="section-description">Compare model size with reconstruction quality, or choose any two scores. Hover, tap, or focus a point for its details.</p>
|
| 47 |
+
<div class="plot-controls">
|
| 48 |
+
<label>Horizontal axis<select id="plot-x"><option value="parameters">Model parameters (billions)</option></select></label>
|
| 49 |
+
<label>Vertical axis<select id="plot-y"></select></label>
|
| 50 |
+
<label>Domain<select id="plot-domain"><option value="all">All five domains</option></select></label>
|
| 51 |
+
<label>Model type<select id="plot-type"><option value="all">All model types</option><option value="VLM">VLM</option><option value="specialist">Specialist</option><option value="classical">Classical</option><option value="hybrid">Hybrid</option><option value="unspecified">Unspecified</option></select></label>
|
| 52 |
+
</div>
|
| 53 |
+
<div class="plot-options"><label><input id="plot-log" type="checkbox" checked> Logarithmic model-size axis</label><span id="plot-count" role="status">Waiting for reviewed results</span></div>
|
| 54 |
+
<div class="plot-layout"><div class="plot-shell"><div id="plot-chart" class="scatter-wrap"></div><div id="plot-empty" class="plot-empty" hidden><h3>Ready to compare the first models.</h3><p>Reviewed submissions will appear here. You can also preview your own saved evaluation summaries below.</p></div></div><aside class="plot-detail" aria-label="Selected model details"><div class="eyebrow">MODEL DETAILS</div><div id="plot-detail"><h3>Select a point</h3><p>Each point represents one submission. Model-size comparisons use declared parameter counts.</p></div><div class="plot-legend"><span><i style="--color:#3968bd"></i> VLM</span><span><i style="--color:#16845c"></i> Specialist</span><span><i style="--color:#b66b20"></i> Classical</span><span><i style="--color:#8261a8"></i> Hybrid</span><span><i style="--color:#859183"></i> Unspecified</span><span class="local-legend">◇ Local previews use outlined points</span></div></aside></div>
|
| 55 |
+
<div class="local-preview"><div><strong>Preview your own results</strong><p>Load summary.json files from the evaluator. Files stay in your browser and are not submitted or added to the rankings.</p></div><div class="preview-actions"><label class="button secondary file-button">Load summaries<input id="plot-files" type="file" accept=".json,application/json" multiple></label><button id="clear-local" type="button" class="text-button" hidden>Clear previews</button></div></div>
|
| 56 |
+
<p id="plot-message" class="caption" role="status"></p>
|
| 57 |
+
<p class="caption">All scores range from 0 to 100; higher is better. Each metric axis includes its aggregation. Models without declared parameter counts are omitted only from size comparisons. Parameter counts are not a measure of latency or compute cost.</p>
|
| 58 |
+
</section>
|
| 59 |
</main>
|
| 60 |
+
<footer><div class="brand"><span class="mark" aria-hidden="true">▦</span> SciKnowOrg</div><div><a href="https://huggingface.co/datasets/SciKnowOrg/Scientific-Table-Reconstruction">Dataset</a><a href="https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/tree/main">Source</a><a href="https://github.com/google-research/google-research/tree/master/deplot">DePlot metric</a></div><span>Built on TabulusBench</span></footer>
|
| 61 |
</div>
|
| 62 |
</body></html>
|
plot-data.mjs
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/** Pure, browser-independent helpers for score comparison and plot geometry. */
|
| 2 |
+
export const MODEL_TYPES = Object.freeze({
|
| 3 |
+
VLM: {label: "VLM", color: "#3968bd"},
|
| 4 |
+
specialist: {label: "Specialist", color: "#16845c"},
|
| 5 |
+
classical: {label: "Classical", color: "#b66b20"},
|
| 6 |
+
hybrid: {label: "Hybrid", color: "#8261a8"},
|
| 7 |
+
unspecified: {label: "Unspecified", color: "#859183"}
|
| 8 |
+
});
|
| 9 |
+
export const SCORE_AXES = [];
|
| 10 |
+
for (const aggregation of ["macro", "micro"]) {
|
| 11 |
+
for (const metric of ["rms", "exact-mapping"]) {
|
| 12 |
+
for (const measure of ["precision", "recall", "f1"]) {
|
| 13 |
+
SCORE_AXES.push({
|
| 14 |
+
key: [aggregation, metric, measure].join("."),
|
| 15 |
+
label: (aggregation === "macro" ? "Macro " : "Micro ") +
|
| 16 |
+
(metric === "rms" ? "RMS " : "Exact Mapping ") +
|
| 17 |
+
(measure === "f1" ? "F1" : measure[0].toUpperCase() + measure.slice(1)) + " (%)"
|
| 18 |
+
});
|
| 19 |
+
}
|
| 20 |
+
}
|
| 21 |
+
}
|
| 22 |
+
export function modelType(submission = {}) {
|
| 23 |
+
return Object.hasOwn(MODEL_TYPES, submission.model_type) ? submission.model_type : "unspecified";
|
| 24 |
+
}
|
| 25 |
+
export function axisLabel(axis) {
|
| 26 |
+
return axis === "parameters" ? "Model parameters (billions)" : SCORE_AXES.find(item => item.key === axis)?.label || axis;
|
| 27 |
+
}
|
| 28 |
+
export function scoreValue(result, axis, domain = "all") {
|
| 29 |
+
if (axis === "parameters") {
|
| 30 |
+
const value = result.submission?.parameter_count_b;
|
| 31 |
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
| 32 |
+
}
|
| 33 |
+
const [aggregation, metric, measure] = axis.split(".");
|
| 34 |
+
const stats = domain === "all" ? result : result.by_domain?.[domain];
|
| 35 |
+
const value = stats?.[aggregation]?.[metric]?.[measure];
|
| 36 |
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 100 ? value : null;
|
| 37 |
+
}
|
| 38 |
+
export function selectPlotPoints(records, {x, y, domain = "all", type = "all"}) {
|
| 39 |
+
let candidates = 0, omitted = 0;
|
| 40 |
+
const points = [];
|
| 41 |
+
for (const record of records) {
|
| 42 |
+
const category = modelType(record.submission);
|
| 43 |
+
if (type !== "all" && category !== type) continue;
|
| 44 |
+
if (domain !== "all" && !record.by_domain?.[domain]) continue;
|
| 45 |
+
candidates++;
|
| 46 |
+
const xValue = scoreValue(record, x, domain), yValue = scoreValue(record, y, domain);
|
| 47 |
+
if (xValue === null || yValue === null) { omitted++; continue; }
|
| 48 |
+
points.push({record, type: category, x: xValue, y: yValue});
|
| 49 |
+
}
|
| 50 |
+
return {points, candidates, omitted};
|
| 51 |
+
}
|
| 52 |
+
export function xScale(points, axis, logarithmic) {
|
| 53 |
+
if (axis !== "parameters") return {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
|
| 54 |
+
const values = points.map(point => point.x);
|
| 55 |
+
if (!values.length) return logarithmic
|
| 56 |
+
? {min: 0.01, max: 100, log: true, ticks: [0.01, 0.1, 1, 10, 100]}
|
| 57 |
+
: {min: 0, max: 100, log: false, ticks: [0, 20, 40, 60, 80, 100]};
|
| 58 |
+
if (!logarithmic) {
|
| 59 |
+
const maxValue = Math.max(...values);
|
| 60 |
+
const power = 10 ** Math.floor(Math.log10(maxValue));
|
| 61 |
+
const max = Math.ceil(maxValue / power) * power;
|
| 62 |
+
return {min: 0, max, log: false, ticks: Array.from({length: 6}, (_, i) => max * i / 5)};
|
| 63 |
+
}
|
| 64 |
+
let lo = Math.floor(Math.log10(Math.min(...values)));
|
| 65 |
+
let hi = Math.ceil(Math.log10(Math.max(...values)));
|
| 66 |
+
if (lo === hi) {lo--; hi++;}
|
| 67 |
+
const ticks = [];
|
| 68 |
+
const step = Math.max(1, Math.ceil((hi - lo) / 8));
|
| 69 |
+
for (let exponent = lo; exponent <= hi; exponent += step) ticks.push(10 ** exponent);
|
| 70 |
+
if (ticks.at(-1) !== 10 ** hi) ticks.push(10 ** hi);
|
| 71 |
+
return {min: 10 ** lo, max: 10 ** hi, log: true, ticks};
|
| 72 |
+
}
|
| 73 |
+
export function fraction(value, scale) {
|
| 74 |
+
const transform = scale.log ? Math.log10 : value => value;
|
| 75 |
+
return (transform(value) - transform(scale.min)) / (transform(scale.max) - transform(scale.min));
|
| 76 |
+
}
|
| 77 |
+
function validateScores(stats, context) {
|
| 78 |
+
for (const axis of SCORE_AXES) {
|
| 79 |
+
if (scoreValue(stats, axis.key) === null) throw new Error(context + ": missing or invalid " + axis.label);
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
export function validateLocalSummary(summary, board) {
|
| 83 |
+
if (!summary || typeof summary !== "object" || Array.isArray(summary)) throw new Error("Expected a summary.json object.");
|
| 84 |
+
if (summary.dataset_fingerprint !== board.dataset_fingerprint) throw new Error("This summary belongs to a different dataset fingerprint.");
|
| 85 |
+
if (summary.protocol_version !== "tabulusbench-step2-v1" ||
|
| 86 |
+
summary.evaluation_mode !== "physical-table" || summary.input_mode !== "image-only")
|
| 87 |
+
throw new Error("Use a physical-table, image-only summary from the published evaluator.");
|
| 88 |
+
if (summary.population?.fragments !== board.tables) throw new Error("The summary must cover all " + board.tables + " tables.");
|
| 89 |
+
if (typeof summary.submission?.submission_id !== "string" || !summary.submission.submission_id ||
|
| 90 |
+
typeof summary.submission?.method_name !== "string" || !summary.submission.method_name)
|
| 91 |
+
throw new Error("Missing submission identity or model name.");
|
| 92 |
+
validateScores(summary, "Overall scores");
|
| 93 |
+
for (const [domain, count] of Object.entries(board.domain_tables)) {
|
| 94 |
+
if (summary.by_domain?.[domain]?.population?.fragments !== count) throw new Error("Incomplete domain population: " + domain);
|
| 95 |
+
validateScores(summary.by_domain[domain], domain);
|
| 96 |
+
}
|
| 97 |
+
const metadata = summary.submission;
|
| 98 |
+
if (metadata.model_type != null && !Object.hasOwn(MODEL_TYPES, metadata.model_type))
|
| 99 |
+
throw new Error("model_type must be VLM, specialist, classical, hybrid or unspecified.");
|
| 100 |
+
if (metadata.parameter_count_b != null &&
|
| 101 |
+
!(typeof metadata.parameter_count_b === "number" && Number.isFinite(metadata.parameter_count_b) && metadata.parameter_count_b > 0))
|
| 102 |
+
throw new Error("parameter_count_b must be a positive number or null.");
|
| 103 |
+
return {...summary, _local: true};
|
| 104 |
+
}
|
plotter.css
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[hidden]{display:none!important}
|
| 2 |
+
.scores{min-width:960px}
|
| 3 |
+
.scores td:nth-child(5){border-right:0}
|
| 4 |
+
.scores td:nth-child(6){border-right:1px solid var(--line)}
|
| 5 |
+
.scores td:nth-child(3),.scores .group-head th.model-type-heading{text-align:left}
|
| 6 |
+
.model-type-badge{display:inline-flex;align-items:center;gap:6px;white-space:nowrap;font-size:10px}
|
| 7 |
+
.model-type-badge i,.plot-legend i{display:inline-block;width:8px;height:8px;background:var(--color);border-radius:50%;flex-shrink:0}
|
| 8 |
+
.plot-controls{display:grid;grid-template-columns:1.25fr 1.25fr 1fr .85fr;gap:14px;margin:24px 0 14px}
|
| 9 |
+
.plot-controls label{font-size:11px;font-weight:600;color:var(--muted);min-width:0}
|
| 10 |
+
.plot-controls select{display:block;width:100%;max-width:none;margin-top:7px;font-size:11px;padding:9px 24px 9px 10px}
|
| 11 |
+
.plot-options{display:flex;align-items:center;justify-content:space-between;gap:15px;margin:0 0 14px;font-size:10px;color:var(--muted)}
|
| 12 |
+
.plot-options label{display:flex;align-items:center;gap:5px}
|
| 13 |
+
.plot-options input{accent-color:var(--ink)}
|
| 14 |
+
.muted-option{opacity:.5}
|
| 15 |
+
.plot-layout{display:grid;grid-template-columns:minmax(0,1fr) 225px;gap:18px}
|
| 16 |
+
.plot-shell{position:relative;min-width:0;border:1px solid var(--line);border-radius:5px;background:var(--white);overflow:hidden}
|
| 17 |
+
.scatter-wrap{overflow-x:auto}
|
| 18 |
+
.scatter-wrap svg{display:block;width:100%;min-width:560px}
|
| 19 |
+
.plot-point{cursor:pointer;transition:r .15s}
|
| 20 |
+
.plot-point:hover,.plot-point:focus{r:10;outline:none;stroke:#193f35;stroke-width:3}
|
| 21 |
+
.plot-empty{position:absolute;inset:24% 9% 26% 16%;padding:15px;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center;background:#fffefaf2;pointer-events:none}
|
| 22 |
+
.plot-empty h3{font:23px/1.2 Georgia,serif;margin:0 0 12px}
|
| 23 |
+
.plot-empty p{font-size:11px;color:var(--muted);max-width:330px;margin:0;line-height:1.7}
|
| 24 |
+
.plot-detail{padding:19px;background:#edf0e780;border:1px solid var(--line);border-radius:5px;overflow-wrap:anywhere}
|
| 25 |
+
.plot-detail .eyebrow{font-size:9px;letter-spacing:1.4px}
|
| 26 |
+
.plot-detail h3{font:21px/1.2 Georgia,serif;margin:18px 0 12px}
|
| 27 |
+
.plot-detail p{font-size:11px;color:var(--muted);line-height:1.7}
|
| 28 |
+
.plot-detail dl{font-size:11px;line-height:1.5;margin:18px 0}
|
| 29 |
+
.plot-detail dt{color:var(--muted);font-size:10px;margin-top:13px}
|
| 30 |
+
.plot-detail dd{margin:3px 0 0;font-weight:600}
|
| 31 |
+
.plot-legend{display:flex;flex-wrap:wrap;gap:9px 15px;padding-top:17px;margin-top:20px;border-top:1px solid var(--line);font-size:10px}
|
| 32 |
+
.plot-legend span{display:flex;align-items:center;gap:6px}
|
| 33 |
+
.plot-legend .local-legend{width:100%;font-size:9px;color:var(--muted);margin-top:5px}
|
| 34 |
+
.local-preview{display:flex;justify-content:space-between;gap:20px;align-items:center;padding:17px 0;border-bottom:1px solid var(--line)}
|
| 35 |
+
.local-preview strong{font-size:12px}
|
| 36 |
+
.local-preview p{margin:5px 0 0;font-size:11px;color:var(--muted);max-width:650px}
|
| 37 |
+
.preview-actions{display:flex;align-items:center;gap:15px;flex-shrink:0}
|
| 38 |
+
.file-button{position:relative;overflow:hidden;cursor:pointer;white-space:nowrap}
|
| 39 |
+
.file-button:focus-within{outline:3px solid var(--accent);outline-offset:4px}
|
| 40 |
+
.file-button input{position:absolute;inset:0;opacity:0;width:100%;cursor:pointer;font-size:0}
|
| 41 |
+
.text-button{background:none;border:0;color:var(--muted);font-size:10px;cursor:pointer;padding:4px}
|
| 42 |
+
.button:disabled{opacity:.45;cursor:default}
|
| 43 |
+
#plot-message{overflow-wrap:anywhere}
|
| 44 |
+
@media(max-width:1000px){.plot-controls{grid-template-columns:1fr 1fr}.plot-layout{grid-template-columns:minmax(0,1fr) 195px}.plot-detail{padding:15px}.local-preview{align-items:start}.preview-actions{flex-direction:column;align-items:end}}
|
| 45 |
+
@media(max-width:760px){.plot-layout{grid-template-columns:1fr}.plot-detail{display:grid;grid-template-columns:1fr 1fr;gap:0 20px}.plot-detail>.eyebrow{grid-column:1/-1}.plot-detail h3{margin-top:12px}.plot-legend{border-top:0;border-left:1px solid var(--line);padding:10px 0 10px 20px;margin:10px 0;align-content:start;gap:10px}.plot-options{flex-wrap:wrap;gap:8px}}
|
| 46 |
+
@media(max-width:640px){header nav{gap:12px}header .brand{max-width:none}.plot-controls{gap:11px}.plot-controls select{font-size:10px}.local-preview{flex-direction:column;gap:13px}.preview-actions{flex-direction:row;align-items:center}.plot-empty{inset:22% 6% 23% 20%}.plot-empty h3{font-size:20px}.plot-detail{grid-template-columns:1fr}.plot-legend{border-left:0;border-top:1px solid var(--line);padding:15px 0 0}.section-top #download-plot{font-size:9px;padding:9px 10px;white-space:nowrap}}
|
plotter.mjs
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import {MODEL_TYPES, SCORE_AXES, axisLabel, modelType, selectPlotPoints, xScale, fraction, validateLocalSummary} from "./plot-data.mjs";
|
| 2 |
+
|
| 3 |
+
const $ = id => document.getElementById(id);
|
| 4 |
+
const NS = "http://www.w3.org/2000/svg";
|
| 5 |
+
const number = value => value >= 0.001 && value < 100000
|
| 6 |
+
? Number(value.toPrecision(4)).toLocaleString("en-US") : value.toExponential(1);
|
| 7 |
+
function svgNode(tag, attributes = {}, text = null) {
|
| 8 |
+
const node = document.createElementNS(NS, tag);
|
| 9 |
+
for (const [key, value] of Object.entries(attributes)) node.setAttribute(key, value);
|
| 10 |
+
if (text !== null) node.textContent = text;
|
| 11 |
+
return node;
|
| 12 |
+
}
|
| 13 |
+
function add(parent, tag, attrs = {}, text = null) {
|
| 14 |
+
const child = svgNode(tag, attrs, text); parent.append(child); return child;
|
| 15 |
+
}
|
| 16 |
+
export function setupPlotter(board) {
|
| 17 |
+
let local = [], currentSVG = null, notice = "";
|
| 18 |
+
for (const axis of SCORE_AXES) {
|
| 19 |
+
for (const id of ["plot-x", "plot-y"]) {
|
| 20 |
+
const option = document.createElement("option"); option.value = axis.key; option.textContent = axis.label; $(id).append(option);
|
| 21 |
+
}
|
| 22 |
+
}
|
| 23 |
+
$("plot-y").value = "macro.rms.f1";
|
| 24 |
+
for (const domain of Object.keys(board.domain_tables)) {
|
| 25 |
+
const option = document.createElement("option"); option.value = domain; option.textContent = domain.replaceAll("_", " "); $("plot-domain").append(option);
|
| 26 |
+
}
|
| 27 |
+
function detail(point, xAxis, yAxis) {
|
| 28 |
+
const root = $("plot-detail"); root.replaceChildren();
|
| 29 |
+
const heading = document.createElement("h3"); heading.textContent = point.record.submission.method_name;
|
| 30 |
+
const type = document.createElement("p"); type.textContent = MODEL_TYPES[point.type].label + " · " +
|
| 31 |
+
(point.record._local ? "Local preview · unreviewed" : "Reviewed submission");
|
| 32 |
+
const values = document.createElement("dl");
|
| 33 |
+
for (const [label, value] of [[axisLabel(xAxis), xAxis === "parameters" ? number(point.x) : point.x.toFixed(2)],
|
| 34 |
+
[axisLabel(yAxis), point.y.toFixed(2)],
|
| 35 |
+
["Evaluation mode", "Physical table · image-only"]]) {
|
| 36 |
+
const term = document.createElement("dt"), definition = document.createElement("dd");
|
| 37 |
+
term.textContent = label; definition.textContent = value; values.append(term, definition);
|
| 38 |
+
}
|
| 39 |
+
const revision = document.createElement("p"); revision.className = "mono"; revision.textContent = point.record.submission.model_revision || "";
|
| 40 |
+
root.append(heading, type, values, revision);
|
| 41 |
+
}
|
| 42 |
+
function render() {
|
| 43 |
+
const xAxis = $("plot-x").value, yAxis = $("plot-y").value, domain = $("plot-domain").value;
|
| 44 |
+
const sizeAxis = xAxis === "parameters";
|
| 45 |
+
$("plot-log").disabled = !sizeAxis;
|
| 46 |
+
$("plot-log").closest("label").classList.toggle("muted-option", !sizeAxis);
|
| 47 |
+
const records = [...board.results, ...local];
|
| 48 |
+
const {points, candidates, omitted} = selectPlotPoints(records, {x: xAxis, y: yAxis, domain, type: $("plot-type").value});
|
| 49 |
+
const scale = xScale(points, xAxis, sizeAxis && $("plot-log").checked);
|
| 50 |
+
const W = 920, H = 460, L = 76, R = 27, T = 28, B = 65, iw = W-L-R, ih = H-T-B;
|
| 51 |
+
const X = value => L + fraction(value, scale) * iw, Y = value => T + (1-value/100) * ih;
|
| 52 |
+
const svg = svgNode("svg", {viewBox: "0 0 920 460", role: "img", "aria-labelledby": "scatter-title scatter-description",
|
| 53 |
+
xmlns: NS, style: "background:#fffefa;font-family:Arial,sans-serif"});
|
| 54 |
+
add(svg, "title", {id: "scatter-title"}, axisLabel(xAxis) + " versus " + axisLabel(yAxis));
|
| 55 |
+
add(svg, "desc", {id: "scatter-description"}, points.length + " model points; color indicates model type. " +
|
| 56 |
+
"All reconstruction scores range from 0 to 100. Local previews use outlined points.");
|
| 57 |
+
for (const value of [0,20,40,60,80,100]) {
|
| 58 |
+
const y = Y(value);
|
| 59 |
+
add(svg,"line",{x1:L,x2:W-R,y1:y,y2:y,stroke:"#e0e5d9","stroke-width":1});
|
| 60 |
+
add(svg,"text",{x:L-12,y:y+4,"text-anchor":"end",fill:"#6b796c","font-size":12},String(value));
|
| 61 |
+
}
|
| 62 |
+
for (const value of scale.ticks) {
|
| 63 |
+
const x = X(value);
|
| 64 |
+
add(svg,"line",{x1:x,x2:x,y1:T,y2:H-B,stroke:"#e8ebdf","stroke-width":1});
|
| 65 |
+
add(svg,"text",{x,y:H-B+23,"text-anchor":"middle",fill:"#6b796c","font-size":12},number(value));
|
| 66 |
+
}
|
| 67 |
+
add(svg,"line",{x1:L,x2:L,y1:T,y2:H-B,stroke:"#82917e","stroke-width":1});
|
| 68 |
+
add(svg,"line",{x1:L,x2:W-R,y1:H-B,y2:H-B,stroke:"#82917e","stroke-width":1});
|
| 69 |
+
add(svg,"text",{x:L+iw/2,y:H-13,"text-anchor":"middle",fill:"#3d5847","font-size":13},
|
| 70 |
+
axisLabel(xAxis) + (scale.log ? " · log scale" : ""));
|
| 71 |
+
add(svg,"text",{transform:"translate(20 "+(T+ih/2)+") rotate(-90)","text-anchor":"middle",fill:"#3d5847","font-size":13},axisLabel(yAxis));
|
| 72 |
+
add(svg,"text",{x:W-R,y:T-10,"text-anchor":"end",fill:"#7a8775","font-size":11},
|
| 73 |
+
sizeAxis ? "Smaller ← · higher score ↑" : "Higher score → ↑");
|
| 74 |
+
const occupied = [], layer = add(svg,"g");
|
| 75 |
+
for (const point of points) {
|
| 76 |
+
const x = X(point.x), y = Y(point.y), color = MODEL_TYPES[point.type].color;
|
| 77 |
+
const circle = add(layer,"circle",{cx:x,cy:y,r:7,fill:point.record._local?"#fffefa":color,stroke:color,
|
| 78 |
+
"stroke-width":point.record._local?3:1.5,tabindex:0,role:"button",
|
| 79 |
+
"aria-label":point.record.submission.method_name + ", " + MODEL_TYPES[point.type].label +
|
| 80 |
+
", " + axisLabel(xAxis) + " " + number(point.x) + ", " + axisLabel(yAxis) + " " + point.y.toFixed(2) +
|
| 81 |
+
(point.record._local ? ", local unreviewed preview" : ", reviewed"),class:"plot-point"});
|
| 82 |
+
add(circle,"title",{},point.record.submission.method_name + " · " + MODEL_TYPES[point.type].label +
|
| 83 |
+
"\n" + axisLabel(xAxis) + ": " + number(point.x) + "\n" + axisLabel(yAxis) + ": " + point.y.toFixed(2));
|
| 84 |
+
const show = () => {detail(point,xAxis,yAxis);};
|
| 85 |
+
circle.addEventListener("pointerenter",show); circle.addEventListener("focus",show); circle.addEventListener("click",show);
|
| 86 |
+
circle.addEventListener("keydown",event=>{if(event.key==="Enter"||event.key===" "){event.preventDefault();show();}});
|
| 87 |
+
const label = point.record.submission.method_name;
|
| 88 |
+
const display = label.length > 28 ? label.slice(0,27)+"…" : label;
|
| 89 |
+
const width = display.length * 6.3;
|
| 90 |
+
const anchor = x+12+width < W-R ? "start" : "end";
|
| 91 |
+
const start = anchor==="start"?x+12:x-12-width;
|
| 92 |
+
const box = {left:start,right:start+width,top:y-10,bottom:y+7};
|
| 93 |
+
const touchesPoint = points.some(other=>other!==point && X(other.x)>box.left-7 && X(other.x)<box.right+7 &&
|
| 94 |
+
Y(other.y)>box.top-7 && Y(other.y)<box.bottom+7);
|
| 95 |
+
if (box.left>L && box.right<W-R && !touchesPoint &&
|
| 96 |
+
!occupied.some(old=>box.left<old.right && box.right>old.left && box.top<old.bottom && box.bottom>old.top)) {
|
| 97 |
+
add(svg,"text",{x:anchor==="start"?x+12:x-12,y:y+4,"text-anchor":anchor,fill:"#3d5847","font-size":12,"pointer-events":"none"},display);
|
| 98 |
+
occupied.push(box);
|
| 99 |
+
}
|
| 100 |
+
}
|
| 101 |
+
$("plot-chart").replaceChildren(svg); currentSVG = svg;
|
| 102 |
+
$("plot-count").textContent = points.length + " plotted · " + board.results.length + " reviewed · " + local.length + " local";
|
| 103 |
+
$("plot-message").textContent = [notice, omitted ? omitted + " of " + candidates +
|
| 104 |
+
" matching submissions omitted: missing " + (sizeAxis ? "parameter count or selected score." : "selected score.") : ""].filter(Boolean).join(" ");
|
| 105 |
+
const empty = $("plot-empty"); empty.hidden = points.length > 0;
|
| 106 |
+
if (!points.length) {
|
| 107 |
+
empty.querySelector("h3").textContent = records.length ? "No points for this view." : "Ready to compare the first models.";
|
| 108 |
+
empty.querySelector("p").textContent = sizeAxis && records.length
|
| 109 |
+
? "Choose a score for the horizontal axis to include models without a declared parameter count, or change the filters."
|
| 110 |
+
: records.length ? "Try another domain or model type." : "Reviewed submissions will appear here. You can also preview your saved evaluation summaries below.";
|
| 111 |
+
}
|
| 112 |
+
$("download-plot").disabled = !points.length; $("clear-local").hidden = !local.length;
|
| 113 |
+
$("plot-detail").replaceChildren();
|
| 114 |
+
const heading = document.createElement("h3"), text = document.createElement("p");
|
| 115 |
+
heading.textContent = "Select a point"; text.textContent = "Hover, tap, or focus a model to see its type, exact scores, and evaluation mode.";
|
| 116 |
+
$("plot-detail").append(heading,text);
|
| 117 |
+
}
|
| 118 |
+
for (const id of ["plot-x","plot-y","plot-domain","plot-type","plot-log"]) $(id).addEventListener("change",render);
|
| 119 |
+
$("plot-files").addEventListener("change", async event => {
|
| 120 |
+
if (event.target.files.length > 50) {notice = "Load at most 50 summaries at a time."; render(); return;}
|
| 121 |
+
let loaded = 0; const errors = [];
|
| 122 |
+
for (const file of event.target.files) {
|
| 123 |
+
try {
|
| 124 |
+
if (file.size > 5*1024*1024) throw new Error("Summary file exceeds 5 MB.");
|
| 125 |
+
const summary = validateLocalSummary(JSON.parse(await file.text()),board);
|
| 126 |
+
local = local.filter(item=>item.submission.submission_id!==summary.submission.submission_id);
|
| 127 |
+
local.push(summary); loaded++;
|
| 128 |
+
} catch(error) {errors.push(file.name+": "+error.message);}
|
| 129 |
+
}
|
| 130 |
+
notice = [loaded ? loaded+" local preview"+(loaded===1?"":"s")+" loaded; rankings are unchanged." : "",...errors].join(" ");
|
| 131 |
+
event.target.value = ""; render();
|
| 132 |
+
});
|
| 133 |
+
$("clear-local").addEventListener("click",()=>{local=[];notice="Local previews cleared.";render();});
|
| 134 |
+
$("download-plot").addEventListener("click",()=>{
|
| 135 |
+
if (!currentSVG) return;
|
| 136 |
+
const clone = currentSVG.cloneNode(true);
|
| 137 |
+
clone.setAttribute("viewBox","0 0 920 490");
|
| 138 |
+
clone.querySelectorAll("[tabindex]").forEach(point=>{point.removeAttribute("tabindex");point.removeAttribute("role");});
|
| 139 |
+
const subtitle = svgNode("text",{x:76,y:17,fill:"#6b796c","font-size":10},$("plot-count").textContent+
|
| 140 |
+
" · "+($("plot-domain").value==="all"?"All domains":$("plot-domain").value.replaceAll("_"," "))+
|
| 141 |
+
(local.length ? " · local previews are unreviewed" : ""));
|
| 142 |
+
clone.append(subtitle);
|
| 143 |
+
const key = svgNode("text",{x:76,y:482,fill:"#6b796c","font-size":10},
|
| 144 |
+
"VLM: blue · Specialist: green · Classical: orange · Hybrid: purple · Unspecified: gray · Outlined: local");
|
| 145 |
+
clone.append(key);
|
| 146 |
+
const blob = new Blob([new XMLSerializer().serializeToString(clone)],{type:"image/svg+xml;charset=utf-8"});
|
| 147 |
+
const url = URL.createObjectURL(blob), link = document.createElement("a");
|
| 148 |
+
link.href=url; link.download="scientific-table-reconstruction-comparison.svg"; link.click();
|
| 149 |
+
setTimeout(()=>URL.revokeObjectURL(url),1000);
|
| 150 |
+
});
|
| 151 |
+
render();
|
| 152 |
+
}
|