jdsouza's picture
Launch free static RMS and Exact Mapping Match leaderboard
9fe20a2 verified
Raw History Blame Contribute Delete
5.77 kB
"""Versioned scoring of complete Step 2 prediction sets."""
from collections import defaultdict
import hashlib
import json
from pathlib import Path
from metric.grid_scoring import aggregate_grid_results, prepare_grid, score_table_grids
PROTOCOL = "tabulusbench-step2-v1"
METRICS = ("rms", "exact-mapping")
EVALUATION_MODE = "physical-table"
INPUT_MODE = "image-only"
def canonical(value):
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
def digest(value):
return hashlib.sha256(canonical(value)).hexdigest()
def key(row):
return row["paper_id"], row["table_id"]
def check_grid(value):
if not isinstance(value, list) or any(
not isinstance(row, list) or any(not isinstance(cell, str) for cell in row) for row in value
):
raise ValueError("rows must be a list of lists of strings")
if len(value) > 10000 or sum(len(row) for row in value) > 1_000_000:
raise ValueError("Prediction exceeds the documented table-size limit")
return value
def validate_submission(records, expected_keys):
predictions = {}
for position, item in enumerate(records, 1):
if not isinstance(item, dict) or set(item) != {"paper_id", "table_id", "status", "rows"}:
raise ValueError(f"Prediction {position} needs exactly paper_id, table_id, status and rows")
if not all(isinstance(item[field], str) and item[field] for field in ("paper_id", "table_id")):
raise ValueError(f"Prediction {position} has an invalid identity")
identity = key(item)
if identity not in expected_keys:
raise ValueError(f"Unknown prediction identity {identity}")
if identity in predictions:
raise ValueError(f"Duplicate prediction identity {identity}")
if item["status"] not in {"ok", "empty", "error"}:
raise ValueError(f"Invalid status for {identity}")
grid = check_grid(item["rows"])
if item["status"] != "ok" and grid:
raise ValueError(f"Failed prediction {identity} must have rows=[]")
predictions[identity] = item
missing = expected_keys - predictions.keys()
if missing:
raise ValueError(f"{len(missing)} missing predictions; first {sorted(missing)[0]}. Use explicit empty/error records.")
return predictions
def validate_metadata(metadata, manifest):
required = ("submission_id", "method_name", "model_revision", "code_url", "code_revision", "prompt",
"parser", "generation_settings", "training_disclosure", "dataset_fingerprint",
"evaluation_mode", "input_mode")
if not isinstance(metadata, dict) or any(
not isinstance(metadata.get(field), str) or not metadata[field].strip() for field in required
):
raise ValueError("Missing submission metadata. See metadata.example.json")
if metadata["dataset_fingerprint"] != manifest["dataset_fingerprint"]:
raise ValueError("Submission targets a different dataset fingerprint")
if metadata["evaluation_mode"] != EVALUATION_MODE or metadata["input_mode"] != INPUT_MODE:
raise ValueError("v1 supports physical-table evaluation with image-only model inputs")
if not metadata["submission_id"].isascii() or any(
char not in "abcdefghijklmnopqrstuvwxyz0123456789-_" for char in metadata["submission_id"]
):
raise ValueError("submission_id must contain lowercase ASCII letters, digits, hyphens or underscores")
def evaluate(gold, records, metadata, manifest, progress=None):
gold = sorted(gold, key=key)
expected_keys = {key(row) for row in gold}
if len(expected_keys) != len(gold) or not gold:
raise ValueError("Gold identities must be unique and nonempty")
validate_metadata(metadata, manifest)
predictions = validate_submission(records, expected_keys)
results = []
by_domain = defaultdict(list)
for index, target in enumerate(gold, 1):
prediction = predictions[key(target)]
observed = prepare_grid(prediction["rows"])
status = "scored"
if prediction["status"] == "error":
status = "unusable"
elif prediction["status"] == "empty" or not observed.nonempty_cells:
status = "empty"
scores = score_table_grids(
target["gold_rows"], observed, metrics=METRICS,
text_threshold=0.5, number_threshold=0.1, status=status,
)
result = {
"paper_id": target["paper_id"], "table_id": target["table_id"], "domain": target["domain"],
"submitted_status": prediction["status"], **scores,
}
results.append(result)
by_domain[target["domain"]].append(result)
if progress:
progress(index, len(gold))
summary = {
"protocol_version": PROTOCOL, "dataset_version": manifest["version"],
"dataset_fingerprint": manifest["dataset_fingerprint"],
"evaluation_mode": EVALUATION_MODE, "input_mode": INPUT_MODE,
"score_scale": "[0,100]", "ranking": "macro.rms.f1",
"submission": metadata, "prediction_sha256": digest(sorted(records, key=key)),
**aggregate_grid_results(results),
"by_domain": {domain: aggregate_grid_results(rows) for domain, rows in sorted(by_domain.items())},
}
return summary, results
def read_jsonl(path):
records = []
with Path(path).open(encoding="utf-8-sig") as handle:
for number, line in enumerate(handle, 1):
if line.strip():
try:
records.append(json.loads(line))
except json.JSONDecodeError as error:
raise ValueError(f"Malformed JSON on line {number}") from error
return records