File size: 5,769 Bytes
9fe20a2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """Versioned scoring of complete Step 2 prediction sets."""
from collections import defaultdict
import hashlib
import json
from pathlib import Path
from metric.grid_scoring import aggregate_grid_results, prepare_grid, score_table_grids
PROTOCOL = "tabulusbench-step2-v1"
METRICS = ("rms", "exact-mapping")
EVALUATION_MODE = "physical-table"
INPUT_MODE = "image-only"
def canonical(value):
return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
def digest(value):
return hashlib.sha256(canonical(value)).hexdigest()
def key(row):
return row["paper_id"], row["table_id"]
def check_grid(value):
if not isinstance(value, list) or any(
not isinstance(row, list) or any(not isinstance(cell, str) for cell in row) for row in value
):
raise ValueError("rows must be a list of lists of strings")
if len(value) > 10000 or sum(len(row) for row in value) > 1_000_000:
raise ValueError("Prediction exceeds the documented table-size limit")
return value
def validate_submission(records, expected_keys):
predictions = {}
for position, item in enumerate(records, 1):
if not isinstance(item, dict) or set(item) != {"paper_id", "table_id", "status", "rows"}:
raise ValueError(f"Prediction {position} needs exactly paper_id, table_id, status and rows")
if not all(isinstance(item[field], str) and item[field] for field in ("paper_id", "table_id")):
raise ValueError(f"Prediction {position} has an invalid identity")
identity = key(item)
if identity not in expected_keys:
raise ValueError(f"Unknown prediction identity {identity}")
if identity in predictions:
raise ValueError(f"Duplicate prediction identity {identity}")
if item["status"] not in {"ok", "empty", "error"}:
raise ValueError(f"Invalid status for {identity}")
grid = check_grid(item["rows"])
if item["status"] != "ok" and grid:
raise ValueError(f"Failed prediction {identity} must have rows=[]")
predictions[identity] = item
missing = expected_keys - predictions.keys()
if missing:
raise ValueError(f"{len(missing)} missing predictions; first {sorted(missing)[0]}. Use explicit empty/error records.")
return predictions
def validate_metadata(metadata, manifest):
required = ("submission_id", "method_name", "model_revision", "code_url", "code_revision", "prompt",
"parser", "generation_settings", "training_disclosure", "dataset_fingerprint",
"evaluation_mode", "input_mode")
if not isinstance(metadata, dict) or any(
not isinstance(metadata.get(field), str) or not metadata[field].strip() for field in required
):
raise ValueError("Missing submission metadata. See metadata.example.json")
if metadata["dataset_fingerprint"] != manifest["dataset_fingerprint"]:
raise ValueError("Submission targets a different dataset fingerprint")
if metadata["evaluation_mode"] != EVALUATION_MODE or metadata["input_mode"] != INPUT_MODE:
raise ValueError("v1 supports physical-table evaluation with image-only model inputs")
if not metadata["submission_id"].isascii() or any(
char not in "abcdefghijklmnopqrstuvwxyz0123456789-_" for char in metadata["submission_id"]
):
raise ValueError("submission_id must contain lowercase ASCII letters, digits, hyphens or underscores")
def evaluate(gold, records, metadata, manifest, progress=None):
gold = sorted(gold, key=key)
expected_keys = {key(row) for row in gold}
if len(expected_keys) != len(gold) or not gold:
raise ValueError("Gold identities must be unique and nonempty")
validate_metadata(metadata, manifest)
predictions = validate_submission(records, expected_keys)
results = []
by_domain = defaultdict(list)
for index, target in enumerate(gold, 1):
prediction = predictions[key(target)]
observed = prepare_grid(prediction["rows"])
status = "scored"
if prediction["status"] == "error":
status = "unusable"
elif prediction["status"] == "empty" or not observed.nonempty_cells:
status = "empty"
scores = score_table_grids(
target["gold_rows"], observed, metrics=METRICS,
text_threshold=0.5, number_threshold=0.1, status=status,
)
result = {
"paper_id": target["paper_id"], "table_id": target["table_id"], "domain": target["domain"],
"submitted_status": prediction["status"], **scores,
}
results.append(result)
by_domain[target["domain"]].append(result)
if progress:
progress(index, len(gold))
summary = {
"protocol_version": PROTOCOL, "dataset_version": manifest["version"],
"dataset_fingerprint": manifest["dataset_fingerprint"],
"evaluation_mode": EVALUATION_MODE, "input_mode": INPUT_MODE,
"score_scale": "[0,100]", "ranking": "macro.rms.f1",
"submission": metadata, "prediction_sha256": digest(sorted(records, key=key)),
**aggregate_grid_results(results),
"by_domain": {domain: aggregate_grid_results(rows) for domain, rows in sorted(by_domain.items())},
}
return summary, results
def read_jsonl(path):
records = []
with Path(path).open(encoding="utf-8-sig") as handle:
for number, line in enumerate(handle, 1):
if line.strip():
try:
records.append(json.loads(line))
except json.JSONDecodeError as error:
raise ValueError(f"Malformed JSON on line {number}") from error
return records
|