File size: 5,769 Bytes
9fe20a2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Versioned scoring of complete Step 2 prediction sets."""
from collections import defaultdict
import hashlib
import json
from pathlib import Path

from metric.grid_scoring import aggregate_grid_results, prepare_grid, score_table_grids

PROTOCOL = "tabulusbench-step2-v1"
METRICS = ("rms", "exact-mapping")
EVALUATION_MODE = "physical-table"
INPUT_MODE = "image-only"


def canonical(value):
    return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")


def digest(value):
    return hashlib.sha256(canonical(value)).hexdigest()


def key(row):
    return row["paper_id"], row["table_id"]


def check_grid(value):
    if not isinstance(value, list) or any(
        not isinstance(row, list) or any(not isinstance(cell, str) for cell in row) for row in value
    ):
        raise ValueError("rows must be a list of lists of strings")
    if len(value) > 10000 or sum(len(row) for row in value) > 1_000_000:
        raise ValueError("Prediction exceeds the documented table-size limit")
    return value


def validate_submission(records, expected_keys):
    predictions = {}
    for position, item in enumerate(records, 1):
        if not isinstance(item, dict) or set(item) != {"paper_id", "table_id", "status", "rows"}:
            raise ValueError(f"Prediction {position} needs exactly paper_id, table_id, status and rows")
        if not all(isinstance(item[field], str) and item[field] for field in ("paper_id", "table_id")):
            raise ValueError(f"Prediction {position} has an invalid identity")
        identity = key(item)
        if identity not in expected_keys:
            raise ValueError(f"Unknown prediction identity {identity}")
        if identity in predictions:
            raise ValueError(f"Duplicate prediction identity {identity}")
        if item["status"] not in {"ok", "empty", "error"}:
            raise ValueError(f"Invalid status for {identity}")
        grid = check_grid(item["rows"])
        if item["status"] != "ok" and grid:
            raise ValueError(f"Failed prediction {identity} must have rows=[]")
        predictions[identity] = item
    missing = expected_keys - predictions.keys()
    if missing:
        raise ValueError(f"{len(missing)} missing predictions; first {sorted(missing)[0]}. Use explicit empty/error records.")
    return predictions


def validate_metadata(metadata, manifest):
    required = ("submission_id", "method_name", "model_revision", "code_url", "code_revision", "prompt",
                "parser", "generation_settings", "training_disclosure", "dataset_fingerprint",
                "evaluation_mode", "input_mode")
    if not isinstance(metadata, dict) or any(
        not isinstance(metadata.get(field), str) or not metadata[field].strip() for field in required
    ):
        raise ValueError("Missing submission metadata. See metadata.example.json")
    if metadata["dataset_fingerprint"] != manifest["dataset_fingerprint"]:
        raise ValueError("Submission targets a different dataset fingerprint")
    if metadata["evaluation_mode"] != EVALUATION_MODE or metadata["input_mode"] != INPUT_MODE:
        raise ValueError("v1 supports physical-table evaluation with image-only model inputs")
    if not metadata["submission_id"].isascii() or any(
        char not in "abcdefghijklmnopqrstuvwxyz0123456789-_" for char in metadata["submission_id"]
    ):
        raise ValueError("submission_id must contain lowercase ASCII letters, digits, hyphens or underscores")


def evaluate(gold, records, metadata, manifest, progress=None):
    gold = sorted(gold, key=key)
    expected_keys = {key(row) for row in gold}
    if len(expected_keys) != len(gold) or not gold:
        raise ValueError("Gold identities must be unique and nonempty")
    validate_metadata(metadata, manifest)
    predictions = validate_submission(records, expected_keys)
    results = []
    by_domain = defaultdict(list)
    for index, target in enumerate(gold, 1):
        prediction = predictions[key(target)]
        observed = prepare_grid(prediction["rows"])
        status = "scored"
        if prediction["status"] == "error":
            status = "unusable"
        elif prediction["status"] == "empty" or not observed.nonempty_cells:
            status = "empty"
        scores = score_table_grids(
            target["gold_rows"], observed, metrics=METRICS,
            text_threshold=0.5, number_threshold=0.1, status=status,
        )
        result = {
            "paper_id": target["paper_id"], "table_id": target["table_id"], "domain": target["domain"],
            "submitted_status": prediction["status"], **scores,
        }
        results.append(result)
        by_domain[target["domain"]].append(result)
        if progress:
            progress(index, len(gold))
    summary = {
        "protocol_version": PROTOCOL, "dataset_version": manifest["version"],
        "dataset_fingerprint": manifest["dataset_fingerprint"],
        "evaluation_mode": EVALUATION_MODE, "input_mode": INPUT_MODE,
        "score_scale": "[0,100]", "ranking": "macro.rms.f1",
        "submission": metadata, "prediction_sha256": digest(sorted(records, key=key)),
        **aggregate_grid_results(results),
        "by_domain": {domain: aggregate_grid_results(rows) for domain, rows in sorted(by_domain.items())},
    }
    return summary, results


def read_jsonl(path):
    records = []
    with Path(path).open(encoding="utf-8-sig") as handle:
        for number, line in enumerate(handle, 1):
            if line.strip():
                try:
                    records.append(json.loads(line))
                except json.JSONDecodeError as error:
                    raise ValueError(f"Malformed JSON on line {number}") from error
    return records