"""Versioned scoring of complete Step 2 prediction sets.""" from collections import defaultdict import hashlib import json from pathlib import Path from metric.grid_scoring import aggregate_grid_results, prepare_grid, score_table_grids PROTOCOL = "tabulusbench-step2-v1" METRICS = ("rms", "exact-mapping") EVALUATION_MODE = "physical-table" INPUT_MODE = "image-only" def canonical(value): return json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8") def digest(value): return hashlib.sha256(canonical(value)).hexdigest() def key(row): return row["paper_id"], row["table_id"] def check_grid(value): if not isinstance(value, list) or any( not isinstance(row, list) or any(not isinstance(cell, str) for cell in row) for row in value ): raise ValueError("rows must be a list of lists of strings") if len(value) > 10000 or sum(len(row) for row in value) > 1_000_000: raise ValueError("Prediction exceeds the documented table-size limit") return value def validate_submission(records, expected_keys): predictions = {} for position, item in enumerate(records, 1): if not isinstance(item, dict) or set(item) != {"paper_id", "table_id", "status", "rows"}: raise ValueError(f"Prediction {position} needs exactly paper_id, table_id, status and rows") if not all(isinstance(item[field], str) and item[field] for field in ("paper_id", "table_id")): raise ValueError(f"Prediction {position} has an invalid identity") identity = key(item) if identity not in expected_keys: raise ValueError(f"Unknown prediction identity {identity}") if identity in predictions: raise ValueError(f"Duplicate prediction identity {identity}") if item["status"] not in {"ok", "empty", "error"}: raise ValueError(f"Invalid status for {identity}") grid = check_grid(item["rows"]) if item["status"] != "ok" and grid: raise ValueError(f"Failed prediction {identity} must have rows=[]") predictions[identity] = item missing = expected_keys - predictions.keys() if missing: raise ValueError(f"{len(missing)} missing predictions; first {sorted(missing)[0]}. Use explicit empty/error records.") return predictions def validate_metadata(metadata, manifest): required = ("submission_id", "method_name", "model_revision", "code_url", "code_revision", "prompt", "parser", "generation_settings", "training_disclosure", "dataset_fingerprint", "evaluation_mode", "input_mode") if not isinstance(metadata, dict) or any( not isinstance(metadata.get(field), str) or not metadata[field].strip() for field in required ): raise ValueError("Missing submission metadata. See metadata.example.json") if metadata["dataset_fingerprint"] != manifest["dataset_fingerprint"]: raise ValueError("Submission targets a different dataset fingerprint") if metadata["evaluation_mode"] != EVALUATION_MODE or metadata["input_mode"] != INPUT_MODE: raise ValueError("v1 supports physical-table evaluation with image-only model inputs") if not metadata["submission_id"].isascii() or any( char not in "abcdefghijklmnopqrstuvwxyz0123456789-_" for char in metadata["submission_id"] ): raise ValueError("submission_id must contain lowercase ASCII letters, digits, hyphens or underscores") def evaluate(gold, records, metadata, manifest, progress=None): gold = sorted(gold, key=key) expected_keys = {key(row) for row in gold} if len(expected_keys) != len(gold) or not gold: raise ValueError("Gold identities must be unique and nonempty") validate_metadata(metadata, manifest) predictions = validate_submission(records, expected_keys) results = [] by_domain = defaultdict(list) for index, target in enumerate(gold, 1): prediction = predictions[key(target)] observed = prepare_grid(prediction["rows"]) status = "scored" if prediction["status"] == "error": status = "unusable" elif prediction["status"] == "empty" or not observed.nonempty_cells: status = "empty" scores = score_table_grids( target["gold_rows"], observed, metrics=METRICS, text_threshold=0.5, number_threshold=0.1, status=status, ) result = { "paper_id": target["paper_id"], "table_id": target["table_id"], "domain": target["domain"], "submitted_status": prediction["status"], **scores, } results.append(result) by_domain[target["domain"]].append(result) if progress: progress(index, len(gold)) summary = { "protocol_version": PROTOCOL, "dataset_version": manifest["version"], "dataset_fingerprint": manifest["dataset_fingerprint"], "evaluation_mode": EVALUATION_MODE, "input_mode": INPUT_MODE, "score_scale": "[0,100]", "ranking": "macro.rms.f1", "submission": metadata, "prediction_sha256": digest(sorted(records, key=key)), **aggregate_grid_results(results), "by_domain": {domain: aggregate_grid_results(rows) for domain, rows in sorted(by_domain.items())}, } return summary, results def read_jsonl(path): records = [] with Path(path).open(encoding="utf-8-sig") as handle: for number, line in enumerate(handle, 1): if line.strip(): try: records.append(json.loads(line)) except json.JSONDecodeError as error: raise ValueError(f"Malformed JSON on line {number}") from error return records