"""Score complete Scientific Table Reconstruction submissions locally on CPU.""" import argparse from importlib.metadata import version import json from pathlib import Path import sys import hashlib from core import digest, evaluate, read_jsonl def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset", default="SciKnowOrg/Scientific-Table-Reconstruction", help="HF dataset ID or local directory containing manifest.json and data/*.parquet") parser.add_argument("--revision", default="v1.0.1") parser.add_argument("--predictions", type=Path, required=True) parser.add_argument("--metadata", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) args = parser.parse_args() if args.out.exists() and any(args.out.iterdir()): parser.error("--out must be an empty or new directory") directory = Path(args.dataset) if not directory.is_dir(): from huggingface_hub import snapshot_download directory = Path(snapshot_download( args.dataset, repo_type="dataset", revision=args.revision, allow_patterns=["manifest.json", "data/*.parquet"], )) manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) implementation = manifest["metric"]["implementation_sha256"] for name, expected_hash in implementation.items(): path = Path(__file__).resolve().parent / name if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash: parser.error(f"Scoring implementation differs from the frozen protocol: {name}") import pyarrow.parquet as pq rows = [] for path in sorted((directory / "data").glob("*.parquet")): rows.extend(pq.read_table(path).to_pylist()) content = [{ "domain": row["domain"], "paper_id": row["paper_id"], "table_id": row["table_id"], "image_sha256": hashlib.sha256(row["image"]["bytes"]).hexdigest(), "gold_sha256": digest(row["gold_rows"]), } for row in rows] if digest(content) != manifest["dataset_fingerprint"]: parser.error("Dataset content does not match its manifest") metadata = json.loads(args.metadata.read_text(encoding="utf-8-sig")) records = read_jsonl(args.predictions) summary, tables = evaluate( rows, records, metadata, manifest, progress=lambda done, total: print(f"Scored {done}/{total}", file=sys.stderr) if done % 50 == 0 else None, ) summary["implementation"] = { "sha256": implementation, "packages": {name: version(name) for name in ("numpy", "scipy", "rapidfuzz", "pyarrow")}, } args.out.mkdir(parents=True, exist_ok=True) (args.out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (args.out / "per_table.jsonl").write_text( "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in tables), encoding="utf-8", ) print(json.dumps({"macro": summary["macro"], "micro": summary["micro"], "coverage": summary["coverage"]}, indent=2)) if __name__ == "__main__": main()