File size: 3,180 Bytes
251b517 9fe20a2 251b517 9fe20a2 251b517 9fe20a2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Score complete Scientific Table Reconstruction submissions locally on CPU."""
import argparse
from importlib.metadata import version
import json
from pathlib import Path
import sys
import hashlib
from core import digest, evaluate, read_jsonl
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset", default="SciKnowOrg/Scientific-Table-Reconstruction",
help="HF dataset ID or local directory containing manifest.json and data/*.parquet")
parser.add_argument("--revision", default="v1.0.1")
parser.add_argument("--predictions", type=Path, required=True)
parser.add_argument("--metadata", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
if args.out.exists() and any(args.out.iterdir()):
parser.error("--out must be an empty or new directory")
directory = Path(args.dataset)
if not directory.is_dir():
from huggingface_hub import snapshot_download
directory = Path(snapshot_download(
args.dataset, repo_type="dataset", revision=args.revision,
allow_patterns=["manifest.json", "data/*.parquet"],
))
manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
implementation = manifest["metric"]["implementation_sha256"]
for name, expected_hash in implementation.items():
path = Path(__file__).resolve().parent / name
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash:
parser.error(f"Scoring implementation differs from the frozen protocol: {name}")
import pyarrow.parquet as pq
rows = []
for path in sorted((directory / "data").glob("*.parquet")):
rows.extend(pq.read_table(path).to_pylist())
content = [{
"domain": row["domain"], "paper_id": row["paper_id"], "table_id": row["table_id"],
"image_sha256": hashlib.sha256(row["image"]["bytes"]).hexdigest(),
"gold_sha256": digest(row["gold_rows"]),
} for row in rows]
if digest(content) != manifest["dataset_fingerprint"]:
parser.error("Dataset content does not match its manifest")
metadata = json.loads(args.metadata.read_text(encoding="utf-8-sig"))
records = read_jsonl(args.predictions)
summary, tables = evaluate(
rows, records, metadata, manifest,
progress=lambda done, total: print(f"Scored {done}/{total}", file=sys.stderr) if done % 50 == 0 else None,
)
summary["implementation"] = {
"sha256": implementation,
"packages": {name: version(name) for name in ("numpy", "scipy", "rapidfuzz", "pyarrow")},
}
args.out.mkdir(parents=True, exist_ok=True)
(args.out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(args.out / "per_table.jsonl").write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in tables), encoding="utf-8",
)
print(json.dumps({"macro": summary["macro"], "micro": summary["micro"], "coverage": summary["coverage"]}, indent=2))
if __name__ == "__main__":
main()
|