Download evaluation/bench.py from SciKnowOrg/Scientific-Table-Reconstruction: direct link, hf CLI and curl.
- Browser
- Download file 3.18 kB
-
https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/resolve/main/evaluation/bench.py
- Command line
-
hf download hf://spaces/SciKnowOrg/Scientific-Table-Reconstruction/evaluation/bench.py
-
curl -L -o bench.py https://huggingface.co/spaces/SciKnowOrg/Scientific-Table-Reconstruction/resolve/main/evaluation/bench.py
3.18 kB
| """Score complete Scientific Table Reconstruction submissions locally on CPU.""" | |
| import argparse | |
| from importlib.metadata import version | |
| import json | |
| from pathlib import Path | |
| import sys | |
| import hashlib | |
| from core import digest, evaluate, read_jsonl | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--dataset", default="SciKnowOrg/Scientific-Table-Reconstruction", | |
| help="HF dataset ID or local directory containing manifest.json and data/*.parquet") | |
| parser.add_argument("--revision", default="v1.0.1") | |
| parser.add_argument("--predictions", type=Path, required=True) | |
| parser.add_argument("--metadata", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| args = parser.parse_args() | |
| if args.out.exists() and any(args.out.iterdir()): | |
| parser.error("--out must be an empty or new directory") | |
| directory = Path(args.dataset) | |
| if not directory.is_dir(): | |
| from huggingface_hub import snapshot_download | |
| directory = Path(snapshot_download( | |
| args.dataset, repo_type="dataset", revision=args.revision, | |
| allow_patterns=["manifest.json", "data/*.parquet"], | |
| )) | |
| manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) | |
| implementation = manifest["metric"]["implementation_sha256"] | |
| for name, expected_hash in implementation.items(): | |
| path = Path(__file__).resolve().parent / name | |
| if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash: | |
| parser.error(f"Scoring implementation differs from the frozen protocol: {name}") | |
| import pyarrow.parquet as pq | |
| rows = [] | |
| for path in sorted((directory / "data").glob("*.parquet")): | |
| rows.extend(pq.read_table(path).to_pylist()) | |
| content = [{ | |
| "domain": row["domain"], "paper_id": row["paper_id"], "table_id": row["table_id"], | |
| "image_sha256": hashlib.sha256(row["image"]["bytes"]).hexdigest(), | |
| "gold_sha256": digest(row["gold_rows"]), | |
| } for row in rows] | |
| if digest(content) != manifest["dataset_fingerprint"]: | |
| parser.error("Dataset content does not match its manifest") | |
| metadata = json.loads(args.metadata.read_text(encoding="utf-8-sig")) | |
| records = read_jsonl(args.predictions) | |
| summary, tables = evaluate( | |
| rows, records, metadata, manifest, | |
| progress=lambda done, total: print(f"Scored {done}/{total}", file=sys.stderr) if done % 50 == 0 else None, | |
| ) | |
| summary["implementation"] = { | |
| "sha256": implementation, | |
| "packages": {name: version(name) for name in ("numpy", "scipy", "rapidfuzz", "pyarrow")}, | |
| } | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| (args.out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (args.out / "per_table.jsonl").write_text( | |
| "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in tables), encoding="utf-8", | |
| ) | |
| print(json.dumps({"macro": summary["macro"], "micro": summary["micro"], "coverage": summary["coverage"]}, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |