File size: 3,180 Bytes
251b517
9fe20a2
 
 
 
 
 
 
 
 
 
 
 
251b517
9fe20a2
251b517
9fe20a2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""Score complete Scientific Table Reconstruction submissions locally on CPU."""
import argparse
from importlib.metadata import version
import json
from pathlib import Path
import sys
import hashlib

from core import digest, evaluate, read_jsonl


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--dataset", default="SciKnowOrg/Scientific-Table-Reconstruction",
                        help="HF dataset ID or local directory containing manifest.json and data/*.parquet")
    parser.add_argument("--revision", default="v1.0.1")
    parser.add_argument("--predictions", type=Path, required=True)
    parser.add_argument("--metadata", type=Path, required=True)
    parser.add_argument("--out", type=Path, required=True)
    args = parser.parse_args()
    if args.out.exists() and any(args.out.iterdir()):
        parser.error("--out must be an empty or new directory")
    directory = Path(args.dataset)
    if not directory.is_dir():
        from huggingface_hub import snapshot_download
        directory = Path(snapshot_download(
            args.dataset, repo_type="dataset", revision=args.revision,
            allow_patterns=["manifest.json", "data/*.parquet"],
        ))
    manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
    implementation = manifest["metric"]["implementation_sha256"]
    for name, expected_hash in implementation.items():
        path = Path(__file__).resolve().parent / name
        if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash:
            parser.error(f"Scoring implementation differs from the frozen protocol: {name}")
    import pyarrow.parquet as pq
    rows = []
    for path in sorted((directory / "data").glob("*.parquet")):
        rows.extend(pq.read_table(path).to_pylist())
    content = [{
        "domain": row["domain"], "paper_id": row["paper_id"], "table_id": row["table_id"],
        "image_sha256": hashlib.sha256(row["image"]["bytes"]).hexdigest(),
        "gold_sha256": digest(row["gold_rows"]),
    } for row in rows]
    if digest(content) != manifest["dataset_fingerprint"]:
        parser.error("Dataset content does not match its manifest")
    metadata = json.loads(args.metadata.read_text(encoding="utf-8-sig"))
    records = read_jsonl(args.predictions)
    summary, tables = evaluate(
        rows, records, metadata, manifest,
        progress=lambda done, total: print(f"Scored {done}/{total}", file=sys.stderr) if done % 50 == 0 else None,
    )
    summary["implementation"] = {
        "sha256": implementation,
        "packages": {name: version(name) for name in ("numpy", "scipy", "rapidfuzz", "pyarrow")},
    }
    args.out.mkdir(parents=True, exist_ok=True)
    (args.out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    (args.out / "per_table.jsonl").write_text(
        "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in tables), encoding="utf-8",
    )
    print(json.dumps({"macro": summary["macro"], "micro": summary["micro"], "coverage": summary["coverage"]}, indent=2))


if __name__ == "__main__":
    main()