File size: 2,031 Bytes
4f675c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | """Summarise a ParseBench run the way the official leaderboard does, next to local open-source parsers.
python scripts/parsebench_summary.py output/docparse [leaderboard.csv]
Headline metric per dimension (parse_bench.analysis.aggregation_report._DEFAULT_METRICS);
Overall is the mean of the five dimensions.
"""
import csv
import json
import sys
from pathlib import Path
HEADLINE = {"table": ("Tables", "grits_trm_composite"), "chart": ("Charts", "rule_pass_rate"),
"text_content": ("Content_Faithfulness", "content_faithfulness"),
"text_formatting": ("Semantic_Formatting", "semantic_formatting"),
"layout": ("Visual_Grounding", "layout_element_rule_pass_rate")}
def summarise(run_dir):
run_dir = Path(run_dir)
out = {}
for cat, (col, metric) in HEADLINE.items():
report = run_dir / cat / "_evaluation_report.json"
if report.exists():
d = json.loads(report.read_text())
m = d["aggregate_metrics"]
out[col] = {"score": round(100 * m.get(f"avg_{metric}", m.get("avg_rule_pass_rate", 0.0)), 2),
"documents": d["total_examples"], "failed": d.get("failed", 0)}
if len(out) == 5:
out["Overall"] = round(sum(v["score"] for v in out.values()) / 5, 2)
return out
def main():
res = summarise(sys.argv[1])
print(json.dumps(res, indent=2))
if len(sys.argv) > 2:
cols = [c for c, _ in HEADLINE.values()]
print("\n| Parser | Overall | " + " | ".join(cols) + " |\n|---|" + "---:|" * 6)
print(f"| **docparse (this run)** | **{res.get('Overall', '-')}** | "
+ " | ".join(str(res[c]["score"]) if c in res else "-" for c in cols) + " |")
for row in csv.DictReader(open(sys.argv[2])):
if row["Category"] == "Open Source - Local":
print(f"| {row['Provider']} | {row['Overall']} | " + " | ".join(row[c] for c in cols) + " |")
if __name__ == "__main__":
main()
|