File size: 1,971 Bytes
6b4a31f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | #!/bin/bash
# Score the Surya checkpoint currently served on :8000 on the four ParseBench
# dimensions this project targets (charts excluded). Usage: [PORT=8000] [CONC=24] bench.sh <tag> [--test]
# Results land in output/<tag>/ and a scorecard in output/<tag>/SCORECARD.txt.
set -u
cd ~/parsebench_rl
TAG="$1"; shift
EXTRA="${*:-}"
export PYTHONPATH=$PWD SURYA_VLLM_URL=http://127.0.0.1:${PORT:-8000}/v1 SURYA_VLLM_MODEL=surya SURYA_PROMPT=${SURYA_PROMPT:-short} SURYA_FIGTEXT=${SURYA_FIGTEXT:-0}
PB=.venv/bin/parse-bench
OUT=output/$TAG
for g in text_formatting text_content table layout; do
$PB run surya_vllm --group $g --output_dir output_run_$TAG --max_concurrent ${CONC:-24} \
--open_report False $EXTRA > output_run_$TAG.$g.log 2>&1
done
mkdir -p $OUT
cp -r output_run_$TAG/surya_vllm/* $OUT/ 2>/dev/null
for g in text_formatting text_content table; do
$PB evaluation run $OUT --group $g --test_cases_dir data > /dev/null 2>&1
cp $OUT/_evaluation_report.json $OUT/_rep_$g.json
done
$PB evaluation run $OUT --group layout --product_type layout_detection --test_cases_dir data > /dev/null 2>&1
cp $OUT/_evaluation_report.json $OUT/_rep_layout.json
.venv/bin/python - "$OUT" <<'PY' | tee $OUT/SCORECARD.txt
import json, sys
out = sys.argv[1]
keys = [("table", "avg_grits_trm_composite", "Tables"),
("text_content", "avg_content_faithfulness", "Content"),
("text_formatting", "avg_semantic_formatting", "SemFmt"),
("layout", "avg_layout_element_rule_pass_rate", "VisGnd")]
vals = []
for g, k, lab in keys:
try:
r = json.load(open(f"{out}/_rep_{g}.json"))
v = r["aggregate_metrics"].get(k, 0) * 100
n = r.get("successful"); f = r.get("failed")
except Exception as e:
v, n, f = float("nan"), "?", str(e)[:40]
vals.append(v)
print(f"{lab:8s} {v:6.2f} (ok={n} failed={f})")
print(f"{'MEAN4':8s} {sum(vals)/4:6.2f} (LlamaParse Agentic Plus = 89.21, Surya OCR 2 published = 75.55)")
PY
|