#!/bin/bash
# Score the Surya checkpoint currently served on :8000 on the four ParseBench
# dimensions this project targets (charts excluded). Usage: [PORT=8000] [CONC=24] bench.sh <tag> [--test]
# Results land in output/<tag>/ and a scorecard in output/<tag>/SCORECARD.txt.
set -u
cd ~/parsebench_rl
TAG="$1"; shift
EXTRA="${*:-}"
export PYTHONPATH=$PWD SURYA_VLLM_URL=http://127.0.0.1:${PORT:-8000}/v1 SURYA_VLLM_MODEL=surya SURYA_PROMPT=${SURYA_PROMPT:-short} SURYA_FIGTEXT=${SURYA_FIGTEXT:-0}
PB=.venv/bin/parse-bench
OUT=output/$TAG
for g in text_formatting text_content table layout; do
  $PB run surya_vllm --group $g --output_dir output_run_$TAG --max_concurrent ${CONC:-24} \
      --open_report False $EXTRA > output_run_$TAG.$g.log 2>&1
done
mkdir -p $OUT
cp -r output_run_$TAG/surya_vllm/* $OUT/ 2>/dev/null
for g in text_formatting text_content table; do
  $PB evaluation run $OUT --group $g --test_cases_dir data > /dev/null 2>&1
  cp $OUT/_evaluation_report.json $OUT/_rep_$g.json
done
$PB evaluation run $OUT --group layout --product_type layout_detection --test_cases_dir data > /dev/null 2>&1
cp $OUT/_evaluation_report.json $OUT/_rep_layout.json
.venv/bin/python - "$OUT" <<'PY' | tee $OUT/SCORECARD.txt
import json, sys
out = sys.argv[1]
keys = [("table", "avg_grits_trm_composite", "Tables"),
        ("text_content", "avg_content_faithfulness", "Content"),
        ("text_formatting", "avg_semantic_formatting", "SemFmt"),
        ("layout", "avg_layout_element_rule_pass_rate", "VisGnd")]
vals = []
for g, k, lab in keys:
    try:
        r = json.load(open(f"{out}/_rep_{g}.json"))
        v = r["aggregate_metrics"].get(k, 0) * 100
        n = r.get("successful"); f = r.get("failed")
    except Exception as e:
        v, n, f = float("nan"), "?", str(e)[:40]
    vals.append(v)
    print(f"{lab:8s} {v:6.2f}   (ok={n} failed={f})")
print(f"{'MEAN4':8s} {sum(vals)/4:6.2f}   (LlamaParse Agentic Plus = 89.21, Surya OCR 2 published = 75.55)")
PY