File size: 3,013 Bytes
81f30c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | """Fase 5, tarea 2 — gold informal: 30 consultas (15 dependientes de figura) sobre
el PDF de ejemplo -> Recall@5 y "cobertura de figura" (la descripción VLM de la
figura contiene las entidades clave). Uso: python scripts/eval_informal_gold.py
"""
from __future__ import annotations
import argparse
import json
import tempfile
from ragsolver.orchestrator.run import wilson
from ragsolver.rag.retrieve import Retriever
from ragsolver.rag.schema import RetrievalRequest
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--gold", default="data/eval/informal_gold.jsonl")
ap.add_argument("--k", type=int, default=5)
ap.add_argument("--out", default="data/eval/results/fase5_informal_gold.json")
args = ap.parse_args()
gold = [json.loads(l) for l in open(args.gold, encoding="utf-8") if l.strip()]
r = Retriever()
r.trace_dir = tempfile.mkdtemp(prefix="ragsolver-eval-")
figures = {f["figure_id"]: f for f in (json.loads(l) for l in open("data/corpus/figures.jsonl", encoding="utf-8"))}
rows, hits_at_k, fig_cov = [], 0, []
for q in gold:
res = r.search(RetrievalRequest(queries=[q["query"]], mode="exemplar", corpora=["informal"], k=args.k, rerank=True, context={"problem_id": q["query_id"], "agent": "eval"}))
items = [h.metadata.get("item_number") for h in res.hits]
hit = any(i in q["relevant_items"] for i in items)
hits_at_k += hit
cov = None
if q["figure_dependent"]:
descs = " ".join((figures[fid]["description"] + " " + " ".join(figures[fid]["labels"])).lower() for h in res.hits for fid in h.metadata.get("figure_ids", []) if fid in figures and h.metadata.get("item_number") in q["relevant_items"])
ents = q["figure_entities"]
cov = sum(1 for e in ents if e.lower() in descs) / len(ents) if ents else None
if cov is not None:
fig_cov.append(cov)
rows.append({"query_id": q["query_id"], "hit@k": hit, "rank": next((i + 1 for i, it in enumerate(items) if it in q["relevant_items"]), None), "top_items": items, "figure_coverage": cov})
print(f"{q['query_id']}: hit={hit} rank={rows[-1]['rank']} top={items} fig_cov={cov}")
n = len(gold)
out = {
"n_queries": n,
f"recall@{args.k}": round(hits_at_k / n, 3),
f"recall@{args.k}_wilson95": wilson(hits_at_k, n),
"figure_dependent": {"n": sum(q["figure_dependent"] for q in gold), f"recall@{args.k}": round(sum(r["hit@k"] for r, q in zip(rows, gold) if q["figure_dependent"]) / max(1, sum(q["figure_dependent"] for q in gold)), 3), "figure_coverage_mean": round(sum(fig_cov) / len(fig_cov), 3) if fig_cov else None},
"mrr": round(sum(1 / r["rank"] for r in rows if r["rank"]) / n, 3),
"rows": rows,
}
json.dump(out, open(args.out, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
print(json.dumps({k: v for k, v in out.items() if k != "rows"}, indent=2))
if __name__ == "__main__":
main()
|