scripts/evaluate.py

"""Run the AgroSense RAG evaluation harness and print a report.

Usage:
    python scripts/evaluate.py                 # uses data/eval_set.json
    python scripts/evaluate.py path/to/set.json --json out.json

Runs fully offline (set AGROSENSE_EMBEDDING_BACKEND=hashing for determinism).
Exit code is non-zero if any report target is missed (handy for CI).
"""
from __future__ import annotations

import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from agrosense import RAGEngine
from agrosense.config import DATA_DIR
from agrosense.evaluation import evaluate, load_cases


def main(argv: list[str]) -> int:
    args = [a for a in argv[1:] if not a.startswith("--")]
    out_json = None
    if "--json" in argv:
        idx = argv.index("--json")
        if idx + 1 < len(argv):
            out_json = argv[idx + 1]

    set_path = Path(args[0]) if args else (DATA_DIR / "eval_set.json")
    print(f"Loading eval set: {set_path}")
    cases = load_cases(set_path)
    print(f"{len(cases)} cases. Building engine...")
    engine = RAGEngine()

    report = evaluate(engine, cases)
    m, t = report["metrics"], report["targets_met"]

    print("\n=== AgroSense RAG Evaluation ===")
    print(f"  In-domain cases     : {m['n_in_domain']}")
    print(f"  Out-of-domain cases : {m['n_out_of_domain']}")
    print("  --- quality ---")
    print(f"  Context relevance   : {m['context_relevance']:.1%}   "
          f"(target >=95%)  {'PASS' if t['context_relevance'] else 'FAIL'}")
    print(f"  Citation MRR        : {m['citation_mrr']:.3f}")
    print(f"  Faithfulness        : {m['faithfulness']:.1%}   "
          f"(target >=95%)  {'PASS' if t['faithfulness'] else 'FAIL'}")
    print(f"  Hallucination rate  : {m['hallucination_rate']:.1%}   "
          f"(target <5%)   {'PASS' if t['hallucination_rate'] else 'FAIL'}")
    print(f"  Answer relevance    : {m['answer_relevance']:.1%}")
    print(f"  Out-of-domain acc.  : "
          + ("n/a" if m['ood_accuracy'] is None else f"{m['ood_accuracy']:.1%}"))
    print("  --- latency ---")
    print(f"  p50 / p95 / max ms  : {m['latency_ms_p50']} / {m['latency_ms_p95']} / "
          f"{m['latency_ms_max']}   (p95 target <=3000)  "
          f"{'PASS' if t['latency_ms_p95'] else 'FAIL'}")
    print(f"\n  ALL TARGETS MET: {report['all_targets_met']}")

    if out_json:
        Path(out_json).write_text(json.dumps(report, indent=2), encoding="utf-8")
        print(f"  (full report written to {out_json})")

    return 0 if report["all_targets_met"] else 1


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))