Spaces:
Sleeping
Sleeping
File size: 1,758 Bytes
b106f38 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | """RAG Evaluation Benchmark Dataset."""
from dataclasses import dataclass
from typing import List, Optional
@dataclass
class EvalSample:
"""A ground-truth question and answer pair for RAG evaluation."""
question: str
expected_answer: str
expected_page: int
expected_filename: str
key_terms: List[str]
# Standard benchmark test cases against the sample research paper
BENCHMARK_DATASET: List[EvalSample] = [
EvalSample(
question="What methodology was used in the research?",
expected_answer="The research evaluated FAISS vector indices with sentence-transformer embeddings.",
expected_page=2,
expected_filename="sample_research.pdf",
key_terms=["FAISS", "sentence-transformer", "methodology", "embeddings"],
),
EvalSample(
question="What was the hit rate achieved in the evaluation?",
expected_answer="Evaluation demonstrated a 94% hit rate on domain queries.",
expected_page=2,
expected_filename="sample_research.pdf",
key_terms=["94%", "hit rate", "domain queries"],
),
EvalSample(
question="What does RAG combine according to the paper?",
expected_answer="RAG combines parametric knowledge in neural models with non-parametric external retrieval.",
expected_page=1,
expected_filename="sample_research.pdf",
key_terms=["parametric", "non-parametric", "external retrieval"],
),
EvalSample(
question="What is the benefit of external retrieval in RAG?",
expected_answer="It prevents hallucinations and provides direct citations.",
expected_page=1,
expected_filename="sample_research.pdf",
key_terms=["hallucinations", "citations"],
),
]
|