| [ |
| { |
| "query": "What is MIMIC-IV?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "electronic health record", |
| "publicly available", |
| "clinical data" |
| ], |
| "expected_evidence_phrases": [ |
| "MIMIC-IV, a publicly available database", |
| "electronic health record dataset", |
| "sourced from the electronic health record" |
| ] |
| }, |
| { |
| "query": "What hospital is MIMIC-IV sourced from?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "Beth Israel Deaconess Medical Center", |
| "BIDMC", |
| "routine clinical care" |
| ], |
| "expected_evidence_phrases": [ |
| "Beth Israel Deaconess Medical Center", |
| "BIDMC", |
| "Data collected at BIDMC" |
| ] |
| }, |
| { |
| "query": "What years of admissions does MIMIC-IV cover?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "2008", |
| "2019", |
| "admissions" |
| ], |
| "expected_evidence_phrases": [ |
| "between 2008–2019", |
| "between 2008-2019", |
| "between 2008 and 2019", |
| "information from 2008" |
| ] |
| }, |
| { |
| "query": "What are the main creation steps for MIMIC-IV?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "acquisition", |
| "transformation", |
| "deidentification" |
| ], |
| "expected_evidence_phrases": [ |
| "three distinct steps", |
| "acquisition, transformation, and deidentification", |
| "data acquisition" |
| ] |
| }, |
| { |
| "query": "How was MIMIC-IV deidentified?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "deidentified", |
| "HIPAA", |
| "identifiers" |
| ], |
| "expected_evidence_phrases": [ |
| "remove HIPAA identifiers", |
| "deidentification pipeline", |
| "dataset to be considered deidentified" |
| ] |
| }, |
| { |
| "query": "What does the HIPAA Safe Harbor provision require for deidentification?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "HIPAA", |
| "Safe Harbor", |
| "18 identifiers" |
| ], |
| "expected_evidence_phrases": [ |
| "HIPAA Safe Harbor", |
| "18 identifiers", |
| "must be removed" |
| ] |
| }, |
| { |
| "query": "Who can access MIMIC-IV data?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "researchers", |
| "training", |
| "data use agreement" |
| ], |
| "expected_evidence_phrases": [ |
| "made available to researchers", |
| "completed training", |
| "signed a data use agreement" |
| ] |
| }, |
| { |
| "query": "What modules are included in MIMIC-IV?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "hosp", |
| "icu", |
| "note" |
| ], |
| "expected_evidence_phrases": [ |
| "Modules in MIMIC-IV include hosp, icu, and note", |
| "include hosp, icu, and note", |
| "modular structure" |
| ] |
| }, |
| { |
| "query": "What does the ICU module in MIMIC-IV contain?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "intravenous infusions", |
| "patient outputs", |
| "charted observations" |
| ], |
| "expected_evidence_phrases": [ |
| "icu module comprises data", |
| "intravenous infusions", |
| "charted observations" |
| ] |
| }, |
| { |
| "query": "What does the note module in MIMIC-IV contain?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "discharge summaries", |
| "radiology reports", |
| "free-text" |
| ], |
| "expected_evidence_phrases": [ |
| "note module comprises", |
| "discharge summaries", |
| "radiology reports" |
| ] |
| }, |
| { |
| "query": "Why is MIMIC-IV useful for clinical research?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "research", |
| "educational material", |
| "clinical research" |
| ], |
| "expected_evidence_phrases": [ |
| "support a wide array of research studies", |
| "educational material", |
| "reduce barriers" |
| ] |
| }, |
| { |
| "query": "Why does MIMIC-IV use a modular organization?", |
| "expected_source": "MIMIC-IV.pdf", |
| "expected_keywords": [ |
| "modular", |
| "linkage", |
| "modalities" |
| ], |
| "expected_evidence_phrases": [ |
| "modular organization", |
| "future linkage", |
| "varying modalities" |
| ] |
| }, |
| { |
| "query": "What is retrieval augmented generation?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "external databases", |
| "knowledge-intensive tasks", |
| "continuous knowledge updates" |
| ], |
| "expected_evidence_phrases": [ |
| "incorporating knowledge from external databases", |
| "knowledge-intensive tasks", |
| "continuous knowledge updates" |
| ] |
| }, |
| { |
| "query": "What problems does RAG address for large language models?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "hallucination", |
| "outdated knowledge", |
| "untraceable" |
| ], |
| "expected_evidence_phrases": [ |
| "hallucination", |
| "outdated knowledge", |
| "untraceable reasoning" |
| ] |
| }, |
| { |
| "query": "What are the main components of RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "Retrieval", |
| "Generation", |
| "Augmentation" |
| ], |
| "expected_evidence_phrases": [ |
| "core stages", |
| "Retrieval", |
| "Generation", |
| "Augmentation" |
| ] |
| }, |
| { |
| "query": "What are the three paradigms of RAG research?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "Naive RAG", |
| "Advanced RAG", |
| "Modular RAG" |
| ], |
| "expected_evidence_phrases": [ |
| "Naive RAG, Advanced RAG, and Modular RAG", |
| "three stages", |
| "three paradigms" |
| ] |
| }, |
| { |
| "query": "How does the naive RAG process work?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "indexing", |
| "retrieval", |
| "generation" |
| ], |
| "expected_evidence_phrases": [ |
| "indexing, retrieval, and generation", |
| "Retrieve-Read", |
| "traditional process" |
| ] |
| }, |
| { |
| "query": "How are documents stored for retrieval in RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "chunks", |
| "vectors", |
| "vector database" |
| ], |
| "expected_evidence_phrases": [ |
| "split into chunks", |
| "encoded into vectors", |
| "stored in a vector database" |
| ] |
| }, |
| { |
| "query": "What happens during the retrieval step in RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "query", |
| "chunks", |
| "semantic similarity" |
| ], |
| "expected_evidence_phrases": [ |
| "Retrieve the Top k chunks", |
| "relevant to the question", |
| "semantic similarity" |
| ] |
| }, |
| { |
| "query": "What is pre-retrieval optimization in advanced RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "pre-retrieval", |
| "indexing", |
| "query optimization" |
| ], |
| "expected_evidence_phrases": [ |
| "Pre-retrieval process", |
| "optimizing the indexing structure", |
| "query optimization" |
| ] |
| }, |
| { |
| "query": "What is iterative retrieval in RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "iterative retrieval", |
| "retrieval", |
| "generation" |
| ], |
| "expected_evidence_phrases": [ |
| "Iterative retrieval", |
| "alternating between retrieval and generation", |
| "multiple retrieval iterations" |
| ] |
| }, |
| { |
| "query": "What is recursive retrieval in RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "recursive retrieval", |
| "sub-problems", |
| "refining" |
| ], |
| "expected_evidence_phrases": [ |
| "Recursive retrieval", |
| "breaking down the problem", |
| "sub-problems" |
| ] |
| }, |
| { |
| "query": "What is adaptive retrieval in RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "adaptive retrieval", |
| "external knowledge", |
| "autonomously determine" |
| ], |
| "expected_evidence_phrases": [ |
| "Adaptive retrieval", |
| "autonomously determine", |
| "external knowledge retrieval" |
| ] |
| }, |
| { |
| "query": "What tools or benchmarks are used to evaluate RAG?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "RAGAS", |
| "ARES", |
| "TruLens" |
| ], |
| "expected_evidence_phrases": [ |
| "RAGAS", |
| "ARES", |
| "TruLens" |
| ] |
| }, |
| { |
| "query": "What metrics are used for RAG evaluation?", |
| "expected_source": "retrieval_augmented_generation.pdf", |
| "expected_keywords": [ |
| "Accuracy", |
| "Recall", |
| "MRR" |
| ], |
| "expected_evidence_phrases": [ |
| "metrics applicable for evaluation", |
| "Accuracy", |
| "MRR" |
| ] |
| } |
| ] |
|
|
