| """Tests for the answer-eval citation parser (eval/run_answers.py).""" |
|
|
| import unittest |
|
|
| from eval.run_answers import cited_ids, is_refusal, lang_ok |
|
|
|
|
| class TestCitedIds(unittest.TestCase): |
| def test_quran_urls(self): |
| answer = ("[[1](https://surahquran.com/aya-255-sora-2.html)] ๏ดฟ...๏ดพ " |
| "[[1](https://quran.ksu.edu.sa/tafseer/qortobi/sura24-aya35.html)]") |
| self.assertEqual(cited_ids(answer, {}), {"2:255", "24:35"}) |
| self.assertEqual(cited_ids("[[1](https://quran.com/2/278)]", {}), {"2:278"}) |
|
|
| def test_english_hadith_refs(self): |
| answer = ("Recorded in Sahih al-Bukhari 6116. Also Sunan Abu Dawood No. 4800 " |
| "and Jami' at-Tirmidhi #2516, Sunan Ibn Majah, hadith 224.") |
| self.assertEqual(cited_ids(answer, {}), |
| {"bukhari_6116", "abudawud_4800", "tirmidhi_2516", "ibnmajah_224"}) |
|
|
| def test_arabic_hadith_refs(self): |
| answer = "ุฑูุงู ุตุญูุญ ุงูุจุฎุงุฑูุ ุฑูู
13. ููู ุตุญูุญ ู
ุณูู
ุ ุฑูู
45. ููู ู
ูุทุฃ ู
ุงููุ ุฑูู
1." |
| self.assertEqual(cited_ids(answer, {}), {"bukhari_13", "muslim_45", "malik_1"}) |
| answer = ("ุงูุชุฑู
ุฐูุ ุฑูู
ุงูุญุฏูุซ 2024. ูุตุญูุญ ู
ุณูู
ุจุฑูู
440. ูุณูู ุงุจู ู
ุงุฌูุ ุงูุฑูู
4303. " |
| "ูุงูุจุฎุงุฑูุ ูุชุงุจ ุงูุฅูู
ุงูุ ุญุฏูุซ ุฑูู
1. ูุฑูุงู ู
ุณูู
ูู ุตุญูุญู (6382).") |
| self.assertEqual(cited_ids(answer, {}), |
| {"tirmidhi_2024", "muslim_440", "ibnmajah_4303", "bukhari_1", "muslim_6382"}) |
|
|
| def test_hadith_ref_needs_adjacent_number(self): |
| |
| self.assertEqual(cited_ids("Sahih al-Bukhari and Sahih Muslim 1907", {}), {"muslim_1907"}) |
| |
| self.assertEqual(cited_ids("ุทูุจ ุงูุนูู
ูุฑูุถุฉ ุนูู ูู ู
ุณูู
3 ู
ุฑุงุช", {}), set()) |
|
|
| def test_fatwa_urls(self): |
| url_to_id = {"https://www.islamweb.net/ar/fatwa/857": "islamweb_857"} |
| answer = "ู
ุฑูุฒ ุงููุชูู [[1](https://www.islamweb.net/ar/fatwa/857/)]." |
| self.assertEqual(cited_ids(answer, url_to_id), {"islamweb_857"}) |
|
|
|
|
| class TestAnswerChecks(unittest.TestCase): |
| def test_refusal_and_language(self): |
| not_found = {"arabic": "ูู
ุฃุฌุฏ", "english": "not found"} |
| self.assertTrue(is_refusal("The dataset does not contain sufficient information.", "en", not_found)) |
| self.assertFalse(is_refusal("Yes, it is obligatory. And Allah knows best.", "en", not_found)) |
| self.assertTrue(lang_ok("ูุนู
ุ ูุฐุง ูุงุฌุจ. ูุงููู ุฃุนูู
.", "ar")) |
| self.assertFalse(lang_ok("Yes, it is obligatory.", "ar")) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
|
|
| class TestRequiredLabels(unittest.TestCase): |
| def test_hit_without_required_source_is_incomplete(self): |
| from eval.run_eval import score_case |
| case = {"id": "x", "category": "quran_topic", "query": "Is alcohol forbidden?", |
| "relevant": ["5:90", "2:219"], "required": ["5:90"]} |
| out = {"results": [{"id": "29:49"}, {"id": "2:219"}], "intent": "general", "top_score": 1.0, |
| "gated": False, "confidence": 0.9, "confidence_source": "reranker"} |
| row = score_case(case, out, 8, {}) |
| self.assertEqual(row["hit@k"], 1) |
| self.assertEqual(row["complete@k"], 0) |
| self.assertEqual(row["missing_required"], ["5:90"]) |
|
|
| def test_required_must_be_subset_of_relevant(self): |
| import tempfile |
| from pathlib import Path |
| from eval.run_eval import load_golden |
| with tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False, encoding="utf-8") as f: |
| f.write('{"id": "x", "category": "c", "query": "q", "relevant": ["2:219"], "required": ["5:90"]}\n') |
| with self.assertRaises(ValueError): |
| load_golden(Path(f.name), None) |
|
|
