Spaces:
Running on Zero
Running on Zero
File size: 3,082 Bytes
3d9ba5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import unittest
from models.extractive.tfidf import CustomTFIDF
from models.extractive.textrank import TextRankSummarizer
from models.extractive.lsa import LSASummarizer
from models.extractive.hybrid_scorer import HybridSummarizer
class TestExtractiveSummarization(unittest.TestCase):
def setUp(self):
self.ar_text = (
"تعد معالجة اللغات الطبيعية فرعا أساسيا من فروع الذكاء الاصطناعي الحديث. "
"تقوم أنظمة التلخيص التلقائي باستخراج وتوليد ملخصات دقيقة للنصوص الطويلة. "
"يعتمد التلخيص الاستخراجي على تحديد أهم الجمل من النص الأصلي بالاعتماد على خوارزميات إحصائية. "
"أما التلخيص التجريدي فيعتمد على شبكات التعلم العميق لتوليد صياغات جديدة للأفكار الرئيسية."
)
self.en_text = (
"Natural language processing is an essential domain in modern artificial intelligence. "
"Automatic text summarization condenses long documents into concise overviews. "
"Extractive summarization extracts the most important sentences directly from the source text. "
"Abstractive summarization creates novel sentences using deep neural networks and attention."
)
def test_tfidf_cosine_similarity(self):
docs = [
["natural", "language", "processing"],
["natural", "language", "understanding"],
["computer", "vision", "images"]
]
tfidf = CustomTFIDF()
vectors = tfidf.fit_transform(docs)
self.assertEqual(len(vectors), 3)
sim_0_1 = CustomTFIDF.cosine_similarity(vectors[0], vectors[1])
sim_0_2 = CustomTFIDF.cosine_similarity(vectors[0], vectors[2])
self.assertGreater(sim_0_1, sim_0_2)
def test_textrank_arabic_and_english(self):
tr = TextRankSummarizer()
ar_res = tr.summarize(self.ar_text, num_sentences=2, lang="ar")
self.assertIn("summary", ar_res)
self.assertEqual(len(ar_res["selected_indices"]), 2)
self.assertGreater(len(ar_res["summary"]), 10)
en_res = tr.summarize(self.en_text, num_sentences=2, lang="en")
self.assertIn("summary", en_res)
self.assertEqual(len(en_res["selected_indices"]), 2)
self.assertGreater(len(en_res["summary"]), 10)
def test_lsa_summarizer(self):
lsa = LSASummarizer()
res = lsa.summarize(self.en_text, num_sentences=2, lang="en")
self.assertEqual(len(res["selected_indices"]), 2)
self.assertGreater(len(res["summary"]), 10)
def test_hybrid_summarizer(self):
hybrid = HybridSummarizer()
res = hybrid.summarize(self.ar_text, num_sentences=2, lang="ar")
self.assertEqual(len(res["selected_indices"]), 2)
self.assertGreater(len(res["summary"]), 10)
if __name__ == "__main__":
unittest.main()
|