| |
| """Topic_Modeling |
| |
| Automatically generated by Colab. |
| |
| Original file is located at |
| https://colab.research.google.com/#fileId=https%3A//storage.googleapis.com/kaggle-colab-exported-notebooks/topic-modeling-bdd24b57-13d3-48b6-af92-3163e201b3d6.ipynb%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com/20250120/auto/storage/goog4_request%26X-Goog-Date%3D20250120T233605Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D4b3e7c096f94ba3002728872dc8f625b31d067ff1ed2658b2dceeb40a8ea2323c967e017902f432a25610f53b5fe9a84b44ae021823e422010301ec4b90e388406008e2d053ff8d2b50e6e08fbf05a79a3503bfa51be6284540b75d946cd5cb5895e5229728ace2cc35fc58e4af09d20a2f945e17e4f4fe7b8652a50f5578e688d184698986ccdbaa2531d5318f95d65cc6251b77b6d85e50352bc58b5b53a8017ec8b86b569c93b18fed9bf69630f8ef42039fa38632512af13df4f86bbbbbe859ea4d471b257d78ab4e9d0cd28ff4ed59b3eb92a1f3e44717c3aa482eaca9f7f17739bab1ee0918fe04decfa14394dfc2bcb7f69dd3e0f881cb7eca5ca5237 |
| |
| Referance : |
| |
| 1- https://chatgpt.com/share/6776a2e7-c510-8008-8d1b-9f8b1df0e113 |
| |
| 11-https://chatgpt.com/c/676d3d51-b468-8008-820f-c2f676a8ba3c |
| |
| 2- https://maartengr.github.io/BERTopic/index.html |
| |
| 3: https://huggingface.co/datasets/inparallel/saudinewsnet `dataset` |
| |
| 4: sentence_transformers : https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#multilingual-models |
| |
| 5: Arabic Tokenizers Leaderboard :https://huggingface.co/spaces/MohamedRashad/arabic-tokenizers-leaderboard |
| |
| 6: New embedding models are released frequently and their performance keeps getting better. To keep track of the best embedding models out there, you can visit the MTEB leaderboard : https://huggingface.co/spaces/mteb/leaderboard |
| |
| 7: AIR-Bench: https://huggingface.co/spaces/AIR-Bench/leaderboard |
| |
| 8:https://chatgpt.com/share/678eca95-a6c8-8008-97c0-9a8d520a8f3f |
| """ |
|
|
| ! pip install datasets bertopic sentence_transformers Arabic-Stopwords==0.4.3 |
|
|
| import numpy as np |
| import pandas as pd |
| import seaborn as sns |
| import matplotlib.pyplot as plt |
| import warnings |
| from warnings import filterwarnings |
| filterwarnings('ignore') |
|
|
| from datasets import load_dataset |
|
|
| |
| import nltk |
| nltk.download('stopwords') |
| nltk.download('punkt') |
| nltk.download('wordnet') |
| nltk.download('omw-1.4') |
| nltk.download('punkt_tab') |
| from nltk.corpus import stopwords |
| from nltk.tokenize import word_tokenize |
| from nltk.stem import WordNetLemmatizer |
| import re |
| import string |
| import re |
|
|
| from bertopic import BERTopic |
| from sentence_transformers import SentenceTransformer |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
| |
|
|
| |
| |
| |
| |
|
|
| |
|
|
| def clean_text(text): |
| if text is None: |
| return '' |
| elif not isinstance(text, str): |
| return '' |
| text=str(text) |
| |
| text = re.sub(r'http\S+|www\S+|https\S+', 'رابط', text, flags=re.MULTILINE) |
|
|
| |
| text = re.sub(r'\d+', 'رقم', text) |
|
|
| |
| text = re.sub(r'\S*@\S*\s?', 'ايميل', text) |
|
|
| |
| text = re.sub(r'\s+', ' ', text).strip() |
|
|
| |
| text=re.sub(r'\s*([.,:;!?])\s*', r'\1', text) |
|
|
| |
| words=word_tokenize(text) |
| text=" ".join([ word for word in words if len(word)>1]) |
|
|
| |
| stop_words=set(stopwords.words('arabic')) |
| words=word_tokenize(text) |
| text=" ".join([ word for word in words if word not in stop_words]) |
|
|
| return text.lower().strip() |
|
|
| """# using Spacy""" |
|
|
| |
| |
| |
|
|
| |
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
| |
| |
|
|
|
|
| |
|
|
| |
| |
|
|
| stop_words=set(stopwords.words('arabic')) |
| print(stop_words) |
|
|
|
|
|
|
| try: |
| dataset=load_dataset("saudinewsnet") |
| except Exception as e: |
| print(e) |
|
|
| dataset |
|
|
| df=dataset['train'].to_pandas() |
|
|
| df.head() |
|
|
| Raw_dataset = df[["source", "date_extracted", "content"]] |
|
|
| Raw_dataset.head() |
|
|
| Raw_dataset=Raw_dataset.sample(frac=1,random_state=101) |
|
|
| dataset["train"]["content"][0] |
|
|
| Raw_dataset["content"] = Raw_dataset["content"].apply(clean_text) |
|
|
| Raw_dataset['content'].head() |
|
|
| Raw_dataset["text_len"]=Raw_dataset["content"].apply(lambda x:len(x.split())) |
|
|
| Raw_dataset.head() |
|
|
| |
| plt.figure(figsize=(10,5)) |
| sns.histplot(Raw_dataset["text_len"],bins=50,kde=True) |
| plt.show() |
|
|
| print (Raw_dataset.shape) |
| Raw_dataset=Raw_dataset[Raw_dataset["text_len"]<=1000] |
| print (Raw_dataset.shape) |
|
|
| """# convert date_extracted into date type""" |
|
|
| |
| Raw_dataset.info() |
|
|
| Raw_dataset["date_extracted"]=pd.to_datetime(Raw_dataset["date_extracted"]) |
|
|
| """# topic_modeling using bert_topic""" |
|
|
| bert_topic=BERTopic(language="arabic",verbose=True) |
| topic,probs=bert_topic.fit_transform(Raw_dataset["content"]) |
|
|
| |
|
|
| |
| |
| |
|
|
| topic_freq=bert_topic.get_topic_info() |
| print (topic_freq) |
|
|
| """# most frequent topic that was generated, topic 0""" |
|
|
| bert_topic.get_topic(216) |
|
|
| """# extract information on a document level, such as their corresponding topics, probabilities,""" |
|
|
| bert_topic.get_document_info(Raw_dataset["content"]) |
|
|
| """# fine_tune representation with keybertinspierd |
| |
| ## represented by contextual keyphrases, not just frequent terms. |
| """ |
|
|
| from bertopic.representation import KeyBERTInspired |
|
|
| topic_model=BERTopic(language="arabic",verbose=True,representation_model=KeyBERTInspired()) |
|
|
| topics,probs=topic_model.fit_transform(Raw_dataset["content"]) |
|
|
| topic_freq=topic_model.get_topic_info() |
| print (topic_freq) |
|
|
| topic_model.get_document_info(Raw_dataset["content"]) |
|
|
| topic_model.get_document_info(Raw_dataset["content"]).loc[3] |
|
|
| """# Comparison of Representation""" |
|
|
| bert_topic.get_topic(170) |
|
|
| bert_topic.get_document_info(Raw_dataset["content"]).loc[3] |
|
|
| |
| topic_model.get_document_info(Raw_dataset["content"]).loc[3] |
|
|
| """the `key_inspierd_representation` is the best : |
| |
| 1. Improved Clarity: Topics are represented by contextual keyphrases, not just |
| frequent terms. |
| |
| 2. Better Interpretability: KeyBERT ensures that the topic representations are more human-readable and understandable. |
| |
| 3. Domain-Specific Insights: For applications like Saudi newspapers, this method captures nuanced topics such as "Vision 2030" or "Al-Ula heritage," making the topics more relevant to the domain. |
| """ |
|
|
| def clean_for_inferance(text): |
| |
| text = re.sub(r'\d+%', '', text) |
| |
| text = re.sub(r'\d+\s*/\s*\d+\s*/\s*\d+', '', text) |
| |
| text = re.sub(r'\d+', '', text) |
| |
| text = re.sub(r'\s+', ' ', text).strip() |
|
|
| return text |
|
|
| |
| new_documents=""" |
| ضبطت الحملات الميدانية المشتركة لمتابعة وضبط مخالفي أنظمة الإقامة والعمل وأمن الحدود، في مناطق السعودية كافة في الفترة من 09 / 01 / 2025 إلى 15 / 01 / 2025، 21485 مخالفاً. |
| |
| وطبقاً لبيان وزارة الداخلية السعودية، فإن هؤلاء المخالفين منهم 13562 مخالفًا لنظام الإقامة، و4853 مخالفًا لنظام أمن الحدود، و3070 مخالفًا لنظام العمل. |
| |
| ويشير البيان أيضاً إلى أن إجمالي الذين ضُبطوا في أثناء محاولتهم عبور الحدود إلى داخل السعودية يصل عددهم نحو 1568 شخصًا 47%، منهم يمنيو الجنسية، و50% إثيوبيو الجنسية، و03% من جنسيات أخرى. |
| |
| وضبطت الحملات الميدانية المشتركة 64 شخصًا لمحاولتهم عبور الحدود إلى خارج المملكة بطريقة غير نظامية، فضلاً عن ضبط 16متورطـًا في نقل وإيواء وتشغيل مخالفي أنظمة الإقامة والعمل وأمن الحدود والتستر عليهم. |
| |
| وأكدت وزارة الداخلية أن كل من يسهل دخول مخالفي نظام أمن الحدود للمملكة أو نقلهم داخلها أو يوفر لهم المأوى أو يقدم لهم أي مساعدة أو خدمة بأي شكل من الأشكال، يعرض نفسه لعقوبات تصل إلى السجن مدة 15 سنة، وغرامة مالية تصل إلى مليون ريال، ومصادرة وسيلة النقل والسكن المستخدم للإيواء، إضافة إلى التشهير به. |
| |
| وأوضحت في الوقت نفسه أن إجمالي الذين يخضعون للإجراءات الخاصة بتنفيذ الأنظمة يصل عددهم إلى نحو 33007 وافدين مخالفين، منهم 30335 رجلاً، و2672 امرأة |
| """ |
|
|
| |
| cleaned_text = clean_for_inferance(new_documents) |
|
|
| print(cleaned_text) |
|
|
| |
| new_topics, new_probs = topic_model.transform(cleaned_text) |
|
|
| |
|
|
| |
| documents = cleaned_text.split('. ') |
|
|
| documents |
|
|
| |
| print(topic_model.get_topic_info(new_topics[0])) |
|
|
| """# Update topic representation""" |
|
|
| topic_model.update_topics(Raw_dataset["content"], n_gram_range=(1, 2)) |
|
|
| |
|
|
| topic_model.get_topic_info() |
|
|
| topic_model.get_document_info(Raw_dataset["content"]) |
|
|
| topic_model.get_document_info(Raw_dataset["content"]).loc[3] |
|
|
| """ |
| #### `Improved Context`: By including bigrams or trigrams, the updated topic representations capture more contextually meaningful phrases (e.g., "Vision 2030" instead of just "Vision"). |
| |
| #### `More Informative Keywords`: Longer n-grams often provide more specific and relevant representations for topics, especially in domains like news, where phrases like "economic transformation" or "football league" are common. |
| |
| #### `Customization`: You can adjust the n-gram range to suit the nature of your data: |
| |
| 1- Use unigrams for general topics. |
| |
| 2-Use bigrams/trigrams for more domain-specific or context-sensitive topics. |
| """ |
|
|
| topic_model.get_topic_freq() |
|
|
| |
| |
|
|
| |
| |
| |
| |
| |
|
|
| """### Generate topic labels without manual topic labeling""" |
|
|
| |
| |
|
|
| |
| |
|
|
| """### Improved Interpretability: `Automatically generated labels` make it easier to understand and communicate the results of topic modeling.""" |
|
|
| |
| |
|
|
| |
| |
|
|
| topic_model.get_topic_info() |
|
|
| topic_model=BERTopic(language="arabic",verbose=True,representation_model=KeyBERTInspired()) |
|
|
| topics,probs=topic_model.fit_transform(Raw_dataset["content"]) |
|
|
| topic_model.find_topics("vehicle") |
|
|
| """# Visualizations""" |
|
|
| |
|
|
| topic_model.visualize_topics() |
|
|
| """## from the previouse visulization we can see the closely similar topic""" |
|
|
| Raw_dataset["content"].head() |
|
|
| |
| Raw_dataset_reset = Raw_dataset.reset_index(drop=True) |
|
|
| Raw_dataset_reset.head() |
|
|
| topic_model.visualize_documents(Raw_dataset_reset["content"]) |
|
|
| |
| |
|
|
| |
|
|
| |
|
|
| """### After we use the package we try build customized topic model using Modularity : |
|  |
| """ |
|
|
| raw_dataset = df[["source", "date_extracted", "content"]] |
|
|
| raw_dataset=raw_dataset.sample(frac=1,random_state=101) |
|
|
| raw_dataset["content"] = raw_dataset["content"].apply(clean_text) |
|
|
| raw_dataset.head(1) |
|
|
| raw_dataset["date_extracted"]=pd.to_datetime(raw_dataset["date_extracted"]).dt.date |
|
|
| raw_dataset["text_len"]=raw_dataset["content"].apply(lambda x:len(x.split())) |
|
|
| print (raw_dataset.shape) |
| raw_dataset=raw_dataset[raw_dataset["text_len"]<=1000] |
| print (raw_dataset.shape) |
|
|
| """# Embedding""" |
|
|
| from sentence_transformers import SentenceTransformer |
|
|
| model = SentenceTransformer('sentence-transformers/LaBSE') |
|
|
| embedding=model.encode(raw_dataset["content"].values,show_progress_bar=True) |
|
|
| |
| |
| |
|
|
| |
| |
|
|
| |
|
|
| embedding.shape |
|
|
| """# Dimensionality Reduction |
| |
| ### you can use UMAP but i here use cuML UMAP |
| """ |
|
|
| |
|
|
| from umap import UMAP |
|
|
| umap_model=UMAP(n_components=15, n_neighbors=15, |
| min_dist=0.0,random_state=101,metric='cosine') |
|
|
| """# Clustering""" |
|
|
| from hdbscan import HDBSCAN |
|
|
| """### A higher min_cluster_size will generate fewer topics |
| ### A lower min_cluster_size will generate more topics. |
| """ |
|
|
| hdbscan_model=HDBSCAN(min_cluster_size=50,metric='euclidean', |
| cluster_selection_method='eom', prediction_data=True) |
|
|
| """# Vectorizer""" |
|
|
| from sklearn.feature_extraction.text import CountVectorizer |
| import arabicstopwords.arabicstopwords as stp |
|
|
| stop_words=stp.stopwords_list() |
|
|
| vectorizer_model = CountVectorizer(min_df=3, |
| stop_words=stop_words, |
| analyzer='word', |
| max_df=0.5, |
| ngram_range=(1,3) |
| ) |
|
|
| |
|
|
| """# Topic Representer""" |
|
|
| from bertopic.representation import KeyBERTInspired |
|
|
| keybert_model = KeyBERTInspired() |
|
|
| representation_model = { |
| "KeyBERT": keybert_model |
| } |
|
|
| """# Let's Go with Customized topic model using Modularity :""" |
|
|
| topic_model = BERTopic(embedding_model=model, |
| umap_model=umap_model, |
| hdbscan_model=hdbscan_model, |
| vectorizer_model=vectorizer_model, |
| representation_model=representation_model, |
|
|
| top_n_words=10, |
| calculate_probabilities=True, |
| verbose=True, |
| ) |
|
|
| topics, probs =topic_model.fit_transform( |
| raw_dataset["content"].values, |
| embedding |
| ) |
|
|
| topic_model.get_topic_info() |
|
|
| raw_dataset.head(10) |
|
|
| raw_dataset["content"].values[19] |
|
|
| topics[19], probs[19] |
|
|
| topic_model.get_topic(11) |
|
|
| topic_model.get_topic_info(11) |
|
|
| raw_dataset["topic"]= topics |
| raw_dataset["prob"]=probs |
|
|
| raw_dataset.head(7) |
|
|
| raw_dataset[ raw_dataset['topic'] == 11 ].head(10) |
|
|
| """# can use in recommendations and improve customer satisfaction |
| |
| # Visualizing BERTopic |
| """ |
|
|
| |
| |
|
|
| topic_model.visualize_topics() |
|
|
| topic_model.visualize_heatmap() |
|
|
| """# Visualize Topics over Time""" |
|
|
| topics_over_time = topic_model.topics_over_time(raw_dataset['content'],raw_dataset['date_extracted']) |
|
|
| topic_model.visualize_topics_over_time(topics_over_time, topics=[10, 11, 12, 14, 15, 16, 17, 18, 19]) |
|
|
| """# Visualize Topics per Class""" |
|
|
| topics_per_class = topic_model.topics_per_class(raw_dataset['content'].values |
| , classes=raw_dataset['source'].values) |
|
|
| topic_model.visualize_topics_per_class(topics_per_class) |
|
|
| """# Visualize documents with Plotly""" |
|
|
| |
| fig = topic_model.visualize_documents(raw_dataset['content'].values, embeddings=embedding) |
| fig.show() |
|
|
| """# Visualize documents with DataMapPlot |
| |
| """ |
|
|
| topic_model.visualize_document_datamap(raw_dataset['content'].values,embeddings=embedding) |
|
|
| |
| |
| |
|
|
| topic_model.get_topic(6) |
|
|
| raw_dataset['content'].values[6] |
|
|
| topics[6] |
|
|
| |
| topic_model.visualize_distribution(probs[6], min_probability=0.01) |
|
|
| print(embedding.shape) |
| print(len(raw_dataset['content'].values)) |
|
|
| |
| topic_distr, topic_token_distr = topic_model.approximate_distribution(raw_dataset['content'].values, calculate_tokens=True) |
|
|
| |
| df = topic_model.visualize_approximate_distribution(raw_dataset['content'].values[7], topic_token_distr[7]) |
| df |
|
|
| """# How It Helps: |
| #### Identify key tokens that strongly influence the topic or label assignment. |
| #### Understand why a document is assigned to a specific topic or class. |
| #### Optimize preprocessing by filtering out low-contributing tokens. |
| #### Focus on tokens that carry the most semantic weight. |
| #### Merge or split topics based on token overlaps. |
| #### Remove outlier tokens that skew topic representation. |
| #### Refine keywords to target specific audiences or improve SEO. |
| #### Debug models by finding tokens that lead to misclassifications. |
| #### Adjust data preprocessing or model training to fix errors. |
| |
| # Terms |
| """ |
|
|
| topic_model.visualize_barchart() |
|
|
| topic_model.visualize_term_rank() |
|
|
| """### Optimize content creation by focusing on terms that define high-priority topics. |
| ### Refine SEO strategies by identifying keywords associated with relevant topics |
| """ |
|
|
| topic_model.get_topic_info(2) |
|
|
| topic_model.visualize_term_rank(log_scale=True) |
|
|
| """# Hierarchy""" |
|
|
| topic_model.visualize_hierarchy() |
|
|
| |
| |
| |
| |
| |
|
|
| |
| |
| |
| |
|
|
|
|
| |
| |
| |
|
|
| |
| |
| |
|
|
| hierarchical_topics = topic_model.hierarchical_topics(raw_dataset['content'].values) |
|
|
| topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics) |
|
|
| topic_model.get_topic(5) |
|
|
| tree = topic_model.get_topic_tree(hierarchical_topics) |
| print(tree) |
|
|
| from umap import UMAP |
|
|
| |
| hierarchical_topics = topic_model.hierarchical_topics(raw_dataset['content'].values) |
|
|
| |
| topic_model.visualize_hierarchical_documents( |
| raw_dataset['content'].values, |
| hierarchical_topics, |
| embeddings=embedding |
| ) |
|
|
| |
| umap = UMAP(n_neighbors=15, n_components=15, min_dist=0.0, metric='cosine', random_state=101) |
| reduced_embeddings = umap.fit_transform(embedding) |
|
|
| |
| topic_model.visualize_hierarchical_documents( |
| raw_dataset['content'].values, |
| hierarchical_topics, |
| reduced_embeddings=reduced_embeddings, |
| hide_document_hover=True |
| ) |
|
|
| raw_dataset.head() |
|
|
| """# Inference""" |
|
|
| story = """ |
| طرحت مؤسسة البترول الكويتية عطاءً؛ لبيع زيت وقود عالي الكبريت للتحميل في الفترة من فبراير/ شباط إلى إبريل/ نيسان. |
| وأوضحت مصادر تجارية، اليوم الخميس، أن المؤسسة تعرض شحنات من زيت الوقود عالي الكبريت تبلغ الواحدة 60 ألف طن لتحميلها من الكويت شهرياً بالفترة المذكورة. |
| """ |
|
|
| _topic, _prob = topic_model.transform([story]) |
|
|
| _topic |
|
|
| topic_model.get_topic(17) |
|
|
| raw_dataset[raw_dataset["topic"]==17].head() |
|
|
| """# that is perfect |
| |
| # Save model |
| """ |
|
|
| model_id="sentence-transformers/LaBSE" |
|
|
| topic_model.save("/kaggle/working/bertopic_dir", serialization="safetensors", save_ctfidf=True, save_embedding_model=model_id) |
|
|
| """# Loading""" |
|
|
| |
| loaded_model = BERTopic.load("/kaggle/working/bertopic_dir") |
|
|
| from huggingface_hub import login |
| login() |
|
|
| from bertopic import BERTopic |
|
|
|
|
|
|
| |
| topic_model.push_to_hf_hub( |
| repo_id="Ah7med/BERTopic_ArXiv", |
| save_ctfidf=True |
| ) |
|
|
|
|
|
|
|
|
