Spaces:
Running
Running
File size: 2,242 Bytes
60e8c0a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """
Topic modeling via TF-IDF + KMeans (scikit-learn).
Auto-discover topik tanpa kategori predefined.
"""
from typing import List, Dict
import re
INDO_STOPWORDS = {
"yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu", "dan",
"atau", "adalah", "akan", "juga", "tidak", "para", "oleh", "sebagai",
"dalam", "tersebut", "ada", "dapat", "bisa", "harus", "lebih", "sangat",
"telah", "sudah", "masih", "hanya", "saja", "karena", "namun", "tetapi",
"tapi", "saat", "ketika", "setelah", "sebelum", "antara", "hingga",
"republika", "okezone", "detik", "kompas", "tribunnews", "cnn", "tempo",
"antaranews", "antara", "merdeka", "kumparan", "news", "com",
}
def _clean(text: str) -> str:
text = re.sub(r"[^a-zA-Z\s]", " ", text.lower())
return re.sub(r"\s+", " ", text).strip()
def discover_topics(items: List, num_topics: int = 8) -> List[Dict]:
if len(items) < num_topics:
num_topics = max(2, len(items) // 2)
if len(items) < 4:
return []
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import numpy as np
docs = [_clean(it.text) for it in items]
ids = [it.id for it in items]
vectorizer = TfidfVectorizer(
max_features=2000,
stop_words=list(INDO_STOPWORDS),
min_df=2,
ngram_range=(1, 2),
)
try:
X = vectorizer.fit_transform(docs)
except ValueError:
return []
if X.shape[1] == 0:
return []
k = min(num_topics, X.shape[0])
km = KMeans(n_clusters=k, random_state=42, n_init=10)
labels = km.fit_predict(X)
terms = vectorizer.get_feature_names_out()
order_centroids = km.cluster_centers_.argsort()[:, ::-1]
topics = []
for topic_id in range(k):
keywords = [terms[ind] for ind in order_centroids[topic_id, :8]]
member_ids = [ids[i] for i in range(len(ids)) if labels[i] == topic_id]
topics.append({
"topic_id": topic_id,
"label": ", ".join(keywords[:3]),
"keywords": keywords,
"article_ids": member_ids,
"size": len(member_ids),
})
topics.sort(key=lambda t: t["size"], reverse=True)
return topics
|