Spaces:
Running
Running
File size: 2,395 Bytes
f39a9fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """
Keyword extraction menggunakan YAKE (Yet Another Keyword Extractor).
Lebih akurat dari TF-IDF manual karena mempertimbangkan posisi kata,
frekuensi, dan co-occurrence.
"""
from typing import List, Dict
import re
# Stopwords Indonesia untuk filtering
STOPWORDS = {
"yang", "di", "ke", "dari", "untuk", "pada", "dengan", "ini", "itu",
"dan", "atau", "adalah", "akan", "juga", "tidak", "para", "oleh",
"sebagai", "dalam", "tersebut", "ada", "dapat", "bisa", "harus",
"lebih", "sangat", "telah", "sudah", "masih", "hanya", "saja",
"republika", "okezone", "detik", "kompas", "tribunnews", "cnn",
"tempo", "antara", "merdeka", "kumparan", "news", "com",
}
def _simple_yake(text: str, top_n: int = 10) -> List[Dict]:
"""
Implementasi YAKE ringan (tanpa library yake).
Scoring: kata yang jarang muncul + tidak di awal/akhir = skor rendah (lebih penting).
"""
text_lower = text.lower()
# Tokenize
words = re.findall(r'\b[a-zA-Z]{3,}\b', text_lower)
if not words:
return []
# Frequency
freq = {}
positions = {}
for i, w in enumerate(words):
if w in STOPWORDS:
continue
freq[w] = freq.get(w, 0) + 1
if w not in positions:
positions[w] = i
if not freq:
return []
max_freq = max(freq.values())
total_words = len(words)
# Score: kombinasi frequency, posisi, dan panjang kata
scored = []
for word, count in freq.items():
# Frequency factor (kata terlalu sering = kurang penting)
freq_score = count / max_freq
# Position factor (kata lebih awal = lebih penting)
pos_score = positions[word] / total_words
# Length factor (kata lebih panjang = lebih bermakna)
len_score = min(1.0, len(word) / 12)
# YAKE-like score (lower = more important)
score = (freq_score * 0.4 + pos_score * 0.3) / (len_score + 0.1)
scored.append({"keyword": word, "score": round(1 - score, 3), "count": count})
# Sort by score descending (higher = more important)
scored.sort(key=lambda x: x["score"], reverse=True)
return scored[:top_n]
def extract_keywords_batch(items: List, top_n: int = 10) -> List[Dict]:
results = []
for item in items:
keywords = _simple_yake(item.text, top_n)
results.append({"id": item.id, "keywords": keywords})
return results
|