slm / extractive.py
primenumbersam's picture
fix: resolve sumy ObjectDocumentModel and ZeroGPU CUDA wrapper initialization
9cc8c20
Raw History Blame Contribute Delete
4.94 kB
import re
import time
from typing import List, Tuple
import nltk
from sklearn.cluster import KMeans
# 형태소 및 NLP 모듈 Lazy 임포트 / 전역 캐시
_kiwi_instance = None
_sbert_models = {}
def get_kiwi():
global _kiwi_instance
if _kiwi_instance is None:
from kiwipiepy import Kiwi
_kiwi_instance = Kiwi()
return _kiwi_instance
def get_sentence_transformer(model_name: str = "intfloat/multilingual-e5-small"):
global _sbert_models
if model_name not in _sbert_models:
from sentence_transformers import SentenceTransformer
# CPU에만 로드하여 ZeroGPU VRAM 소모 차단
_sbert_models[model_name] = SentenceTransformer(model_name, device="cpu")
return _sbert_models[model_name]
def is_korean_text(text: str, threshold: float = 0.20) -> bool:
if not text:
return False
korean_chars = len(re.findall(r"[\uac00-\ud7a3]", text))
total_letters = len(re.findall(r"[a-zA-Z\uac00-\ud7a3]", text))
if total_letters == 0:
return False
return (korean_chars / total_letters) >= threshold
def split_sentences(text: str, is_korean: bool) -> List[str]:
clean_text = text.strip()
if not clean_text:
return []
if is_korean:
try:
kiwi = get_kiwi()
sents = kiwi.split_into_sents(clean_text)
return [s.text.strip() for s in sents if s.text.strip()]
except Exception:
pass
# 영어 및 기본 분리기
try:
nltk.data.find("tokenizers/punkt_tab")
return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()]
except Exception:
try:
nltk.download("punkt", quiet=True)
nltk.download("punkt_tab", quiet=True)
return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()]
except Exception:
# 정규식 폴백
raw = re.split(r"(?<=[.!?])\s+", clean_text)
return [s.strip() for s in raw if s.strip()]
class SimpleTokenizer:
def __init__(self, tokenize_fn):
self._tokenize_fn = tokenize_fn
def to_words(self, text: str):
return self._tokenize_fn(text)
def lexrank_summary(text: str, sentence_count: int = 3) -> Tuple[str, float]:
t0 = time.perf_counter()
is_ko = is_korean_text(text)
sentences = split_sentences(text, is_ko)
if len(sentences) <= sentence_count:
elapsed_ms = (time.perf_counter() - t0) * 1000
return " ".join(sentences), elapsed_ms
from sumy.models.dom import ObjectDocumentModel, Paragraph, Sentence
from sumy.summarizers.lex_rank import LexRankSummarizer
if is_ko:
kiwi = get_kiwi()
tok = SimpleTokenizer(lambda t: [token.form for token in kiwi.tokenize(t)])
else:
tok = SimpleTokenizer(lambda t: re.findall(r"\w+", t.lower()))
sumy_sentences = [Sentence(s, tok) for s in sentences]
doc = ObjectDocumentModel([Paragraph(sumy_sentences)])
summarizer = LexRankSummarizer()
selected_sumy = summarizer(doc, sentence_count)
selected_texts = []
for s_obj in selected_sumy:
try:
idx = sumy_sentences.index(s_obj)
selected_texts.append((idx, sentences[idx]))
except ValueError:
pass
selected_texts.sort(key=lambda x: x[0])
result = " ".join([t[1] for t in selected_texts]) if selected_texts else " ".join(sentences[:sentence_count])
elapsed_ms = (time.perf_counter() - t0) * 1000
return result, elapsed_ms
def embedding_summary(text: str, sentence_count: int = 3, model_name: str = "intfloat/multilingual-e5-small") -> Tuple[str, float]:
t0 = time.perf_counter()
is_ko = is_korean_text(text)
sentences = split_sentences(text, is_ko)
if len(sentences) <= sentence_count:
elapsed_ms = (time.perf_counter() - t0) * 1000
return " ".join(sentences), elapsed_ms
embedder = get_sentence_transformer(model_name)
# e5 모델 접두사 호환성
if "e5" in model_name.lower():
encoded_inputs = [f"passage: {s}" for s in sentences]
else:
encoded_inputs = sentences
embeddings = embedder.encode(encoded_inputs, normalize_embeddings=True)
k = min(sentence_count, len(sentences))
kmeans = KMeans(n_clusters=k, random_state=42, n_init="auto")
kmeans.fit(embeddings)
selected_indices = set()
for center in kmeans.cluster_centers_:
dists = ((embeddings - center) ** 2).sum(axis=1)
# 이미 선택된 인덱스 제외하고 가장 가까운 것 선별
sorted_idx = dists.argsort()
for idx in sorted_idx:
if idx not in selected_indices:
selected_indices.add(int(idx))
break
sorted_indices = sorted(list(selected_indices))
result = " ".join([sentences[i] for i in sorted_indices])
elapsed_ms = (time.perf_counter() - t0) * 1000
return result, elapsed_ms