import re import time from typing import List, Tuple import nltk from sklearn.cluster import KMeans # 형태소 및 NLP 모듈 Lazy 임포트 / 전역 캐시 _kiwi_instance = None _sbert_models = {} def get_kiwi(): global _kiwi_instance if _kiwi_instance is None: from kiwipiepy import Kiwi _kiwi_instance = Kiwi() return _kiwi_instance def get_sentence_transformer(model_name: str = "intfloat/multilingual-e5-small"): global _sbert_models if model_name not in _sbert_models: from sentence_transformers import SentenceTransformer # CPU에만 로드하여 ZeroGPU VRAM 소모 차단 _sbert_models[model_name] = SentenceTransformer(model_name, device="cpu") return _sbert_models[model_name] def is_korean_text(text: str, threshold: float = 0.20) -> bool: if not text: return False korean_chars = len(re.findall(r"[\uac00-\ud7a3]", text)) total_letters = len(re.findall(r"[a-zA-Z\uac00-\ud7a3]", text)) if total_letters == 0: return False return (korean_chars / total_letters) >= threshold def split_sentences(text: str, is_korean: bool) -> List[str]: clean_text = text.strip() if not clean_text: return [] if is_korean: try: kiwi = get_kiwi() sents = kiwi.split_into_sents(clean_text) return [s.text.strip() for s in sents if s.text.strip()] except Exception: pass # 영어 및 기본 분리기 try: nltk.data.find("tokenizers/punkt_tab") return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()] except Exception: try: nltk.download("punkt", quiet=True) nltk.download("punkt_tab", quiet=True) return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()] except Exception: # 정규식 폴백 raw = re.split(r"(?<=[.!?])\s+", clean_text) return [s.strip() for s in raw if s.strip()] class SimpleTokenizer: def __init__(self, tokenize_fn): self._tokenize_fn = tokenize_fn def to_words(self, text: str): return self._tokenize_fn(text) def lexrank_summary(text: str, sentence_count: int = 3) -> Tuple[str, float]: t0 = time.perf_counter() is_ko = is_korean_text(text) sentences = split_sentences(text, is_ko) if len(sentences) <= sentence_count: elapsed_ms = (time.perf_counter() - t0) * 1000 return " ".join(sentences), elapsed_ms from sumy.models.dom import ObjectDocumentModel, Paragraph, Sentence from sumy.summarizers.lex_rank import LexRankSummarizer if is_ko: kiwi = get_kiwi() tok = SimpleTokenizer(lambda t: [token.form for token in kiwi.tokenize(t)]) else: tok = SimpleTokenizer(lambda t: re.findall(r"\w+", t.lower())) sumy_sentences = [Sentence(s, tok) for s in sentences] doc = ObjectDocumentModel([Paragraph(sumy_sentences)]) summarizer = LexRankSummarizer() selected_sumy = summarizer(doc, sentence_count) selected_texts = [] for s_obj in selected_sumy: try: idx = sumy_sentences.index(s_obj) selected_texts.append((idx, sentences[idx])) except ValueError: pass selected_texts.sort(key=lambda x: x[0]) result = " ".join([t[1] for t in selected_texts]) if selected_texts else " ".join(sentences[:sentence_count]) elapsed_ms = (time.perf_counter() - t0) * 1000 return result, elapsed_ms def embedding_summary(text: str, sentence_count: int = 3, model_name: str = "intfloat/multilingual-e5-small") -> Tuple[str, float]: t0 = time.perf_counter() is_ko = is_korean_text(text) sentences = split_sentences(text, is_ko) if len(sentences) <= sentence_count: elapsed_ms = (time.perf_counter() - t0) * 1000 return " ".join(sentences), elapsed_ms embedder = get_sentence_transformer(model_name) # e5 모델 접두사 호환성 if "e5" in model_name.lower(): encoded_inputs = [f"passage: {s}" for s in sentences] else: encoded_inputs = sentences embeddings = embedder.encode(encoded_inputs, normalize_embeddings=True) k = min(sentence_count, len(sentences)) kmeans = KMeans(n_clusters=k, random_state=42, n_init="auto") kmeans.fit(embeddings) selected_indices = set() for center in kmeans.cluster_centers_: dists = ((embeddings - center) ** 2).sum(axis=1) # 이미 선택된 인덱스 제외하고 가장 가까운 것 선별 sorted_idx = dists.argsort() for idx in sorted_idx: if idx not in selected_indices: selected_indices.add(int(idx)) break sorted_indices = sorted(list(selected_indices)) result = " ".join([sentences[i] for i in sorted_indices]) elapsed_ms = (time.perf_counter() - t0) * 1000 return result, elapsed_ms