Spaces:
Sleeping
Sleeping
Download extractive.py from primenumbersam/slm: direct link, hf CLI and curl.
- Browser
- Download file 4.94 kB
-
https://huggingface.co/spaces/primenumbersam/slm/resolve/main/extractive.py
- Command line
-
hf download hf://spaces/primenumbersam/slm/extractive.py
-
curl -L -o extractive.py https://huggingface.co/spaces/primenumbersam/slm/resolve/main/extractive.py
4.94 kB
| import re | |
| import time | |
| from typing import List, Tuple | |
| import nltk | |
| from sklearn.cluster import KMeans | |
| # 형태소 및 NLP 모듈 Lazy 임포트 / 전역 캐시 | |
| _kiwi_instance = None | |
| _sbert_models = {} | |
| def get_kiwi(): | |
| global _kiwi_instance | |
| if _kiwi_instance is None: | |
| from kiwipiepy import Kiwi | |
| _kiwi_instance = Kiwi() | |
| return _kiwi_instance | |
| def get_sentence_transformer(model_name: str = "intfloat/multilingual-e5-small"): | |
| global _sbert_models | |
| if model_name not in _sbert_models: | |
| from sentence_transformers import SentenceTransformer | |
| # CPU에만 로드하여 ZeroGPU VRAM 소모 차단 | |
| _sbert_models[model_name] = SentenceTransformer(model_name, device="cpu") | |
| return _sbert_models[model_name] | |
| def is_korean_text(text: str, threshold: float = 0.20) -> bool: | |
| if not text: | |
| return False | |
| korean_chars = len(re.findall(r"[\uac00-\ud7a3]", text)) | |
| total_letters = len(re.findall(r"[a-zA-Z\uac00-\ud7a3]", text)) | |
| if total_letters == 0: | |
| return False | |
| return (korean_chars / total_letters) >= threshold | |
| def split_sentences(text: str, is_korean: bool) -> List[str]: | |
| clean_text = text.strip() | |
| if not clean_text: | |
| return [] | |
| if is_korean: | |
| try: | |
| kiwi = get_kiwi() | |
| sents = kiwi.split_into_sents(clean_text) | |
| return [s.text.strip() for s in sents if s.text.strip()] | |
| except Exception: | |
| pass | |
| # 영어 및 기본 분리기 | |
| try: | |
| nltk.data.find("tokenizers/punkt_tab") | |
| return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()] | |
| except Exception: | |
| try: | |
| nltk.download("punkt", quiet=True) | |
| nltk.download("punkt_tab", quiet=True) | |
| return [s.strip() for s in nltk.sent_tokenize(clean_text) if s.strip()] | |
| except Exception: | |
| # 정규식 폴백 | |
| raw = re.split(r"(?<=[.!?])\s+", clean_text) | |
| return [s.strip() for s in raw if s.strip()] | |
| class SimpleTokenizer: | |
| def __init__(self, tokenize_fn): | |
| self._tokenize_fn = tokenize_fn | |
| def to_words(self, text: str): | |
| return self._tokenize_fn(text) | |
| def lexrank_summary(text: str, sentence_count: int = 3) -> Tuple[str, float]: | |
| t0 = time.perf_counter() | |
| is_ko = is_korean_text(text) | |
| sentences = split_sentences(text, is_ko) | |
| if len(sentences) <= sentence_count: | |
| elapsed_ms = (time.perf_counter() - t0) * 1000 | |
| return " ".join(sentences), elapsed_ms | |
| from sumy.models.dom import ObjectDocumentModel, Paragraph, Sentence | |
| from sumy.summarizers.lex_rank import LexRankSummarizer | |
| if is_ko: | |
| kiwi = get_kiwi() | |
| tok = SimpleTokenizer(lambda t: [token.form for token in kiwi.tokenize(t)]) | |
| else: | |
| tok = SimpleTokenizer(lambda t: re.findall(r"\w+", t.lower())) | |
| sumy_sentences = [Sentence(s, tok) for s in sentences] | |
| doc = ObjectDocumentModel([Paragraph(sumy_sentences)]) | |
| summarizer = LexRankSummarizer() | |
| selected_sumy = summarizer(doc, sentence_count) | |
| selected_texts = [] | |
| for s_obj in selected_sumy: | |
| try: | |
| idx = sumy_sentences.index(s_obj) | |
| selected_texts.append((idx, sentences[idx])) | |
| except ValueError: | |
| pass | |
| selected_texts.sort(key=lambda x: x[0]) | |
| result = " ".join([t[1] for t in selected_texts]) if selected_texts else " ".join(sentences[:sentence_count]) | |
| elapsed_ms = (time.perf_counter() - t0) * 1000 | |
| return result, elapsed_ms | |
| def embedding_summary(text: str, sentence_count: int = 3, model_name: str = "intfloat/multilingual-e5-small") -> Tuple[str, float]: | |
| t0 = time.perf_counter() | |
| is_ko = is_korean_text(text) | |
| sentences = split_sentences(text, is_ko) | |
| if len(sentences) <= sentence_count: | |
| elapsed_ms = (time.perf_counter() - t0) * 1000 | |
| return " ".join(sentences), elapsed_ms | |
| embedder = get_sentence_transformer(model_name) | |
| # e5 모델 접두사 호환성 | |
| if "e5" in model_name.lower(): | |
| encoded_inputs = [f"passage: {s}" for s in sentences] | |
| else: | |
| encoded_inputs = sentences | |
| embeddings = embedder.encode(encoded_inputs, normalize_embeddings=True) | |
| k = min(sentence_count, len(sentences)) | |
| kmeans = KMeans(n_clusters=k, random_state=42, n_init="auto") | |
| kmeans.fit(embeddings) | |
| selected_indices = set() | |
| for center in kmeans.cluster_centers_: | |
| dists = ((embeddings - center) ** 2).sum(axis=1) | |
| # 이미 선택된 인덱스 제외하고 가장 가까운 것 선별 | |
| sorted_idx = dists.argsort() | |
| for idx in sorted_idx: | |
| if idx not in selected_indices: | |
| selected_indices.add(int(idx)) | |
| break | |
| sorted_indices = sorted(list(selected_indices)) | |
| result = " ".join([sentences[i] for i in sorted_indices]) | |
| elapsed_ms = (time.perf_counter() - t0) * 1000 | |
| return result, elapsed_ms | |