Spaces:
Sleeping
Sleeping
| import math | |
| import numpy as np | |
| from typing import List, Dict, Tuple, Optional | |
| from nlp_core.tokenizer import BilingualTokenizer | |
| from .tfidf import CustomTFIDF | |
| class LSASummarizer: | |
| """ | |
| Latent Semantic Analysis (LSA) Extractive Summarizer. | |
| Uses Singular Value Decomposition (SVD) on the Term-Sentence matrix | |
| to uncover latent semantic topics and extract sentences with the highest | |
| topic representation energy. | |
| """ | |
| def __init__(self): | |
| self.tokenizer = BilingualTokenizer() | |
| def summarize( | |
| self, | |
| text: str, | |
| num_sentences: Optional[int] = None, | |
| ratio: Optional[float] = None, | |
| lang: Optional[str] = None | |
| ) -> Dict[str, any]: | |
| """ | |
| Summarizes text using Latent Semantic Analysis (SVD). | |
| """ | |
| if not text or not text.strip(): | |
| return { | |
| "summary": "", | |
| "selected_indices": [], | |
| "sentence_scores": [], | |
| "original_sentences": [], | |
| "language": lang or "en" | |
| } | |
| if lang is None: | |
| lang = self.tokenizer.lang_detector.detect_language(text) | |
| raw_sentences = self.tokenizer.split_sentences(text, lang=lang) | |
| total_sents = len(raw_sentences) | |
| if total_sents <= 2: | |
| return { | |
| "summary": ' '.join(raw_sentences), | |
| "selected_indices": list(range(total_sents)), | |
| "sentence_scores": [(s, 1.0) for s in raw_sentences], | |
| "original_sentences": raw_sentences, | |
| "language": lang | |
| } | |
| # Tokenize and clean | |
| tokenized_sentences = [ | |
| self.tokenizer.tokenize_words(s, lang=lang, remove_stopwords=True, stem=True) | |
| for s in raw_sentences | |
| ] | |
| # TF-IDF fit | |
| tfidf = CustomTFIDF() | |
| sparse_vectors = tfidf.fit_transform(tokenized_sentences) | |
| vocab_size = len(tfidf.vocabulary) | |
| if vocab_size == 0: | |
| return { | |
| "summary": ' '.join(raw_sentences[:2]), | |
| "selected_indices": list(range(min(2, total_sents))), | |
| "sentence_scores": [(s, 1.0) for s in raw_sentences], | |
| "original_sentences": raw_sentences, | |
| "language": lang | |
| } | |
| # Build Term-Sentence Matrix A (shape: vocab_size x num_sentences) | |
| A = np.zeros((vocab_size, total_sents), dtype=np.float64) | |
| for sent_idx, vec in enumerate(sparse_vectors): | |
| for term_idx, weight in vec.items(): | |
| A[term_idx, sent_idx] = weight | |
| # Singular Value Decomposition: A = U * S * Vt | |
| try: | |
| U, S, Vt = np.linalg.svd(A, full_matrices=False) | |
| # S is singular values, Vt is sentence loadings across concepts (shape: concepts x num_sentences) | |
| num_topics = min(len(S), total_sents) | |
| # Weight sentence components by singular values (Steinberger & Jezek approach) | |
| weighted_vt = S[:num_topics, np.newaxis] * Vt[:num_topics, :] | |
| # Compute Euclidean norm score for each sentence across top topics | |
| scores = np.linalg.norm(weighted_vt, axis=0) | |
| # Normalize scores to [0, 1] | |
| max_score = np.max(scores) if np.max(scores) > 0 else 1.0 | |
| scores = (scores / max_score).tolist() | |
| except Exception: | |
| # Fallback if SVD fails on degenerate matrix | |
| scores = [1.0 / total_sents] * total_sents | |
| # Determine target summary size | |
| if num_sentences is not None: | |
| k = max(1, min(num_sentences, total_sents)) | |
| elif ratio is not None: | |
| k = max(1, int(round(total_sents * ratio))) | |
| else: | |
| k = max(1, min(3, total_sents)) | |
| ranked_indices = sorted(range(total_sents), key=lambda idx: scores[idx], reverse=True) | |
| top_k_indices = ranked_indices[:k] | |
| selected_indices = sorted(top_k_indices) | |
| selected_sentences = [raw_sentences[i] for i in selected_indices] | |
| summary = ' '.join(selected_sentences) | |
| sentence_scores = [(raw_sentences[i], float(scores[i])) for i in range(total_sents)] | |
| return { | |
| "summary": summary, | |
| "selected_indices": selected_indices, | |
| "sentence_scores": sentence_scores, | |
| "original_sentences": raw_sentences, | |
| "language": lang | |
| } | |