import math import numpy as np from typing import List, Dict, Tuple, Optional from nlp_core.tokenizer import BilingualTokenizer from .tfidf import CustomTFIDF class LSASummarizer: """ Latent Semantic Analysis (LSA) Extractive Summarizer. Uses Singular Value Decomposition (SVD) on the Term-Sentence matrix to uncover latent semantic topics and extract sentences with the highest topic representation energy. """ def __init__(self): self.tokenizer = BilingualTokenizer() def summarize( self, text: str, num_sentences: Optional[int] = None, ratio: Optional[float] = None, lang: Optional[str] = None ) -> Dict[str, any]: """ Summarizes text using Latent Semantic Analysis (SVD). """ if not text or not text.strip(): return { "summary": "", "selected_indices": [], "sentence_scores": [], "original_sentences": [], "language": lang or "en" } if lang is None: lang = self.tokenizer.lang_detector.detect_language(text) raw_sentences = self.tokenizer.split_sentences(text, lang=lang) total_sents = len(raw_sentences) if total_sents <= 2: return { "summary": ' '.join(raw_sentences), "selected_indices": list(range(total_sents)), "sentence_scores": [(s, 1.0) for s in raw_sentences], "original_sentences": raw_sentences, "language": lang } # Tokenize and clean tokenized_sentences = [ self.tokenizer.tokenize_words(s, lang=lang, remove_stopwords=True, stem=True) for s in raw_sentences ] # TF-IDF fit tfidf = CustomTFIDF() sparse_vectors = tfidf.fit_transform(tokenized_sentences) vocab_size = len(tfidf.vocabulary) if vocab_size == 0: return { "summary": ' '.join(raw_sentences[:2]), "selected_indices": list(range(min(2, total_sents))), "sentence_scores": [(s, 1.0) for s in raw_sentences], "original_sentences": raw_sentences, "language": lang } # Build Term-Sentence Matrix A (shape: vocab_size x num_sentences) A = np.zeros((vocab_size, total_sents), dtype=np.float64) for sent_idx, vec in enumerate(sparse_vectors): for term_idx, weight in vec.items(): A[term_idx, sent_idx] = weight # Singular Value Decomposition: A = U * S * Vt try: U, S, Vt = np.linalg.svd(A, full_matrices=False) # S is singular values, Vt is sentence loadings across concepts (shape: concepts x num_sentences) num_topics = min(len(S), total_sents) # Weight sentence components by singular values (Steinberger & Jezek approach) weighted_vt = S[:num_topics, np.newaxis] * Vt[:num_topics, :] # Compute Euclidean norm score for each sentence across top topics scores = np.linalg.norm(weighted_vt, axis=0) # Normalize scores to [0, 1] max_score = np.max(scores) if np.max(scores) > 0 else 1.0 scores = (scores / max_score).tolist() except Exception: # Fallback if SVD fails on degenerate matrix scores = [1.0 / total_sents] * total_sents # Determine target summary size if num_sentences is not None: k = max(1, min(num_sentences, total_sents)) elif ratio is not None: k = max(1, int(round(total_sents * ratio))) else: k = max(1, min(3, total_sents)) ranked_indices = sorted(range(total_sents), key=lambda idx: scores[idx], reverse=True) top_k_indices = ranked_indices[:k] selected_indices = sorted(top_k_indices) selected_sentences = [raw_sentences[i] for i in selected_indices] summary = ' '.join(selected_sentences) sentence_scores = [(raw_sentences[i], float(scores[i])) for i in range(total_sents)] return { "summary": summary, "selected_indices": selected_indices, "sentence_scores": sentence_scores, "original_sentences": raw_sentences, "language": lang }