Spaces:
Sleeping
Sleeping
File size: 4,414 Bytes
3d9ba5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import math
import numpy as np
from typing import List, Dict, Tuple, Optional
from nlp_core.tokenizer import BilingualTokenizer
from .tfidf import CustomTFIDF
class LSASummarizer:
"""
Latent Semantic Analysis (LSA) Extractive Summarizer.
Uses Singular Value Decomposition (SVD) on the Term-Sentence matrix
to uncover latent semantic topics and extract sentences with the highest
topic representation energy.
"""
def __init__(self):
self.tokenizer = BilingualTokenizer()
def summarize(
self,
text: str,
num_sentences: Optional[int] = None,
ratio: Optional[float] = None,
lang: Optional[str] = None
) -> Dict[str, any]:
"""
Summarizes text using Latent Semantic Analysis (SVD).
"""
if not text or not text.strip():
return {
"summary": "",
"selected_indices": [],
"sentence_scores": [],
"original_sentences": [],
"language": lang or "en"
}
if lang is None:
lang = self.tokenizer.lang_detector.detect_language(text)
raw_sentences = self.tokenizer.split_sentences(text, lang=lang)
total_sents = len(raw_sentences)
if total_sents <= 2:
return {
"summary": ' '.join(raw_sentences),
"selected_indices": list(range(total_sents)),
"sentence_scores": [(s, 1.0) for s in raw_sentences],
"original_sentences": raw_sentences,
"language": lang
}
# Tokenize and clean
tokenized_sentences = [
self.tokenizer.tokenize_words(s, lang=lang, remove_stopwords=True, stem=True)
for s in raw_sentences
]
# TF-IDF fit
tfidf = CustomTFIDF()
sparse_vectors = tfidf.fit_transform(tokenized_sentences)
vocab_size = len(tfidf.vocabulary)
if vocab_size == 0:
return {
"summary": ' '.join(raw_sentences[:2]),
"selected_indices": list(range(min(2, total_sents))),
"sentence_scores": [(s, 1.0) for s in raw_sentences],
"original_sentences": raw_sentences,
"language": lang
}
# Build Term-Sentence Matrix A (shape: vocab_size x num_sentences)
A = np.zeros((vocab_size, total_sents), dtype=np.float64)
for sent_idx, vec in enumerate(sparse_vectors):
for term_idx, weight in vec.items():
A[term_idx, sent_idx] = weight
# Singular Value Decomposition: A = U * S * Vt
try:
U, S, Vt = np.linalg.svd(A, full_matrices=False)
# S is singular values, Vt is sentence loadings across concepts (shape: concepts x num_sentences)
num_topics = min(len(S), total_sents)
# Weight sentence components by singular values (Steinberger & Jezek approach)
weighted_vt = S[:num_topics, np.newaxis] * Vt[:num_topics, :]
# Compute Euclidean norm score for each sentence across top topics
scores = np.linalg.norm(weighted_vt, axis=0)
# Normalize scores to [0, 1]
max_score = np.max(scores) if np.max(scores) > 0 else 1.0
scores = (scores / max_score).tolist()
except Exception:
# Fallback if SVD fails on degenerate matrix
scores = [1.0 / total_sents] * total_sents
# Determine target summary size
if num_sentences is not None:
k = max(1, min(num_sentences, total_sents))
elif ratio is not None:
k = max(1, int(round(total_sents * ratio)))
else:
k = max(1, min(3, total_sents))
ranked_indices = sorted(range(total_sents), key=lambda idx: scores[idx], reverse=True)
top_k_indices = ranked_indices[:k]
selected_indices = sorted(top_k_indices)
selected_sentences = [raw_sentences[i] for i in selected_indices]
summary = ' '.join(selected_sentences)
sentence_scores = [(raw_sentences[i], float(scores[i])) for i in range(total_sents)]
return {
"summary": summary,
"selected_indices": selected_indices,
"sentence_scores": sentence_scores,
"original_sentences": raw_sentences,
"language": lang
}
|