Spaces:
Runtime error
Runtime error
File size: 4,604 Bytes
6e99b69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | import spacy
import pandas as pd
from typing import List, Dict, Tuple
import re
import numpy as np
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer
class TextPreprocessor:
def __init__(self):
self.nlp = spacy.load("en_core_web_sm")
# Mathematical terms and symbols patterns
self.math_patterns = {
'numbers': r'\d+\.?\d*',
'operations': r'[\+\-\*\/\=\<\>]',
'fractions': r'\d+\/\d+',
'math_terms': ['equation', 'fraction', 'decimal', 'multiply', 'divide',
'add', 'subtract', 'equal', 'greater', 'less', 'square', 'root']
}
# Initialize topic modeling components
self.vectorizer = CountVectorizer(max_features=1000, stop_words='english')
self.lda = LatentDirichletAllocation(
n_components=5,
random_state=42,
learning_method='batch'
)
self.topic_keywords = None
def preprocess_text(self, text: str) -> Dict:
"""Preprocess text using spaCy with mathematical feature extraction"""
doc = self.nlp(text.lower())
# Extract basic NLP features
features = {
'tokens': [token.text for token in doc if not token.is_stop and not token.is_punct],
'lemmas': [token.lemma_ for token in doc if not token.is_stop and not token.is_punct],
'pos_tags': [token.pos_ for token in doc],
'entities': [ent.text for ent in doc.ents],
'text_length': len(doc),
'num_entities': len(doc.ents),
'vector': doc.vector.tolist() # Add document vector
}
# Extract mathematical features
features.update({
'numbers_found': len(re.findall(self.math_patterns['numbers'], text)),
'operations_found': len(re.findall(self.math_patterns['operations'], text)),
'fractions_found': len(re.findall(self.math_patterns['fractions'], text)),
'math_terms_found': sum(1 for term in self.math_patterns['math_terms']
if term in text.lower())
})
# Calculate feature ratios
features['math_density'] = (features['numbers_found'] +
features['operations_found'] +
features['fractions_found']) / max(len(doc), 1)
return features
def get_document_embedding(self, text: str) -> np.ndarray:
"""Get document embedding using spaCy"""
doc = self.nlp(text.lower())
return doc.vector
def train_topic_model(self, texts: List[str]) -> Tuple[np.ndarray, List[str]]:
"""Train LDA topic model and return topic distributions"""
# Create document-term matrix
dtm = self.vectorizer.fit_transform(texts)
# Train LDA model
doc_topics = self.lda.fit_transform(dtm)
# Get feature names
feature_names = self.vectorizer.get_feature_names_out()
# Get top words for each topic
topic_keywords = []
for topic_idx, topic in enumerate(self.lda.components_):
top_words_idx = topic.argsort()[:-5-1:-1] # Get top 5 words
top_words = [feature_names[i] for i in top_words_idx]
topic_keywords.append(', '.join(top_words))
self.topic_keywords = topic_keywords
return doc_topics, topic_keywords
def process_batch(self, texts: List[str]) -> pd.DataFrame:
"""Process a batch of texts with advanced NLP features"""
features_list = []
for text in texts:
features = self.preprocess_text(text)
features_list.append(features)
# Convert list features to string for DataFrame compatibility
df = pd.DataFrame(features_list)
for col in ['tokens', 'lemmas', 'pos_tags', 'entities']:
df[col] = df[col].apply(lambda x: ' '.join(x))
# Add topic modeling if enough samples
if len(texts) >= 3: # Minimum documents for meaningful topics
doc_topics, topic_keywords = self.train_topic_model(texts)
# Add topic distributions to features
for i, topic_dist in enumerate(doc_topics):
for topic_idx, prob in enumerate(topic_dist):
df.loc[i, f'topic_{topic_idx}'] = prob
return df
def extract_features(self, df: pd.DataFrame, text_column: str) -> pd.DataFrame:
"""Extract features from text column"""
return self.process_batch(df[text_column].tolist()) |