import spacy import pandas as pd from typing import List, Dict, Tuple import re import numpy as np from sklearn.decomposition import LatentDirichletAllocation from sklearn.feature_extraction.text import CountVectorizer class TextPreprocessor: def __init__(self): self.nlp = spacy.load("en_core_web_sm") # Mathematical terms and symbols patterns self.math_patterns = { 'numbers': r'\d+\.?\d*', 'operations': r'[\+\-\*\/\=\<\>]', 'fractions': r'\d+\/\d+', 'math_terms': ['equation', 'fraction', 'decimal', 'multiply', 'divide', 'add', 'subtract', 'equal', 'greater', 'less', 'square', 'root'] } # Initialize topic modeling components self.vectorizer = CountVectorizer(max_features=1000, stop_words='english') self.lda = LatentDirichletAllocation( n_components=5, random_state=42, learning_method='batch' ) self.topic_keywords = None def preprocess_text(self, text: str) -> Dict: """Preprocess text using spaCy with mathematical feature extraction""" doc = self.nlp(text.lower()) # Extract basic NLP features features = { 'tokens': [token.text for token in doc if not token.is_stop and not token.is_punct], 'lemmas': [token.lemma_ for token in doc if not token.is_stop and not token.is_punct], 'pos_tags': [token.pos_ for token in doc], 'entities': [ent.text for ent in doc.ents], 'text_length': len(doc), 'num_entities': len(doc.ents), 'vector': doc.vector.tolist() # Add document vector } # Extract mathematical features features.update({ 'numbers_found': len(re.findall(self.math_patterns['numbers'], text)), 'operations_found': len(re.findall(self.math_patterns['operations'], text)), 'fractions_found': len(re.findall(self.math_patterns['fractions'], text)), 'math_terms_found': sum(1 for term in self.math_patterns['math_terms'] if term in text.lower()) }) # Calculate feature ratios features['math_density'] = (features['numbers_found'] + features['operations_found'] + features['fractions_found']) / max(len(doc), 1) return features def get_document_embedding(self, text: str) -> np.ndarray: """Get document embedding using spaCy""" doc = self.nlp(text.lower()) return doc.vector def train_topic_model(self, texts: List[str]) -> Tuple[np.ndarray, List[str]]: """Train LDA topic model and return topic distributions""" # Create document-term matrix dtm = self.vectorizer.fit_transform(texts) # Train LDA model doc_topics = self.lda.fit_transform(dtm) # Get feature names feature_names = self.vectorizer.get_feature_names_out() # Get top words for each topic topic_keywords = [] for topic_idx, topic in enumerate(self.lda.components_): top_words_idx = topic.argsort()[:-5-1:-1] # Get top 5 words top_words = [feature_names[i] for i in top_words_idx] topic_keywords.append(', '.join(top_words)) self.topic_keywords = topic_keywords return doc_topics, topic_keywords def process_batch(self, texts: List[str]) -> pd.DataFrame: """Process a batch of texts with advanced NLP features""" features_list = [] for text in texts: features = self.preprocess_text(text) features_list.append(features) # Convert list features to string for DataFrame compatibility df = pd.DataFrame(features_list) for col in ['tokens', 'lemmas', 'pos_tags', 'entities']: df[col] = df[col].apply(lambda x: ' '.join(x)) # Add topic modeling if enough samples if len(texts) >= 3: # Minimum documents for meaningful topics doc_topics, topic_keywords = self.train_topic_model(texts) # Add topic distributions to features for i, topic_dist in enumerate(doc_topics): for topic_idx, prob in enumerate(topic_dist): df.loc[i, f'topic_{topic_idx}'] = prob return df def extract_features(self, df: pd.DataFrame, text_column: str) -> pd.DataFrame: """Extract features from text column""" return self.process_batch(df[text_column].tolist())