MISCONCEPTION / preprocessing.py
harrycool12's picture
Upload 4 files
6e99b69 verified
Raw History Blame Contribute Delete
4.6 kB
import spacy
import pandas as pd
from typing import List, Dict, Tuple
import re
import numpy as np
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer
class TextPreprocessor:
def __init__(self):
self.nlp = spacy.load("en_core_web_sm")
# Mathematical terms and symbols patterns
self.math_patterns = {
'numbers': r'\d+\.?\d*',
'operations': r'[\+\-\*\/\=\<\>]',
'fractions': r'\d+\/\d+',
'math_terms': ['equation', 'fraction', 'decimal', 'multiply', 'divide',
'add', 'subtract', 'equal', 'greater', 'less', 'square', 'root']
}
# Initialize topic modeling components
self.vectorizer = CountVectorizer(max_features=1000, stop_words='english')
self.lda = LatentDirichletAllocation(
n_components=5,
random_state=42,
learning_method='batch'
)
self.topic_keywords = None
def preprocess_text(self, text: str) -> Dict:
"""Preprocess text using spaCy with mathematical feature extraction"""
doc = self.nlp(text.lower())
# Extract basic NLP features
features = {
'tokens': [token.text for token in doc if not token.is_stop and not token.is_punct],
'lemmas': [token.lemma_ for token in doc if not token.is_stop and not token.is_punct],
'pos_tags': [token.pos_ for token in doc],
'entities': [ent.text for ent in doc.ents],
'text_length': len(doc),
'num_entities': len(doc.ents),
'vector': doc.vector.tolist() # Add document vector
}
# Extract mathematical features
features.update({
'numbers_found': len(re.findall(self.math_patterns['numbers'], text)),
'operations_found': len(re.findall(self.math_patterns['operations'], text)),
'fractions_found': len(re.findall(self.math_patterns['fractions'], text)),
'math_terms_found': sum(1 for term in self.math_patterns['math_terms']
if term in text.lower())
})
# Calculate feature ratios
features['math_density'] = (features['numbers_found'] +
features['operations_found'] +
features['fractions_found']) / max(len(doc), 1)
return features
def get_document_embedding(self, text: str) -> np.ndarray:
"""Get document embedding using spaCy"""
doc = self.nlp(text.lower())
return doc.vector
def train_topic_model(self, texts: List[str]) -> Tuple[np.ndarray, List[str]]:
"""Train LDA topic model and return topic distributions"""
# Create document-term matrix
dtm = self.vectorizer.fit_transform(texts)
# Train LDA model
doc_topics = self.lda.fit_transform(dtm)
# Get feature names
feature_names = self.vectorizer.get_feature_names_out()
# Get top words for each topic
topic_keywords = []
for topic_idx, topic in enumerate(self.lda.components_):
top_words_idx = topic.argsort()[:-5-1:-1] # Get top 5 words
top_words = [feature_names[i] for i in top_words_idx]
topic_keywords.append(', '.join(top_words))
self.topic_keywords = topic_keywords
return doc_topics, topic_keywords
def process_batch(self, texts: List[str]) -> pd.DataFrame:
"""Process a batch of texts with advanced NLP features"""
features_list = []
for text in texts:
features = self.preprocess_text(text)
features_list.append(features)
# Convert list features to string for DataFrame compatibility
df = pd.DataFrame(features_list)
for col in ['tokens', 'lemmas', 'pos_tags', 'entities']:
df[col] = df[col].apply(lambda x: ' '.join(x))
# Add topic modeling if enough samples
if len(texts) >= 3: # Minimum documents for meaningful topics
doc_topics, topic_keywords = self.train_topic_model(texts)
# Add topic distributions to features
for i, topic_dist in enumerate(doc_topics):
for topic_idx, prob in enumerate(topic_dist):
df.loc[i, f'topic_{topic_idx}'] = prob
return df
def extract_features(self, df: pd.DataFrame, text_column: str) -> pd.DataFrame:
"""Extract features from text column"""
return self.process_batch(df[text_column].tolist())