Spaces:
Runtime error
Runtime error
Download preprocessing.py from harrycool12/MISCONCEPTION: direct link, hf CLI and curl.
- Browser
- Download file 4.6 kB
-
https://huggingface.co/spaces/harrycool12/MISCONCEPTION/resolve/main/preprocessing.py
- Command line
-
hf download hf://spaces/harrycool12/MISCONCEPTION/preprocessing.py
-
curl -L -o preprocessing.py https://huggingface.co/spaces/harrycool12/MISCONCEPTION/resolve/main/preprocessing.py
4.6 kB
| import spacy | |
| import pandas as pd | |
| from typing import List, Dict, Tuple | |
| import re | |
| import numpy as np | |
| from sklearn.decomposition import LatentDirichletAllocation | |
| from sklearn.feature_extraction.text import CountVectorizer | |
| class TextPreprocessor: | |
| def __init__(self): | |
| self.nlp = spacy.load("en_core_web_sm") | |
| # Mathematical terms and symbols patterns | |
| self.math_patterns = { | |
| 'numbers': r'\d+\.?\d*', | |
| 'operations': r'[\+\-\*\/\=\<\>]', | |
| 'fractions': r'\d+\/\d+', | |
| 'math_terms': ['equation', 'fraction', 'decimal', 'multiply', 'divide', | |
| 'add', 'subtract', 'equal', 'greater', 'less', 'square', 'root'] | |
| } | |
| # Initialize topic modeling components | |
| self.vectorizer = CountVectorizer(max_features=1000, stop_words='english') | |
| self.lda = LatentDirichletAllocation( | |
| n_components=5, | |
| random_state=42, | |
| learning_method='batch' | |
| ) | |
| self.topic_keywords = None | |
| def preprocess_text(self, text: str) -> Dict: | |
| """Preprocess text using spaCy with mathematical feature extraction""" | |
| doc = self.nlp(text.lower()) | |
| # Extract basic NLP features | |
| features = { | |
| 'tokens': [token.text for token in doc if not token.is_stop and not token.is_punct], | |
| 'lemmas': [token.lemma_ for token in doc if not token.is_stop and not token.is_punct], | |
| 'pos_tags': [token.pos_ for token in doc], | |
| 'entities': [ent.text for ent in doc.ents], | |
| 'text_length': len(doc), | |
| 'num_entities': len(doc.ents), | |
| 'vector': doc.vector.tolist() # Add document vector | |
| } | |
| # Extract mathematical features | |
| features.update({ | |
| 'numbers_found': len(re.findall(self.math_patterns['numbers'], text)), | |
| 'operations_found': len(re.findall(self.math_patterns['operations'], text)), | |
| 'fractions_found': len(re.findall(self.math_patterns['fractions'], text)), | |
| 'math_terms_found': sum(1 for term in self.math_patterns['math_terms'] | |
| if term in text.lower()) | |
| }) | |
| # Calculate feature ratios | |
| features['math_density'] = (features['numbers_found'] + | |
| features['operations_found'] + | |
| features['fractions_found']) / max(len(doc), 1) | |
| return features | |
| def get_document_embedding(self, text: str) -> np.ndarray: | |
| """Get document embedding using spaCy""" | |
| doc = self.nlp(text.lower()) | |
| return doc.vector | |
| def train_topic_model(self, texts: List[str]) -> Tuple[np.ndarray, List[str]]: | |
| """Train LDA topic model and return topic distributions""" | |
| # Create document-term matrix | |
| dtm = self.vectorizer.fit_transform(texts) | |
| # Train LDA model | |
| doc_topics = self.lda.fit_transform(dtm) | |
| # Get feature names | |
| feature_names = self.vectorizer.get_feature_names_out() | |
| # Get top words for each topic | |
| topic_keywords = [] | |
| for topic_idx, topic in enumerate(self.lda.components_): | |
| top_words_idx = topic.argsort()[:-5-1:-1] # Get top 5 words | |
| top_words = [feature_names[i] for i in top_words_idx] | |
| topic_keywords.append(', '.join(top_words)) | |
| self.topic_keywords = topic_keywords | |
| return doc_topics, topic_keywords | |
| def process_batch(self, texts: List[str]) -> pd.DataFrame: | |
| """Process a batch of texts with advanced NLP features""" | |
| features_list = [] | |
| for text in texts: | |
| features = self.preprocess_text(text) | |
| features_list.append(features) | |
| # Convert list features to string for DataFrame compatibility | |
| df = pd.DataFrame(features_list) | |
| for col in ['tokens', 'lemmas', 'pos_tags', 'entities']: | |
| df[col] = df[col].apply(lambda x: ' '.join(x)) | |
| # Add topic modeling if enough samples | |
| if len(texts) >= 3: # Minimum documents for meaningful topics | |
| doc_topics, topic_keywords = self.train_topic_model(texts) | |
| # Add topic distributions to features | |
| for i, topic_dist in enumerate(doc_topics): | |
| for topic_idx, prob in enumerate(topic_dist): | |
| df.loc[i, f'topic_{topic_idx}'] = prob | |
| return df | |
| def extract_features(self, df: pd.DataFrame, text_column: str) -> pd.DataFrame: | |
| """Extract features from text column""" | |
| return self.process_batch(df[text_column].tolist()) |