File size: 4,604 Bytes
6e99b69
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import spacy
import pandas as pd
from typing import List, Dict, Tuple
import re
import numpy as np
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer

class TextPreprocessor:
    def __init__(self):
        self.nlp = spacy.load("en_core_web_sm")
        # Mathematical terms and symbols patterns
        self.math_patterns = {
            'numbers': r'\d+\.?\d*',
            'operations': r'[\+\-\*\/\=\<\>]',
            'fractions': r'\d+\/\d+',
            'math_terms': ['equation', 'fraction', 'decimal', 'multiply', 'divide', 
                          'add', 'subtract', 'equal', 'greater', 'less', 'square', 'root']
        }
        # Initialize topic modeling components
        self.vectorizer = CountVectorizer(max_features=1000, stop_words='english')
        self.lda = LatentDirichletAllocation(
            n_components=5,
            random_state=42,
            learning_method='batch'
        )
        self.topic_keywords = None

    def preprocess_text(self, text: str) -> Dict:
        """Preprocess text using spaCy with mathematical feature extraction"""
        doc = self.nlp(text.lower())

        # Extract basic NLP features
        features = {
            'tokens': [token.text for token in doc if not token.is_stop and not token.is_punct],
            'lemmas': [token.lemma_ for token in doc if not token.is_stop and not token.is_punct],
            'pos_tags': [token.pos_ for token in doc],
            'entities': [ent.text for ent in doc.ents],
            'text_length': len(doc),
            'num_entities': len(doc.ents),
            'vector': doc.vector.tolist()  # Add document vector
        }

        # Extract mathematical features
        features.update({
            'numbers_found': len(re.findall(self.math_patterns['numbers'], text)),
            'operations_found': len(re.findall(self.math_patterns['operations'], text)),
            'fractions_found': len(re.findall(self.math_patterns['fractions'], text)),
            'math_terms_found': sum(1 for term in self.math_patterns['math_terms'] 
                                  if term in text.lower())
        })

        # Calculate feature ratios
        features['math_density'] = (features['numbers_found'] + 
                                  features['operations_found'] + 
                                  features['fractions_found']) / max(len(doc), 1)

        return features

    def get_document_embedding(self, text: str) -> np.ndarray:
        """Get document embedding using spaCy"""
        doc = self.nlp(text.lower())
        return doc.vector

    def train_topic_model(self, texts: List[str]) -> Tuple[np.ndarray, List[str]]:
        """Train LDA topic model and return topic distributions"""
        # Create document-term matrix
        dtm = self.vectorizer.fit_transform(texts)

        # Train LDA model
        doc_topics = self.lda.fit_transform(dtm)

        # Get feature names
        feature_names = self.vectorizer.get_feature_names_out()

        # Get top words for each topic
        topic_keywords = []
        for topic_idx, topic in enumerate(self.lda.components_):
            top_words_idx = topic.argsort()[:-5-1:-1]  # Get top 5 words
            top_words = [feature_names[i] for i in top_words_idx]
            topic_keywords.append(', '.join(top_words))

        self.topic_keywords = topic_keywords
        return doc_topics, topic_keywords

    def process_batch(self, texts: List[str]) -> pd.DataFrame:
        """Process a batch of texts with advanced NLP features"""
        features_list = []
        for text in texts:
            features = self.preprocess_text(text)
            features_list.append(features)

        # Convert list features to string for DataFrame compatibility
        df = pd.DataFrame(features_list)
        for col in ['tokens', 'lemmas', 'pos_tags', 'entities']:
            df[col] = df[col].apply(lambda x: ' '.join(x))

        # Add topic modeling if enough samples
        if len(texts) >= 3:  # Minimum documents for meaningful topics
            doc_topics, topic_keywords = self.train_topic_model(texts)

            # Add topic distributions to features
            for i, topic_dist in enumerate(doc_topics):
                for topic_idx, prob in enumerate(topic_dist):
                    df.loc[i, f'topic_{topic_idx}'] = prob

        return df

    def extract_features(self, df: pd.DataFrame, text_column: str) -> pd.DataFrame:
        """Extract features from text column"""
        return self.process_batch(df[text_column].tolist())