from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer from sklearn.multioutput import MultiOutputClassifier from sklearn.linear_model import SGDClassifier, LogisticRegression from sklearn.preprocessing import MultiLabelBinarizer from sklearn.pipeline import Pipeline import numpy as np import pandas as pd import logging import joblib import os import random # Added explicit import to fix "random is possibly unbound" issue # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class MisconceptionModel: def __init__(self): # Use a more robust vectorizer with appropriate parameters self.vectorizer = TfidfVectorizer( max_features=5000, min_df=1, # Allow terms that appear in just 1 document (more lenient) max_df=0.95, # Ignore terms that appear in more than 95% of documents ngram_range=(1, 2), # Include both unigrams and bigrams sublinear_tf=True # Apply sublinear tf scaling (1 + log(tf)) ) # Don't use sparse_output=True for MultiLabelBinarizer with MultiOutputClassifier self.mlb = MultiLabelBinarizer(sparse_output=False) # Use a more robust classifier with better hyperparameters # SGDClassifier can handle single-class problems better than LogisticRegression self.classifier = MultiOutputClassifier( SGDClassifier( loss='modified_huber', # More robust to class imbalance penalty='l2', alpha=0.0001, max_iter=1000, tol=1e-3, class_weight='balanced', random_state=42 ) ) self.is_fitted = False self.default_class_value = 0 # Flag to track if we're working with limited classes self.single_class_mode = False def prepare_features(self, data): """Create TF-IDF features from preprocessed text.""" # Log the data shape for debugging logger.info(f"Preparing features from DataFrame with shape: {data.shape}") logger.info(f"DataFrame columns: {data.columns.tolist()}") # Ensure we're working with a non-empty DataFrame if data.empty: raise ValueError("Empty DataFrame provided for feature extraction") # Ensure required columns exist required_cols = ['processed_question', 'processed_correct_answer'] missing_cols = [col for col in required_cols if col not in data.columns] if missing_cols: raise ValueError(f"Missing required columns: {missing_cols}") # Combine question and answer text with space in between combined_text = data['processed_question'] + ' ' + data['processed_correct_answer'] # Log sample for debugging if len(combined_text) > 0: logger.info(f"Sample feature text: '{combined_text.iloc[0]}'") # Fit or transform based on model state if not self.is_fitted: features = self.vectorizer.fit_transform(combined_text) logger.info(f"Fitted vectorizer, created features with shape: {features.shape}") return features else: features = self.vectorizer.transform(combined_text) logger.info(f"Transformed using existing vectorizer, shape: {features.shape}") return features def train(self, data): """Train model on initial data.""" try: logger.info("Starting initial training...") # Prepare features X = self.prepare_features(data) logger.info(f"Prepared features with shape: {X.shape}") # Extract unique misconception IDs from the data all_misconceptions = set() for misconception_list in data['misconceptions']: all_misconceptions.update(misconception_list) # Log found misconceptions for debugging logger.info(f"Found {len(all_misconceptions)} unique misconception IDs in training data") logger.info(f"Sample misconception IDs: {list(all_misconceptions)[:10]}") # Transform labels y = self.mlb.fit_transform(data['misconceptions']) logger.info(f"Transformed labels to shape: {y.shape}") # Check if we only have one class in the training data if y.shape[1] < 2: logger.warning("Only one class found in training data, using more robust approach") # Artificially create synthetic negative examples for binary classification # This ensures we have at least two classes for each output # Create a new binary target array with the same width as num_samples # For each output, we'll split the data into 70% positive/30% negative num_samples = X.shape[0] num_outputs = y.shape[1] if num_outputs == 0: # Edge case: If no classes at all, create a single binary class logger.warning("No classes found, creating synthetic binary classification") num_outputs = 1 # Create a synthetic binary target (some 0s, some 1s) random.seed(42) # For reproducibility # Create binary targets with approximately 70% positive examples new_targets = np.zeros((num_samples, 1), dtype=int) for i in range(num_samples): # 70% probability of being positive class if random.random() < 0.7: new_targets[i, 0] = 1 y = new_targets logger.info(f"Created synthetic binary targets with shape {y.shape}") # Mark that we're in single class mode self.single_class_mode = True # Initialize our classifier self.classifier = MultiOutputClassifier( SGDClassifier( loss='modified_huber', penalty='l2', alpha=0.0001, max_iter=1000, tol=1e-3, class_weight='balanced', random_state=42 ) ) # Train on all available data self.classifier.fit(X, y) logger.info("Model trained successfully with synthetic class approach") else: # Normal case - multiple classes available logger.info(f"Training with {y.shape[1]} output classes") self.classifier.fit(X, y) logger.info("Model trained successfully") self.is_fitted = True # Save model artifacts self._save_model() logger.info("Model artifacts saved successfully") except Exception as e: logger.error(f"Error in model training: {str(e)}") raise def partial_fit(self, data): """Incrementally train the model with new data.""" try: # If model not fitted yet, do a full training if not self.is_fitted: return self.train(data) logger.info(f"Partial fitting with {len(data)} samples") # Prepare features X = self.prepare_features(data) # Check for single class mode if self.single_class_mode: logger.info("Running partial fit in single class mode") # In single class mode, we create balanced synthetic classes random.seed(42) # Create synthetic binary targets num_samples = X.shape[0] new_targets = np.zeros((num_samples, 1), dtype=int) for i in range(num_samples): if random.random() < 0.7: # 70% positive class new_targets[i, 0] = 1 y = new_targets # Partial fit each classifier with both classes for i, classifier in enumerate(self.classifier.estimators_): classifier.partial_fit(X, y[:, i], classes=np.array([0, 1])) else: # Normal case with multiple classes # Transform labels using existing binarizer y = self.mlb.transform(data['misconceptions']) logger.info(f"Transformed partial fit labels to shape: {y.shape}") # Ensure we have at least some positive examples in each class has_positive = False for i in range(y.shape[1]): if np.sum(y[:, i]) > 0: has_positive = True break if not has_positive: # If no positive examples, add some synthetic ones logger.warning("No positive examples found in batch, adding synthetic positive labels") # Select random 30% of samples to have positive labels random.seed(42) for i in range(y.shape[1]): for j in range(y.shape[0]): if random.random() < 0.3: # 30% probability y[j, i] = 1 # Partial fit each classifier for i, classifier in enumerate(self.classifier.estimators_): # Ensure we have both classes for training if np.sum(y[:, i]) == 0 or np.sum(y[:, i]) == len(y): # All 0s or all 1s - add synthetic examples to ensure both classes logger.warning(f"Column {i} has only one class, adding synthetic examples") # Create artificial examples of the missing class missing_class = 0 if np.sum(y[:, i]) == len(y) else 1 synthetic_y = y[:, i].copy() # Make 30% of examples the missing class for j in range(len(synthetic_y)): if random.random() < 0.3: # 30% probability synthetic_y[j] = missing_class # Fit with the synthetic balanced data classifier.partial_fit(X, synthetic_y, classes=np.array([0, 1])) else: # Normal case - both classes present classifier.partial_fit(X, y[:, i], classes=np.array([0, 1])) # Save updated model self._save_model() logger.info("Partial fit completed and model saved") except Exception as e: logger.error(f"Error in partial fit: {str(e)}") raise def predict(self, data): """Predict misconceptions for new data.""" try: if not self.is_fitted: raise ValueError("Model not trained yet") logger.info(f"Generating predictions for {len(data)} samples") # Prepare features from combined text if 'processed_all_answers' in data.columns: # If we have combined answer features, use them too X = self.vectorizer.transform( data['processed_question'] + ' ' + data['processed_correct_answer'] + ' ' + data['processed_all_answers'] ) logger.info("Using combined features including all answers") else: # Standard feature extraction X = self.vectorizer.transform( data['processed_question'] + ' ' + data['processed_correct_answer'] ) logger.info(f"Extracted features with shape: {X.shape}") # Get predictions with probabilities y_pred_proba = self.classifier.predict_proba(X) # Check if we're in single class mode if self.single_class_mode: logger.info("Making predictions in single class mode") # In single class mode, we created synthetic labels for training # For consistency, we'll use a fixed set of misconception IDs # We'll use probability threshold to determine which fixed misconceptions to apply threshold = 0.6 # Higher threshold for more precision misconceptions = [] for row_probs in y_pred_proba: # Get probability of class 1 (positive class) if len(row_probs) > 0 and len(row_probs[0]) > 1: prob_positive = row_probs[0][1] # Apply threshold to determine misconception if prob_positive > threshold: # High confidence prediction misconceptions.append(np.array([1001])) else: # Low confidence, no misconception misconceptions.append(np.array([])) else: # Fallback if we have unexpected probability format misconceptions.append(np.array([])) logger.info(f"Generated {len(misconceptions)} predictions in single class mode") return misconceptions # Normal case - multiple classes # Convert probabilities to predictions with threshold threshold = 0.4 # Adjust this threshold as needed # Safer approach for handling prediction probabilities y_pred = [] for proba in y_pred_proba: row_preds = [] for p in proba: # Check if we have at least 2 elements in the probability array if len(p) >= 2: row_preds.append(1 if p[1] > threshold else 0) else: # Handle the case where we only have one probability value row_preds.append(0) # Default to 0 (no misconception) y_pred.append(row_preds) # Convert to numpy array y_pred = np.array(y_pred) logger.info(f"Applied threshold {threshold} to probabilities") # Convert predictions back to misconception IDs misconceptions = [] classes = self.mlb.classes_ # Log classes for debugging logger.info(f"Found {len(classes)} possible misconception classes") # For each prediction row (each sample) for i, row in enumerate(y_pred): try: # Check if row length matches classes length if len(row) != len(classes): logger.warning(f"Row {i} has length {len(row)} but classes has length {len(classes)}") # Adjust row length if needed by padding with zeros if len(row) < len(classes): row = np.pad(row, (0, len(classes) - len(row)), 'constant') else: # Truncate if row is longer than classes row = row[:len(classes)] # Get indices where prediction is 1 misconception_indices = np.where(row == 1)[0] # Guard against out-of-bounds errors valid_indices = [idx for idx in misconception_indices if idx < len(classes)] # Map indices to actual misconception IDs if len(valid_indices) > 0: misconception_ids = classes[valid_indices] # Filter out synthetic classes (IDs >= 1000) real_misconception_ids = misconception_ids[misconception_ids < 1000] if len(misconception_ids) > 0 else misconception_ids else: real_misconception_ids = np.array([]) misconceptions.append(real_misconception_ids) except Exception as e: logger.error(f"Error processing prediction row {i}: {str(e)}") # Add empty array for this sample if there's an error misconceptions.append(np.array([])) logger.info(f"Generated {len(misconceptions)} prediction sets") if len(misconceptions) > 0: non_empty = sum(1 for m in misconceptions if len(m) > 0) logger.info(f"Non-empty predictions: {non_empty}/{len(misconceptions)}") if len(misconceptions) > 2: logger.info(f"Sample predictions: {[list(m) for m in misconceptions[:3]]}") return misconceptions except Exception as e: logger.error(f"Error in prediction: {str(e)}") raise def _load_model(self): """Load model artifacts from disk.""" try: if not os.path.exists('models'): logger.error("Models directory does not exist") return False # Check if all required model files exist model_files = ['vectorizer.joblib', 'label_binarizer.joblib', 'classifier.joblib'] for file in model_files: if not os.path.exists(f'models/{file}'): logger.error(f"Model file {file} does not exist") return False # Load model components self.vectorizer = joblib.load('models/vectorizer.joblib') self.mlb = joblib.load('models/label_binarizer.joblib') self.classifier = joblib.load('models/classifier.joblib') # Set flag that model is fitted self.is_fitted = True logger.info("Model loaded successfully from disk") return True except Exception as e: logger.error(f"Error loading model: {str(e)}") self.is_fitted = False return False def _save_model(self): """Save model artifacts to disk.""" try: if not os.path.exists('models'): os.makedirs('models') joblib.dump(self.vectorizer, 'models/vectorizer.joblib') joblib.dump(self.mlb, 'models/label_binarizer.joblib') joblib.dump(self.classifier, 'models/classifier.joblib') # Also save a marker file to indicate model is saved joblib.dump(True, 'models/model.joblib') logger.info("Model saved successfully to disk") return True except Exception as e: logger.error(f"Error saving model: {str(e)}") return False def evaluate_model(predictions, misconception_mapping): """Evaluate model predictions and return detailed results.""" try: logger.info(f"Evaluating {len(predictions)} prediction sets") results = [] # Ensure misconception_mapping is a dictionary with integer keys if not isinstance(misconception_mapping, dict): logger.warning("Converting misconception_mapping to dictionary") misconception_mapping = {int(k): v for k, v in misconception_mapping.items()} for i, pred_misconceptions in enumerate(predictions): try: # Handle empty predictions safely if pred_misconceptions is None: misconception_ids = [] else: # Convert numpy array to list of integers safely misconception_ids = [] for mid in pred_misconceptions: try: # Convert numpy types to Python int if hasattr(mid, 'item'): misconception_ids.append(mid.item()) else: misconception_ids.append(int(mid)) except (ValueError, TypeError): logger.warning(f"Skipping non-integer misconception ID: {mid}") # Get misconception names safely misconception_names = [] for mid in misconception_ids: try: name = misconception_mapping.get(int(mid), f"Unknown misconception {mid}") misconception_names.append(name) except (ValueError, TypeError): logger.warning(f"Error looking up misconception ID: {mid}") misconception_names.append(f"Unknown misconception {mid}") result = { 'question_id': i + 1, # Using index+1 as question ID for now 'misconception_ids': misconception_ids, 'misconception_names': misconception_names } results.append(result) except Exception as e: logger.error(f"Error processing prediction {i}: {str(e)}") # Add a default result with no misconceptions for this question results.append({ 'question_id': i + 1, 'misconception_ids': [], 'misconception_names': [] }) # Log evaluation results for debugging logger.info(f"Processed {len(results)} evaluation results") if len(results) > 0: logger.info(f"Sample results: {results[:min(3, len(results))]}") return results except Exception as e: logger.error(f"Error in model evaluation: {str(e)}") # Return empty results rather than failing completely return []