Spaces:
Runtime error
Runtime error
Download model.py from harrycool12/MISCONCEPTION: direct link, hf CLI and curl.
- Browser
- Download file 4.37 kB
-
https://huggingface.co/spaces/harrycool12/MISCONCEPTION/resolve/main/model.py
- Command line
-
hf download hf://spaces/harrycool12/MISCONCEPTION/model.py
-
curl -L -o model.py https://huggingface.co/spaces/harrycool12/MISCONCEPTION/resolve/main/model.py
4.37 kB
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.ensemble import RandomForestClassifier | |
| from sklearn.pipeline import Pipeline | |
| from sklearn.model_selection import train_test_split, cross_val_score | |
| from sklearn.metrics import classification_report, confusion_matrix | |
| import numpy as np | |
| import pandas as pd | |
| import joblib | |
| from datetime import datetime | |
| class MisconceptionClassifier: | |
| def __init__(self): | |
| self.pipeline = Pipeline([ | |
| ('tfidf', TfidfVectorizer( | |
| max_features=1000, | |
| ngram_range=(1, 2), # Include bigrams | |
| stop_words='english' | |
| )), | |
| ('classifier', RandomForestClassifier( | |
| n_estimators=100, | |
| max_depth=10, | |
| random_state=42 | |
| )) | |
| ]) | |
| self.cv_scores = None | |
| self.classification_report = None | |
| self.confusion_mat = None | |
| self.training_history = [] | |
| def train(self, X: pd.Series, y: pd.Series, is_retraining: bool = False): | |
| """Train the model with cross-validation""" | |
| # Train the model | |
| self.pipeline.fit(X, y) | |
| # Get class distribution | |
| class_counts = y.value_counts() | |
| min_class_size = class_counts.min() | |
| n_classes = len(class_counts) | |
| # Initialize empty metrics | |
| self.cv_scores = np.array([]) | |
| # Only perform cross-validation if we have enough samples | |
| if min_class_size >= 3 and len(X) >= n_classes * 3: | |
| try: | |
| # Use number of splits based on minimum class size | |
| n_splits = min(3, min_class_size - 1) | |
| self.cv_scores = cross_val_score( | |
| self.pipeline, X, y, cv=n_splits, scoring='accuracy' | |
| ) | |
| except Exception as e: | |
| print(f"Cross-validation failed: {str(e)}") | |
| # Continue without CV scores | |
| # Generate classification metrics | |
| y_pred = self.pipeline.predict(X) | |
| self.classification_report = classification_report(y, y_pred, output_dict=True) | |
| self.confusion_mat = confusion_matrix(y, y_pred) | |
| # Record training history | |
| history_entry = { | |
| 'timestamp': datetime.now().isoformat(), | |
| 'samples': len(X), | |
| 'class_distribution': class_counts.to_dict(), | |
| 'metrics': self.get_model_metrics(), | |
| 'is_retraining': is_retraining | |
| } | |
| self.training_history.append(history_entry) | |
| def predict(self, X: pd.Series) -> np.ndarray: | |
| """Make predictions""" | |
| return self.pipeline.predict(X) | |
| def predict_proba(self, X: pd.Series) -> np.ndarray: | |
| """Get prediction probabilities""" | |
| return self.pipeline.predict_proba(X) | |
| def get_feature_importance(self) -> dict: | |
| """Get feature importance scores""" | |
| tfidf = self.pipeline.named_steps['tfidf'] | |
| clf = self.pipeline.named_steps['classifier'] | |
| feature_importance = dict(zip( | |
| tfidf.get_feature_names_out(), | |
| clf.feature_importances_ | |
| )) | |
| return feature_importance | |
| def get_model_metrics(self) -> dict: | |
| """Get model performance metrics""" | |
| metrics = { | |
| 'classification_report': self.classification_report, | |
| 'confusion_matrix': self.confusion_mat.tolist() if self.confusion_mat is not None else None | |
| } | |
| if self.cv_scores is not None and len(self.cv_scores) > 0: | |
| metrics.update({ | |
| 'cv_scores': self.cv_scores.tolist(), | |
| 'cv_mean': self.cv_scores.mean(), | |
| 'cv_std': self.cv_scores.std() | |
| }) | |
| return metrics | |
| def save_model(self, path: str = 'models/current_model.joblib'): | |
| """Save the current model to disk""" | |
| # Create models directory if it doesn't exist | |
| import os | |
| os.makedirs(os.path.dirname(path), exist_ok=True) | |
| joblib.dump(self, path) | |
| def load_model(cls, path: str = 'models/current_model.joblib') -> 'MisconceptionClassifier': | |
| """Load a saved model from disk""" | |
| try: | |
| return joblib.load(path) | |
| except: | |
| return cls() | |
| def get_training_history(self) -> list: | |
| """Get the model's training history""" | |
| return self.training_history |