Download evaluate_models.py from Erenuo/API: direct link, hf CLI and curl.
- Browser
- Download file 5.29 kB
-
https://huggingface.co/spaces/Erenuo/API/resolve/main/evaluate_models.py
- Command line
-
hf download hf://spaces/Erenuo/API/evaluate_models.py
-
curl -L -o evaluate_models.py https://huggingface.co/spaces/Erenuo/API/resolve/main/evaluate_models.py
5.29 kB
| from sklearn.metrics import classification_report, confusion_matrix, accuracy_score | |
| from transformers import BertTokenizer, BertModel | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| import pandas as pd | |
| import numpy as np | |
| import joblib | |
| import torch | |
| import nltk | |
| import os | |
| TEST_DATA_PATH = 'test.csv' | |
| MODELS_DIR = 'saved_models' | |
| RESULTS_DIR = 'evaluation_results' | |
| # Helper Function for Encoders | |
| def get_bert_embeddings(texts, model, tokenizer): | |
| """Generates BERT embeddings for a list of texts.""" | |
| model.eval() # Put model in evaluation mode | |
| embeddings = [] | |
| for text in texts: | |
| inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512) | |
| with torch.no_grad(): | |
| outputs = model(**inputs) | |
| # Use the mean of the last hidden state as the sentence embedding | |
| embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy()) | |
| return np.array(embeddings) | |
| def evaluate_all_models(): | |
| """ | |
| Loads all trained models and evaluates them against the test dataset. | |
| """ | |
| # Create results directory | |
| if not os.path.exists(RESULTS_DIR): | |
| os.makedirs(RESULTS_DIR) | |
| print(f"Created directory: {RESULTS_DIR}") | |
| # Load test data | |
| try: | |
| test_df = pd.read_csv(TEST_DATA_PATH) | |
| print(f"\nLoaded test data from '{TEST_DATA_PATH}' ({len(test_df)} rows)") | |
| except FileNotFoundError: | |
| print(f"Error: Test data not found at '{TEST_DATA_PATH}'") | |
| print("Please run the 'split_dataset.py' script first.") | |
| return | |
| # Find all model files and load the single label encoder | |
| model_files = [f for f in os.listdir(MODELS_DIR) if f.endswith('_model.joblib')] | |
| if not model_files: | |
| print(f"Error: No models found in '{MODELS_DIR}' directory.") | |
| return | |
| print(f"Found {len(model_files)} models to evaluate.") | |
| try: | |
| # Load the single, shared label encoder | |
| le_path = os.path.join(MODELS_DIR, 'label_encoder.joblib') | |
| le = joblib.load(le_path) | |
| print(f"Successfully loaded shared label encoder from '{le_path}'") | |
| except FileNotFoundError: | |
| print(f"Error: The shared 'label_encoder.joblib' was not found in '{MODELS_DIR}'.") | |
| print("Please ensure at least one training script has been run to create it.") | |
| return | |
| # Pre-load BERT model and tokenizer to avoid loading them in the loop | |
| print("\nLoading BERT model (this may take a moment)...") | |
| bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') | |
| bert_model = BertModel.from_pretrained('bert-base-uncased') | |
| nltk.download('punkt', quiet=True) | |
| # Loop through and evaluate each model | |
| for model_file in sorted(model_files): | |
| model_name = model_file.replace('_model.joblib', '') | |
| print(f"\n--- Evaluating Model: {model_name} ---") | |
| try: | |
| # Load the model | |
| model = joblib.load(os.path.join(MODELS_DIR, model_file)) | |
| # Encode the true labels from the test set using the shared encoder | |
| y_true = le.transform(test_df['intent']) | |
| X_test_text = test_df['utterance'] | |
| # Preprocess test data based on model type | |
| if 'char_tfidf' in model_name: | |
| vectorizer_path = os.path.join(MODELS_DIR, 'char_tfidf_vectorizer.joblib') | |
| vectorizer = joblib.load(vectorizer_path) | |
| X_test = vectorizer.transform(X_test_text) | |
| elif 'tfidf' in model_name: | |
| vectorizer_path = os.path.join(MODELS_DIR, 'tfidf_vectorizer.joblib') | |
| vectorizer = joblib.load(vectorizer_path) | |
| X_test = vectorizer.transform(X_test_text) | |
| elif 'bert' in model_name: | |
| X_test = get_bert_embeddings(X_test_text.tolist(), bert_model, bert_tokenizer) | |
| else: | |
| print(f"Warning: Unknown model type for {model_name}. Skipping.") | |
| continue | |
| # Make predictions | |
| y_pred = model.predict(X_test) | |
| # Generate and print classification report | |
| print("Classification Report:") | |
| report = classification_report(y_true, y_pred, target_names=le.classes_) | |
| print(report) | |
| # Generate and save confusion matrix | |
| cm = confusion_matrix(y_true, y_pred) | |
| plt.figure(figsize=(10, 8)) | |
| sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=le.classes_, yticklabels=le.classes_) | |
| plt.title(f'Confusion Matrix - {model_name}') | |
| plt.ylabel('Actual Intent') | |
| plt.xlabel('Predicted Intent') | |
| plt.xticks(rotation=45, ha='right') | |
| plt.yticks(rotation=0) | |
| plt.tight_layout() | |
| cm_path = os.path.join(RESULTS_DIR, f'{model_name}_confusion_matrix.png') | |
| plt.savefig(cm_path) | |
| plt.close() | |
| print(f"Confusion matrix saved to: {cm_path}") | |
| except FileNotFoundError as e: | |
| print(f"Error loading files for {model_name}: {e}. Skipping.") | |
| except Exception as e: | |
| print(f"An unexpected error occurred while evaluating {model_name}: {e}. Skipping.") | |
| if __name__ == '__main__': | |
| evaluate_all_models() | |