File size: 5,294 Bytes
bd6bce1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from transformers import BertTokenizer, BertModel
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import joblib
import torch
import nltk
import os
TEST_DATA_PATH = 'test.csv'
MODELS_DIR = 'saved_models'
RESULTS_DIR = 'evaluation_results'
# Helper Function for Encoders
def get_bert_embeddings(texts, model, tokenizer):
"""Generates BERT embeddings for a list of texts."""
model.eval() # Put model in evaluation mode
embeddings = []
for text in texts:
inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
with torch.no_grad():
outputs = model(**inputs)
# Use the mean of the last hidden state as the sentence embedding
embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy())
return np.array(embeddings)
def evaluate_all_models():
"""
Loads all trained models and evaluates them against the test dataset.
"""
# Create results directory
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
print(f"Created directory: {RESULTS_DIR}")
# Load test data
try:
test_df = pd.read_csv(TEST_DATA_PATH)
print(f"\nLoaded test data from '{TEST_DATA_PATH}' ({len(test_df)} rows)")
except FileNotFoundError:
print(f"Error: Test data not found at '{TEST_DATA_PATH}'")
print("Please run the 'split_dataset.py' script first.")
return
# Find all model files and load the single label encoder
model_files = [f for f in os.listdir(MODELS_DIR) if f.endswith('_model.joblib')]
if not model_files:
print(f"Error: No models found in '{MODELS_DIR}' directory.")
return
print(f"Found {len(model_files)} models to evaluate.")
try:
# Load the single, shared label encoder
le_path = os.path.join(MODELS_DIR, 'label_encoder.joblib')
le = joblib.load(le_path)
print(f"Successfully loaded shared label encoder from '{le_path}'")
except FileNotFoundError:
print(f"Error: The shared 'label_encoder.joblib' was not found in '{MODELS_DIR}'.")
print("Please ensure at least one training script has been run to create it.")
return
# Pre-load BERT model and tokenizer to avoid loading them in the loop
print("\nLoading BERT model (this may take a moment)...")
bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
bert_model = BertModel.from_pretrained('bert-base-uncased')
nltk.download('punkt', quiet=True)
# Loop through and evaluate each model
for model_file in sorted(model_files):
model_name = model_file.replace('_model.joblib', '')
print(f"\n--- Evaluating Model: {model_name} ---")
try:
# Load the model
model = joblib.load(os.path.join(MODELS_DIR, model_file))
# Encode the true labels from the test set using the shared encoder
y_true = le.transform(test_df['intent'])
X_test_text = test_df['utterance']
# Preprocess test data based on model type
if 'char_tfidf' in model_name:
vectorizer_path = os.path.join(MODELS_DIR, 'char_tfidf_vectorizer.joblib')
vectorizer = joblib.load(vectorizer_path)
X_test = vectorizer.transform(X_test_text)
elif 'tfidf' in model_name:
vectorizer_path = os.path.join(MODELS_DIR, 'tfidf_vectorizer.joblib')
vectorizer = joblib.load(vectorizer_path)
X_test = vectorizer.transform(X_test_text)
elif 'bert' in model_name:
X_test = get_bert_embeddings(X_test_text.tolist(), bert_model, bert_tokenizer)
else:
print(f"Warning: Unknown model type for {model_name}. Skipping.")
continue
# Make predictions
y_pred = model.predict(X_test)
# Generate and print classification report
print("Classification Report:")
report = classification_report(y_true, y_pred, target_names=le.classes_)
print(report)
# Generate and save confusion matrix
cm = confusion_matrix(y_true, y_pred)
plt.figure(figsize=(10, 8))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=le.classes_, yticklabels=le.classes_)
plt.title(f'Confusion Matrix - {model_name}')
plt.ylabel('Actual Intent')
plt.xlabel('Predicted Intent')
plt.xticks(rotation=45, ha='right')
plt.yticks(rotation=0)
plt.tight_layout()
cm_path = os.path.join(RESULTS_DIR, f'{model_name}_confusion_matrix.png')
plt.savefig(cm_path)
plt.close()
print(f"Confusion matrix saved to: {cm_path}")
except FileNotFoundError as e:
print(f"Error loading files for {model_name}: {e}. Skipping.")
except Exception as e:
print(f"An unexpected error occurred while evaluating {model_name}: {e}. Skipping.")
if __name__ == '__main__':
evaluate_all_models()
|