API / evaluate_models.py
Erenuo's picture
Upload initial application files
bd6bce1
Raw History Blame Contribute Delete
5.29 kB
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from transformers import BertTokenizer, BertModel
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import joblib
import torch
import nltk
import os
TEST_DATA_PATH = 'test.csv'
MODELS_DIR = 'saved_models'
RESULTS_DIR = 'evaluation_results'
# Helper Function for Encoders
def get_bert_embeddings(texts, model, tokenizer):
"""Generates BERT embeddings for a list of texts."""
model.eval() # Put model in evaluation mode
embeddings = []
for text in texts:
inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
with torch.no_grad():
outputs = model(**inputs)
# Use the mean of the last hidden state as the sentence embedding
embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy())
return np.array(embeddings)
def evaluate_all_models():
"""
Loads all trained models and evaluates them against the test dataset.
"""
# Create results directory
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
print(f"Created directory: {RESULTS_DIR}")
# Load test data
try:
test_df = pd.read_csv(TEST_DATA_PATH)
print(f"\nLoaded test data from '{TEST_DATA_PATH}' ({len(test_df)} rows)")
except FileNotFoundError:
print(f"Error: Test data not found at '{TEST_DATA_PATH}'")
print("Please run the 'split_dataset.py' script first.")
return
# Find all model files and load the single label encoder
model_files = [f for f in os.listdir(MODELS_DIR) if f.endswith('_model.joblib')]
if not model_files:
print(f"Error: No models found in '{MODELS_DIR}' directory.")
return
print(f"Found {len(model_files)} models to evaluate.")
try:
# Load the single, shared label encoder
le_path = os.path.join(MODELS_DIR, 'label_encoder.joblib')
le = joblib.load(le_path)
print(f"Successfully loaded shared label encoder from '{le_path}'")
except FileNotFoundError:
print(f"Error: The shared 'label_encoder.joblib' was not found in '{MODELS_DIR}'.")
print("Please ensure at least one training script has been run to create it.")
return
# Pre-load BERT model and tokenizer to avoid loading them in the loop
print("\nLoading BERT model (this may take a moment)...")
bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
bert_model = BertModel.from_pretrained('bert-base-uncased')
nltk.download('punkt', quiet=True)
# Loop through and evaluate each model
for model_file in sorted(model_files):
model_name = model_file.replace('_model.joblib', '')
print(f"\n--- Evaluating Model: {model_name} ---")
try:
# Load the model
model = joblib.load(os.path.join(MODELS_DIR, model_file))
# Encode the true labels from the test set using the shared encoder
y_true = le.transform(test_df['intent'])
X_test_text = test_df['utterance']
# Preprocess test data based on model type
if 'char_tfidf' in model_name:
vectorizer_path = os.path.join(MODELS_DIR, 'char_tfidf_vectorizer.joblib')
vectorizer = joblib.load(vectorizer_path)
X_test = vectorizer.transform(X_test_text)
elif 'tfidf' in model_name:
vectorizer_path = os.path.join(MODELS_DIR, 'tfidf_vectorizer.joblib')
vectorizer = joblib.load(vectorizer_path)
X_test = vectorizer.transform(X_test_text)
elif 'bert' in model_name:
X_test = get_bert_embeddings(X_test_text.tolist(), bert_model, bert_tokenizer)
else:
print(f"Warning: Unknown model type for {model_name}. Skipping.")
continue
# Make predictions
y_pred = model.predict(X_test)
# Generate and print classification report
print("Classification Report:")
report = classification_report(y_true, y_pred, target_names=le.classes_)
print(report)
# Generate and save confusion matrix
cm = confusion_matrix(y_true, y_pred)
plt.figure(figsize=(10, 8))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=le.classes_, yticklabels=le.classes_)
plt.title(f'Confusion Matrix - {model_name}')
plt.ylabel('Actual Intent')
plt.xlabel('Predicted Intent')
plt.xticks(rotation=45, ha='right')
plt.yticks(rotation=0)
plt.tight_layout()
cm_path = os.path.join(RESULTS_DIR, f'{model_name}_confusion_matrix.png')
plt.savefig(cm_path)
plt.close()
print(f"Confusion matrix saved to: {cm_path}")
except FileNotFoundError as e:
print(f"Error loading files for {model_name}: {e}. Skipping.")
except Exception as e:
print(f"An unexpected error occurred while evaluating {model_name}: {e}. Skipping.")
if __name__ == '__main__':
evaluate_all_models()