File size: 5,294 Bytes
bd6bce1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from transformers import BertTokenizer, BertModel
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import joblib
import torch
import nltk
import os

TEST_DATA_PATH = 'test.csv'
MODELS_DIR = 'saved_models'
RESULTS_DIR = 'evaluation_results'

# Helper Function for Encoders
def get_bert_embeddings(texts, model, tokenizer):
    """Generates BERT embeddings for a list of texts."""
    model.eval() # Put model in evaluation mode
    embeddings = []
    for text in texts:
        inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
        with torch.no_grad():
            outputs = model(**inputs)
        # Use the mean of the last hidden state as the sentence embedding
        embeddings.append(outputs.last_hidden_state.mean(dim=1).squeeze().numpy())
    return np.array(embeddings)


def evaluate_all_models():
    """
    Loads all trained models and evaluates them against the test dataset.
    """
    # Create results directory
    if not os.path.exists(RESULTS_DIR):
        os.makedirs(RESULTS_DIR)
        print(f"Created directory: {RESULTS_DIR}")

    # Load test data
    try:
        test_df = pd.read_csv(TEST_DATA_PATH)
        print(f"\nLoaded test data from '{TEST_DATA_PATH}' ({len(test_df)} rows)")
    except FileNotFoundError:
        print(f"Error: Test data not found at '{TEST_DATA_PATH}'")
        print("Please run the 'split_dataset.py' script first.")
        return

    # Find all model files and load the single label encoder
    model_files = [f for f in os.listdir(MODELS_DIR) if f.endswith('_model.joblib')]
    if not model_files:
        print(f"Error: No models found in '{MODELS_DIR}' directory.")
        return
        
    print(f"Found {len(model_files)} models to evaluate.")

    try:
        # Load the single, shared label encoder
        le_path = os.path.join(MODELS_DIR, 'label_encoder.joblib')
        le = joblib.load(le_path)
        print(f"Successfully loaded shared label encoder from '{le_path}'")
    except FileNotFoundError:
        print(f"Error: The shared 'label_encoder.joblib' was not found in '{MODELS_DIR}'.")
        print("Please ensure at least one training script has been run to create it.")
        return

    # Pre-load BERT model and tokenizer to avoid loading them in the loop
    print("\nLoading BERT model (this may take a moment)...")
    bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    bert_model = BertModel.from_pretrained('bert-base-uncased')
    nltk.download('punkt', quiet=True)


    # Loop through and evaluate each model
    for model_file in sorted(model_files):
        model_name = model_file.replace('_model.joblib', '')
        print(f"\n--- Evaluating Model: {model_name} ---")

        try:
            # Load the model
            model = joblib.load(os.path.join(MODELS_DIR, model_file))

            # Encode the true labels from the test set using the shared encoder
            y_true = le.transform(test_df['intent'])
            X_test_text = test_df['utterance']
            
            # Preprocess test data based on model type
            if 'char_tfidf' in model_name:
                vectorizer_path = os.path.join(MODELS_DIR, 'char_tfidf_vectorizer.joblib')
                vectorizer = joblib.load(vectorizer_path)
                X_test = vectorizer.transform(X_test_text)
            
            elif 'tfidf' in model_name:
                vectorizer_path = os.path.join(MODELS_DIR, 'tfidf_vectorizer.joblib')
                vectorizer = joblib.load(vectorizer_path)
                X_test = vectorizer.transform(X_test_text)

            elif 'bert' in model_name:
                X_test = get_bert_embeddings(X_test_text.tolist(), bert_model, bert_tokenizer)

            else:
                print(f"Warning: Unknown model type for {model_name}. Skipping.")
                continue

            # Make predictions
            y_pred = model.predict(X_test)

            # Generate and print classification report
            print("Classification Report:")
            report = classification_report(y_true, y_pred, target_names=le.classes_)
            print(report)

            # Generate and save confusion matrix
            cm = confusion_matrix(y_true, y_pred)
            plt.figure(figsize=(10, 8))
            sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=le.classes_, yticklabels=le.classes_)
            plt.title(f'Confusion Matrix - {model_name}')
            plt.ylabel('Actual Intent')
            plt.xlabel('Predicted Intent')
            plt.xticks(rotation=45, ha='right')
            plt.yticks(rotation=0)
            plt.tight_layout()
            cm_path = os.path.join(RESULTS_DIR, f'{model_name}_confusion_matrix.png')
            plt.savefig(cm_path)
            plt.close()
            print(f"Confusion matrix saved to: {cm_path}")

        except FileNotFoundError as e:
            print(f"Error loading files for {model_name}: {e}. Skipping.")
        except Exception as e:
            print(f"An unexpected error occurred while evaluating {model_name}: {e}. Skipping.")


if __name__ == '__main__':
    evaluate_all_models()