File size: 8,932 Bytes
ae77244
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
"""
Cross-Dataset Evaluation
Tests the ModernBERT model on the legacy Kaggle Composite dataset.
"""

import os
import argparse
import subprocess
from pathlib import Path

import pandas as pd
import numpy as np
import torch
import mlflow
import dagshub
from dotenv import load_dotenv
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import matplotlib.pyplot as plt
import seaborn as sns

REPO_ROOT = Path(__file__).resolve().parents[1]
BRANCH_STAGE_MAP = {
    "feature/phase-2-audio-asr": "feature/phase-2-audio-asr",
    "feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization",
    "feature/phase-1.5-ultimate-dataset": "model-modernbert-universal",
    "model-long-context": "model-modernbert-universal",
    "model-distilbert": "model-distilbert",
    "main": "main",
}


def detect_branch(default="main"):
    env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH")
    if env_branch:
        return env_branch.strip()
    result = subprocess.run(
        ["git", "branch", "--show-current"],
        cwd=REPO_ROOT,
        capture_output=True,
        text=True,
        check=False,
    )
    branch = result.stdout.strip()
    return branch or default


def stage_for_branch(branch):
    return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main")

def get_device():
    if torch.backends.mps.is_available():
        return "mps"
    if torch.cuda.is_available():
        return "cuda"
    return "cpu"

def load_legacy_data():
    """Downloads and merges the entire legacy composite dataset from DagsHub S3."""
    repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
    repo_name = os.getenv("DAGSHUB_REPO_NAME")
    
    if not (repo_owner and repo_name):
        raise ValueError("Missing DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME in .env")
        
    s3_client = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}")
    os.makedirs("data/legacy_composite", exist_ok=True)
    
    csv1 = "data/legacy_composite/composite_train.csv"
    csv2 = "data/legacy_composite/composite_test.csv"
    
    print("Downloading legacy datasets from DagsHub S3...")
    s3_client.download_file(repo_name, csv1, csv1)
    s3_client.download_file(repo_name, csv2, csv2)
    
    df1 = pd.read_csv(csv1)
    df2 = pd.read_csv(csv2)
    df_all = pd.concat([df1, df2], ignore_index=True)
    
    print(f"Loaded legacy dataset: {len(df_all)} rows.")
    return df_all

def main():
    load_dotenv()
    parser = argparse.ArgumentParser()
    parser.add_argument("--model_dir", type=str, default="./scam-classifier-model", help="Path to model directory or 'mlflow' to pull from DagsHub")
    parser.add_argument("--registry_name", type=str, default=None, help="Name of the model in DagsHub MLflow Registry (if --model_dir=mlflow)")
    args = parser.parse_args()
    branch = detect_branch()
    stage = stage_for_branch(branch)
    stage_slug = stage.replace("/", "-")
    registry_name = args.registry_name or f"ModernBERT-Scam-Classifier-{stage_slug}"
    
    username = os.getenv("MLFLOW_TRACKING_USERNAME")
    password = os.getenv("MLFLOW_TRACKING_PASSWORD")
    repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
    repo_name = os.getenv("DAGSHUB_REPO_NAME")
    
    if username and password:
        os.environ["DAGSHUB_USER"] = username
        os.environ["DAGSHUB_TOKEN"] = password
        dagshub.auth.add_app_token(password)
        
    if repo_owner and repo_name:
        dagshub.init(repo_name=repo_name, repo_owner=repo_owner, mlflow=True)
        os.environ["MLFLOW_S3_ENDPOINT_URL"] = "https://dagshub.com"
        os.environ["MLFLOW_S3_IGNORE_TLS"] = "true"
        os.environ["AWS_ACCESS_KEY_ID"] = username
        os.environ["AWS_SECRET_ACCESS_KEY"] = password
        
    device = get_device()
    print(f"Using device: {device}")
    
    # 1. Download & Load Data
    try:
        df = load_legacy_data()
    except Exception as e:
        print(f"Failed to load legacy data: {e}")
        return
        
    if "text" not in df.columns or "label" not in df.columns:
        print("Dataset missing 'text' or 'label' columns.")
        return
        
    texts = df["text"].astype(str).tolist()
    true_labels = df["label"].astype(int).tolist()
    
    # 2. Load Model
    if args.model_dir == "mlflow":
        print("Fetching latest model from DagsHub MLflow registry...")
        mlflow.set_experiment(f"scam-detection/{stage}/eval")
        
        # Load directly from the Model Registry
        model_uri = f"models:/{registry_name}/latest"
        print(f"Loading model from registry: {model_uri}")
        
        try:
            pipeline = mlflow.transformers.load_model(model_uri, return_type="components")
            model = pipeline["model"]
            tokenizer = pipeline["tokenizer"]
        except Exception as e:
            print(f"ERROR: Failed to load model from registry ({e}). Please ensure you have trained and registered it.")
            return
    else:
        print(f"Loading model locally from {args.model_dir}...")
        tokenizer = AutoTokenizer.from_pretrained(args.model_dir)
        model = AutoModelForSequenceClassification.from_pretrained(args.model_dir)
        
    model.to(device)
    model.eval()
    
    # 3. Inference (Batched for speed and memory efficiency)
    batch_size = 4  # Reduced batch size to prevent OOM
    
    max_len = tokenizer.model_max_length
    if max_len > 100000:
        max_len = 8192
        
    print(f"Tokenization max_length set to: {max_len}")
    
    # Sort by length to minimize padding overhead in batches
    print("Sorting dataset by text length to optimize memory usage...")
    lengths = [len(str(t)) for t in texts]
    sorted_indices = np.argsort(lengths)
    
    sorted_texts = [texts[i] for i in sorted_indices]
    sorted_labels = [true_labels[i] for i in sorted_indices]
    
    all_preds_sorted = []
    
    print(f"Starting batched inference on {len(sorted_texts)} samples...")
    with torch.no_grad():
        for i in range(0, len(sorted_texts), batch_size):
            batch_texts = sorted_texts[i:i+batch_size]
            inputs = tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=max_len).to(device)
            
            if "token_type_ids" in inputs:
                del inputs["token_type_ids"]
                
            outputs = model(**inputs)
            logits = outputs.logits
            preds = torch.argmax(logits, dim=-1).cpu().tolist()
            all_preds_sorted.extend(preds)
            
            # Clear CUDA cache periodically to prevent memory fragmentation
            if i % 100 == 0 and device == "cuda":
                torch.cuda.empty_cache()
            
            if (i // batch_size) % 50 == 0:
                print(f"Processed {i}/{len(sorted_texts)}...")
                
    # Unsort predictions to match original true_labels order
    unsorted_preds = [0] * len(all_preds_sorted)
    for sorted_idx, original_idx in enumerate(sorted_indices):
        unsorted_preds[original_idx] = all_preds_sorted[sorted_idx]
        
    all_preds = unsorted_preds
                
    # 4. Metrics
    print("\n--- Cross-Dataset Evaluation Results ---")
    acc = accuracy_score(true_labels, all_preds)
    precision, recall, f1, _ = precision_recall_fscore_support(true_labels, all_preds, average="binary")
    cm = confusion_matrix(true_labels, all_preds)
    
    print(f"Accuracy:  {acc:.4f}")
    print(f"Precision: {precision:.4f}")
    print(f"Recall:    {recall:.4f}")
    print(f"F1 Score:  {f1:.4f}")
    
    print("\nConfusion Matrix:")
    print(cm)
    
    # Plotting and saving the Confusion Matrix
    plt.figure(figsize=(8, 6))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['Legit', 'Scam'], yticklabels=['Legit', 'Scam'])
    plt.xlabel('Predicted')
    plt.ylabel('Actual')
    plt.title(f'Confusion Matrix - {registry_name}')
    
    report_dir = Path("reports") / stage
    report_dir.mkdir(parents=True, exist_ok=True)
    cm_path = report_dir / f"confusion_matrix_{registry_name}.png"
    plt.savefig(cm_path)
    print(f"\nSaved confusion matrix plot to {cm_path}")
    
    # Log the figure to MLflow if tracking is enabled
    if args.model_dir == "mlflow":
        with mlflow.start_run(run_name=f"{stage}-eval-{registry_name}"):
            mlflow.log_figure(plt.gcf(), "confusion_matrix.png")
            mlflow.log_metric("eval_accuracy_cross", acc)
            mlflow.log_metric("eval_f1_cross", f1)
            mlflow.set_tag("project_stage", stage)
            mlflow.set_tag("git_branch", branch)
            print("Logged evaluation metrics and confusion matrix to MLflow.")
            
    plt.close()
    
    print("\n----------------------------------------")
    
if __name__ == "__main__":
    main()