File size: 8,932 Bytes
ae77244 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 | """
Cross-Dataset Evaluation
Tests the ModernBERT model on the legacy Kaggle Composite dataset.
"""
import os
import argparse
import subprocess
from pathlib import Path
import pandas as pd
import numpy as np
import torch
import mlflow
import dagshub
from dotenv import load_dotenv
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import matplotlib.pyplot as plt
import seaborn as sns
REPO_ROOT = Path(__file__).resolve().parents[1]
BRANCH_STAGE_MAP = {
"feature/phase-2-audio-asr": "feature/phase-2-audio-asr",
"feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization",
"feature/phase-1.5-ultimate-dataset": "model-modernbert-universal",
"model-long-context": "model-modernbert-universal",
"model-distilbert": "model-distilbert",
"main": "main",
}
def detect_branch(default="main"):
env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH")
if env_branch:
return env_branch.strip()
result = subprocess.run(
["git", "branch", "--show-current"],
cwd=REPO_ROOT,
capture_output=True,
text=True,
check=False,
)
branch = result.stdout.strip()
return branch or default
def stage_for_branch(branch):
return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main")
def get_device():
if torch.backends.mps.is_available():
return "mps"
if torch.cuda.is_available():
return "cuda"
return "cpu"
def load_legacy_data():
"""Downloads and merges the entire legacy composite dataset from DagsHub S3."""
repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
repo_name = os.getenv("DAGSHUB_REPO_NAME")
if not (repo_owner and repo_name):
raise ValueError("Missing DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME in .env")
s3_client = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}")
os.makedirs("data/legacy_composite", exist_ok=True)
csv1 = "data/legacy_composite/composite_train.csv"
csv2 = "data/legacy_composite/composite_test.csv"
print("Downloading legacy datasets from DagsHub S3...")
s3_client.download_file(repo_name, csv1, csv1)
s3_client.download_file(repo_name, csv2, csv2)
df1 = pd.read_csv(csv1)
df2 = pd.read_csv(csv2)
df_all = pd.concat([df1, df2], ignore_index=True)
print(f"Loaded legacy dataset: {len(df_all)} rows.")
return df_all
def main():
load_dotenv()
parser = argparse.ArgumentParser()
parser.add_argument("--model_dir", type=str, default="./scam-classifier-model", help="Path to model directory or 'mlflow' to pull from DagsHub")
parser.add_argument("--registry_name", type=str, default=None, help="Name of the model in DagsHub MLflow Registry (if --model_dir=mlflow)")
args = parser.parse_args()
branch = detect_branch()
stage = stage_for_branch(branch)
stage_slug = stage.replace("/", "-")
registry_name = args.registry_name or f"ModernBERT-Scam-Classifier-{stage_slug}"
username = os.getenv("MLFLOW_TRACKING_USERNAME")
password = os.getenv("MLFLOW_TRACKING_PASSWORD")
repo_owner = os.getenv("DAGSHUB_REPO_OWNER")
repo_name = os.getenv("DAGSHUB_REPO_NAME")
if username and password:
os.environ["DAGSHUB_USER"] = username
os.environ["DAGSHUB_TOKEN"] = password
dagshub.auth.add_app_token(password)
if repo_owner and repo_name:
dagshub.init(repo_name=repo_name, repo_owner=repo_owner, mlflow=True)
os.environ["MLFLOW_S3_ENDPOINT_URL"] = "https://dagshub.com"
os.environ["MLFLOW_S3_IGNORE_TLS"] = "true"
os.environ["AWS_ACCESS_KEY_ID"] = username
os.environ["AWS_SECRET_ACCESS_KEY"] = password
device = get_device()
print(f"Using device: {device}")
# 1. Download & Load Data
try:
df = load_legacy_data()
except Exception as e:
print(f"Failed to load legacy data: {e}")
return
if "text" not in df.columns or "label" not in df.columns:
print("Dataset missing 'text' or 'label' columns.")
return
texts = df["text"].astype(str).tolist()
true_labels = df["label"].astype(int).tolist()
# 2. Load Model
if args.model_dir == "mlflow":
print("Fetching latest model from DagsHub MLflow registry...")
mlflow.set_experiment(f"scam-detection/{stage}/eval")
# Load directly from the Model Registry
model_uri = f"models:/{registry_name}/latest"
print(f"Loading model from registry: {model_uri}")
try:
pipeline = mlflow.transformers.load_model(model_uri, return_type="components")
model = pipeline["model"]
tokenizer = pipeline["tokenizer"]
except Exception as e:
print(f"ERROR: Failed to load model from registry ({e}). Please ensure you have trained and registered it.")
return
else:
print(f"Loading model locally from {args.model_dir}...")
tokenizer = AutoTokenizer.from_pretrained(args.model_dir)
model = AutoModelForSequenceClassification.from_pretrained(args.model_dir)
model.to(device)
model.eval()
# 3. Inference (Batched for speed and memory efficiency)
batch_size = 4 # Reduced batch size to prevent OOM
max_len = tokenizer.model_max_length
if max_len > 100000:
max_len = 8192
print(f"Tokenization max_length set to: {max_len}")
# Sort by length to minimize padding overhead in batches
print("Sorting dataset by text length to optimize memory usage...")
lengths = [len(str(t)) for t in texts]
sorted_indices = np.argsort(lengths)
sorted_texts = [texts[i] for i in sorted_indices]
sorted_labels = [true_labels[i] for i in sorted_indices]
all_preds_sorted = []
print(f"Starting batched inference on {len(sorted_texts)} samples...")
with torch.no_grad():
for i in range(0, len(sorted_texts), batch_size):
batch_texts = sorted_texts[i:i+batch_size]
inputs = tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=max_len).to(device)
if "token_type_ids" in inputs:
del inputs["token_type_ids"]
outputs = model(**inputs)
logits = outputs.logits
preds = torch.argmax(logits, dim=-1).cpu().tolist()
all_preds_sorted.extend(preds)
# Clear CUDA cache periodically to prevent memory fragmentation
if i % 100 == 0 and device == "cuda":
torch.cuda.empty_cache()
if (i // batch_size) % 50 == 0:
print(f"Processed {i}/{len(sorted_texts)}...")
# Unsort predictions to match original true_labels order
unsorted_preds = [0] * len(all_preds_sorted)
for sorted_idx, original_idx in enumerate(sorted_indices):
unsorted_preds[original_idx] = all_preds_sorted[sorted_idx]
all_preds = unsorted_preds
# 4. Metrics
print("\n--- Cross-Dataset Evaluation Results ---")
acc = accuracy_score(true_labels, all_preds)
precision, recall, f1, _ = precision_recall_fscore_support(true_labels, all_preds, average="binary")
cm = confusion_matrix(true_labels, all_preds)
print(f"Accuracy: {acc:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall: {recall:.4f}")
print(f"F1 Score: {f1:.4f}")
print("\nConfusion Matrix:")
print(cm)
# Plotting and saving the Confusion Matrix
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['Legit', 'Scam'], yticklabels=['Legit', 'Scam'])
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.title(f'Confusion Matrix - {registry_name}')
report_dir = Path("reports") / stage
report_dir.mkdir(parents=True, exist_ok=True)
cm_path = report_dir / f"confusion_matrix_{registry_name}.png"
plt.savefig(cm_path)
print(f"\nSaved confusion matrix plot to {cm_path}")
# Log the figure to MLflow if tracking is enabled
if args.model_dir == "mlflow":
with mlflow.start_run(run_name=f"{stage}-eval-{registry_name}"):
mlflow.log_figure(plt.gcf(), "confusion_matrix.png")
mlflow.log_metric("eval_accuracy_cross", acc)
mlflow.log_metric("eval_f1_cross", f1)
mlflow.set_tag("project_stage", stage)
mlflow.set_tag("git_branch", branch)
print("Logged evaluation metrics and confusion matrix to MLflow.")
plt.close()
print("\n----------------------------------------")
if __name__ == "__main__":
main()
|