Download scripts/03_evaluate_models.py from AbhijithNusamNu/scam_dectect_sheid: direct link, hf CLI and curl.
- Browser
- Download file 8.93 kB
-
https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/03_evaluate_models.py
- Command line
-
hf download hf://spaces/AbhijithNusamNu/scam_dectect_sheid/scripts/03_evaluate_models.py
-
curl -L -o 03_evaluate_models.py https://huggingface.co/spaces/AbhijithNusamNu/scam_dectect_sheid/resolve/main/scripts/03_evaluate_models.py
8.93 kB
| """ | |
| Cross-Dataset Evaluation | |
| Tests the ModernBERT model on the legacy Kaggle Composite dataset. | |
| """ | |
| import os | |
| import argparse | |
| import subprocess | |
| from pathlib import Path | |
| import pandas as pd | |
| import numpy as np | |
| import torch | |
| import mlflow | |
| import dagshub | |
| from dotenv import load_dotenv | |
| from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| REPO_ROOT = Path(__file__).resolve().parents[1] | |
| BRANCH_STAGE_MAP = { | |
| "feature/phase-2-audio-asr": "feature/phase-2-audio-asr", | |
| "feature/phase-3-serving-quantization": "feature/phase-3-serving-quantization", | |
| "feature/phase-1.5-ultimate-dataset": "model-modernbert-universal", | |
| "model-long-context": "model-modernbert-universal", | |
| "model-distilbert": "model-distilbert", | |
| "main": "main", | |
| } | |
| def detect_branch(default="main"): | |
| env_branch = os.getenv("DAGSHUB_BRANCH") or os.getenv("GIT_BRANCH") | |
| if env_branch: | |
| return env_branch.strip() | |
| result = subprocess.run( | |
| ["git", "branch", "--show-current"], | |
| cwd=REPO_ROOT, | |
| capture_output=True, | |
| text=True, | |
| check=False, | |
| ) | |
| branch = result.stdout.strip() | |
| return branch or default | |
| def stage_for_branch(branch): | |
| return BRANCH_STAGE_MAP.get(branch, branch.replace("/", "-") or "main") | |
| def get_device(): | |
| if torch.backends.mps.is_available(): | |
| return "mps" | |
| if torch.cuda.is_available(): | |
| return "cuda" | |
| return "cpu" | |
| def load_legacy_data(): | |
| """Downloads and merges the entire legacy composite dataset from DagsHub S3.""" | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| if not (repo_owner and repo_name): | |
| raise ValueError("Missing DAGSHUB_REPO_OWNER or DAGSHUB_REPO_NAME in .env") | |
| s3_client = dagshub.get_repo_bucket_client(f"{repo_owner}/{repo_name}") | |
| os.makedirs("data/legacy_composite", exist_ok=True) | |
| csv1 = "data/legacy_composite/composite_train.csv" | |
| csv2 = "data/legacy_composite/composite_test.csv" | |
| print("Downloading legacy datasets from DagsHub S3...") | |
| s3_client.download_file(repo_name, csv1, csv1) | |
| s3_client.download_file(repo_name, csv2, csv2) | |
| df1 = pd.read_csv(csv1) | |
| df2 = pd.read_csv(csv2) | |
| df_all = pd.concat([df1, df2], ignore_index=True) | |
| print(f"Loaded legacy dataset: {len(df_all)} rows.") | |
| return df_all | |
| def main(): | |
| load_dotenv() | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--model_dir", type=str, default="./scam-classifier-model", help="Path to model directory or 'mlflow' to pull from DagsHub") | |
| parser.add_argument("--registry_name", type=str, default=None, help="Name of the model in DagsHub MLflow Registry (if --model_dir=mlflow)") | |
| args = parser.parse_args() | |
| branch = detect_branch() | |
| stage = stage_for_branch(branch) | |
| stage_slug = stage.replace("/", "-") | |
| registry_name = args.registry_name or f"ModernBERT-Scam-Classifier-{stage_slug}" | |
| username = os.getenv("MLFLOW_TRACKING_USERNAME") | |
| password = os.getenv("MLFLOW_TRACKING_PASSWORD") | |
| repo_owner = os.getenv("DAGSHUB_REPO_OWNER") | |
| repo_name = os.getenv("DAGSHUB_REPO_NAME") | |
| if username and password: | |
| os.environ["DAGSHUB_USER"] = username | |
| os.environ["DAGSHUB_TOKEN"] = password | |
| dagshub.auth.add_app_token(password) | |
| if repo_owner and repo_name: | |
| dagshub.init(repo_name=repo_name, repo_owner=repo_owner, mlflow=True) | |
| os.environ["MLFLOW_S3_ENDPOINT_URL"] = "https://dagshub.com" | |
| os.environ["MLFLOW_S3_IGNORE_TLS"] = "true" | |
| os.environ["AWS_ACCESS_KEY_ID"] = username | |
| os.environ["AWS_SECRET_ACCESS_KEY"] = password | |
| device = get_device() | |
| print(f"Using device: {device}") | |
| # 1. Download & Load Data | |
| try: | |
| df = load_legacy_data() | |
| except Exception as e: | |
| print(f"Failed to load legacy data: {e}") | |
| return | |
| if "text" not in df.columns or "label" not in df.columns: | |
| print("Dataset missing 'text' or 'label' columns.") | |
| return | |
| texts = df["text"].astype(str).tolist() | |
| true_labels = df["label"].astype(int).tolist() | |
| # 2. Load Model | |
| if args.model_dir == "mlflow": | |
| print("Fetching latest model from DagsHub MLflow registry...") | |
| mlflow.set_experiment(f"scam-detection/{stage}/eval") | |
| # Load directly from the Model Registry | |
| model_uri = f"models:/{registry_name}/latest" | |
| print(f"Loading model from registry: {model_uri}") | |
| try: | |
| pipeline = mlflow.transformers.load_model(model_uri, return_type="components") | |
| model = pipeline["model"] | |
| tokenizer = pipeline["tokenizer"] | |
| except Exception as e: | |
| print(f"ERROR: Failed to load model from registry ({e}). Please ensure you have trained and registered it.") | |
| return | |
| else: | |
| print(f"Loading model locally from {args.model_dir}...") | |
| tokenizer = AutoTokenizer.from_pretrained(args.model_dir) | |
| model = AutoModelForSequenceClassification.from_pretrained(args.model_dir) | |
| model.to(device) | |
| model.eval() | |
| # 3. Inference (Batched for speed and memory efficiency) | |
| batch_size = 4 # Reduced batch size to prevent OOM | |
| max_len = tokenizer.model_max_length | |
| if max_len > 100000: | |
| max_len = 8192 | |
| print(f"Tokenization max_length set to: {max_len}") | |
| # Sort by length to minimize padding overhead in batches | |
| print("Sorting dataset by text length to optimize memory usage...") | |
| lengths = [len(str(t)) for t in texts] | |
| sorted_indices = np.argsort(lengths) | |
| sorted_texts = [texts[i] for i in sorted_indices] | |
| sorted_labels = [true_labels[i] for i in sorted_indices] | |
| all_preds_sorted = [] | |
| print(f"Starting batched inference on {len(sorted_texts)} samples...") | |
| with torch.no_grad(): | |
| for i in range(0, len(sorted_texts), batch_size): | |
| batch_texts = sorted_texts[i:i+batch_size] | |
| inputs = tokenizer(batch_texts, return_tensors="pt", padding=True, truncation=True, max_length=max_len).to(device) | |
| if "token_type_ids" in inputs: | |
| del inputs["token_type_ids"] | |
| outputs = model(**inputs) | |
| logits = outputs.logits | |
| preds = torch.argmax(logits, dim=-1).cpu().tolist() | |
| all_preds_sorted.extend(preds) | |
| # Clear CUDA cache periodically to prevent memory fragmentation | |
| if i % 100 == 0 and device == "cuda": | |
| torch.cuda.empty_cache() | |
| if (i // batch_size) % 50 == 0: | |
| print(f"Processed {i}/{len(sorted_texts)}...") | |
| # Unsort predictions to match original true_labels order | |
| unsorted_preds = [0] * len(all_preds_sorted) | |
| for sorted_idx, original_idx in enumerate(sorted_indices): | |
| unsorted_preds[original_idx] = all_preds_sorted[sorted_idx] | |
| all_preds = unsorted_preds | |
| # 4. Metrics | |
| print("\n--- Cross-Dataset Evaluation Results ---") | |
| acc = accuracy_score(true_labels, all_preds) | |
| precision, recall, f1, _ = precision_recall_fscore_support(true_labels, all_preds, average="binary") | |
| cm = confusion_matrix(true_labels, all_preds) | |
| print(f"Accuracy: {acc:.4f}") | |
| print(f"Precision: {precision:.4f}") | |
| print(f"Recall: {recall:.4f}") | |
| print(f"F1 Score: {f1:.4f}") | |
| print("\nConfusion Matrix:") | |
| print(cm) | |
| # Plotting and saving the Confusion Matrix | |
| plt.figure(figsize=(8, 6)) | |
| sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['Legit', 'Scam'], yticklabels=['Legit', 'Scam']) | |
| plt.xlabel('Predicted') | |
| plt.ylabel('Actual') | |
| plt.title(f'Confusion Matrix - {registry_name}') | |
| report_dir = Path("reports") / stage | |
| report_dir.mkdir(parents=True, exist_ok=True) | |
| cm_path = report_dir / f"confusion_matrix_{registry_name}.png" | |
| plt.savefig(cm_path) | |
| print(f"\nSaved confusion matrix plot to {cm_path}") | |
| # Log the figure to MLflow if tracking is enabled | |
| if args.model_dir == "mlflow": | |
| with mlflow.start_run(run_name=f"{stage}-eval-{registry_name}"): | |
| mlflow.log_figure(plt.gcf(), "confusion_matrix.png") | |
| mlflow.log_metric("eval_accuracy_cross", acc) | |
| mlflow.log_metric("eval_f1_cross", f1) | |
| mlflow.set_tag("project_stage", stage) | |
| mlflow.set_tag("git_branch", branch) | |
| print("Logged evaluation metrics and confusion matrix to MLflow.") | |
| plt.close() | |
| print("\n----------------------------------------") | |
| if __name__ == "__main__": | |
| main() | |