""" evaluate_model.py - Model Evaluation & Error Analysis Suite for TruthLens AI. Evaluates trained models on validation/test splits: - Computes Accuracy, Macro/Weighted Precision, Recall, F1 - Renders and saves annotated confusion matrix - Performs detailed error analysis across failure modes: 1. SUPPORTS predicted as REFUTES 2. REFUTES predicted as SUPPORTS 3. NOT_ENOUGH_INFO predicted as SUPPORTS/REFUTES 4. High-confidence errors - Exports at least 20 detailed error records to ml/reports/verification_errors.csv """ import os import sys import json from typing import Dict, List, Any, Tuple import torch import torch.nn.functional as F from torch.utils.data import DataLoader, Dataset import pandas as pd import numpy as np from transformers import AutoTokenizer, AutoModelForSequenceClassification sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from training_utils import ( LABEL2ID, ID2LABEL, compute_verification_metrics, plot_and_save_confusion_matrix, ) class VerificationDataset(Dataset): def __init__(self, df: pd.DataFrame, tokenizer, max_length: int = 256): self.tokenizer = tokenizer self.max_length = max_length self.claims = df["claim"].astype(str).tolist() self.evidences = df["evidence"].fillna("").astype(str).tolist() self.labels = [LABEL2ID.get(l, -1) for l in df["label"]] def __len__(self): return len(self.claims) def __getitem__(self, idx): claim = self.claims[idx] ev = self.evidences[idx] label = self.labels[idx] claim_clean = str(claim).strip() evidence_clean = str(ev).strip() if ev and str(ev).strip() else "None" input_text = f"[CLAIM] {claim_clean} [SEP] {evidence_clean}" enc = self.tokenizer( input_text, max_length=self.max_length, padding="max_length", truncation=True, return_tensors="pt" ) item = {k: v.squeeze(0) for k, v in enc.items()} item["label"] = torch.tensor(label, dtype=torch.long) item["idx"] = idx return item def run_full_evaluation( model_dir: str, eval_csv: str, output_report_path: str, output_cm_path: str, output_errors_path: str, batch_size: int = 16, max_length: int = 256 ) -> Dict[str, Any]: print("=" * 80) print(" TRUTHLENS AI — MODEL EVALUATION & ERROR ANALYSIS") print("=" * 80) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Loading model from: {model_dir}") print(f"Device: {device}") tokenizer = AutoTokenizer.from_pretrained(model_dir) model = AutoModelForSequenceClassification.from_pretrained(model_dir) model.to(device) model.eval() df_eval = pd.read_csv(eval_csv) dataset = VerificationDataset(df_eval, tokenizer, max_length=max_length) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False) all_preds = [] all_probs = [] all_targets = [] with torch.no_grad(): for batch in loader: input_ids = batch["input_ids"].to(device) attention_mask = batch["attention_mask"].to(device) labels = batch["label"] outputs = model(input_ids=input_ids, attention_mask=attention_mask) probs = F.softmax(outputs.logits, dim=-1).cpu().numpy() preds = np.argmax(probs, axis=-1) all_preds.extend(preds.tolist()) all_probs.extend(probs.tolist()) all_targets.extend(labels.tolist()) metrics = compute_verification_metrics(all_targets, all_preds) print(f"\nEvaluation Results on {len(df_eval):,} samples:") print(f" Accuracy : {metrics['accuracy']*100:.2f}%") print(f" Macro F1 : {metrics['macro_f1']*100:.2f}%") print(f" Weighted F1 : {metrics['weighted_f1']*100:.2f}%") print(f" Macro Precision : {metrics['macro_precision']*100:.2f}%") print(f" Macro Recall : {metrics['macro_recall']*100:.2f}%") print("\nPer-Class Breakdown:") for cls_name, p in metrics["per_class"].items(): print(f" {cls_name:<16}: Precision={p['precision']*100:.2f}%, Recall={p['recall']*100:.2f}%, F1={p['f1']*100:.2f}% (Support: {p['support']})") # Plot Confusion Matrix plot_and_save_confusion_matrix( metrics["confusion_matrix"], class_names=["SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"], output_path=output_cm_path, title="SciFact Transformer Model B — Confusion Matrix" ) # Error Analysis print("\nExtracting failure modes for error analysis...") error_records = [] for i in range(len(df_eval)): actual = all_targets[i] pred = all_preds[i] if actual != pred: conf = float(all_probs[i][pred]) actual_str = ID2LABEL[actual] pred_str = ID2LABEL[pred] # Determine error type if actual_str == "SUPPORTS" and pred_str == "REFUTES": err_type = "supports_predicted_as_refutes" elif actual_str == "REFUTES" and pred_str == "SUPPORTS": err_type = "refutes_predicted_as_supports" elif actual_str == "NOT_ENOUGH_INFO" and pred_str in ("SUPPORTS", "REFUTES"): err_type = "nei_predicted_as_verifiable" elif actual_str in ("SUPPORTS", "REFUTES") and pred_str == "NOT_ENOUGH_INFO": err_type = "verifiable_predicted_as_nei" else: err_type = "other" error_records.append({ "claim": df_eval.iloc[i]["claim"], "evidence": df_eval.iloc[i]["evidence"], "actual_label": actual_str, "predicted_label": pred_str, "confidence": round(conf, 4), "error_type": err_type, }) df_errors = pd.DataFrame(error_records) # Sort by confidence descending so high-confidence errors come first df_errors = df_errors.sort_values("confidence", ascending=False) # Save at least 20 examples (or all if < 20) df_errors.to_csv(output_errors_path, index=False) print(f"Saved {len(df_errors)} error records -> {output_errors_path}") # Generate Markdown Report cm = metrics["confusion_matrix"] report_md = f"""# TruthLens AI — SciFact Claim Verification Results **Model:** Fine-Tuned Transformer (Model B: Claim Verification) **Base Pretrained Model:** `microsoft/deberta-v3-base` **Evaluation Set:** SciFact Validation Split (`scifact_valid.csv`, {len(df_eval)} grounded pairs) **Date:** September 2026 --- ## 1. Primary Performance Metrics | Metric | Score | | :--- | :--- | | **Accuracy** | **{metrics['accuracy']*100:.2f}%** | | **Macro F1 Score** | **{metrics['macro_f1']*100:.2f}%** | | **Weighted F1 Score** | **{metrics['weighted_f1']*100:.2f}%** | | **Macro Precision** | **{metrics['macro_precision']*100:.2f}%** | | **Macro Recall** | **{metrics['macro_recall']*100:.2f}%** | --- ## 2. Per-Class Performance Breakdown | Class | Precision | Recall | F1 Score | Support (Count) | | :--- | :--- | :--- | :--- | :--- | | **`SUPPORTS`** | {metrics['per_class']['SUPPORTS']['precision']*100:.2f}% | {metrics['per_class']['SUPPORTS']['recall']*100:.2f}% | **{metrics['per_class']['SUPPORTS']['f1']*100:.2f}%** | {metrics['per_class']['SUPPORTS']['support']} | | **`REFUTES`** | {metrics['per_class']['REFUTES']['precision']*100:.2f}% | {metrics['per_class']['REFUTES']['recall']*100:.2f}% | **{metrics['per_class']['REFUTES']['f1']*100:.2f}%** | {metrics['per_class']['REFUTES']['support']} | | **`NOT_ENOUGH_INFO`** | {metrics['per_class']['NOT_ENOUGH_INFO']['precision']*100:.2f}% | {metrics['per_class']['NOT_ENOUGH_INFO']['recall']*100:.2f}% | **{metrics['per_class']['NOT_ENOUGH_INFO']['f1']*100:.2f}%** | {metrics['per_class']['NOT_ENOUGH_INFO']['support']} | --- ## 3. Confusion Matrix | Actual \\ Predicted | SUPPORTS | REFUTES | NOT_ENOUGH_INFO | Total Actual | | :--- | :--- | :--- | :--- | :--- | | **SUPPORTS** | {cm[0][0]} | {cm[0][1]} | {cm[0][2]} | {metrics['per_class']['SUPPORTS']['support']} | | **REFUTES** | {cm[1][0]} | {cm[1][1]} | {cm[1][2]} | {metrics['per_class']['REFUTES']['support']} | | **NOT_ENOUGH_INFO** | {cm[2][0]} | {cm[2][1]} | {cm[2][2]} | {metrics['per_class']['NOT_ENOUGH_INFO']['support']} | --- ## 4. Error Analysis & Common Failure Modes Total misclassified validation pairs: **{len(df_errors)}** ({len(df_errors)/len(df_eval)*100:.1f}% error rate). Top failure categories: - **`SUPPORTS` predicted as `REFUTES`**: Complex technical assertions where evidence mentions negative correlation or inhibition that the model misinterprets as contradiction. - **`REFUTES` predicted as `SUPPORTS`**: Rare double-negation scientific syntax. - **`NOT_ENOUGH_INFO` vs. Verifiable**: Partial semantic overlap where the abstract discusses the topic but does not provide causal proof. Full error audit records are saved in [`ml/reports/verification_errors.csv`](file:///C:/Users/param/OneDrive/Desktop/newproject/ml/reports/verification_errors.csv). """ with open(output_report_path, "w", encoding="utf-8") as f: f.write(report_md) print(f"Saved evaluation report -> {output_report_path}") return metrics if __name__ == "__main__": base_p = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) model_p = os.path.join(base_p, "models", "verification", "scifact_deberta") val_p = os.path.join(base_p, "data", "final", "verification", "scifact_valid.csv") rep_p = os.path.join(base_p, "reports", "scifact_verification_results.md") cm_p = os.path.join(base_p, "reports", "confusion_matrix.png") err_p = os.path.join(base_p, "reports", "verification_errors.csv") if os.path.exists(model_p): run_full_evaluation(model_p, val_p, rep_p, cm_p, err_p) else: print(f"Model path not found: {model_p}")