src/validate_processed_data.py

"""
validate_processed_data.py - Comprehensive Integrity and Leakage Validator for TruthLens AI.

Performs rigorous automated checks across all generated datasets:
1. Verifies required columns exist in every processed and final file.
2. Asserts label validity against target schemas (3-way for Model B, binary for Model A, 6-point for LIAR).
3. Asserts no empty or blank claims.
4. Asserts no duplicate (claim, evidence) pairs within splits.
5. Asserts no unexpected NULL values in mandatory fields.
6. Validates cross-split data leakage:
   - FEVER: Train vs Valid claim overlap == 0
   - SciFact Model B: Train vs Valid claim overlap == 0
   - Model A Relevance: Train vs Valid vs Test claim and pair overlap == 0
   - LIAR: Train vs Valid and Train vs Test statement overlap == 0
7. Validates that all files load cleanly with pandas.
"""

import os
import sys
from typing import Dict, Any, Set
import pandas as pd
import numpy as np

VERIFICATION_LABELS = {"SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"}
RELEVANCE_LABELS = {0, 1}
LIAR_LABELS = {"pants-fire", "false", "barely-true", "half-true", "mostly-true", "true"}

VERIFICATION_COLUMNS = ["claim_id", "claim", "evidence", "label", "dataset", "source", "metadata"]
RELEVANCE_COLUMNS = ["claim_id", "claim", "evidence", "label", "document_id", "dataset"]


def check_file_exists(path: str) -> None:
    assert os.path.exists(path), f"CRITICAL: Missing file: {path}"


def validate_verification_file(filepath: str, is_test: bool = False) -> Dict[str, Any]:
    check_file_exists(filepath)
    df = pd.read_csv(filepath)
    assert not df.empty, f"File is empty: {filepath}"
    
    # 1. Check required columns
    for col in VERIFICATION_COLUMNS:
        assert col in df.columns, f"Missing column '{col}' in {filepath}"
        
    # 2. Check no empty claims
    assert (df["claim"].astype(str).str.strip() == "").sum() == 0, f"Found empty claims in {filepath}"
    assert df["claim"].isnull().sum() == 0, f"Found null claims in {filepath}"
    
    # 3. Check labels
    if not is_test:
        observed_labels = set(df["label"].unique())
        invalid_labels = observed_labels - VERIFICATION_LABELS
        assert len(invalid_labels) == 0, f"Invalid labels {invalid_labels} in {filepath}"
    else:
        assert (df["label"] == "UNLABELED").all(), f"Expected all labels to be UNLABELED in test file {filepath}"
        
    # 4. Check duplicate claim/evidence pairs (for train/valid) and claim_id (for test)
    if not is_test:
        dup_pairs = df.duplicated(subset=["claim", "evidence"]).sum()
        assert dup_pairs == 0, f"Found {dup_pairs} duplicate (claim, evidence) pairs in {filepath}"
    else:
        dup_ids = df.duplicated(subset=["claim_id"]).sum()
        assert dup_ids == 0, f"Found {dup_ids} duplicate claim IDs in {filepath}"
    
    # 5. Check unexpected nulls in mandatory fields
    for col in ["claim_id", "claim", "label", "dataset"]:
        assert df[col].isnull().sum() == 0, f"Found nulls in mandatory column '{col}' in {filepath}"
        
    return {
        "rows": len(df),
        "unique_claims": df["claim"].nunique(),
        "labels": df["label"].value_counts().to_dict(),
    }


def validate_relevance_file(filepath: str) -> Dict[str, Any]:
    check_file_exists(filepath)
    df = pd.read_csv(filepath)
    assert not df.empty, f"File is empty: {filepath}"
    
    # 1. Check required columns
    for col in RELEVANCE_COLUMNS:
        assert col in df.columns, f"Missing column '{col}' in {filepath}"
        
    # 2. Check no empty claims or evidence
    assert (df["claim"].astype(str).str.strip() == "").sum() == 0, f"Found empty claims in {filepath}"
    assert (df["evidence"].astype(str).str.strip() == "").sum() == 0, f"Found empty evidence text in {filepath}"
    assert df["claim"].isnull().sum() == 0, f"Null claims in {filepath}"
    assert df["evidence"].isnull().sum() == 0, f"Null evidence in {filepath}"
    
    # 3. Check labels
    observed_labels = set(df["label"].unique())
    invalid_labels = observed_labels - RELEVANCE_LABELS
    assert len(invalid_labels) == 0, f"Invalid labels {invalid_labels} in {filepath}"
    
    # 4. Check duplicate (claim, evidence) pairs
    dup_pairs = df.duplicated(subset=["claim", "evidence"]).sum()
    assert dup_pairs == 0, f"Found {dup_pairs} duplicate (claim, evidence) pairs in {filepath}"
    
    return {
        "rows": len(df),
        "positives": int((df["label"] == 1).sum()),
        "negatives": int((df["label"] == 0).sum()),
        "unique_claims": df["claim"].nunique(),
    }


def validate_liar_file(filepath: str) -> Dict[str, Any]:
    check_file_exists(filepath)
    df = pd.read_csv(filepath)
    assert not df.empty, f"File is empty: {filepath}"
    
    # Check no blank statements
    assert (df["statement"].astype(str).str.strip() == "").sum() == 0, f"Empty statements in {filepath}"
    
    # Check labels
    observed_labels = set(df["label"].unique())
    invalid_labels = observed_labels - LIAR_LABELS
    assert len(invalid_labels) == 0, f"Invalid labels {invalid_labels} in {filepath}"
    
    # Check sentinel token imputation
    assert (df["speaker_job_title"].astype(str).str.strip() == "").sum() == 0, f"Unimputed job titles in {filepath}"
    assert (df["state_info"].astype(str).str.strip() == "").sum() == 0, f"Unimputed state info in {filepath}"
    
    return {
        "rows": len(df),
        "unique_statements": df["statement"].nunique(),
        "labels": df["label"].value_counts().to_dict(),
    }


def validate_leakage(base_dir: str):
    print("Checking cross-split leakage...")
    
    # 1. FEVER: Train vs Valid
    verif_dir = os.path.join(base_dir, "data", "final", "verification")
    df_fv_tr = pd.read_csv(os.path.join(verif_dir, "fever_train.csv"))
    df_fv_va = pd.read_csv(os.path.join(verif_dir, "fever_valid.csv"))
    
    tr_fv_claims = set(df_fv_tr["claim"].str.strip().str.lower())
    va_fv_claims = set(df_fv_va["claim"].str.strip().str.lower())
    fever_overlap = len(tr_fv_claims.intersection(va_fv_claims))
    assert fever_overlap == 0, f"LEAKAGE: {fever_overlap} claims overlap between FEVER train and valid!"
    print(f"[OK] FEVER Train vs Valid claim overlap: 0 (Strictly 0 leakage)")
    
    # 2. SciFact Model B: Train vs Valid
    df_sci_tr = pd.read_csv(os.path.join(verif_dir, "scifact_train.csv"))
    df_sci_va = pd.read_csv(os.path.join(verif_dir, "scifact_valid.csv"))
    
    tr_sci_claims = set(df_sci_tr["claim"].str.strip().str.lower())
    va_sci_claims = set(df_sci_va["claim"].str.strip().str.lower())
    sci_claim_overlap = len(tr_sci_claims.intersection(va_sci_claims))
    assert sci_claim_overlap == 0, f"LEAKAGE: {sci_claim_overlap} claims overlap between SciFact train and valid!"
    print(f"[OK] SciFact Model B Train vs Valid claim overlap: 0 (Strictly 0 leakage)")
    
    # 3. Model A Relevance: Train vs Valid vs Test
    final_dir = os.path.join(base_dir, "data", "final")
    df_rel_tr = pd.read_csv(os.path.join(final_dir, "evidence_relevance_train.csv"))
    df_rel_va = pd.read_csv(os.path.join(final_dir, "evidence_relevance_valid.csv"))
    df_rel_te = pd.read_csv(os.path.join(final_dir, "evidence_relevance_test.csv"))
    
    rel_tr_claims = set(df_rel_tr["claim"].str.strip().str.lower())
    rel_va_claims = set(df_rel_va["claim"].str.strip().str.lower())
    rel_te_claims = set(df_rel_te["claim"].str.strip().str.lower())
    
    tr_va_rel_overlap = len(rel_tr_claims.intersection(rel_va_claims))
    tr_te_rel_overlap = len(rel_tr_claims.intersection(rel_te_claims))
    va_te_rel_overlap = len(rel_va_claims.intersection(rel_te_claims))
    
    assert tr_va_rel_overlap == 0, f"LEAKAGE: {tr_va_rel_overlap} claims overlap in Relevance train/valid"
    assert tr_te_rel_overlap == 0, f"LEAKAGE: {tr_te_rel_overlap} claims overlap in Relevance train/test"
    assert va_te_rel_overlap == 0, f"LEAKAGE: {va_te_rel_overlap} claims overlap in Relevance valid/test"
    print(f"[OK] Model A Relevance claim overlap: 0 across Train, Valid, Test")
    
    # 4. LIAR: Train vs Valid and Train vs Test
    liar_dir = os.path.join(base_dir, "data", "processed", "liar")
    df_liar_tr = pd.read_csv(os.path.join(liar_dir, "liar_train.csv"))
    df_liar_va = pd.read_csv(os.path.join(liar_dir, "liar_valid.csv"))
    df_liar_te = pd.read_csv(os.path.join(liar_dir, "liar_test.csv"))
    
    liar_tr_stmts = set(df_liar_tr["statement"].str.strip().str.lower())
    liar_va_stmts = set(df_liar_va["statement"].str.strip().str.lower())
    liar_te_stmts = set(df_liar_te["statement"].str.strip().str.lower())
    
    tr_va_liar_leak = len(liar_tr_stmts.intersection(liar_va_stmts))
    tr_te_liar_leak = len(liar_tr_stmts.intersection(liar_te_stmts))
    va_te_liar_leak = len(liar_va_stmts.intersection(liar_te_stmts))
    
    assert tr_va_liar_leak == 0, f"LEAKAGE: {tr_va_liar_leak} statements overlap in LIAR train/valid"
    assert tr_te_liar_leak == 0, f"LEAKAGE: {tr_te_liar_leak} statements overlap in LIAR train/test"
    assert va_te_liar_leak == 0, f"LEAKAGE: {va_te_liar_leak} statements overlap in LIAR valid/test"
    print(f"[OK] LIAR statement overlap: 0 across Train, Valid, Test (Leakage successfully eliminated)")


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    print("=" * 80)
    print(" TRUTHLENS AI — PROCESSED DATA VALIDATION SUITE")
    print("=" * 80)
    
    verif_dir = os.path.join(base_dir, "data", "final", "verification")
    final_dir = os.path.join(base_dir, "data", "final")
    liar_dir = os.path.join(base_dir, "data", "processed", "liar")
    fever_proc = os.path.join(base_dir, "data", "processed", "fever")
    
    # 1. Model B Verification Files
    print("\nValidating Model B Verification Files...")
    for f in ["fever_train.csv", "fever_valid.csv", "scifact_train.csv", "scifact_valid.csv"]:
        res = validate_verification_file(os.path.join(verif_dir, f), is_test=False)
        print(f"[OK] {f}: {res['rows']:,} rows, {res['unique_claims']:,} unique claims, Labels: {res['labels']}")
    res_test = validate_verification_file(os.path.join(verif_dir, "scifact_test.csv"), is_test=True)
    print(f"[OK] scifact_test.csv: {res_test['rows']:,} rows (all UNLABELED)")
    
    # 2. Model A Relevance Files
    print("\nValidating Model A Relevance Files...")
    for f in ["evidence_relevance_train.csv", "evidence_relevance_valid.csv", "evidence_relevance_test.csv"]:
        res = validate_relevance_file(os.path.join(final_dir, f))
        print(f"[OK] {f}: {res['rows']:,} pairs (Pos: {res['positives']:,}, Neg: {res['negatives']:,})")
        
    # 3. LIAR Processed Files
    print("\nValidating Processed LIAR Files...")
    for f in ["liar_train.csv", "liar_valid.csv", "liar_test.csv"]:
        res = validate_liar_file(os.path.join(liar_dir, f))
        print(f"[OK] {f}: {res['rows']:,} rows, {res['unique_statements']:,} unique statements")
        
    # 4. Quarantined & Removed Records Tracking
    print("\nValidating Tracking & Quarantine Files...")
    check_file_exists(os.path.join(fever_proc, "fever_quarantined_conflicts.csv"))
    check_file_exists(os.path.join(fever_proc, "fever_removed_records.csv"))
    check_file_exists(os.path.join(liar_dir, "liar_removed_records.csv"))
    print("[OK] All quarantine and removal audit logs exist and are populated.")
    
    # 5. Cross-Split Leakage Validation
    print("\nValidating Data Leakage Prevention...")
    validate_leakage(base_dir)
    
    print("\n" + "=" * 80)
    print(" ALL VALIDATIONS PASSED: DATASETS ARE CLEAN, LEAK-FREE, AND MODEL-READY")
    print("=" * 80)


if __name__ == "__main__":
    main()