Spaces:

Param0304
/

truthlens-ml-api

Paused

truthlens-ml-api / src /train_baseline.py

Deploy TruthLens ML API to Hugging Face Space

3abf967 about 2 months ago

8.95 kB

"""
train_baseline.py - FEVER Claim-Only Baseline Model for TruthLens AI.
Task Definition:
FEVER CLAIM-ONLY BASELINE: claim -> label (SUPPORTS, REFUTES, NOT_ENOUGH_INFO)
*Explicit Note: This baseline operates strictly on claim text alone because
Wikipedia evidence text is not provided in raw FEVER files. It establishes a
claim-only linguistic prior baseline, NOT a full claim+evidence verification model.*
Architecture:
- Feature Extraction: Sublinear TF-IDF (unigrams + bigrams, 25,000 features)
- Classifier: Multinomial Logistic Regression with balanced class weights
- Evaluation: Stratified FEVER validation split (13,477 claims)
"""

import os
import sys
import json
import time
from typing import Dict, Any
import numpy as np
import pandas as pd
import joblib
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from training_utils import (
    LABEL2ID,
    ID2LABEL,
    set_seed,
    compute_verification_metrics,
    plot_and_save_confusion_matrix,
)


def run_baseline_training(
    base_dir: str,
    max_features: int = 25000,
    random_state: int = 42
) -> Dict[str, Any]:
    print("=" * 80)
    print(" TRUTHLENS AI — FEVER CLAIM-ONLY BASELINE EXPERIMENT")
    print("=" * 80)
    
    set_seed(random_state)
    start_time = time.time()
    
    data_dir = os.path.join(base_dir, "data", "final", "verification")
    models_dir = os.path.join(base_dir, "models", "baseline")
    reports_dir = os.path.join(base_dir, "reports")
    os.makedirs(models_dir, exist_ok=True)
    os.makedirs(reports_dir, exist_ok=True)
    
    train_path = os.path.join(data_dir, "fever_train.csv")
    valid_path = os.path.join(data_dir, "fever_valid.csv")
    
    print(f"Loading train claims: {train_path}")
    df_train = pd.read_csv(train_path)
    print(f"Loading valid claims: {valid_path}")
    df_valid = pd.read_csv(valid_path)
    
    X_train = df_train["claim"].astype(str).tolist()
    y_train = df_train["label"].map(LABEL2ID).tolist()
    
    X_valid = df_valid["claim"].astype(str).tolist()
    y_valid = df_valid["label"].map(LABEL2ID).tolist()
    
    print(f"Train samples: {len(X_train):,} | Valid samples: {len(X_valid):,}")
    
    # 1. Feature Extraction: TF-IDF
    print("\nFitting TF-IDF Vectorizer (unigrams + bigrams)...")
    tfidf = TfidfVectorizer(
        max_features=max_features,
        ngram_range=(1, 2),
        sublinear_tf=True,
        strip_accents="unicode"
    )
    X_tr_tfidf = tfidf.fit_transform(X_train)
    X_va_tfidf = tfidf.transform(X_valid)
    print(f"Vocabulary size: {len(tfidf.vocabulary_):,} features")
    
    # 2. Classifier: Logistic Regression (Balanced class weights)
    print("\nTraining Multinomial Logistic Regression with balanced class weights...")
    clf = LogisticRegression(
        C=1.0,
        max_iter=1000,
        class_weight="balanced",
        solver="lbfgs",
        random_state=random_state
    )
    clf.fit(X_tr_tfidf, y_train)
    
    # 3. Evaluation on Validation Set
    print("\nEvaluating on FEVER Validation Split...")
    y_pred = clf.predict(X_va_tfidf)
    metrics = compute_verification_metrics(y_valid, y_pred.tolist())
    
    elapsed_time = time.time() - start_time
    print(f"Baseline Training & Evaluation Runtime: {elapsed_time:.2f} seconds")
    print(f"Validation Accuracy : {metrics['accuracy']*100:.2f}%")
    print(f"Macro F1 Score      : {metrics['macro_f1']*100:.2f}%")
    print(f"Weighted F1 Score   : {metrics['weighted_f1']*100:.2f}%")
    
    print("\nPer-Class Breakdown:")
    for cls_name, p_metrics in metrics["per_class"].items():
        print(f"  {cls_name:<16}: Precision={p_metrics['precision']*100:.2f}%, Recall={p_metrics['recall']*100:.2f}%, F1={p_metrics['f1']*100:.2f}% (Support: {p_metrics['support']:,})")
        
    # 4. Save Model Artifacts
    vec_path = os.path.join(models_dir, "tfidf_vectorizer.joblib")
    model_path = os.path.join(models_dir, "logistic_regression_model.joblib")
    joblib.dump(tfidf, vec_path)
    joblib.dump(clf, model_path)
    print(f"\nSaved model artifacts -> {models_dir}")
    
    # Save config
    config = {
        "experiment_name": "FEVER CLAIM-ONLY BASELINE",
        "model_type": "TF-IDF + Logistic Regression",
        "input_features": "claim_only (no evidence text available in raw FEVER)",
        "train_samples": len(X_train),
        "valid_samples": len(X_valid),
        "max_features": max_features,
        "class_weight": "balanced",
        "random_state": random_state,
        "metrics": metrics,
        "runtime_seconds": round(elapsed_time, 2),
    }
    with open(os.path.join(models_dir, "baseline_config.json"), "w", encoding="utf-8") as f:
        json.dump(config, f, indent=2)
        
    # 5. Plot Confusion Matrix
    cm_plot_path = os.path.join(reports_dir, "baseline_confusion_matrix.png")
    plot_and_save_confusion_matrix(
        metrics["confusion_matrix"],
        class_names=["SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"],
        output_path=cm_plot_path,
        title="FEVER Claim-Only Baseline — Confusion Matrix"
    )
    
    # 6. Generate Baseline Report Markdown
    report_path = os.path.join(reports_dir, "baseline_results.md")
    report_content = f"""# TruthLens AI — FEVER Claim-Only Baseline Results
**Experiment:** FEVER CLAIM-ONLY BASELINE  
**Task Definition:** Predict veracity verdict solely from statement text (`claim -> label`)  
**Architecture:** TF-IDF (25,000 features, n-grams 1-2) + Logistic Regression (Balanced)  
**Date:** September 2026  
**Runtime:** {elapsed_time:.2f} seconds  
---
## 1. Experimental Framing & Limitation Notice
> **IMPORTANT ARCHITECTURAL NOTICE:**  
> This baseline evaluates claim-only predictability. Because raw FEVER files contain Wikipedia sentence indices rather than embedded text, this model establishes the linguistic prior baseline (e.g. lexical cues and annotator artifacts in synthetic claims). It is **not** the final evidence-based verification model.
---
## 2. Quantitative Results (Validation Split: 13,477 claims)
| Metric | Score |
| :--- | :--- |
| **Accuracy** | **{metrics['accuracy']*100:.2f}%** |
| **Macro Precision** | **{metrics['macro_precision']*100:.2f}%** |
| **Macro Recall** | **{metrics['macro_recall']*100:.2f}%** |
| **Macro F1 Score** | **{metrics['macro_f1']*100:.2f}%** |
| **Weighted F1 Score** | **{metrics['weighted_f1']*100:.2f}%** |
---
## 3. Per-Class Performance
| Class | Precision | Recall | F1 Score | Support (Validation) |
| :--- | :--- | :--- | :--- | :--- |
| **`SUPPORTS`** | {metrics['per_class']['SUPPORTS']['precision']*100:.2f}% | {metrics['per_class']['SUPPORTS']['recall']*100:.2f}% | **{metrics['per_class']['SUPPORTS']['f1']*100:.2f}%** | {metrics['per_class']['SUPPORTS']['support']:,} |
| **`REFUTES`** | {metrics['per_class']['REFUTES']['precision']*100:.2f}% | {metrics['per_class']['REFUTES']['recall']*100:.2f}% | **{metrics['per_class']['REFUTES']['f1']*100:.2f}%** | {metrics['per_class']['REFUTES']['support']:,} |
| **`NOT_ENOUGH_INFO`** | {metrics['per_class']['NOT_ENOUGH_INFO']['precision']*100:.2f}% | {metrics['per_class']['NOT_ENOUGH_INFO']['recall']*100:.2f}% | **{metrics['per_class']['NOT_ENOUGH_INFO']['f1']*100:.2f}%** | {metrics['per_class']['NOT_ENOUGH_INFO']['support']:,} |
---
## 4. Confusion Matrix
| Actual \\ Predicted | SUPPORTS | REFUTES | NOT_ENOUGH_INFO | Total Actual |
| :--- | :--- | :--- | :--- | :--- |
| **SUPPORTS** | {metrics['confusion_matrix'][0][0]:,} | {metrics['confusion_matrix'][0][1]:,} | {metrics['confusion_matrix'][0][2]:,} | {metrics['per_class']['SUPPORTS']['support']:,} |
| **REFUTES** | {metrics['confusion_matrix'][1][0]:,} | {metrics['confusion_matrix'][1][1]:,} | {metrics['confusion_matrix'][1][2]:,} | {metrics['per_class']['REFUTES']['support']:,} |
| **NOT_ENOUGH_INFO** | {metrics['confusion_matrix'][2][0]:,} | {metrics['confusion_matrix'][2][1]:,} | {metrics['confusion_matrix'][2][2]:,} | {metrics['per_class']['NOT_ENOUGH_INFO']['support']:,} |
---
## 5. Key Findings & Baseline Benchmark
1. **Linguistic Priors in FEVER:** The claim-only baseline achieves notable predictive power above majority-class random guessing (~33-54%), confirming known crowd-worker linguistic artifacts (e.g. specific negation words used when mutating claims to `REFUTES`).
2. **Need for Grounded Evidence:** While claim-only signals exist, true fact-checking requires external evidence verification. This baseline serves as the benchmark against which the Transformer Claim+Evidence model will be compared.
"""
    with open(report_path, "w", encoding="utf-8") as f:
        f.write(report_content)
    print(f"Generated baseline report -> {report_path}")
    
    return metrics


if __name__ == "__main__":
    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    run_baseline_training(root_dir)

	"""
	train_baseline.py - FEVER Claim-Only Baseline Model for TruthLens AI.

	Task Definition:
	FEVER CLAIM-ONLY BASELINE: claim -> label (SUPPORTS, REFUTES, NOT_ENOUGH_INFO)
	*Explicit Note: This baseline operates strictly on claim text alone because
	Wikipedia evidence text is not provided in raw FEVER files. It establishes a
	claim-only linguistic prior baseline, NOT a full claim+evidence verification model.*

	Architecture:
	- Feature Extraction: Sublinear TF-IDF (unigrams + bigrams, 25,000 features)
	- Classifier: Multinomial Logistic Regression with balanced class weights
	- Evaluation: Stratified FEVER validation split (13,477 claims)
	"""

	import os
	import sys
	import json
	import time
	from typing import Dict, Any
	import numpy as np
	import pandas as pd
	import joblib
	from sklearn.feature_extraction.text import TfidfVectorizer
	from sklearn.linear_model import LogisticRegression

	sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
	from training_utils import (
	LABEL2ID,
	ID2LABEL,
	set_seed,
	compute_verification_metrics,
	plot_and_save_confusion_matrix,
	)


	def run_baseline_training(
	base_dir: str,
	max_features: int = 25000,
	random_state: int = 42
	) -> Dict[str, Any]:
	print("=" * 80)
	print(" TRUTHLENS AI — FEVER CLAIM-ONLY BASELINE EXPERIMENT")
	print("=" * 80)

	set_seed(random_state)
	start_time = time.time()

	data_dir = os.path.join(base_dir, "data", "final", "verification")
	models_dir = os.path.join(base_dir, "models", "baseline")
	reports_dir = os.path.join(base_dir, "reports")
	os.makedirs(models_dir, exist_ok=True)
	os.makedirs(reports_dir, exist_ok=True)

	train_path = os.path.join(data_dir, "fever_train.csv")
	valid_path = os.path.join(data_dir, "fever_valid.csv")

	print(f"Loading train claims: {train_path}")
	df_train = pd.read_csv(train_path)
	print(f"Loading valid claims: {valid_path}")
	df_valid = pd.read_csv(valid_path)

	X_train = df_train["claim"].astype(str).tolist()
	y_train = df_train["label"].map(LABEL2ID).tolist()

	X_valid = df_valid["claim"].astype(str).tolist()
	y_valid = df_valid["label"].map(LABEL2ID).tolist()

	print(f"Train samples: {len(X_train):,} \| Valid samples: {len(X_valid):,}")

	# 1. Feature Extraction: TF-IDF
	print("\nFitting TF-IDF Vectorizer (unigrams + bigrams)...")
	tfidf = TfidfVectorizer(
	max_features=max_features,
	ngram_range=(1, 2),
	sublinear_tf=True,
	strip_accents="unicode"
	)
	X_tr_tfidf = tfidf.fit_transform(X_train)
	X_va_tfidf = tfidf.transform(X_valid)
	print(f"Vocabulary size: {len(tfidf.vocabulary_):,} features")

	# 2. Classifier: Logistic Regression (Balanced class weights)
	print("\nTraining Multinomial Logistic Regression with balanced class weights...")
	clf = LogisticRegression(
	C=1.0,
	max_iter=1000,
	class_weight="balanced",
	solver="lbfgs",
	random_state=random_state
	)
	clf.fit(X_tr_tfidf, y_train)

	# 3. Evaluation on Validation Set
	print("\nEvaluating on FEVER Validation Split...")
	y_pred = clf.predict(X_va_tfidf)
	metrics = compute_verification_metrics(y_valid, y_pred.tolist())

	elapsed_time = time.time() - start_time
	print(f"Baseline Training & Evaluation Runtime: {elapsed_time:.2f} seconds")
	print(f"Validation Accuracy : {metrics['accuracy']*100:.2f}%")
	print(f"Macro F1 Score : {metrics['macro_f1']*100:.2f}%")
	print(f"Weighted F1 Score : {metrics['weighted_f1']*100:.2f}%")

	print("\nPer-Class Breakdown:")
	for cls_name, p_metrics in metrics["per_class"].items():
	print(f" {cls_name:<16}: Precision={p_metrics['precision']100:.2f}%, Recall={p_metrics['recall']100:.2f}%, F1={p_metrics['f1']*100:.2f}% (Support: {p_metrics['support']:,})")

	# 4. Save Model Artifacts
	vec_path = os.path.join(models_dir, "tfidf_vectorizer.joblib")
	model_path = os.path.join(models_dir, "logistic_regression_model.joblib")
	joblib.dump(tfidf, vec_path)
	joblib.dump(clf, model_path)
	print(f"\nSaved model artifacts -> {models_dir}")

	# Save config
	config = {
	"experiment_name": "FEVER CLAIM-ONLY BASELINE",
	"model_type": "TF-IDF + Logistic Regression",
	"input_features": "claim_only (no evidence text available in raw FEVER)",
	"train_samples": len(X_train),
	"valid_samples": len(X_valid),
	"max_features": max_features,
	"class_weight": "balanced",
	"random_state": random_state,
	"metrics": metrics,
	"runtime_seconds": round(elapsed_time, 2),
	}
	with open(os.path.join(models_dir, "baseline_config.json"), "w", encoding="utf-8") as f:
	json.dump(config, f, indent=2)

	# 5. Plot Confusion Matrix
	cm_plot_path = os.path.join(reports_dir, "baseline_confusion_matrix.png")
	plot_and_save_confusion_matrix(
	metrics["confusion_matrix"],
	class_names=["SUPPORTS", "REFUTES", "NOT_ENOUGH_INFO"],
	output_path=cm_plot_path,
	title="FEVER Claim-Only Baseline — Confusion Matrix"
	)

	# 6. Generate Baseline Report Markdown
	report_path = os.path.join(reports_dir, "baseline_results.md")
	report_content = f"""# TruthLens AI — FEVER Claim-Only Baseline Results

	Experiment: FEVER CLAIM-ONLY BASELINE
	Task Definition: Predict veracity verdict solely from statement text (`claim -> label`)
	Architecture: TF-IDF (25,000 features, n-grams 1-2) + Logistic Regression (Balanced)
	Date: September 2026
	Runtime: {elapsed_time:.2f} seconds

	---

	## 1. Experimental Framing & Limitation Notice
	> IMPORTANT ARCHITECTURAL NOTICE:
	> This baseline evaluates claim-only predictability. Because raw FEVER files contain Wikipedia sentence indices rather than embedded text, this model establishes the linguistic prior baseline (e.g. lexical cues and annotator artifacts in synthetic claims). It is not the final evidence-based verification model.

	---

	## 2. Quantitative Results (Validation Split: 13,477 claims)

	\| Metric \| Score \|
	\| :--- \| :--- \|
	\| Accuracy \| *{metrics['accuracy']100:.2f}%** \|
	\| Macro Precision \| *{metrics['macro_precision']100:.2f}%** \|
	\| Macro Recall \| *{metrics['macro_recall']100:.2f}%** \|
	\| Macro F1 Score \| *{metrics['macro_f1']100:.2f}%** \|
	\| Weighted F1 Score \| *{metrics['weighted_f1']100:.2f}%** \|

	---

	## 3. Per-Class Performance

	\| Class \| Precision \| Recall \| F1 Score \| Support (Validation) \|
	\| :--- \| :--- \| :--- \| :--- \| :--- \|
	\| `SUPPORTS` \| {metrics['per_class']['SUPPORTS']['precision']100:.2f}% \| {metrics['per_class']['SUPPORTS']['recall']100:.2f}% \| *{metrics['per_class']['SUPPORTS']['f1']100:.2f}%** \| {metrics['per_class']['SUPPORTS']['support']:,} \|
	\| `REFUTES` \| {metrics['per_class']['REFUTES']['precision']100:.2f}% \| {metrics['per_class']['REFUTES']['recall']100:.2f}% \| *{metrics['per_class']['REFUTES']['f1']100:.2f}%** \| {metrics['per_class']['REFUTES']['support']:,} \|
	\| `NOT_ENOUGH_INFO` \| {metrics['per_class']['NOT_ENOUGH_INFO']['precision']100:.2f}% \| {metrics['per_class']['NOT_ENOUGH_INFO']['recall']100:.2f}% \| *{metrics['per_class']['NOT_ENOUGH_INFO']['f1']100:.2f}%** \| {metrics['per_class']['NOT_ENOUGH_INFO']['support']:,} \|

	---

	## 4. Confusion Matrix

	\| Actual \\ Predicted \| SUPPORTS \| REFUTES \| NOT_ENOUGH_INFO \| Total Actual \|
	\| :--- \| :--- \| :--- \| :--- \| :--- \|
	\| SUPPORTS \| {metrics['confusion_matrix'][0][0]:,} \| {metrics['confusion_matrix'][0][1]:,} \| {metrics['confusion_matrix'][0][2]:,} \| {metrics['per_class']['SUPPORTS']['support']:,} \|
	\| REFUTES \| {metrics['confusion_matrix'][1][0]:,} \| {metrics['confusion_matrix'][1][1]:,} \| {metrics['confusion_matrix'][1][2]:,} \| {metrics['per_class']['REFUTES']['support']:,} \|
	\| NOT_ENOUGH_INFO \| {metrics['confusion_matrix'][2][0]:,} \| {metrics['confusion_matrix'][2][1]:,} \| {metrics['confusion_matrix'][2][2]:,} \| {metrics['per_class']['NOT_ENOUGH_INFO']['support']:,} \|

	---

	## 5. Key Findings & Baseline Benchmark
	1. Linguistic Priors in FEVER: The claim-only baseline achieves notable predictive power above majority-class random guessing (~33-54%), confirming known crowd-worker linguistic artifacts (e.g. specific negation words used when mutating claims to `REFUTES`).
	2. Need for Grounded Evidence: While claim-only signals exist, true fact-checking requires external evidence verification. This baseline serves as the benchmark against which the Transformer Claim+Evidence model will be compared.
	"""
	with open(report_path, "w", encoding="utf-8") as f:
	f.write(report_content)
	print(f"Generated baseline report -> {report_path}")

	return metrics


	if __name__ == "__main__":
	root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
	run_baseline_training(root_dir)