Spaces:
Paused
Paused
File size: 4,361 Bytes
3abf967 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
preprocessing.py - Text Cleaning and Schema Normalization for TruthLens AI.
This module provides preprocessing utilities for the TruthLens AI pipeline,
including text normalization, label mapping, and schema formatting across
FEVER, LIAR, and SciFact.
"""
import re
import html
from typing import Dict, List, Any, Optional
import pandas as pd
def clean_text(text: str) -> str:
"""
Cleans raw text by normalizing unicode characters, removing unescaped entities,
collapsing redundant whitespace, and handling quotes.
"""
if not isinstance(text, str):
return ""
# Unescape HTML entities (e.g., & -> &)
text = html.unescape(text)
# Replace non-breaking spaces and other special spaces
text = text.replace("\u00a0", " ").replace("\u200b", "")
# Standardize curly single/double quotes to standard quotes
text = re.sub(r"[\u2018\u2019\u201a\u201b]", "'", text)
text = re.sub(r"[\u201c\u201d\u201e\u201f]", '"', text)
# Standardize dashes
text = re.sub(r"[\u2013\u2014]", "-", text)
# Remove control characters except standard tabs/newlines
text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", text)
# Collapse multiple spaces into one
text = re.sub(r"\s+", " ", text).strip()
return text
def map_scifact_label(raw_label: str) -> str:
"""
Maps SciFact raw labels ('SUPPORT', 'CONTRADICT', '') to standardized
verification labels ('SUPPORTS', 'REFUTES', 'NOT_ENOUGH_INFO').
"""
label_norm = str(raw_label).strip().upper()
mapping = {
"SUPPORT": "SUPPORTS",
"CONTRADICT": "REFUTES",
"NOT_ENOUGH_INFO": "NOT_ENOUGH_INFO",
"NEI": "NOT_ENOUGH_INFO",
"": "NOT_ENOUGH_INFO",
}
return mapping.get(label_norm, label_norm)
def map_liar_label_to_3way(raw_label: str) -> str:
"""
Proposes a mapping of LIAR's 6-point scale into a 3-way verification schema:
- 'true', 'mostly-true' -> 'SUPPORTS'
- 'barely-true', 'false', 'pants-fire' -> 'REFUTES'
- 'half-true' -> 'NOT_ENOUGH_INFO' (ambiguous / mixed claims)
NOTE: LIAR does NOT contain evidence text in its raw tsv files.
This mapping is intended for downstream claim classification or when
retrieving external news evidence.
"""
label_norm = str(raw_label).strip().lower()
mapping = {
"true": "SUPPORTS",
"mostly-true": "SUPPORTS",
"half-true": "NOT_ENOUGH_INFO",
"barely-true": "REFUTES",
"false": "REFUTES",
"pants-fire": "REFUTES",
}
return mapping.get(label_norm, "UNKNOWN")
def map_liar_label_to_binary(raw_label: str) -> Optional[str]:
"""
Binary mapping commonly used in political fact-checking literature:
- 'true', 'mostly-true' -> 'TRUE'
- 'barely-true', 'false', 'pants-fire' -> 'FALSE'
- 'half-true' -> Ambiguous (often excluded or evaluated separately)
"""
label_norm = str(raw_label).strip().lower()
if label_norm in ("true", "mostly-true"):
return "TRUE"
elif label_norm in ("barely-true", "false", "pants-fire"):
return "FALSE"
return None
def format_to_unified_schema(
claim_id: str,
claim: str,
evidence: str,
label: str,
source: str,
domain: str,
dataset: str,
metadata: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Formats a fact-checking record into the unified TruthLens schema.
Schema Fields:
- claim_id (str): Unique globally qualified identifier (e.g., 'fever_75397')
- claim (str): Normalized claim statement
- evidence (str): Concatenated text of verified evidence or rationale
- label (str): Normalized 3-way label ('SUPPORTS', 'REFUTES', 'NOT_ENOUGH_INFO')
- source (str): Citation, document ID, Wikipedia title, or speaker
- domain (str): Subject domain ('encyclopedic', 'political', 'biomedical')
- dataset (str): Origin dataset ('fever', 'liar', 'scifact')
- metadata (dict): Dataset-specific attributes preserved in full
"""
return {
"claim_id": str(claim_id),
"claim": clean_text(claim),
"evidence": clean_text(evidence),
"label": label,
"source": str(source) if source else "",
"domain": domain,
"dataset": dataset,
"metadata": metadata or {},
}
|