added feature

.env.example

@@ -0,0 +1 @@
+DATABASE_URL=postgresql://user:password@localhost:5432/factcheck

.gitignore

@@ -16,3 +16,6 @@ build/
 # OS
 .DS_Store
 Thumbs.db
+
+# Environment
+.env

backend/evaluate_model.py

@@ -18,6 +18,7 @@ import re
 import time
 import numpy as np
 from textblob import TextBlob
+import textstat
 
 PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_ROOT)
@@ -65,15 +66,182 @@ def clean_text(text):
 # ───────────────────────────────────────────────────────────
 
 
+# ── Word lists for linguistic features ──
+FIRST_PERSON_PRONOUNS = {
+    "i",
+    "me",
+    "my",
+    "mine",
+    "myself",
+    "we",
+    "us",
+    "our",
+    "ours",
+    "ourselves",
+    "ako",
+    "ko",
+    "akin",
+    "aking",
+    "natin",
+    "atin",
+    "namin",
+    "amin",
+    "tayo",
+    "kami",
+    "ta",
+}
+
+AUXILIARY_VERBS = {
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "shall",
+    "should",
+    "may",
+    "might",
+    "can",
+    "could",
+    "must",
+    "am",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "ay",
+    "dapat",
+    "mayroon",
+    "meron",
+    "maaari",
+    "pwede",
+    "kailangan",
+}
+
+ANALYTICAL_WORDS = {
+    "the",
+    "a",
+    "an",
+    "of",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "with",
+    "by",
+    "from",
+    "about",
+    "between",
+    "through",
+    "during",
+    "before",
+    "after",
+    "ang",
+    "ng",
+    "sa",
+    "mga",
+    "nang",
+    "para",
+    "tungkol",
+    "mula",
+}
+
+CERTAINTY_WORDS = {
+    "always",
+    "never",
+    "absolutely",
+    "definitely",
+    "certainly",
+    "undoubtedly",
+    "clearly",
+    "obviously",
+    "without doubt",
+    "guaranteed",
+    "proven",
+    "fact",
+    "undeniable",
+    "indisputable",
+    "every",
+    "all",
+    "palagi",
+    "sigurado",
+    "tiyak",
+    "talaga",
+    "totoo",
+    "lagi",
+    "walang duda",
+}
+
+TENTATIVE_WORDS = {
+    "perhaps",
+    "maybe",
+    "possibly",
+    "might",
+    "could",
+    "likely",
+    "unlikely",
+    "suggests",
+    "appears",
+    "seems",
+    "allegedly",
+    "reportedly",
+    "according",
+    "probable",
+    "approximately",
+    "estimated",
+    "siguro",
+    "marahil",
+    "maaaring",
+    "mukhang",
+    "parang",
+    "umano",
+    "diumano",
+}
+
+CLOUT_WORDS = {
+    "must",
+    "demand",
+    "require",
+    "order",
+    "command",
+    "insist",
+    "decree",
+    "mandate",
+    "authority",
+    "power",
+    "control",
+    "dominant",
+    "superior",
+    "we must",
+    "you must",
+    "kailangan",
+    "dapat",
+    "utos",
+    "kapangyarihan",
+    "kontrol",
+    "mando",
+}
+
+
 def extract_stylometric_features(text):
-    """Extract 9 stylometric features from text (matches train.py)."""
+    """Extract 22 stylometric features from text (matches train.py)."""
     if not text or not isinstance(text, str):
-        return [0.0] * 9
+        return [0.0] * 22
 
     words = text.split()
     token_count = len(words)
     if token_count == 0:
-        return [0.0] * 9
+        return [0.0] * 22
+
+    words_lower = [w.lower() for w in words]
+    text_len = len(text)
 
     exclamation_density = text.count("!") / token_count
     question_count = text.count("?")
@@ -90,9 +258,9 @@ def extract_stylometric_features(text):
     )
 
     punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
-    punctuation_density = (punct_chars / len(text)) * 100 if len(text) > 0 else 0
+    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0
 
-    unique_words = len(set(w.lower() for w in words))
+    unique_words = len(set(words_lower))
     unique_word_ratio = unique_words / token_count
 
     avg_word_length = sum(len(w) for w in words) / token_count
@@ -102,6 +270,54 @@ def extract_stylometric_features(text):
     except Exception:
         subjectivity = 0.0
 
+    try:
+        flesch_reading_ease = textstat.flesch_reading_ease(text)
+        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
+        coleman_liau_index = textstat.coleman_liau_index(text)
+        ari = textstat.automated_readability_index(text)
+    except Exception:
+        flesch_reading_ease = 0.0
+        flesch_kincaid_grade = 0.0
+        coleman_liau_index = 0.0
+        ari = 0.0
+
+    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
+    first_person_ratio = first_person_count / token_count
+
+    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
+    auxiliary_verb_ratio = aux_count / token_count
+
+    try:
+        gunning_fog_index = textstat.gunning_fog(text)
+    except Exception:
+        gunning_fog_index = 0.0
+
+    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
+    analytical_thinking = analytical_count / token_count
+
+    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
+    certainty_score = certainty_count / token_count
+
+    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
+    tentative_score = tentative_count / token_count
+
+    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
+    clout_score = clout_count / token_count
+
+    comma_period_count = text.count(",") + text.count(".")
+    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0
+
+    informal_count = (
+        text.count("(")
+        + text.count(")")
+        + text.count("—")
+        + text.count("–")
+        + text.count("-")
+        + text.count("...")
+        + text.count("…")
+    )
+    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0
+
     return [
         float(exclamation_density),
         float(question_count),
@@ -112,6 +328,19 @@ def extract_stylometric_features(text):
         float(unique_word_ratio),
         float(avg_word_length),
         float(subjectivity),
+        float(flesch_reading_ease),
+        float(flesch_kincaid_grade),
+        float(coleman_liau_index),
+        float(ari),
+        float(first_person_ratio),
+        float(auxiliary_verb_ratio),
+        float(gunning_fog_index),
+        float(analytical_thinking),
+        float(certainty_score),
+        float(tentative_score),
+        float(clout_score),
+        float(comma_period_density),
+        float(informal_punct_density),
     ]
 
 
@@ -125,6 +354,19 @@ STYLOMETRIC_FEATURE_NAMES = [
     "unique_word_ratio",
     "avg_word_length",
     "subjectivity",
+    "flesch_reading_ease",
+    "flesch_kincaid_grade",
+    "coleman_liau_index",
+    "ari",
+    "first_person_ratio",
+    "auxiliary_verb_ratio",
+    "gunning_fog_index",
+    "analytical_thinking",
+    "certainty_score",
+    "tentative_score",
+    "clout_score",
+    "comma_period_density",
+    "informal_punct_density",
 ]
 
 

backend/train.py

@@ -27,6 +27,7 @@ sys.path.insert(0, PROJECT_ROOT)
 import pandas as pd
 import joblib
 from textblob import TextBlob
+import textstat
 from scipy.sparse import hstack, csr_matrix
 from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
 from sklearn.feature_extraction.text import TfidfVectorizer
@@ -59,30 +60,222 @@ def clean_text(text):
 # ───────────────────────────────────────────────────────────
 
 
+# ── Word lists for linguistic features ──
+FIRST_PERSON_PRONOUNS = {
+    # English
+    "i",
+    "me",
+    "my",
+    "mine",
+    "myself",
+    "we",
+    "us",
+    "our",
+    "ours",
+    "ourselves",
+    # Filipino
+    "ako",
+    "ko",
+    "akin",
+    "aking",
+    "natin",
+    "atin",
+    "namin",
+    "amin",
+    "tayo",
+    "kami",
+    "ta",
+}
+
+AUXILIARY_VERBS = {
+    # English
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "shall",
+    "should",
+    "may",
+    "might",
+    "can",
+    "could",
+    "must",
+    "am",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    # Filipino
+    "ay",
+    "dapat",
+    "mayroon",
+    "meron",
+    "maaari",
+    "pwede",
+    "kailangan",
+}
+
+ANALYTICAL_WORDS = {
+    # English articles + prepositions
+    "the",
+    "a",
+    "an",
+    "of",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "with",
+    "by",
+    "from",
+    "about",
+    "between",
+    "through",
+    "during",
+    "before",
+    "after",
+    # Filipino
+    "ang",
+    "ng",
+    "sa",
+    "mga",
+    "nang",
+    "para",
+    "tungkol",
+    "mula",
+}
+
+CERTAINTY_WORDS = {
+    # English
+    "always",
+    "never",
+    "absolutely",
+    "definitely",
+    "certainly",
+    "undoubtedly",
+    "clearly",
+    "obviously",
+    "without doubt",
+    "guaranteed",
+    "proven",
+    "fact",
+    "undeniable",
+    "indisputable",
+    "every",
+    "all",
+    # Filipino
+    "palagi",
+    "sigurado",
+    "tiyak",
+    "talaga",
+    "totoo",
+    "lagi",
+    "walang duda",
+}
+
+TENTATIVE_WORDS = {
+    # English
+    "perhaps",
+    "maybe",
+    "possibly",
+    "might",
+    "could",
+    "likely",
+    "unlikely",
+    "suggests",
+    "appears",
+    "seems",
+    "allegedly",
+    "reportedly",
+    "according",
+    "probable",
+    "approximately",
+    "estimated",
+    # Filipino
+    "siguro",
+    "marahil",
+    "maaaring",
+    "mukhang",
+    "parang",
+    "umano",
+    "diumano",
+}
+
+CLOUT_WORDS = {
+    # English — authority and dominance markers
+    "must",
+    "demand",
+    "require",
+    "order",
+    "command",
+    "insist",
+    "decree",
+    "mandate",
+    "authority",
+    "power",
+    "control",
+    "dominant",
+    "superior",
+    "we must",
+    "you must",
+    # Filipino
+    "kailangan",
+    "dapat",
+    "utos",
+    "kapangyarihan",
+    "kontrol",
+    "mando",
+}
+
+
 def extract_stylometric_features(text):
     """Extract linguistic style features from text.
 
-    Features (9 total — Hybrid Feature Set):
-        1. exclamation_density  — Exclamation marks per word
-        2. question_count       — Number of '?' characters
-        3. caps_ratio           — Uppercase Ratio (ALL CAPS words / total words)
-        4. avg_sentence_length  — Average number of words per sentence
-        5. punctuation_density  — Punctuation chars per 100 chars
-        6. token_count          — Total word count (metadata feature)
-        7. unique_word_ratio    — Unique words / total words (vocabulary richness)
-        8. avg_word_length      — Average word length in characters
-        9. subjectivity         — TextBlob subjectivity score (0=objective, 1=subjective)
+    Features (22 total — Hybrid Feature Set):
+        1.  exclamation_density    — Exclamation marks per word
+        2.  question_count         — Number of '?' characters
+        3.  caps_ratio             — Uppercase Ratio (ALL CAPS words / total words)
+        4.  avg_sentence_length    — Average number of words per sentence
+        5.  punctuation_density    — Punctuation chars per 100 chars
+        6.  token_count            — Total word count (metadata feature)
+        7.  unique_word_ratio      — Unique words / total words (vocabulary richness)
+        8.  avg_word_length        — Average word length in characters
+        9.  subjectivity           — TextBlob subjectivity score (0=objective, 1=subjective)
+       10.  flesch_reading_ease    — Flesch Reading Ease (higher = easier to read)
+       11.  flesch_kincaid_grade   — Flesch-Kincaid Grade Level
+       12.  coleman_liau_index     — Coleman-Liau Index
+       13.  ari                    — Automated Readability Index
+       14.  first_person_ratio     — First-person pronouns / total words
+       15.  auxiliary_verb_ratio   — Auxiliary/linking verbs / total words
+       16.  gunning_fog_index      — Gunning Fog readability index
+       17.  analytical_thinking    — Articles + prepositions / total words
+       18.  certainty_score        — High-certainty words / total words
+       19.  tentative_score        — Hedge/tentative words / total words
+       20.  clout_score            — Dominance/authority words / total words
+       21.  comma_period_density   — (commas + periods) per 100 chars
+       22.  informal_punct_density — (parentheses + dashes + ellipses) per 100 chars
 
     Returns:
-        List of 9 float values.
+        List of 22 float values.
     """
     if not text or not isinstance(text, str):
-        return [0.0] * 9
+        return [0.0] * 22
 
     words = text.split()
     token_count = len(words)
     if token_count == 0:
-        return [0.0] * 9
+        return [0.0] * 22
+
+    words_lower = [w.lower() for w in words]
+    text_len = len(text)
 
     # 1. Exclamation Mark Density (per word)
     exclamation_density = text.count("!") / token_count
@@ -105,13 +298,13 @@ def extract_stylometric_features(text):
 
     # 5. Punctuation density (per 100 chars)
     punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
-    punctuation_density = (punct_chars / len(text)) * 100 if len(text) > 0 else 0
+    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0
 
     # 6. Token Count (article length — real news is typically longer)
     # (already computed as token_count)
 
     # 7. Unique word ratio (vocabulary richness)
-    unique_words = len(set(w.lower() for w in words))
+    unique_words = len(set(words_lower))
     unique_word_ratio = unique_words / token_count
 
     # 8. Average word length
@@ -123,6 +316,64 @@ def extract_stylometric_features(text):
     except Exception:
         subjectivity = 0.0
 
+    # 10-13. Readability Scores (textstat)
+    try:
+        flesch_reading_ease = textstat.flesch_reading_ease(text)
+        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
+        coleman_liau_index = textstat.coleman_liau_index(text)
+        ari = textstat.automated_readability_index(text)
+    except Exception:
+        flesch_reading_ease = 0.0
+        flesch_kincaid_grade = 0.0
+        coleman_liau_index = 0.0
+        ari = 0.0
+
+    # 14. First-person pronoun ratio
+    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
+    first_person_ratio = first_person_count / token_count
+
+    # 15. Auxiliary verb ratio
+    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
+    auxiliary_verb_ratio = aux_count / token_count
+
+    # 16. Gunning Fog Index
+    try:
+        gunning_fog_index = textstat.gunning_fog(text)
+    except Exception:
+        gunning_fog_index = 0.0
+
+    # 17. Analytical thinking (articles + prepositions ratio)
+    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
+    analytical_thinking = analytical_count / token_count
+
+    # 18. Certainty score
+    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
+    certainty_score = certainty_count / token_count
+
+    # 19. Tentative score
+    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
+    tentative_score = tentative_count / token_count
+
+    # 20. Clout score (dominance/authority markers)
+    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
+    clout_score = clout_count / token_count
+
+    # 21. Comma + period density (per 100 chars)
+    comma_period_count = text.count(",") + text.count(".")
+    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0
+
+    # 22. Informal punctuation density — parentheses, dashes, ellipses (per 100 chars)
+    informal_count = (
+        text.count("(")
+        + text.count(")")
+        + text.count("—")
+        + text.count("–")
+        + text.count("-")
+        + text.count("...")
+        + text.count("…")
+    )
+    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0
+
     return [
         float(exclamation_density),
         float(question_count),
@@ -133,6 +384,19 @@ def extract_stylometric_features(text):
         float(unique_word_ratio),
         float(avg_word_length),
         float(subjectivity),
+        float(flesch_reading_ease),
+        float(flesch_kincaid_grade),
+        float(coleman_liau_index),
+        float(ari),
+        float(first_person_ratio),
+        float(auxiliary_verb_ratio),
+        float(gunning_fog_index),
+        float(analytical_thinking),
+        float(certainty_score),
+        float(tentative_score),
+        float(clout_score),
+        float(comma_period_density),
+        float(informal_punct_density),
     ]
 
 
@@ -146,6 +410,19 @@ STYLOMETRIC_FEATURE_NAMES = [
     "unique_word_ratio",
     "avg_word_length",
     "subjectivity",
+    "flesch_reading_ease",
+    "flesch_kincaid_grade",
+    "coleman_liau_index",
+    "ari",
+    "first_person_ratio",
+    "auxiliary_verb_ratio",
+    "gunning_fog_index",
+    "analytical_thinking",
+    "certainty_score",
+    "tentative_score",
+    "clout_score",
+    "comma_period_density",
+    "informal_punct_density",
 ]
 
 
@@ -155,23 +432,70 @@ STYLOMETRIC_FEATURE_NAMES = [
 
 
 def load_fake_news_dataset():
-    """Load the fake_news_filipino dataset from local CSV."""
-    print("Loading fake_news_filipino dataset...")
+    """Load and merge fake news datasets.
+
+    Merges:
+      1. jcblaise/fake_news_filipino (data/raw/fakenews/fakenews/full.csv)
+      2. Philippine Fake News Corpus (data/raw/philippine_corpus/Philippine Fake News Corpus.csv)
 
-    csv_path = os.path.join(
-        PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv"
+    Deduplicates by a fingerprint of article length + first 200 chars.
+    """
+    print("Loading datasets...")
+
+    frames = []
+
+    # ── Dataset 1: jcblaise/fake_news_filipino ──
+    csv1 = os.path.join(PROJECT_ROOT, "data", "raw", "fakenews", "fakenews", "full.csv")
+    if os.path.exists(csv1):
+        df1 = pd.read_csv(csv1)
+        # Columns: article, label (0=Real, 1=Fake)
+        df1 = df1[["article", "label"]].copy()
+        print(f"  [1] jcblaise/fake_news_filipino: {len(df1)} articles")
+        frames.append(df1)
+    else:
+        print(f"  [1] jcblaise dataset not found at {csv1}, skipping.")
+
+    # ── Dataset 2: Philippine Fake News Corpus ──
+    csv2 = os.path.join(
+        PROJECT_ROOT,
+        "data",
+        "raw",
+        "philippine_corpus",
+        "Philippine Fake News Corpus.csv",
     )
-    if not os.path.exists(csv_path):
+    if os.path.exists(csv2):
+        df2 = pd.read_csv(csv2)
+        # Map: Content → article, Label → label (Credible=0, Not Credible=1)
+        df2 = df2.rename(columns={"Content": "article"})
+        df2["label"] = df2["Label"].map({"Credible": 0, "Not Credible": 1})
+        df2 = df2[["article", "label"]].copy()
+        print(f"  [2] Philippine Fake News Corpus: {len(df2)} articles")
+        frames.append(df2)
+    else:
+        print(f"  [2] Philippine Fake News Corpus not found at {csv2}, skipping.")
+
+    if not frames:
         raise FileNotFoundError(
-            f"Dataset not found at {csv_path}. "
-            "Download fakenews.zip from huggingface.co/datasets/jcblaise/fake_news_filipino"
+            "No datasets found! Place at least one dataset in data/raw/."
         )
 
-    df = pd.read_csv(csv_path)
+    # ── Merge and deduplicate ──
+    df = pd.concat(frames, ignore_index=True)
+    df = df.dropna(subset=["article"]).copy()
+    df = df[df["article"].str.len() > 0].copy()
+
+    # Deduplicate by fingerprint (length + first 200 chars)
+    before = len(df)
+    df["_fingerprint"] = df["article"].apply(lambda x: f"{len(str(x))}_{str(x)[:200]}")
+    df = df.drop_duplicates(subset=["_fingerprint"]).drop(columns=["_fingerprint"])
+    after = len(df)
+    if before > after:
+        print(f"  Removed {before - after} duplicate articles")
+
     label_map = {0: "Real", 1: "Fake"}
     df["label_name"] = df["label"].map(label_map)
 
-    print(f"  Loaded {len(df)} articles")
+    print(f"\n  MERGED TOTAL: {len(df)} articles")
     print(f"  Distribution:\n{df['label_name'].value_counts().to_string()}")
     return df
 

checker/internal/core.py

@@ -12,6 +12,7 @@ import numpy as np
 import joblib
 from scipy.sparse import hstack, csr_matrix
 from textblob import TextBlob
+import textstat
 from checker.internal.bias_analyzer import analyze_bias
 
 
@@ -32,15 +33,182 @@ def clean_text(text):
     return text.strip()
 
 
+# ── Word lists for linguistic features ──
+FIRST_PERSON_PRONOUNS = {
+    "i",
+    "me",
+    "my",
+    "mine",
+    "myself",
+    "we",
+    "us",
+    "our",
+    "ours",
+    "ourselves",
+    "ako",
+    "ko",
+    "akin",
+    "aking",
+    "natin",
+    "atin",
+    "namin",
+    "amin",
+    "tayo",
+    "kami",
+    "ta",
+}
+
+AUXILIARY_VERBS = {
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "shall",
+    "should",
+    "may",
+    "might",
+    "can",
+    "could",
+    "must",
+    "am",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "ay",
+    "dapat",
+    "mayroon",
+    "meron",
+    "maaari",
+    "pwede",
+    "kailangan",
+}
+
+ANALYTICAL_WORDS = {
+    "the",
+    "a",
+    "an",
+    "of",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "with",
+    "by",
+    "from",
+    "about",
+    "between",
+    "through",
+    "during",
+    "before",
+    "after",
+    "ang",
+    "ng",
+    "sa",
+    "mga",
+    "nang",
+    "para",
+    "tungkol",
+    "mula",
+}
+
+CERTAINTY_WORDS = {
+    "always",
+    "never",
+    "absolutely",
+    "definitely",
+    "certainly",
+    "undoubtedly",
+    "clearly",
+    "obviously",
+    "without doubt",
+    "guaranteed",
+    "proven",
+    "fact",
+    "undeniable",
+    "indisputable",
+    "every",
+    "all",
+    "palagi",
+    "sigurado",
+    "tiyak",
+    "talaga",
+    "totoo",
+    "lagi",
+    "walang duda",
+}
+
+TENTATIVE_WORDS = {
+    "perhaps",
+    "maybe",
+    "possibly",
+    "might",
+    "could",
+    "likely",
+    "unlikely",
+    "suggests",
+    "appears",
+    "seems",
+    "allegedly",
+    "reportedly",
+    "according",
+    "probable",
+    "approximately",
+    "estimated",
+    "siguro",
+    "marahil",
+    "maaaring",
+    "mukhang",
+    "parang",
+    "umano",
+    "diumano",
+}
+
+CLOUT_WORDS = {
+    "must",
+    "demand",
+    "require",
+    "order",
+    "command",
+    "insist",
+    "decree",
+    "mandate",
+    "authority",
+    "power",
+    "control",
+    "dominant",
+    "superior",
+    "we must",
+    "you must",
+    "kailangan",
+    "dapat",
+    "utos",
+    "kapangyarihan",
+    "kontrol",
+    "mando",
+}
+
+
 def extract_stylometric_features(text):
-    """Extract the same 9 stylometric features used during training."""
+    """Extract the same 22 stylometric features used during training."""
     if not text or not isinstance(text, str):
-        return [0.0] * 9
+        return [0.0] * 22
 
     words = text.split()
     token_count = len(words)
     if token_count == 0:
-        return [0.0] * 9
+        return [0.0] * 22
+
+    words_lower = [w.lower() for w in words]
+    text_len = len(text)
 
     exclamation_density = text.count("!") / token_count
     question_count = text.count("?")
@@ -57,9 +225,9 @@ def extract_stylometric_features(text):
     )
 
     punct_chars = sum(1 for c in text if c in ".,;:!?-\"'()[]{}...")
-    punctuation_density = (punct_chars / len(text)) * 100 if len(text) > 0 else 0
+    punctuation_density = (punct_chars / text_len) * 100 if text_len > 0 else 0
 
-    unique_words = len(set(w.lower() for w in words))
+    unique_words = len(set(words_lower))
     unique_word_ratio = unique_words / token_count
 
     avg_word_length = sum(len(w) for w in words) / token_count
@@ -69,6 +237,54 @@ def extract_stylometric_features(text):
     except Exception:
         subjectivity = 0.0
 
+    try:
+        flesch_reading_ease = textstat.flesch_reading_ease(text)
+        flesch_kincaid_grade = textstat.flesch_kincaid_grade(text)
+        coleman_liau_index = textstat.coleman_liau_index(text)
+        ari = textstat.automated_readability_index(text)
+    except Exception:
+        flesch_reading_ease = 0.0
+        flesch_kincaid_grade = 0.0
+        coleman_liau_index = 0.0
+        ari = 0.0
+
+    first_person_count = sum(1 for w in words_lower if w in FIRST_PERSON_PRONOUNS)
+    first_person_ratio = first_person_count / token_count
+
+    aux_count = sum(1 for w in words_lower if w in AUXILIARY_VERBS)
+    auxiliary_verb_ratio = aux_count / token_count
+
+    try:
+        gunning_fog_index = textstat.gunning_fog(text)
+    except Exception:
+        gunning_fog_index = 0.0
+
+    analytical_count = sum(1 for w in words_lower if w in ANALYTICAL_WORDS)
+    analytical_thinking = analytical_count / token_count
+
+    certainty_count = sum(1 for w in words_lower if w in CERTAINTY_WORDS)
+    certainty_score = certainty_count / token_count
+
+    tentative_count = sum(1 for w in words_lower if w in TENTATIVE_WORDS)
+    tentative_score = tentative_count / token_count
+
+    clout_count = sum(1 for w in words_lower if w in CLOUT_WORDS)
+    clout_score = clout_count / token_count
+
+    comma_period_count = text.count(",") + text.count(".")
+    comma_period_density = (comma_period_count / text_len) * 100 if text_len > 0 else 0
+
+    informal_count = (
+        text.count("(")
+        + text.count(")")
+        + text.count("—")
+        + text.count("–")
+        + text.count("-")
+        + text.count("...")
+        + text.count("…")
+    )
+    informal_punct_density = (informal_count / text_len) * 100 if text_len > 0 else 0
+
     return [
         float(exclamation_density),
         float(question_count),
@@ -79,6 +295,19 @@ def extract_stylometric_features(text):
         float(unique_word_ratio),
         float(avg_word_length),
         float(subjectivity),
+        float(flesch_reading_ease),
+        float(flesch_kincaid_grade),
+        float(coleman_liau_index),
+        float(ari),
+        float(first_person_ratio),
+        float(auxiliary_verb_ratio),
+        float(gunning_fog_index),
+        float(analytical_thinking),
+        float(certainty_score),
+        float(tentative_score),
+        float(clout_score),
+        float(comma_period_density),
+        float(informal_punct_density),
     ]
 
 

data/embeddings.npz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a20cfa3b8b8338fa8a7ffa09f91f73ff7a1f6c4010bcb0003b5ac1d2fa1047c4
-size 2274678
+oid sha256:d06963218662e1331c1c63015736827c44076d4f7fab8d32ed192afd785927dd
+size 3735378

data/{news.db → raw/Philippine_Fake_News_Corpus.zip}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9950e8bc416a38ae8a16781ba46d32fc156625feec8cbdbfe64c6343bc55330e
-size 421888
+oid sha256:d114013042deb8d19c461766c5e3ad8cb99760df4c0238853c96d2c04a33db23
+size 23734636

data/raw/philippine_corpus/Philippine Fake News Corpus.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b84427e60ea159b2f67f6b88aa9d2c7187f684f1d40f8035a54405605d043a70
+size 60680690

data_models/rf_fakenews_model.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a1feab8d853cdc56041b39d6b9670fc4aa76486299ddb4269e83f44407b737e
-size 3407993
+oid sha256:aa06afbff4dbfe6d61de77948dc2332df9fe08e201f28bc2250c056ce88b1999
+size 14827833

data_models/stylo_scaler.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d81dd464ac6d1334d7d6136382e476fda543dca67c877def3d8013ec2a2c777
-size 783
+oid sha256:20432d5d492d214b693bd1f23f99e436aa55b3eb96238f29c35ba7a8a4a6bad8
+size 1111

data_models/tfidf_fakenews.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cca784a4d98bd9bccd319eea61102b0a70ec8d783b1aabb5b4b4ad55c367fbec
-size 579116
+oid sha256:d0f3d21fa1e93a71c51188f6983447882f571f74e91d30c044106337d843730d
+size 577702

db/database.py

@@ -1,28 +1,33 @@
-import sqlite3
 import os
+import psycopg2
+import psycopg2.extras
 from datetime import datetime
+from dotenv import load_dotenv
 
-DB_DIR = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data"
+# Load .env from project root
+load_dotenv(
+    os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env")
+)
+
+DATABASE_URL = os.getenv(
+    "DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/factcheck"
 )
-DB_PATH = os.path.join(DB_DIR, "news.db")
 
 
 def get_connection():
-    """Get a connection to the SQLite database."""
-    os.makedirs(DB_DIR, exist_ok=True)
-    conn = sqlite3.connect(DB_PATH)
-    conn.row_factory = sqlite3.Row
+    """Get a connection to the PostgreSQL database."""
+    conn = psycopg2.connect(DATABASE_URL)
     return conn
 
 
 def init_db():
     """Create the articles table if it doesn't exist."""
     conn = get_connection()
-    conn.execute(
+    cursor = conn.cursor()
+    cursor.execute(
         """
         CREATE TABLE IF NOT EXISTS articles (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            id SERIAL PRIMARY KEY,
             title TEXT NOT NULL,
             url TEXT NOT NULL UNIQUE,
             source TEXT NOT NULL,
@@ -45,8 +50,9 @@ def insert_articles(articles):
     inserted = 0
     for article in articles:
         cursor.execute(
-            "INSERT OR IGNORE INTO articles (title, url, source, published, scraped_at) "
-            "VALUES (?, ?, ?, ?, ?)",
+            "INSERT INTO articles (title, url, source, published, scraped_at) "
+            "VALUES (%s, %s, %s, %s, %s) "
+            "ON CONFLICT (url) DO NOTHING",
             (
                 article["title"],
                 article["url"],
@@ -65,17 +71,21 @@ def insert_articles(articles):
 def get_all_articles():
     """Return all articles from the database."""
     conn = get_connection()
-    rows = conn.execute("SELECT * FROM articles ORDER BY scraped_at DESC").fetchall()
+    cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cursor.execute("SELECT * FROM articles ORDER BY scraped_at DESC")
+    rows = cursor.fetchall()
     conn.close()
     return [dict(row) for row in rows]
 
 
 def search_articles(query):
-    """Search articles by keyword in title."""
+    """Search articles by keyword in title (case-insensitive)."""
     conn = get_connection()
-    rows = conn.execute(
-        "SELECT * FROM articles WHERE title LIKE ? ORDER BY scraped_at DESC",
+    cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
+    cursor.execute(
+        "SELECT * FROM articles WHERE title ILIKE %s ORDER BY scraped_at DESC",
         (f"%{query}%",),
-    ).fetchall()
+    )
+    rows = cursor.fetchall()
     conn.close()
     return [dict(row) for row in rows]

migrate_to_postgres.py

@@ -0,0 +1,77 @@
+"""
+One-time migration: SQLite → PostgreSQL
+=========================================
+Copies all articles from data/news.db into the PostgreSQL factcheck database,
+then deletes the old SQLite file.
+
+Usage:
+    python migrate_to_postgres.py
+"""
+
+import sqlite3
+import os
+from db.database import get_connection, init_db
+
+SQLITE_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "data", "news.db"
+)
+
+
+def migrate():
+    # 1. Ensure PostgreSQL table exists
+    init_db()
+
+    # 2. Check if SQLite DB exists
+    if not os.path.exists(SQLITE_PATH):
+        print(f"No SQLite database found at {SQLITE_PATH}. Nothing to migrate.")
+        return
+
+    # 3. Read all articles from SQLite
+    sqlite_conn = sqlite3.connect(SQLITE_PATH)
+    sqlite_conn.row_factory = sqlite3.Row
+    rows = sqlite_conn.execute("SELECT * FROM articles").fetchall()
+    articles = [dict(row) for row in rows]
+    sqlite_conn.close()
+
+    print(f"Found {len(articles)} articles in SQLite database.")
+
+    if not articles:
+        print("No articles to migrate.")
+        return
+
+    # 4. Insert into PostgreSQL
+    pg_conn = get_connection()
+    cursor = pg_conn.cursor()
+    inserted = 0
+
+    for article in articles:
+        cursor.execute(
+            "INSERT INTO articles (title, url, source, published, scraped_at) "
+            "VALUES (%s, %s, %s, %s, %s) "
+            "ON CONFLICT (url) DO NOTHING",
+            (
+                article["title"],
+                article["url"],
+                article["source"],
+                article.get("published", "N/A"),
+                article.get("scraped_at", ""),
+            ),
+        )
+        if cursor.rowcount > 0:
+            inserted += 1
+
+    pg_conn.commit()
+    pg_conn.close()
+
+    print(f"Migrated {inserted} articles to PostgreSQL.")
+    if inserted < len(articles):
+        print(f"  ({len(articles) - inserted} duplicates skipped)")
+
+    # 5. Delete old SQLite database
+    os.remove(SQLITE_PATH)
+    print(f"Deleted old SQLite database: {SQLITE_PATH}")
+    print("\nMigration complete!")
+
+
+if __name__ == "__main__":
+    migrate()

requirements.txt

@@ -9,3 +9,6 @@ datasets
 scikit-learn
 pandas
 textblob
+psycopg2-binary
+python-dotenv
+textstat