preprocess.py

# ============================================================
# preprocess.py
# This file cleans resume and job description text
# so the AI can understand it better.
# ============================================================

import re
import nltk

# Download required NLTK data (run once)
nltk.download('stopwords', quiet=True)
nltk.download('wordnet', quiet=True)
nltk.download('punkt', quiet=True)

from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

# Load English stopwords (common words like "the", "is", "and")
STOPWORDS = set(stopwords.words('english'))

# Lemmatizer converts words to their base form (e.g. "running" → "run")
lemmatizer = WordNetLemmatizer()


def clean_text(text):
    """
    Cleans a single piece of text (resume or job description).
    Steps: lowercase → remove URLs → remove emails →
           remove punctuation → remove numbers → remove extra spaces
    """
    if not isinstance(text, str):
        return ""  # return empty string if text is missing/NaN

    # Step 1: Convert to lowercase
    text = text.lower()

    # Step 2: Remove URLs (like https://linkedin.com/...)
    text = re.sub(r'http\S+|www\S+', '', text)

    # Step 3: Remove email addresses
    text = re.sub(r'\S+@\S+', '', text)

    # Step 4: Remove punctuation (keep only letters and spaces)
    text = re.sub(r'[^a-z\s]', '', text)

    # Step 5: Remove extra whitespace
    text = re.sub(r'\s+', ' ', text).strip()

    return text


def remove_stopwords(text):
    """
    Removes common English words that don't add meaning.
    Example: 'i am a data scientist' → 'data scientist'
    """
    words = text.split()
    filtered = [word for word in words if word not in STOPWORDS]
    return ' '.join(filtered)


def lemmatize_text(text):
    """
    Reduces words to their base/root form.
    Example: 'developed building created' → 'develop build create'
    """
    words = text.split()
    lemmatized = [lemmatizer.lemmatize(word) for word in words]
    return ' '.join(lemmatized)


def full_preprocess(text):
    """
    Runs the complete NLP preprocessing pipeline:
    clean → remove stopwords → lemmatize
    Use this function on any resume or job description text.
    """
    text = clean_text(text)
    text = remove_stopwords(text)
    text = lemmatize_text(text)
    return text