engine/parser.py

import os
import re
import sys
import traceback

import pdfplumber
from docx import Document


# =============================================================================
# INTERNAL HELPER — Text Cleaning
# =============================================================================

def clean_text(text: str) -> str:
    """
    Clean raw extracted text for downstream processing.

    What this does:
    - Removes non-printable / control characters (common in PDFs)
    - Collapses multiple blank lines into a single blank line
    - Strips leading/trailing whitespace from each line
    - Preserves paragraph structure (single blank line between sections)
    - Does NOT remove punctuation or change any words — extractor.py needs raw content

    Args:
        text: Raw string from pdfplumber or python-docx

    Returns:
        Cleaned string, ready for extractor.py
    """
    if not text:
        return ""

    # Step 1: Remove non-printable characters (null bytes, form feeds etc)
    # Keep: normal chars, newlines, tabs
    text = re.sub(r'[^\x09\x0A\x0D\x20-\x7E\u00A0-\uFFFF]', ' ', text)

    # Step 2: Normalise line endings (Windows \r\n → \n)
    text = text.replace('\r\n', '\n').replace('\r', '\n')

    # Step 3: Strip trailing whitespace from each line
    lines = [line.rstrip() for line in text.split('\n')]

    # Step 4: Collapse 3+ consecutive blank lines into 2
    # This preserves section gaps while removing excessive whitespace
    cleaned_lines = []
    blank_count = 0
    for line in lines:
        if line.strip() == '':
            blank_count += 1
            if blank_count <= 2:
                cleaned_lines.append('')
        else:
            blank_count = 0
            cleaned_lines.append(line)

    # Step 5: Remove leading/trailing blank lines from the whole document
    result = '\n'.join(cleaned_lines).strip()

    return result


# =============================================================================
# PDF EXTRACTION
# =============================================================================

def extract_text_from_pdf(file_path: str) -> str:
    """
    Extract all text from a PDF resume using pdfplumber.

    Handles:
    - Multi-page PDFs (reads all pages)
    - Single and multi-column layouts
    - Tables inside PDFs (extracts cell text)
    - PDFs with mixed text + table sections

    Does NOT handle:
    - Scanned/image PDFs (these have no embedded text — returns empty string)
    - Password-protected PDFs (raises an error)

    Args:
        file_path: Absolute or relative path to the PDF file

    Returns:
        Cleaned text string. Empty string if no text found (likely scanned PDF).

    Raises:
        FileNotFoundError: If file does not exist
        Exception: If pdfplumber cannot open the file
    """
    if not os.path.exists(file_path):
        raise FileNotFoundError(f"PDF not found: {file_path}")

    all_text = []

    with pdfplumber.open(file_path) as pdf:
        for page_num, page in enumerate(pdf.pages, start=1):
            page_text_parts = []

            # --- Extract regular text ---
            text = page.extract_text()
            if text:
                page_text_parts.append(text)

            # --- Extract text from tables ---
            # Some resumes put skills or experience in HTML-style tables
            # pdfplumber reads these separately from regular text
            tables = page.extract_tables()
            for table in tables:
                for row in table:
                    # Each row is a list of cell values (some may be None)
                    row_text = '  '.join(
                        cell.strip() for cell in row
                        if cell and cell.strip()
                    )
                    if row_text:
                        page_text_parts.append(row_text)

            # Combine text from this page
            if page_text_parts:
                page_combined = '\n'.join(page_text_parts)
                all_text.append(f"[Page {page_num}]\n{page_combined}")

    raw_text = '\n\n'.join(all_text)
    return clean_text(raw_text)


# =============================================================================
# DOCX EXTRACTION
# =============================================================================

def extract_text_from_docx(file_path: str) -> str:
    """
    Extract all text from a DOCX resume using python-docx.

    Handles:
    - All paragraph text (normal, headings, bullets)
    - Tables inside DOCX (common in formatted resumes)
    - Bold/italic text (text content preserved, formatting ignored)
    - Multi-section documents

    Args:
        file_path: Absolute or relative path to the DOCX file

    Returns:
        Cleaned text string.

    Raises:
        FileNotFoundError: If file does not exist
    """
    if not os.path.exists(file_path):
        raise FileNotFoundError(f"DOCX not found: {file_path}")

    doc = Document(file_path)
    all_text = []

    # --- Extract paragraph text ---
    for para in doc.paragraphs:
        text = para.text.strip()
        if text:
            all_text.append(text)

    # --- Extract table text ---
    # Many DOCX resumes use tables for layout (two-column skill sections etc)
    for table in doc.tables:
        for row in table.rows:
            row_parts = []
            for cell in row.cells:
                cell_text = cell.text.strip()
                if cell_text:
                    row_parts.append(cell_text)
            if row_parts:
                all_text.append('  |  '.join(row_parts))

    raw_text = '\n'.join(all_text)
    return clean_text(raw_text)


# =============================================================================
# MAIN ROUTER — Use This Function in All Other Files
# =============================================================================

def extract_text(file_path: str) -> str:
    """
    Auto-detect file type and extract text from a resume.

    This is the only function you need to call from outside this file.
    It detects .pdf vs .docx and routes to the correct extractor.

    Args:
        file_path: Path to a .pdf or .docx resume file

    Returns:
        Cleaned text string ready for extractor.py

    Raises:
        ValueError:  If file extension is not .pdf or .docx
        FileNotFoundError: If file does not exist
    """
    # Check extension first — unsupported format is a config error, not a missing file
    ext = os.path.splitext(file_path)[1].lower()
    if ext not in ('.pdf', '.docx'):
        raise ValueError(
            f"Unsupported file format: '{ext}'\n"
            f"Supported formats: .pdf, .docx\n"
            f"File: {file_path}"
        )

    # Then check existence
    if not os.path.exists(file_path):
        raise FileNotFoundError(
            f"File not found: {file_path}\n"
            f"Make sure the file is saved in data/resumes/"
        )

    if ext == '.pdf':
        return extract_text_from_pdf(file_path)
    else:
        return extract_text_from_docx(file_path)


# =============================================================================
# BATCH LOADER — Processes an Entire Folder
# =============================================================================

def load_resumes_from_folder(folder_path: str,
                              verbose: bool = True) -> list:
    """
    Load and parse all PDF and DOCX resumes from a folder.

    Processes every supported file in the folder.
    Skips unsupported files silently.
    Logs a warning for files that fail to parse (does not crash).

    Args:
        folder_path: Path to folder containing resume files
        verbose:     If True, prints progress for each file

    Returns:
        List of dicts, one per successfully parsed resume:
        [
            {
                "file_name" : "resume_1.pdf",
                "file_path" : "/full/path/to/resume_1.pdf",
                "raw_text"  : "Priya Sharma\nData Analyst...",
                "char_count": 3241,
                "word_count": 512,
                "status"    : "ok"
            },
            ...
        ]
        Failed files are included with status "failed" and empty raw_text,
        so you can see what went wrong without crashing the pipeline.
    """
    if not os.path.exists(folder_path):
        raise FileNotFoundError(
            f"Resume folder not found: {folder_path}\n"
            f"Create it and add PDF/DOCX files, then run again."
        )

    # Find all supported files in the folder
    all_files = os.listdir(folder_path)
    supported = [
        f for f in sorted(all_files)
        if os.path.splitext(f)[1].lower() in ('.pdf', '.docx')
    ]
    skipped = [
        f for f in all_files
        if f not in supported and not f.startswith('.')
    ]

    if verbose:
        print(f"\n[Parser] Found {len(supported)} supported file(s) "
              f"in '{os.path.basename(folder_path)}'")
        if skipped:
            print(f"[Parser] Skipping {len(skipped)} unsupported file(s): "
                  f"{', '.join(skipped[:5])}")

    results = []

    for i, fname in enumerate(supported, start=1):
        fpath = os.path.join(folder_path, fname)

        if verbose:
            print(f"[Parser] ({i}/{len(supported)}) Parsing: {fname} ...", end=' ')

        try:
            text = extract_text(fpath)

            # Quality check — warn if text is suspiciously short
            word_count = len(text.split())
            char_count = len(text)

            if word_count < 30:
                status = "warning_short"
                if verbose:
                    print(f"⚠ Only {word_count} words extracted — "
                          f"may be a scanned or image-based PDF")
            else:
                status = "ok"
                if verbose:
                    print(f"✓  {char_count:,} chars | {word_count:,} words")

            results.append({
                "file_name"  : fname,
                "file_path"  : os.path.abspath(fpath),
                "raw_text"   : text,
                "char_count" : char_count,
                "word_count" : word_count,
                "status"     : status,
            })

        except Exception as e:
            if verbose:
                print(f"✗ FAILED — {type(e).__name__}: {e}")
            results.append({
                "file_name"  : fname,
                "file_path"  : os.path.abspath(fpath),
                "raw_text"   : "",
                "char_count" : 0,
                "word_count" : 0,
                "status"     : f"failed: {type(e).__name__}: {e}",
            })

    # Summary
    ok_count   = sum(1 for r in results if r["status"] == "ok")
    fail_count = sum(1 for r in results if r["status"].startswith("failed"))
    warn_count = sum(1 for r in results if r["status"] == "warning_short")

    if verbose:
        print(f"\n[Parser] Done — "
              f"{ok_count} parsed OK | "
              f"{warn_count} warnings | "
              f"{fail_count} failed\n")

    return results


# =============================================================================
# QUICK SELF-TEST
# Run this file directly to test parser on your resumes:
#   python engine/parser.py
# =============================================================================

if __name__ == "__main__":

    # Add project root to path so config.py can be imported
    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

    try:
        import config
        resume_folder = config.RESUME_DIR
    except ImportError:
        # Fallback if run outside project structure
        resume_folder = os.path.join(
            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
            "data", "../data/resumes"
        )

    print("\n" + "="*60)
    print("  PARSER SELF-TEST")
    print("="*60)
    print(f"  Resume folder: {resume_folder}")

    if not os.path.exists(resume_folder):
        print(f"\n  Folder does not exist yet.")
        print(f"  Create it and add PDF/DOCX files:\n  {resume_folder}\n")
        sys.exit(0)

    resumes = load_resumes_from_folder(resume_folder, verbose=True)

    if not resumes:
        print("\n  No resumes found. Add PDF/DOCX files to the folder above.\n")
        sys.exit(0)

    # Show a preview of the first successfully parsed resume
    ok_resumes = [r for r in resumes if r["status"] == "ok"]
    if ok_resumes:
        first = ok_resumes[0]
        print("="*60)
        print(f"  PREVIEW — {first['file_name']}")
        print("="*60)
        preview_lines = first["raw_text"].split('\n')[:20]
        for line in preview_lines:
            if line.strip():
                print(f"  {line}")
        print(f"\n  ... ({first['word_count']} total words)\n")