Blame for services/pdf_extractor.py

import re
import pdfplumber


# Section keywords to detect standard resume sections
SECTION_KEYWORDS = {
    "experience":  ["experience", "work experience", "employment", "work history"],
    "education":   ["education", "academic background", "qualifications"],
    "skills":      ["skills", "technical skills", "core competencies", "expertise"],
    "summary":     ["summary", "objective", "profile", "about me"],
    "contact":     ["contact", "personal information", "personal details"],
    "projects":    ["projects", "personal projects", "key projects"],
    "certifications": ["certifications", "certificates", "licenses"],
}


def extract_resume_from_stream(stream) -> dict:
    """
    Opens a PDF with pdfplumber from a stream, extracts text page by page,
    detects basic metadata, and identifies resume sections.
    """
    text_pages = []
    hidden_pages = []
    has_tables = False
    has_images = False
    page_count = 0
    font_names = set()
    font_sizes = []
    has_multiple_columns = False

    with pdfplumber.open(stream) as pdf:
        page_count = len(pdf.pages)

        for page in pdf.pages:
            # Detect tables
            if page.find_tables():
                has_tables = True

            # Detect images
            if page.images:
                has_images = True

            has_multiple_columns = has_multiple_columns or _detect_multiple_columns(page)

            for char in page.chars:
                font_name = _normalize_font_name(str(char.get("fontname", "")))
                if font_name:
                    font_names.add(font_name)
                size = char.get("size")
                if isinstance(size, (int, float)) and size > 0:
                    font_sizes.append(float(size))

            page_text, page_hidden = _split_hidden(page)
            if page_hidden:
                hidden_pages.append(page_hidden)
            if _is_sparse_page_text(page_text, page):
                page_text = _extract_words_as_lines(page) or page_text
            text_pages.append(page_text)

    full_text = "\n".join(text_pages).strip()
    hidden_text = "\n".join(hidden_pages).strip()

    # Confidence: penalise if text is very short relative to page count
    expected_min_chars = page_count * 200
    confidence = min(1.0, len(full_text) / max(expected_min_chars, 1))
    confidence = round(confidence, 2)

    sections = _detect_sections(full_text)
    non_empty_lines = [line.strip() for line in full_text.splitlines() if line.strip()]
    long_line_count = sum(len(line) > 120 for line in non_empty_lines)
    bullet_count = sum(
        bool(re.match(r"^[\u2022\u25e6\u25aa\u25cf\-\u2013\u2014*]\s+", line))
        for line in non_empty_lines
    )

    return {
        "text": full_text,
        "sections": sections,
        "metadata": {
            "hasTables":  has_tables,
            "hasImages":  has_images,
            "pageCount":  page_count,
            "confidence": confidence,
            "hasMultipleColumns": has_multiple_columns,
            "fontCount": len(font_names),
            "minFontSize": round(min(font_sizes), 1) if font_sizes else None,
            "averageFontSize": round(sum(font_sizes) / len(font_sizes), 1) if font_sizes else None,
            "averageCharsPerPage": round(len(full_text) / max(page_count, 1)),
            "longLineCount": long_line_count,
            "bulletCount": bullet_count,
            # Text the page draws but nobody can read. The backend treats this
            # as a high-severity integrity finding rather than as CV content:
            # skills only present here were written for the scorer, not for the
            # person doing the hiring.
            "hiddenTextFound": bool(hidden_text),
            "hiddenTextSample": hidden_text[:300] or None,
        },
    }


def extract_docx_from_stream(stream) -> dict:
    """Extract ordered text and layout signals from a DOCX document."""
    from docx import Document
    from docx.oxml.ns import qn

    document = Document(stream)
    text_lines = []
    hidden_runs = []
    font_names = set()
    font_sizes = []

    for paragraph in document.paragraphs:
        # Word has a literal hidden-text attribute (w:vanish) as well as the
        # white-on-white trick, and python-docx exposes both. A run hidden by
        # either is dropped from the text and kept as evidence: it is not
        # content the candidate is showing the employer.
        visible_runs = []
        for run in paragraph.runs:
            if _is_hidden_run(run):
                if run.text.strip():
                    hidden_runs.append(run.text.strip())
                continue
            visible_runs.append(run)
            if run.font.name:
                font_names.add(_normalize_font_name(run.font.name))
            if run.font.size:
                font_sizes.append(float(run.font.size.pt))

        text = ("".join(run.text for run in visible_runs) or paragraph.text).strip()
        if hidden_runs and not visible_runs:
            text = ""
        if text:
            text_lines.append(text)

    for table in document.tables:
        for row in table.rows:
            cells = [cell.text.strip() for cell in row.cells if cell.text.strip()]
            if cells:
                text_lines.append(" | ".join(cells))

    full_text = "\n".join(text_lines).strip()
    hidden_text = "\n".join(hidden_runs).strip()
    estimated_pages = max(1, round(len(full_text) / 3500))
    has_multiple_columns = False
    for section in document.sections:
        columns = section._sectPr.xpath('./w:cols')
        if columns:
            count = columns[0].get(qn('w:num'))
            if count and int(count) > 1:
                has_multiple_columns = True

    long_line_count = sum(len(line) > 120 for line in text_lines)
    bullet_count = sum(
        bool(re.match(r"^[\u2022\u25e6\u25aa\u25cf\-\u2013\u2014*]\s+", line))
        for line in text_lines
    )
    confidence = 1.0 if len(full_text) >= 100 else round(len(full_text) / 100, 2)

    return {
        "text": full_text,
        "sections": _detect_sections(full_text),
        "metadata": {
            "hasTables": bool(document.tables),
            "hasImages": bool(document.inline_shapes),
            "pageCount": estimated_pages,
            "confidence": confidence,
            "hasMultipleColumns": has_multiple_columns,
            "fontCount": len(font_names),
            "minFontSize": round(min(font_sizes), 1) if font_sizes else None,
            "averageFontSize": round(sum(font_sizes) / len(font_sizes), 1) if font_sizes else None,
            "averageCharsPerPage": round(len(full_text) / estimated_pages),
            "longLineCount": long_line_count,
            "bulletCount": bullet_count,
            "hiddenTextFound": bool(hidden_text),
            "hiddenTextSample": hidden_text[:300] or None,
        },
    }


# ----------------------------------------------------------------- hiding --

# Below this, nothing is readable on a screen or on paper. A CV does not contain
# text nobody is meant to read unless somebody meant it.
_UNREADABLE_PT = 3.5

# How close to the page colour counts as invisible. Not exact equality: a CV
# built to hide text often uses 0.98 grey rather than pure white, precisely
# because an exact-match check is the obvious one to write.
_INK_EPSILON = 0.04


def _is_background_colour(color) -> bool:
    """True when a character is drawn in (near) the page's own colour.

    pdfplumber reports `non_stroking_color` in whatever colourspace the PDF
    used, so this has to handle all of them:

        None            unknown - treated as visible, never as evidence
        0.97            greyscale, 1 is white
        (0.99,1,1)      RGB
        (0,0,0,0)       CMYK, where 0 ink is white
    """
    if color is None:
        return False
    if isinstance(color, (int, float)):
        return float(color) >= 1 - _INK_EPSILON
    try:
        channels = [float(c) for c in color]
    except (TypeError, ValueError):
        return False
    if not channels:
        return False
    if len(channels) == 4:  # CMYK: no ink at all
        return all(c <= _INK_EPSILON for c in channels)
    return all(c >= 1 - _INK_EPSILON for c in channels)


def _is_hidden_char(char) -> bool:
    """A character the page renders but a person cannot read."""
    size = char.get("size")
    if isinstance(size, (int, float)) and 0 < float(size) < _UNREADABLE_PT:
        return True
    return _is_background_colour(char.get("non_stroking_color"))


def _is_hidden_run(run) -> bool:
    """A DOCX run Word will render invisibly, or not at all."""
    font = run.font
    # w:vanish - Word's own "hidden text" checkbox.
    if getattr(font, "hidden", None):
        return True
    size = getattr(font, "size", None)
    if size is not None and getattr(size, "pt", None) and float(size.pt) < _UNREADABLE_PT:
        return True
    color = getattr(font, "color", None)
    rgb = getattr(color, "rgb", None) if color is not None else None
    if rgb is None:
        return False
    try:
        r, g, b = int(str(rgb)[0:2], 16), int(str(rgb)[2:4], 16), int(str(rgb)[4:6], 16)
    except (ValueError, IndexError):
        return False
    return r >= 245 and g >= 245 and b >= 245


def _split_hidden(page):
    """Return (visible_text, hidden_text) for one page.

    The important subtlety is the RATIO. A CV designed with a dark background
    and white type is a legitimate design choice, and on such a page nearly
    every character is 'white' - flagging it would accuse a graphic designer of
    fraud for owning a portfolio template. Hiding, by contrast, is a small block
    of invisible text on an otherwise ordinary page.

    So white text is only treated as hidden when it is the MINORITY of the page.
    Above half, the page is read as a design and nothing is stripped.
    """
    chars = page.chars or []
    if not chars:
        return page.extract_text() or "", ""

    hidden = [c for c in chars if _is_hidden_char(c)]
    if not hidden:
        return page.extract_text() or "", ""

    if len(hidden) > len(chars) * 0.5:
        # A dark-background or light-on-colour design, not a hiding place.
        return page.extract_text() or "", ""

    hidden_text = "".join(str(c.get("text", "")) for c in hidden).strip()
    visible = page.filter(
        lambda obj: obj.get("object_type") != "char" or not _is_hidden_char(obj)
    )
    return (visible.extract_text() or ""), hidden_text


def _normalize_font_name(font_name: str) -> str:
    """Collapse embedded font prefixes and style suffixes into a font family."""
    family = font_name.split("+")[-1].lower()
    family = re.sub(r"(bold|italic|oblique|regular|medium|light|semibold|mt)", "", family)
    return re.sub(r"[^a-z0-9]", "", family)


def _detect_multiple_columns(page) -> bool:
    """Detect repeated, independently populated left/right text regions."""
    words = page.extract_words(use_text_flow=False, keep_blank_chars=False)
    if len(words) < 20 or not page.width:
        return False

    rows: dict[int, list[dict]] = {}
    for word in words:
        row_key = round(float(word.get("top", 0)) / 5)
        rows.setdefault(row_key, []).append(word)

    midpoint = float(page.width) / 2
    split_rows = 0
    for row_words in rows.values():
        left = [w for w in row_words if float(w.get("x1", 0)) < midpoint]
        right = [w for w in row_words if float(w.get("x0", 0)) > midpoint]
        if not left or not right:
            continue

        left_text = " ".join(str(w.get("text", "")) for w in left)
        right_text = " ".join(str(w.get("text", "")) for w in right)
        gap = min(float(w.get("x0", 0)) for w in right) - max(
            float(w.get("x1", 0)) for w in left
        )
        if len(left_text) >= 8 and len(right_text) >= 8 and gap > float(page.width) * 0.08:
            split_rows += 1

    return split_rows >= max(4, round(len(rows) * 0.15))


def _is_sparse_page_text(page_text: str, page) -> bool:
    """True when the regular extractor produced much less text than the word layer."""
    words = page.extract_words(use_text_flow=False, keep_blank_chars=False)
    if not words:
        return False
    word_chars = sum(len(str(word.get("text", ""))) for word in words)
    return len(page_text.strip()) < max(80, word_chars * 0.35)


def _extract_words_as_lines(page) -> str:
    """Rebuild readable lines from positioned words when extract_text is sparse."""
    words = page.extract_words(use_text_flow=False, keep_blank_chars=False)
    if not words:
        return ""

    rows: dict[int, list[dict]] = {}
    for word in words:
        row_key = round(float(word.get("top", 0)) / 4)
        rows.setdefault(row_key, []).append(word)

    lines = []
    for _, row_words in sorted(rows.items(), key=lambda item: item[0]):
        ordered = sorted(row_words, key=lambda word: float(word.get("x0", 0)))
        line = " ".join(str(word.get("text", "")).strip() for word in ordered).strip()
        if line:
            lines.append(line)

    return "\n".join(lines)


def _detect_sections(text: str) -> dict:
    """
    Splits the resume text into named sections by detecting section headers.
    Returns a dict of section_name → list of content lines.
    """
    lines = text.split("\n")
    sections: dict[str, list[str]] = {}
    current_section: str | None = None

    for line in lines:
        stripped = line.strip()
        if not stripped:
            continue

        detected = _match_section_header(stripped)

        if detected:
            current_section = detected
            if current_section not in sections:
                sections[current_section] = []
        elif current_section:
            sections[current_section].append(stripped)

    return sections


def _match_section_header(line: str) -> str | None:
    """
    Returns the canonical section name if the line looks like a section header,
    otherwise returns None.
    A line is treated as a header if it's short (≤50 chars) and matches a keyword.
    """
    if len(line) > 50:
        return None

    lower = line.lower().rstrip(":").strip()

    for section_name, keywords in SECTION_KEYWORDS.items():
        for kw in keywords:
            # Exact match or the line IS the keyword phrase
            if re.fullmatch(re.escape(kw), lower):
                return section_name

    return None