deploy: extract only the text a person can see (hidden/white/1pt detection)

services/pdf_extractor.py

@@ -20,6 +20,7 @@ def extract_resume_from_stream(stream) -> dict:
     detects basic metadata, and identifies resume sections.
     """
     text_pages = []
+    hidden_pages = []
     has_tables = False
     has_images = False
     page_count = 0
@@ -49,12 +50,15 @@ def extract_resume_from_stream(stream) -> dict:
                 if isinstance(size, (int, float)) and size > 0:
                     font_sizes.append(float(size))
 
-            page_text = page.extract_text() or ""
+            page_text, page_hidden = _split_hidden(page)
+            if page_hidden:
+                hidden_pages.append(page_hidden)
             if _is_sparse_page_text(page_text, page):
                 page_text = _extract_words_as_lines(page) or page_text
             text_pages.append(page_text)
 
     full_text = "\n".join(text_pages).strip()
+    hidden_text = "\n".join(hidden_pages).strip()
 
     # Confidence: penalise if text is very short relative to page count
     expected_min_chars = page_count * 200
@@ -84,6 +88,12 @@ def extract_resume_from_stream(stream) -> dict:
             "averageCharsPerPage": round(len(full_text) / max(page_count, 1)),
             "longLineCount": long_line_count,
             "bulletCount": bullet_count,
+            # Text the page draws but nobody can read. The backend treats this
+            # as a high-severity integrity finding rather than as CV content:
+            # skills only present here were written for the scorer, not for the
+            # person doing the hiring.
+            "hiddenTextFound": bool(hidden_text),
+            "hiddenTextSample": hidden_text[:300] or None,
         },
     }
 
@@ -95,19 +105,33 @@ def extract_docx_from_stream(stream) -> dict:
 
     document = Document(stream)
     text_lines = []
+    hidden_runs = []
     font_names = set()
     font_sizes = []
 
     for paragraph in document.paragraphs:
-        text = paragraph.text.strip()
-        if text:
-            text_lines.append(text)
+        # Word has a literal hidden-text attribute (w:vanish) as well as the
+        # white-on-white trick, and python-docx exposes both. A run hidden by
+        # either is dropped from the text and kept as evidence: it is not
+        # content the candidate is showing the employer.
+        visible_runs = []
         for run in paragraph.runs:
+            if _is_hidden_run(run):
+                if run.text.strip():
+                    hidden_runs.append(run.text.strip())
+                continue
+            visible_runs.append(run)
             if run.font.name:
                 font_names.add(_normalize_font_name(run.font.name))
             if run.font.size:
                 font_sizes.append(float(run.font.size.pt))
 
+        text = ("".join(run.text for run in visible_runs) or paragraph.text).strip()
+        if hidden_runs and not visible_runs:
+            text = ""
+        if text:
+            text_lines.append(text)
+
     for table in document.tables:
         for row in table.rows:
             cells = [cell.text.strip() for cell in row.cells if cell.text.strip()]
@@ -115,6 +139,7 @@ def extract_docx_from_stream(stream) -> dict:
                 text_lines.append(" | ".join(cells))
 
     full_text = "\n".join(text_lines).strip()
+    hidden_text = "\n".join(hidden_runs).strip()
     estimated_pages = max(1, round(len(full_text) / 3500))
     has_multiple_columns = False
     for section in document.sections:
@@ -146,10 +171,109 @@ def extract_docx_from_stream(stream) -> dict:
             "averageCharsPerPage": round(len(full_text) / estimated_pages),
             "longLineCount": long_line_count,
             "bulletCount": bullet_count,
+            "hiddenTextFound": bool(hidden_text),
+            "hiddenTextSample": hidden_text[:300] or None,
         },
     }
 
 
+# ----------------------------------------------------------------- hiding --
+
+# Below this, nothing is readable on a screen or on paper. A CV does not contain
+# text nobody is meant to read unless somebody meant it.
+_UNREADABLE_PT = 3.5
+
+# How close to the page colour counts as invisible. Not exact equality: a CV
+# built to hide text often uses 0.98 grey rather than pure white, precisely
+# because an exact-match check is the obvious one to write.
+_INK_EPSILON = 0.04
+
+
+def _is_background_colour(color) -> bool:
+    """True when a character is drawn in (near) the page's own colour.
+
+    pdfplumber reports `non_stroking_color` in whatever colourspace the PDF
+    used, so this has to handle all of them:
+
+        None            unknown - treated as visible, never as evidence
+        0.97            greyscale, 1 is white
+        (0.99,1,1)      RGB
+        (0,0,0,0)       CMYK, where 0 ink is white
+    """
+    if color is None:
+        return False
+    if isinstance(color, (int, float)):
+        return float(color) >= 1 - _INK_EPSILON
+    try:
+        channels = [float(c) for c in color]
+    except (TypeError, ValueError):
+        return False
+    if not channels:
+        return False
+    if len(channels) == 4:  # CMYK: no ink at all
+        return all(c <= _INK_EPSILON for c in channels)
+    return all(c >= 1 - _INK_EPSILON for c in channels)
+
+
+def _is_hidden_char(char) -> bool:
+    """A character the page renders but a person cannot read."""
+    size = char.get("size")
+    if isinstance(size, (int, float)) and 0 < float(size) < _UNREADABLE_PT:
+        return True
+    return _is_background_colour(char.get("non_stroking_color"))
+
+
+def _is_hidden_run(run) -> bool:
+    """A DOCX run Word will render invisibly, or not at all."""
+    font = run.font
+    # w:vanish - Word's own "hidden text" checkbox.
+    if getattr(font, "hidden", None):
+        return True
+    size = getattr(font, "size", None)
+    if size is not None and getattr(size, "pt", None) and float(size.pt) < _UNREADABLE_PT:
+        return True
+    color = getattr(font, "color", None)
+    rgb = getattr(color, "rgb", None) if color is not None else None
+    if rgb is None:
+        return False
+    try:
+        r, g, b = int(str(rgb)[0:2], 16), int(str(rgb)[2:4], 16), int(str(rgb)[4:6], 16)
+    except (ValueError, IndexError):
+        return False
+    return r >= 245 and g >= 245 and b >= 245
+
+
+def _split_hidden(page):
+    """Return (visible_text, hidden_text) for one page.
+
+    The important subtlety is the RATIO. A CV designed with a dark background
+    and white type is a legitimate design choice, and on such a page nearly
+    every character is 'white' - flagging it would accuse a graphic designer of
+    fraud for owning a portfolio template. Hiding, by contrast, is a small block
+    of invisible text on an otherwise ordinary page.
+
+    So white text is only treated as hidden when it is the MINORITY of the page.
+    Above half, the page is read as a design and nothing is stripped.
+    """
+    chars = page.chars or []
+    if not chars:
+        return page.extract_text() or "", ""
+
+    hidden = [c for c in chars if _is_hidden_char(c)]
+    if not hidden:
+        return page.extract_text() or "", ""
+
+    if len(hidden) > len(chars) * 0.5:
+        # A dark-background or light-on-colour design, not a hiding place.
+        return page.extract_text() or "", ""
+
+    hidden_text = "".join(str(c.get("text", "")) for c in hidden).strip()
+    visible = page.filter(
+        lambda obj: obj.get("object_type") != "char" or not _is_hidden_char(obj)
+    )
+    return (visible.extract_text() or ""), hidden_text
+
+
 def _normalize_font_name(font_name: str) -> str:
     """Collapse embedded font prefixes and style suffixes into a font family."""
     family = font_name.split("+")[-1].lower()

tests/test_pdf_extractor.py

@@ -71,3 +71,102 @@ class PdfExtractorTests(unittest.TestCase):
 
 if __name__ == '__main__':
     unittest.main()
+
+
+class FakeCharPage:
+    """A page whose characters we control, for the hidden-text splitter."""
+
+    def __init__(self, chars):
+        self.chars = chars
+
+    def extract_text(self):
+        return "".join(str(c.get("text", "")) for c in self.chars)
+
+    def filter(self, predicate):
+        kept = [c for c in self.chars if predicate({**c, "object_type": "char"})]
+        return FakeCharPage(kept)
+
+
+def _char(text, size=11.0, color=(0, 0, 0)):
+    return {"text": text, "size": size, "non_stroking_color": color, "object_type": "char"}
+
+
+class HiddenTextTests(unittest.TestCase):
+    """White-on-white and 1pt text is written for the scorer, not the reader."""
+
+    def test_background_colour_detection_across_colourspaces(self):
+        from services.pdf_extractor import _is_background_colour
+
+        self.assertTrue(_is_background_colour(1))                # greyscale white
+        self.assertTrue(_is_background_colour((1, 1, 1)))        # RGB white
+        self.assertTrue(_is_background_colour((0.99, 1, 0.98)))  # near-white, the sneaky one
+        self.assertTrue(_is_background_colour((0, 0, 0, 0)))     # CMYK, no ink
+        self.assertFalse(_is_background_colour((0, 0, 0)))       # black
+        self.assertFalse(_is_background_colour(0.2))             # dark grey
+        self.assertFalse(_is_background_colour(None))            # unknown is never evidence
+
+    def test_unreadably_small_text_counts_as_hidden(self):
+        from services.pdf_extractor import _is_hidden_char
+
+        self.assertTrue(_is_hidden_char(_char("x", size=1.0)))
+        self.assertFalse(_is_hidden_char(_char("x", size=9.0)))
+
+    def test_white_block_is_split_out_of_the_visible_text(self):
+        from services.pdf_extractor import _split_hidden
+
+        # The hidden block has to be the MINORITY of the page, which is what a
+        # real attack looks like: an ordinary CV with a small invisible payload.
+        body = "Frontend Developer at Orchard Labs building the customer dashboard"
+        visible = [_char(c) for c in body]
+        hidden = [_char(c, color=(1, 1, 1)) for c in "KubernetesTerraform"]
+        page = FakeCharPage(visible + hidden)
+
+        text, hidden_text = _split_hidden(page)
+        self.assertEqual(text, body)
+        self.assertEqual(hidden_text, "KubernetesTerraform")
+        self.assertNotIn("Kubernetes", text)
+
+    def test_a_dark_background_design_is_not_treated_as_hiding(self):
+        """Mostly-white type means a dark template, not a hiding place.
+
+        Flagging it would accuse a designer of fraud for owning a portfolio CV.
+        """
+        from services.pdf_extractor import _split_hidden
+
+        page = FakeCharPage([_char(c, color=(1, 1, 1)) for c in "Creative Director"])
+        text, hidden_text = _split_hidden(page)
+        self.assertEqual(text, "Creative Director")
+        self.assertEqual(hidden_text, "")
+
+    def test_an_ordinary_page_reports_nothing_hidden(self):
+        from services.pdf_extractor import _split_hidden
+
+        page = FakeCharPage([_char(c) for c in "Registered Nurse"])
+        text, hidden_text = _split_hidden(page)
+        self.assertEqual(text, "Registered Nurse")
+        self.assertEqual(hidden_text, "")
+
+
+class HiddenDocxRunTests(unittest.TestCase):
+    def _run(self, hidden=None, size_pt=None, rgb=None):
+        run = Mock()
+        run.font.hidden = hidden
+        run.font.size = Mock(pt=size_pt) if size_pt is not None else None
+        run.font.color = Mock(rgb=rgb) if rgb is not None else None
+        return run
+
+    def test_words_own_hidden_attribute_is_respected(self):
+        from services.pdf_extractor import _is_hidden_run
+
+        self.assertTrue(_is_hidden_run(self._run(hidden=True)))
+
+    def test_white_font_colour_is_hidden(self):
+        from services.pdf_extractor import _is_hidden_run
+
+        self.assertTrue(_is_hidden_run(self._run(rgb="FFFFFF")))
+        self.assertFalse(_is_hidden_run(self._run(rgb="222222")))
+
+    def test_normal_run_is_visible(self):
+        from services.pdf_extractor import _is_hidden_run
+
+        self.assertFalse(_is_hidden_run(self._run(size_pt=11.0, rgb="000000")))