Spaces:
Sleeping
Sleeping
File size: 6,653 Bytes
327b929 be87d41 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 | import unittest
import sys
import io
from unittest.mock import Mock
# The layout helper tests do not open PDFs, so keep them runnable before the
# optional service dependencies are installed in a fresh clone.
sys.modules.setdefault('pdfplumber', Mock())
from services.pdf_extractor import (
_detect_multiple_columns,
_detect_sections,
_normalize_font_name,
extract_docx_from_stream,
)
class FakePage:
width = 600
def __init__(self, words):
self._words = words
def extract_words(self, **_kwargs):
return self._words
class PdfExtractorTests(unittest.TestCase):
def test_normalizes_embedded_font_variants(self):
self.assertEqual(_normalize_font_name('ABCDEF+Arial-BoldMT'), 'arial')
def test_detects_repeated_left_and_right_regions(self):
words = []
for row in range(8):
top = row * 20
words.extend([
{'top': top, 'x0': 30, 'x1': 140, 'text': 'Left column content'},
{'top': top, 'x0': 390, 'x1': 550, 'text': 'Right column content'},
])
# Ensure the fixture passes the minimum word-count guard.
words.extend({'top': 200 + i, 'x0': 30, 'x1': 60, 'text': 'word'} for i in range(5))
self.assertTrue(_detect_multiple_columns(FakePage(words)))
def test_extracts_named_sections(self):
sections = _detect_sections('Jane Doe\nSUMMARY\nEngineer\nSKILLS\nPython\nEDUCATION\nBS CS')
self.assertEqual(sections['summary'], ['Engineer'])
self.assertEqual(sections['skills'], ['Python'])
self.assertEqual(sections['education'], ['BS CS'])
def test_extracts_docx_text_and_layout_metadata(self):
from docx import Document
from docx.shared import Pt
document = Document()
heading = document.add_paragraph('Jane Doe')
heading.runs[0].font.name = 'Arial'
heading.runs[0].font.size = Pt(16)
document.add_paragraph('SUMMARY')
document.add_paragraph('Software engineer building reliable products.')
document.add_paragraph('SKILLS')
document.add_paragraph('Python, TypeScript')
stream = io.BytesIO()
document.save(stream)
stream.seek(0)
result = extract_docx_from_stream(stream)
self.assertIn('Software engineer', result['text'])
self.assertEqual(result['sections']['skills'], ['Python, TypeScript'])
self.assertIn('fontCount', result['metadata'])
if __name__ == '__main__':
unittest.main()
class FakeCharPage:
"""A page whose characters we control, for the hidden-text splitter."""
def __init__(self, chars):
self.chars = chars
def extract_text(self):
return "".join(str(c.get("text", "")) for c in self.chars)
def filter(self, predicate):
kept = [c for c in self.chars if predicate({**c, "object_type": "char"})]
return FakeCharPage(kept)
def _char(text, size=11.0, color=(0, 0, 0)):
return {"text": text, "size": size, "non_stroking_color": color, "object_type": "char"}
class HiddenTextTests(unittest.TestCase):
"""White-on-white and 1pt text is written for the scorer, not the reader."""
def test_background_colour_detection_across_colourspaces(self):
from services.pdf_extractor import _is_background_colour
self.assertTrue(_is_background_colour(1)) # greyscale white
self.assertTrue(_is_background_colour((1, 1, 1))) # RGB white
self.assertTrue(_is_background_colour((0.99, 1, 0.98))) # near-white, the sneaky one
self.assertTrue(_is_background_colour((0, 0, 0, 0))) # CMYK, no ink
self.assertFalse(_is_background_colour((0, 0, 0))) # black
self.assertFalse(_is_background_colour(0.2)) # dark grey
self.assertFalse(_is_background_colour(None)) # unknown is never evidence
def test_unreadably_small_text_counts_as_hidden(self):
from services.pdf_extractor import _is_hidden_char
self.assertTrue(_is_hidden_char(_char("x", size=1.0)))
self.assertFalse(_is_hidden_char(_char("x", size=9.0)))
def test_white_block_is_split_out_of_the_visible_text(self):
from services.pdf_extractor import _split_hidden
# The hidden block has to be the MINORITY of the page, which is what a
# real attack looks like: an ordinary CV with a small invisible payload.
body = "Frontend Developer at Orchard Labs building the customer dashboard"
visible = [_char(c) for c in body]
hidden = [_char(c, color=(1, 1, 1)) for c in "KubernetesTerraform"]
page = FakeCharPage(visible + hidden)
text, hidden_text = _split_hidden(page)
self.assertEqual(text, body)
self.assertEqual(hidden_text, "KubernetesTerraform")
self.assertNotIn("Kubernetes", text)
def test_a_dark_background_design_is_not_treated_as_hiding(self):
"""Mostly-white type means a dark template, not a hiding place.
Flagging it would accuse a designer of fraud for owning a portfolio CV.
"""
from services.pdf_extractor import _split_hidden
page = FakeCharPage([_char(c, color=(1, 1, 1)) for c in "Creative Director"])
text, hidden_text = _split_hidden(page)
self.assertEqual(text, "Creative Director")
self.assertEqual(hidden_text, "")
def test_an_ordinary_page_reports_nothing_hidden(self):
from services.pdf_extractor import _split_hidden
page = FakeCharPage([_char(c) for c in "Registered Nurse"])
text, hidden_text = _split_hidden(page)
self.assertEqual(text, "Registered Nurse")
self.assertEqual(hidden_text, "")
class HiddenDocxRunTests(unittest.TestCase):
def _run(self, hidden=None, size_pt=None, rgb=None):
run = Mock()
run.font.hidden = hidden
run.font.size = Mock(pt=size_pt) if size_pt is not None else None
run.font.color = Mock(rgb=rgb) if rgb is not None else None
return run
def test_words_own_hidden_attribute_is_respected(self):
from services.pdf_extractor import _is_hidden_run
self.assertTrue(_is_hidden_run(self._run(hidden=True)))
def test_white_font_colour_is_hidden(self):
from services.pdf_extractor import _is_hidden_run
self.assertTrue(_is_hidden_run(self._run(rgb="FFFFFF")))
self.assertFalse(_is_hidden_run(self._run(rgb="222222")))
def test_normal_run_is_visible(self):
from services.pdf_extractor import _is_hidden_run
self.assertFalse(_is_hidden_run(self._run(size_pt=11.0, rgb="000000")))
|