Spaces:
Running
Running
File size: 1,506 Bytes
62d8a6c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """
Masking must give the same person the same token in every document of a pack.
Needs Presidio and a spaCy English model; skipped when they are not installed.
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
pytest.importorskip("presidio_analyzer")
spacy_util = pytest.importorskip("spacy.util")
if not (spacy_util.is_package("en_core_web_sm") or spacy_util.is_package("en_core_web_lg")):
pytest.skip("no spaCy English model installed", allow_module_level=True)
from privacy import PIIMasker # noqa: E402
def test_same_person_same_token_across_documents():
masker = PIIMasker()
schedule, s_map = masker.mask(
"Policyholder: Alex Morgan. Named drivers: Priya Shah and Jordan Reed."
)
certificate, c_map = masker.mask(
"Persons entitled to drive: Jordan Reed, Alex Morgan, Priya Shah."
)
s_tokens = {v: k for k, v in s_map.items()}
c_tokens = {" ".join(v.split()): k for k, v in c_map.items()}
for name in ("Alex Morgan", "Priya Shah", "Jordan Reed"):
if name in s_tokens and name in c_tokens:
assert s_tokens[name] == c_tokens[name], name
assert "Alex Morgan" not in certificate
def test_reset_starts_numbering_again():
masker = PIIMasker()
first, _ = masker.mask("My name is Alex Morgan.")
masker.reset()
second, _ = masker.mask("My name is Priya Shah.")
assert first.strip() == second.strip()
|