File size: 2,924 Bytes
2bda233 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | from __future__ import annotations
import unittest
from br_sovereign_llm.corpus import (
NormalizationConfig,
NormalizationError,
canonicalize_locator,
make_document_id,
normalize_text,
normalized_content_sha256,
)
class NormalizationTests(unittest.TestCase):
def test_normalization_is_unicode_and_newline_stable(self) -> None:
raw = " Cafe\u0301\tbrasileiro \r\n\r\n\r\n Lei\u00a0aberta. \u0007"
normalized = normalize_text(raw)
self.assertEqual(normalized, "Café brasileiro\n\nLei aberta.")
self.assertEqual(normalize_text(normalized), normalized)
def test_canonically_equivalent_text_has_same_content_hash(self) -> None:
composed = "Constituição"
decomposed = "Constituic\u0327a\u0303o"
self.assertEqual(
normalized_content_sha256(composed),
normalized_content_sha256(decomposed),
)
def test_nul_is_rejected_instead_of_silently_removed(self) -> None:
with self.assertRaises(NormalizationError):
normalize_text("texto\x00oculto")
def test_locator_canonicalization_removes_fragment_and_default_port(self) -> None:
self.assertEqual(
canonicalize_locator("HTTPS://PT.WIKIPEDIA.ORG:443/wiki/Brasil#História"),
"https://pt.wikipedia.org/wiki/Brasil",
)
def test_project_fixture_locator_is_canonicalized(self) -> None:
self.assertEqual(
canonicalize_locator("PROJECT://FIXTURE/001#ignored"),
"project://fixture/001",
)
def test_document_id_uses_canonical_locator_and_normalized_text(self) -> None:
first = make_document_id(
source_id="wikimedia_ptwiki",
canonical_url="https://PT.WIKIPEDIA.org:443/wiki/Brasil#section",
text="Brasil\r\n soberano",
)
second = make_document_id(
source_id="wikimedia_ptwiki",
canonical_url="https://pt.wikipedia.org/wiki/Brasil",
text="Brasil\n\tsoberano ",
)
self.assertEqual(first, second)
self.assertRegex(first, r"^doc_[0-9a-f]{64}$")
def test_normalization_version_changes_document_id(self) -> None:
nfc_config = NormalizationConfig(normalization_id="nfc-v1", unicode_form="NFC")
nfkc_config = NormalizationConfig(
normalization_id="nfkc-v1",
unicode_form="NFKC",
)
first = make_document_id(
source_id="project_fixture",
canonical_url="urn:br-sovereign-llm:test:one",
text="Texto",
config=nfc_config,
)
second = make_document_id(
source_id="project_fixture",
canonical_url="urn:br-sovereign-llm:test:one",
text="Texto",
config=nfkc_config,
)
self.assertNotEqual(first, second)
if __name__ == "__main__":
unittest.main()
|