| from __future__ import annotations |
|
|
| import unittest |
|
|
| from br_sovereign_llm.corpus import ( |
| NormalizationConfig, |
| NormalizationError, |
| canonicalize_locator, |
| make_document_id, |
| normalize_text, |
| normalized_content_sha256, |
| ) |
|
|
|
|
| class NormalizationTests(unittest.TestCase): |
| def test_normalization_is_unicode_and_newline_stable(self) -> None: |
| raw = " Cafe\u0301\tbrasileiro \r\n\r\n\r\n Lei\u00a0aberta. \u0007" |
|
|
| normalized = normalize_text(raw) |
|
|
| self.assertEqual(normalized, "Café brasileiro\n\nLei aberta.") |
| self.assertEqual(normalize_text(normalized), normalized) |
|
|
| def test_canonically_equivalent_text_has_same_content_hash(self) -> None: |
| composed = "Constituição" |
| decomposed = "Constituic\u0327a\u0303o" |
|
|
| self.assertEqual( |
| normalized_content_sha256(composed), |
| normalized_content_sha256(decomposed), |
| ) |
|
|
| def test_nul_is_rejected_instead_of_silently_removed(self) -> None: |
| with self.assertRaises(NormalizationError): |
| normalize_text("texto\x00oculto") |
|
|
| def test_locator_canonicalization_removes_fragment_and_default_port(self) -> None: |
| self.assertEqual( |
| canonicalize_locator("HTTPS://PT.WIKIPEDIA.ORG:443/wiki/Brasil#História"), |
| "https://pt.wikipedia.org/wiki/Brasil", |
| ) |
|
|
| def test_project_fixture_locator_is_canonicalized(self) -> None: |
| self.assertEqual( |
| canonicalize_locator("PROJECT://FIXTURE/001#ignored"), |
| "project://fixture/001", |
| ) |
|
|
| def test_document_id_uses_canonical_locator_and_normalized_text(self) -> None: |
| first = make_document_id( |
| source_id="wikimedia_ptwiki", |
| canonical_url="https://PT.WIKIPEDIA.org:443/wiki/Brasil#section", |
| text="Brasil\r\n soberano", |
| ) |
| second = make_document_id( |
| source_id="wikimedia_ptwiki", |
| canonical_url="https://pt.wikipedia.org/wiki/Brasil", |
| text="Brasil\n\tsoberano ", |
| ) |
|
|
| self.assertEqual(first, second) |
| self.assertRegex(first, r"^doc_[0-9a-f]{64}$") |
|
|
| def test_normalization_version_changes_document_id(self) -> None: |
| nfc_config = NormalizationConfig(normalization_id="nfc-v1", unicode_form="NFC") |
| nfkc_config = NormalizationConfig( |
| normalization_id="nfkc-v1", |
| unicode_form="NFKC", |
| ) |
|
|
| first = make_document_id( |
| source_id="project_fixture", |
| canonical_url="urn:br-sovereign-llm:test:one", |
| text="Texto", |
| config=nfc_config, |
| ) |
| second = make_document_id( |
| source_id="project_fixture", |
| canonical_url="urn:br-sovereign-llm:test:one", |
| text="Texto", |
| config=nfkc_config, |
| ) |
|
|
| self.assertNotEqual(first, second) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
