| """Unit tests for the employer-policy ingester (canlex/conduct.py). Offline.""" |
| import json |
| import unittest |
|
|
| from canlex.conduct import DOCS, OUT_FILE, parse |
|
|
| _HTML = """<html><head><title>x</title></head><body> |
| <main> |
| <h1>Canada Border Services Agency Code of Conduct</h1> |
| <h2>On this page</h2><ul><li>Chapter 1</li><li>Chapter 2</li></ul> |
| <section> |
| <h2>Chapter 2: Conflict of Interest</h2> |
| <p>Intro paragraph long enough to count as body text for the chapter heading, comfortably past the threshold.</p> |
| <h3>C. Conflict of Interest During Employment</h3> |
| <p>Employees must not engage in outside employment that conflicts with their duties at the Agency.</p> |
| <ul><li>Disclose outside activities.<ul><li>1.1 Nested item.</li></ul></li><li>Seek approval before acting.</li></ul> |
| <h4>1. Outside Employment</h4> |
| <p>Short.</p> |
| </section> |
| <h2>Footnotes</h2><p>Footnote text that is long enough to be a body but must be skipped anyway ok.</p> |
| <dl><dt>Date modified:</dt><dd><time property="dateModified">2024-05-14</time></dd></dl> |
| </main></body></html>""" |
|
|
|
|
| class ParseTests(unittest.TestCase): |
| def setUp(self): |
| self.chunks = parse(_HTML, DOCS[0]) |
|
|
| def test_one_chunk_per_heading_with_body(self): |
| titles = [c["section"] for c in self.chunks] |
| self.assertEqual(titles, ["Chapter 2: Conflict of Interest", |
| "C. Conflict of Interest During Employment"]) |
|
|
| def test_navigation_and_footnotes_are_skipped(self): |
| titles = " ".join(c["section"] for c in self.chunks) |
| self.assertNotIn("On this page", titles) |
| self.assertNotIn("Footnotes", titles) |
|
|
| def test_lists_become_bullets_and_chapter_is_the_part(self): |
| c = self.chunks[1] |
| self.assertIn("- Disclose outside activities.", c["text"]) |
| self.assertIn("- 1.1 Nested item.", c["text"]) |
| self.assertNotIn("activities. 1.1", c["text"]) |
| self.assertEqual(c["part"], "Chapter 2: Conflict of Interest") |
| self.assertEqual(c["doc_type"], "policy") |
| self.assertEqual(c["act_code"], "CBSA-CODE") |
|
|
| def test_currency_comes_from_date_modified(self): |
| self.assertEqual(self.chunks[0]["current_to"], "2024-05-14") |
| |
| |
| self.assertEqual(self.chunks[0]["citation"], "CBSA Code of Conduct") |
| self.assertEqual(self.chunks[0]["marginal_note"], |
| "Chapter 2: Conflict of Interest") |
|
|
| def test_empty_page_yields_nothing(self): |
| self.assertEqual(parse("<html><main></main></html>", DOCS[0]), []) |
|
|
| def test_multi_page_numbering_continues_and_keeps_page_url(self): |
| doc = {"code": "X-DOC", "short": "X", "name": "X doc", "raw": "x", |
| "urls": ["https://a.test/1", "https://a.test/2"]} |
| first = parse(_HTML, doc, url=doc["urls"][0]) |
| second = parse(_HTML, doc, url=doc["urls"][1], start=len(first)) |
| self.assertEqual([c["id"] for c in first], ["policy-x-doc-1", "policy-x-doc-2"]) |
| self.assertEqual([c["id"] for c in second], ["policy-x-doc-3", "policy-x-doc-4"]) |
| self.assertEqual(second[0]["source_url"], "https://a.test/2") |
|
|
| def test_french_twin_carries_lang_and_fr_infix(self): |
| doc = dict(DOCS[0], lang="fr") |
| c = parse(_HTML, doc)[0] |
| self.assertEqual(c["lang"], "fr") |
| self.assertTrue(c["id"].startswith("policy-fr-cbsa-code-")) |
|
|
| def test_doc_type_override(self): |
| doc = dict(DOCS[0], doc_type="memorandum") |
| self.assertEqual(parse(_HTML, doc)[0]["doc_type"], "memorandum") |
|
|
|
|
| _FORM_PAGE = """<html><body><main> |
| <h1>Application for Criminal Rehabilitation (IMM 1444)</h1> |
| <h2>Download the form</h2> |
| <h3> How to download, save and open this form</h3> |
| <ul><li>Use your computer. The form may not open on tablets or mobile phones.</li> |
| <li>Save the form on your computer in a place you can remember.</li></ul> |
| <h3> Open this form in Acrobat Reader</h3> |
| <ul><li>Open Acrobat Reader. You need Acrobat Reader version 10 or higher to open our forms.</li></ul> |
| <h2>Complete the form</h2> |
| <h3>Section A</h3> |
| <p>1. Print your name as it appears in your passport or on your travel document or other valid identity document.</p> |
| <dl><dt>Date modified:</dt><dd><time property="dateModified">2025-10-21</time></dd></dl> |
| </main></body></html>""" |
|
|
|
|
| class IrccRehabilitationPagesTests(unittest.TestCase): |
| CODES = {"IMM-1444", "IRCC-PUB-OCC", "IRCC-PUB-DEEMED"} |
|
|
| def test_pages_are_configured_as_guidance_fetched_through_powershell(self): |
| docs = {d["code"]: d for d in DOCS if d["code"] in self.CODES} |
| self.assertEqual(set(docs), self.CODES) |
| for d in docs.values(): |
| self.assertEqual(d["doc_type"], "memorandum") |
| self.assertTrue(d["powershell"]) |
| self.assertTrue(d["url"].startswith( |
| "https://www.canada.ca/en/immigration-refugees-citizenship/")) |
| |
| self.assertFalse(d["url"].endswith(".pdf")) |
| self.assertIn("applicant-facing", d["name"]) |
|
|
| def test_form_software_instructions_are_skipped(self): |
| doc = next(d for d in DOCS if d["code"] == "IMM-1444") |
| chunks = parse(_FORM_PAGE, doc) |
| self.assertEqual([c["section"] for c in chunks], ["Section A"]) |
| self.assertEqual(chunks[0]["part"], "Complete the form") |
| self.assertEqual(chunks[0]["id"], "policy-imm-1444-1") |
| self.assertEqual(chunks[0]["current_to"], "2025-10-21") |
|
|
| def test_canada_ca_pages_use_powershell(self): |
| |
| |
| for d in DOCS: |
| for url in d.get("urls") or [d["url"]]: |
| if url.startswith("https://www.canada.ca/"): |
| self.assertTrue(d.get("powershell"), d["code"]) |
|
|
| @unittest.skipUnless(OUT_FILE.exists(), "policy.json not built") |
| def test_processed_corpus_carries_the_pages(self): |
| chunks = json.loads(OUT_FILE.read_text(encoding="utf-8")) |
| by = {} |
| for c in chunks: |
| by.setdefault(c["act_code"], []).append(c) |
| for code in self.CODES: |
| self.assertIn(code, by) |
| self.assertTrue(all(c["current_to"] for c in by[code]), code) |
| self.assertTrue(all(c["doc_type"] == "memorandum" for c in by[code])) |
| self.assertFalse(any("Acrobat" in c["section"] for c in by["IMM-1444"])) |
|
|
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
