Spaces:

Beemer0
/

CanLex

Running

App Files Files Community
CanLex / tests /test_ingest.py
Beemer0
Beemer
claude opus 5 (1m context)
Beemer
claude fable 5
Super-squash branch 'main' using huggingface_hub
ed26ce9 2 days ago
Raw History Blame Contribute Delete
31.3 kB
"""Unit tests for Justice Laws XML parsing (canlex/ingest.py).
Offline: builds a tiny Justice Laws-shaped XML document and parses it, so no
network or real corpus is touched. Focuses on the per-section in-force date and
repeal-status metadata.
"""
import contextlib
import io
import json
import tempfile
import unittest
from pathlib import Path
from unittest import mock

from canlex import ingest
from canlex import _common as ingest_common


_XML = b"""<?xml version="1.0" encoding="UTF-8"?>
<Statute xmlns:lims="http://justice.gc.ca/lims" lims:current-date="2026-03-31">
  <Body>
    <Section lims:inforce-start-date="2015-01-01" lims:lastAmendedDate="2023-06-22">
      <Label>25</Label>
      <MarginalNote>Humanitarian and compassionate considerations</MarginalNote>
      <Text>The Minister may grant relief.</Text>
    </Section>
    <Section lims:inforce-start-date="2018-12-13" lims:lastAmendedDate="2018-12-13">
      <Label>13</Label>
      <Text><Repealed>[Repealed, 2018, c. 27, s. 174]</Repealed></Text>
    </Section>
    <Section lims:inforce-start-date="2001-06-28">
      <Label>40</Label>
      <MarginalNote>Live section with a repealed subsection</MarginalNote>
      <Text>Intro.</Text>
      <Subsection><Label>(1)</Label><Text>Still in force.</Text></Subsection>
      <Subsection><Label>(2)</Label><Text><Repealed>[Repealed, 2020, c. 1]</Repealed></Text></Subsection>
    </Section>
  </Body>
  <Schedule id="RelatedProvs">
    <BillPiece><RelatedOrNotInForce>
      <Heading level="5" style="nifrp"><TitleText>- 1991, c. 43, s. 10</TitleText></Heading>
      <Section><Label>10</Label><Text>A transitional related provision.</Text></Section>
    </RelatedOrNotInForce></BillPiece>
  </Schedule>
  <Schedule id="NifProvs">
    <BillPiece><RelatedOrNotInForce>
      <Heading level="5" style="nifrp"><TitleText>- 2026, c. 8, s. 3</TitleText></Heading>
      <Section><MarginalNote>Coercive control</MarginalNote><Label>3</Label>
        <Text>Section 264 is amended by adding a coercive control offence.</Text>
      </Section>
    </RelatedOrNotInForce></BillPiece>
  </Schedule>
</Statute>"""


# One fixture file for the module, written once and removed in
# tearDownModule: the old per-call NamedTemporaryFile(delete=False) leaked six
# tmp*.xml files into %TEMP% on every run (2026-09-21 audit: 618 had piled up).
_TMP = None
_FIXTURE = None


def setUpModule():
    global _TMP, _FIXTURE
    _TMP = tempfile.TemporaryDirectory()
    _FIXTURE = Path(_TMP.name) / "fixture.xml"
    _FIXTURE.write_bytes(_XML)


def tearDownModule():
    global _TMP, _FIXTURE
    if _TMP is not None:
        _TMP.cleanup()
    _TMP = _FIXTURE = None


def _path():
    return _FIXTURE


def _write(xml, name="case.xml"):
    """A second fixture in the module's temporary directory."""
    p = Path(_TMP.name) / name
    p.write_bytes(xml)
    return p


def _parse():
    # I-2.5 is a real SOURCES code; parse_legislation only reads its short/
    # name/web_base for citation building. Keyed by section, so the NIF chunk
    # (section="") is excluded here; NotInForceTests reads it by id.
    return {c["section"]: c for c in ingest.parse_legislation(_path(), "I-2.5")
            if c["section"]}


class InForceTests(unittest.TestCase):
    def test_every_section_carries_its_in_force_date(self):
        by_sec = _parse()
        self.assertEqual(by_sec["25"]["in_force"], "2015-01-01")
        self.assertEqual(by_sec["40"]["in_force"], "2001-06-28")


class TempFileHygieneTests(unittest.TestCase):
    def test_the_fixture_lives_in_the_module_directory_not_the_system_temp(self):
        self.assertEqual(_path().parent, Path(_TMP.name))
        self.assertTrue(_path().exists())


class AmendmentStubTests(unittest.TestCase):
    def test_matches_bare_amendment_markers(self):
        for s in ["30 [Amendment]", "162 [Amendments]", "5 [Related provision]",
                  "5.1 [Related provisions]"]:
            self.assertTrue(ingest._AMENDMENT_STUB.match(s), s)

    def test_matches_the_french_markers(self):
        # 2026-09-21 audit: the French twins kept 314 such stubs as live
        # provisions (C-1.4-fr 117, C-24.5-fr 66, SOR-2018-108-fr 31, ...).
        # ranges too (the audit follow-up of 2026-09-21: 104 survived the one-token rule)
        for s in ["38 [Modifications]", "55 [Modification]", "17 [Disposition connexe]",
                  "204 et 205 [Modifications]", "211 à 214 [Modifications]",
                  "30 to 32 [Amendments]", "30 and 31 [Amendments]", "2.1 et 2.2 [Modification]",
                  "17.1 [Dispositions connexes]"]:
            self.assertTrue(ingest._AMENDMENT_STUB.match(s), s)

    def test_does_not_match_real_or_repealed_content(self):
        for s in ["13 [Repealed, 2018, c. 27, s. 174]",
                  "13 [Abrogé, 2018, ch. 27, art. 174]",
                  "34 The Minister may [Amendment] later amend...",
                  "40 Inadmissibility on security grounds",
                  "38 Modifications corrélatives"]:
            self.assertIsNone(ingest._AMENDMENT_STUB.match(s), s)

    def test_a_french_stub_section_is_dropped_at_parse(self):
        xml = _XML.replace(
            b"<Text>The Minister may grant relief.</Text>",
            b"<Text>The Minister may grant relief.</Text></Section>\n"
            b'    <Section lims:inforce-start-date="2020-01-01"><Label>38</Label>'
            b"<Text>[Modifications]</Text>", 1)
        chunks = ingest.parse_legislation(_write(xml, "fr-stub.xml"), "I-2.5")
        self.assertNotIn("38", {c["section"] for c in chunks})
        self.assertIn("25", {c["section"] for c in chunks})


class LabelFootnoteTests(unittest.TestCase):
    """Justice Laws puts <FootnoteRef>*</FootnoteRef> inside the Label of a
    provision whose in-force date is footnoted (IRPA ss. 189-193, s. 188(4)).
    2026-09-21 audit: 70 chunks were stored as section '*189', cited 's. *189'
    and linked to a section-*189.html that does not exist."""

    XML = _XML.replace(
        b"<Label>25</Label>",
        b'<Label><FootnoteRef idref="I-2.5_en_2">*</FootnoteRef>25</Label>', 1
    ).replace(
        b"<Subsection><Label>(1)</Label>",
        b'<Subsection><Label><FootnoteRef idref="I-2.5_en_1">*</FootnoteRef>(1)</Label>', 1)

    def setUp(self):
        self.chunks = ingest.parse_legislation(_write(self.XML, "footnote.xml"), "I-2.5")
        self.by_sec = {c["section"]: c for c in self.chunks if c["section"]}

    def test_the_section_number_is_bare(self):
        self.assertIn("25", self.by_sec)
        self.assertNotIn("*25", self.by_sec)
        c = self.by_sec["25"]
        self.assertEqual(c["id"], "I-2.5-s25")
        self.assertEqual(c["citation"], "IRPA, s. 25")
        self.assertTrue(c["source_url"].endswith("/section-25.html"), c["source_url"])

    def test_a_subsection_label_is_bare_too(self):
        text = self.by_sec["40"]["text"]
        self.assertIn("(1) Still in force.", text)
        self.assertNotIn("*(1)", text)

    def test_label_text_helper(self):
        import xml.etree.ElementTree as ET
        el = ET.fromstring('<Label><FootnoteRef idref="x">*</FootnoteRef>189</Label>')
        self.assertEqual(ingest._label_text(el), "189")
        el = ET.fromstring("<Label>12.1</Label>")
        self.assertEqual(ingest._label_text(el), "12.1")
        el = ET.fromstring("<Label>PART <Emphasis>I</Emphasis></Label>")
        self.assertEqual(ingest._label_text(el), "PART I")

    def test_a_footnote_marker_in_body_text_is_kept(self):
        # only the Label reads strip it: C-46 carries FootnoteRef inside <Text>
        xml = _XML.replace(b"<Text>The Minister may grant relief.</Text>",
                           b'<Text>The Minister<FootnoteRef idref="f">*</FootnoteRef> may '
                           b"grant relief.</Text>", 1)
        chunks = ingest.parse_legislation(_write(xml, "bodyfn.xml"), "I-2.5")
        c = next(c for c in chunks if c["section"] == "25")
        self.assertIn("Minister* may", c["text"])


class RawCacheTests(unittest.TestCase):
    """2026-09-21 audit: fetch_xml cached whatever 200 body Justice Laws
    returned (a maintenance page), and parse_legislation then failed the same
    way on every plain run with nothing saying the cache was the cause."""

    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.raw = Path(self.tmp.name)
        self.addCleanup(self.tmp.cleanup)

    def _serve(self, body):
        class _Resp(io.BytesIO):
            def __enter__(self_inner):
                return self_inner

            def __exit__(self_inner, *exc):
                return False
        return mock.patch.object(ingest.urllib.request, "urlopen",
                                 lambda req, timeout=None: _Resp(body))

    def test_a_non_xml_body_is_refused_and_not_cached(self):
        dest = self.raw / "U-2.xml"
        with self._serve(b"<html><body>Maintenance</body></html>"), \
                mock.patch.object(ingest.time, "sleep"):
            with self.assertRaises(ValueError) as cm:
                ingest._download_xml("http://x/U-2.xml", dest)
        self.assertIn("not Justice Laws XML", str(cm.exception))
        self.assertEqual(list(self.raw.iterdir()), [])

    def test_a_consolidation_is_cached_atomically(self):
        dest = self.raw / "I-2.5.xml"
        with self._serve(_XML), mock.patch.object(ingest.time, "sleep"):
            ingest._download_xml("http://x/I-2.5.xml", dest)
        self.assertEqual(dest.read_bytes(), _XML)
        self.assertEqual([p.name for p in self.raw.iterdir()], ["I-2.5.xml"])

    def test_a_bom_prefixed_regulation_passes_the_root_check(self):
        dest = self.raw / "SOR.xml"
        body = b"\xef\xbb\xbf" + _XML.replace(b"<Statute", b"<Regulation").replace(
            b"</Statute>", b"</Regulation>")
        with self._serve(body), mock.patch.object(ingest.time, "sleep"):
            ingest._download_xml("http://x/SOR.xml", dest)
        self.assertTrue(dest.exists())

    def test_a_poisoned_cache_is_removed_when_it_cannot_be_parsed(self):
        dest = self.raw / "U-2.xml"
        dest.write_bytes(b"<html><body>Maintenance</body></html>")
        with mock.patch.object(ingest, "RAW_DIR", self.raw), \
                mock.patch.object(ingest, "PROCESSED_DIR", self.raw / "out"), \
                contextlib.redirect_stdout(io.StringIO()):
            with self.assertRaises(ValueError) as cm:
                ingest.ingest("U-2")
        self.assertIn("cached U-2.xml removed", str(cm.exception))
        self.assertFalse(dest.exists())

    def test_a_truncated_cache_is_removed_too(self):
        dest = self.raw / "U-2.xml"
        dest.write_bytes(_XML[:200])           # cut mid-element: a ParseError
        with mock.patch.object(ingest, "RAW_DIR", self.raw), \
                mock.patch.object(ingest, "PROCESSED_DIR", self.raw / "out"), \
                contextlib.redirect_stdout(io.StringIO()):
            with self.assertRaises(ingest.ET.ParseError):
                ingest.ingest("U-2")
        self.assertFalse(dest.exists())


class NotInForceTests(unittest.TestCase):
    def test_nif_schedule_entry_ingested_as_pending_law(self):
        chunks = ingest.parse_legislation(_path(), "I-2.5")
        nif = [c for c in chunks if c["id"].startswith("I-2.5-nif-")]
        self.assertEqual(len(nif), 1)
        c = nif[0]
        self.assertEqual(c["status"], "not-in-force")
        self.assertEqual(c["section"], "")          # never satisfies a lookup
        self.assertIn("2026, c. 8, s. 3", c["history"])
        self.assertIn("coercive control", c["text"].lower())
        self.assertIn("not in force", c["citation"].lower())

    def test_related_provisions_schedule_is_not_ingested(self):
        chunks = ingest.parse_legislation(_path(), "I-2.5")
        self.assertFalse(any("transitional related provision" in c["text"].lower()
                             for c in chunks))


class RepealStatusTests(unittest.TestCase):
    def test_wholly_repealed_section_is_flagged(self):
        self.assertEqual(_parse()["13"]["status"], "repealed")

    def test_live_section_with_a_repealed_subsection_is_not_flagged(self):
        # The <Repealed> sits inside a Subsection, not the section's own Text,
        # so the section itself is still in force.
        self.assertEqual(_parse()["40"]["status"], "in force")

    def test_ordinary_section_is_in_force(self):
        self.assertEqual(_parse()["25"]["status"], "in force")


class WriteGuardTests(unittest.TestCase):
    """A Justice Laws schema change would parse an Act down to a handful of
    sections; the write must refuse rather than gut the statute. The guard
    itself is covered by tests/test_common.py -- these pin the wiring."""

    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.dir = Path(self.tmp.name)
        self.addCleanup(self.tmp.cleanup)

    def _run(self, parsed, stored=None, **kwargs):
        target = self.dir / "C-46.json"
        if stored is not None:
            target.write_text(json.dumps(stored), encoding="utf-8")
        with mock.patch.object(ingest, "PROCESSED_DIR", self.dir), \
                mock.patch.object(ingest, "fetch_xml",
                                  return_value=Path("unused.xml")), \
                mock.patch.object(ingest, "parse_legislation",
                                  return_value=parsed), \
                contextlib.redirect_stdout(io.StringIO()):
            return ingest.ingest("C-46", **kwargs), target

    def test_a_collapsed_act_is_refused_and_the_file_survives(self):
        stored = [{"id": str(i)} for i in range(2184)]
        with self.assertRaises(RuntimeError):
            self._run([{"id": "only-one"}], stored=stored)
        kept = json.loads((self.dir / "C-46.json").read_text(encoding="utf-8"))
        self.assertEqual(len(kept), 2184)

    def test_allow_shrink_lets_a_understood_drop_through(self):
        self._run([{"id": "a"}], stored=[{"id": str(i)} for i in range(100)],
                  allow_shrink=True)
        kept = json.loads((self.dir / "C-46.json").read_text(encoding="utf-8"))
        self.assertEqual(len(kept), 1)

    def test_a_normal_run_writes(self):
        chunks = [{"id": str(i)} for i in range(2184)]
        _result, target = self._run(chunks, stored=chunks)
        self.assertEqual(len(json.loads(target.read_text(encoding="utf-8"))),
                         2184)

    def test_first_ever_ingest_writes(self):
        _result, target = self._run([{"id": "a"}])
        self.assertTrue(target.exists())

    def test_refusal_leaves_no_temp_file_behind(self):
        with self.assertRaises(RuntimeError):
            self._run([{"id": "a"}], stored=[{"id": str(i)} for i in range(100)])
        self.assertEqual(list(self.dir.glob("*.tmp")), [])


class DownloadRetryTests(unittest.TestCase):
    """Audit follow-up (2026-09-22), missing item 12: fetch_xml retries a
    transient failure once. Before, one 503 from Justice Laws failed the Act
    outright; a 404 still fails at once."""

    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.dest = Path(self.tmp.name) / "I-2.5.xml"
        for mod in (ingest, ingest_common):
            patcher = mock.patch.object(mod.time, "sleep")
            patcher.start()
            self.addCleanup(patcher.stop)

    def _serve(self, answers):
        calls = []

        class _Resp(io.BytesIO):
            def __enter__(self_inner):
                return self_inner

            def __exit__(self_inner, *exc):
                return False

        def _urlopen(req, timeout=None):
            calls.append(req.full_url)
            answer = answers[min(len(calls), len(answers)) - 1]
            if isinstance(answer, Exception):
                raise answer
            return _Resp(answer)
        return calls, mock.patch.object(ingest.urllib.request, "urlopen", _urlopen)

    def _http(self, code):
        exc = ingest.urllib.error.HTTPError("http://x", code, "status", {},
                                            io.BytesIO(b""))
        self.addCleanup(exc.close)
        return exc

    def test_a_503_then_the_consolidation_is_cached(self):
        calls, serve = self._serve([self._http(503), _XML])
        with serve, contextlib.redirect_stdout(io.StringIO()) as out:
            ingest._download_xml("http://x/I-2.5.xml", self.dest)
        self.assertEqual(len(calls), 2)
        self.assertEqual(self.dest.read_bytes(), _XML)
        self.assertIn("retrying", out.getvalue())

    def test_a_404_is_not_retried(self):
        calls, serve = self._serve([self._http(404), _XML])
        with serve, self.assertRaises(ingest.urllib.error.HTTPError):
            ingest._download_xml("http://x/I-2.5.xml", self.dest)
        self.assertEqual(len(calls), 1)
        self.assertFalse(self.dest.exists())

    def test_a_non_xml_body_is_not_retried(self):
        calls, serve = self._serve([b"<html>Maintenance</html>", _XML])
        with serve, self.assertRaises(ValueError):
            ingest._download_xml("http://x/I-2.5.xml", self.dest)
        self.assertEqual(len(calls), 1)
        self.assertEqual(list(self.dest.parent.iterdir()), [])


# A French consolidation in miniature: a range label, a provision long enough
# to split, a schedule and a not-in-force entry -- every structural label the
# parser writes itself.
_LONG = "\n".join(
    f"<Subsection><Label>({k})</Label><Text>{'Le ministre peut prendre des mesures. ' * 14}"
    f"</Text></Subsection>" for k in range(1, 7))
_FR_XML = f"""<?xml version="1.0" encoding="UTF-8"?>
<Statute xmlns:lims="http://justice.gc.ca/lims" lims:current-date="2026-03-31">
  <Body>
    <Section lims:inforce-start-date="2002-06-28"><Label>55 et 56</Label>
      <Text>Texte des articles 55 et 56.</Text></Section>
    <Section lims:inforce-start-date="2002-06-28"><Label>36</Label>
      <MarginalNote>Grande criminalité</MarginalNote>{_LONG}</Section>
  </Body>
  <Schedule><ScheduleFormHeading><Label>ANNEXE 1</Label>
      <TitleText>Liste des infractions</TitleText></ScheduleFormHeading>
    <List><Item><Text>Première infraction de la liste.</Text></Item></List>
  </Schedule>
  <Schedule id="NifProvs">
    <BillPiece><RelatedOrNotInForce>
      <Heading level="5" style="nifrp"><TitleText>- 2026, ch. 8, art. 3</TitleText></Heading>
      <Section><Label>3</Label><Text>L’article 264 est modifié.</Text></Section>
    </RelatedOrNotInForce></BillPiece>
  </Schedule>
</Statute>""".encode("utf-8")


class FrenchLabelTests(unittest.TestCase):
    """Audit follow-up (2026-09-22), missing item 12 (ingest-16): the French
    twins carried the parser's own English labels -- 'SCHEDULES' (845 French
    chunks), 'Amendments not in force' (452), '(part k of n)' -- and had no
    way to pair a '55 et 56' with its English '55 and 56'. The English parse
    is untouched."""

    @classmethod
    def setUpClass(cls):
        tmp = tempfile.TemporaryDirectory()
        cls.addClassCleanup(tmp.cleanup)
        path = Path(tmp.name) / "fr.xml"
        path.write_bytes(_FR_XML)
        cls.en = ingest.parse_legislation(path, "I-2.5")
        cls.fr = {c["id"]: c for c in ingest._frenchify(cls.en, "I-2.5")}

    def test_before_the_labels_were_english(self):
        by_id = {c["id"]: c for c in self.en}
        self.assertEqual(by_id["I-2.5-sched1"]["part"], "SCHEDULES")
        self.assertEqual(by_id["I-2.5-nif-1"]["heading"], "Amendments not in force")
        self.assertTrue(by_id["I-2.5-s36-p1"]["citation"].endswith("(part 1 of 2)"))

    def test_the_schedules_part(self):
        self.assertEqual(self.fr["I-2.5-fr-sched1"]["part"], "ANNEXES")

    def test_the_not_in_force_entry(self):
        nif = self.fr["I-2.5-fr-nif-1"]
        self.assertEqual(nif["part"], "MODIFICATIONS NON EN VIGUEUR")
        self.assertEqual(nif["heading"], "Modifications non en vigueur")
        self.assertEqual(nif["marginal_note"],
                         "Modification non en vigueur — 2026, ch. 8, art. 3")
        self.assertEqual(nif["citation"],
                         "IRPA — modification non en vigueur (2026, ch. 8, art. 3)")
        self.assertEqual(nif["status"], "not-in-force")          # untouched

    def test_the_part_note_of_a_split_article(self):
        self.assertEqual(self.fr["I-2.5-fr-s36-p1"]["citation"],
                         "IRPA, art. 36 (partie 1 de 2)")
        self.assertEqual(self.fr["I-2.5-fr-s36-p2"]["citation"],
                         "IRPA, art. 36 (partie 2 de 2)")

    def test_section_en_pairs_each_french_label(self):
        self.assertEqual(self.fr["I-2.5-fr-s55 et 56"]["section_en"], "55 and 56")
        self.assertEqual(self.fr["I-2.5-fr-s36-p1"]["section_en"], "36")
        self.assertEqual(self.fr["I-2.5-fr-sched1"]["section_en"], "Schedule 1")
        self.assertEqual(self.fr["I-2.5-fr-nif-1"]["section_en"], "")

    def test_section_en_table(self):
        for fr, en in (("211 à 214", "211 to 214"), ("Formule 6", "Form 6"),
                       ("Annexes Iv Et V", "Schedules Iv And V"),
                       ("Préambule", "Preamble"),
                       ("Annexe [De La Partie Xxv]", "Schedule [To Part Xxv]"),
                       ("36", "36")):
            with self.subTest(fr=fr):
                self.assertEqual(ingest.section_en(fr), en)

    def test_the_fallback_labels(self):
        chunk = {"id": "X-pre-1", "section": "Preamble", "marginal_note": "Preamble",
                 "citation": "X, Preamble", "source_url": "u"}
        fr = ingest._frenchify([chunk], "X")[0]
        self.assertEqual((fr["section"], fr["marginal_note"], fr["citation"],
                          fr["section_en"]),
                         ("Préambule", "Préambule", "X, Préambule", "Preamble"))
        sched = {"id": "X-sched3", "section": "Schedule (3)", "heading": "Schedule (3)",
                 "marginal_note": "Schedule (3)", "citation": "X, Schedule (3)",
                 "source_url": "u"}
        fr = ingest._frenchify([sched], "X")[0]
        self.assertEqual((fr["section"], fr["heading"], fr["citation"], fr["section_en"]),
                         ("Annexe (3)", "Annexe (3)", "X, annexe (3)", "Schedule (3)"))

    def test_the_english_chunks_are_not_touched(self):
        before = json.dumps(self.en, sort_keys=True)
        ingest._frenchify(self.en, "I-2.5")
        self.assertEqual(json.dumps(self.en, sort_keys=True), before)
        self.assertFalse(any("section_en" in c for c in self.en))


# The shapes of the aviation-security ingest of 2026-10-12, taken from the CASR
# 2012 XML: a table inside a section (s. 78(2): who may carry a weapon at an
# aerodrome, one row's conditions a nested list), a schedule titled only on its
# table ('Class 1 Aerodromes'), a heading whose sections are cross-references,
# a row whose last cell is empty, and a French 'SECTION' division heading.
_TABLE_XML = """<?xml version="1.0" encoding="UTF-8"?>
<Regulation xmlns:lims="http://justice.gc.ca/lims" lims:current-date="2025-04-14">
  <Body>
    <Heading level="2"><Label>SECTION 1</Label><TitleText>Dispositions générales</TitleText></Heading>
    <Section><Label>78</Label><MarginalNote>Weapons</MarginalNote>
      <Subsection><Label>(1)</Label><Text>No person shall carry a weapon.</Text></Subsection>
      <Subsection><Label>(2)</Label><MarginalNote>Exceptions</MarginalNote>
        <Text>Subsection (1) does not apply to a person set out in column 1.</Text>
        <TableGroup><Caption><Emphasis style="smallcaps">table</Emphasis></Caption>
          <table><tgroup cols="4"><thead>
            <row><entry>Item</entry><entry>Person</entry><entry>Permitted Goods</entry><entry>Conditions</entry></row>
          </thead><tbody>
            <row><entry>2</entry><entry>a peace officer</entry><entry>a weapon and ammunition</entry><entry>the peace officer is acting in the course of their duties</entry></row>
            <row><entry>12</entry><entry>a border services officer</entry><entry>an agency firearm</entry><entry><Provision><List><Item><Label>(a)</Label><Text>the officer is in uniform</Text></Item><Item><Label>(b)</Label><Text>the officer is on duty</Text></Item></List></Provision></entry></row>
          </tbody></tgroup></table>
        </TableGroup>
      </Subsection>
    </Section>
  </Body>
  <Schedule>
    <ScheduleFormHeading><Label>SCHEDULE 1</Label><OriginatingRef>(Paragraph 2(d))</OriginatingRef></ScheduleFormHeading>
    <TableGroup><table><title>Class 1 Aerodromes</title><tgroup cols="2"><tbody>
      <row><entry>Vancouver</entry><entry>CYVR</entry></row>
    </tbody></tgroup></table></TableGroup>
  </Schedule>
  <Schedule>
    <ScheduleFormHeading><Label>SCHEDULE 4</Label><OriginatingRef>(Sections <XRefInternal>797</XRefInternal> and <XRefInternal>798</XRefInternal>)</OriginatingRef></ScheduleFormHeading>
    <TableGroup><table><title>Designated Provisions</title><tgroup cols="3"><tbody>
      <row><entry>Section 14</entry><entry>5,000</entry><entry></entry></row>
      <row><entry>Subsection 5(1)</entry><entry></entry><entry>25,000</entry></row>
    </tbody></tgroup></table></TableGroup>
  </Schedule>
</Regulation>""".encode("utf-8")


def _parse_tables():
    # SOR-2011-318 is the real SOURCES code; only its short/name/web_base are read.
    chunks = ingest.parse_legislation(_write(_TABLE_XML, "tables.xml"), "SOR-2011-318")
    return {c["section"]: c for c in chunks}


class TableInSectionTests(unittest.TestCase):
    """A <TableGroup> inside a section dropped every cell's own text (2026-10-12):
    CASR s. 78(2) kept only the lists nested in its cells, which then read as
    paragraphs 78(2)(a)-(d). IRPR s. 7.01, the Cannabis Regulations' s. 21 table,
    SFCR s. 375 and 46 Food and Drug Regulations sections lost text the same way."""

    def test_a_table_in_a_section_keeps_its_cells_one_row_per_line(self):
        # Arrange / Act
        text = _parse_tables()["78"]["text"]

        # Assert
        self.assertIn("Item | Person | Permitted Goods | Conditions", text)
        self.assertIn("2 | a peace officer | a weapon and ammunition | the peace officer is "
                      "acting in the course of their duties", text)

    def test_a_list_nested_in_a_cell_stays_on_its_row(self):
        # Arrange / Act
        lines = _parse_tables()["78"]["text"].splitlines()

        # Assert
        row = next(l for l in lines if l.startswith("12 | "))
        self.assertIn("| (a) the officer is in uniform (b) the officer is on duty", row)
        self.assertFalse(any(l.startswith("(a)") for l in lines), lines)


class TableCellTests(unittest.TestCase):
    def test_a_cell_keeps_every_word_of_its_images_and_footnotes(self):
        # Arrange -- SOR/96-263 Schedule I: the vicuña mark's wording is in an
        # ImageGroup's caption and alternate text, beside a footnote, in one cell
        import xml.etree.ElementTree as ET
        entry = ET.fromstring(
            "<entry><Provision><Label>(i)</Label><Text>the mark must take this form:</Text>"
            "<ImageGroup><Caption>VICUÑA [COUNTRY OF ORIGIN]</Caption>"
            "<AlternateText>letters stating “VICUÑA [PAÍS DE ORIGEN]”</AlternateText>"
            "<Image source='x.png'/></ImageGroup></Provision>"
            "<Footnote><Label>A3</Label><Text>Exclusive purpose.</Text></Footnote></entry>")

        # Act
        text = ingest._cell_text(entry)

        # Assert
        self.assertEqual(text, "(i) the mark must take this form: VICUÑA [COUNTRY OF ORIGIN] "
                               "letters stating “VICUÑA [PAÍS DE ORIGEN]” A3 Exclusive purpose.")

    def test_a_plain_cell_reads_as_its_text_did(self):
        # Arrange
        import xml.etree.ElementTree as ET
        entry = ET.fromstring("<entry>an <Emphasis>agency</Emphasis> firearm<FootnoteRef>*</FootnoteRef></entry>")

        # Act / Assert -- inline elements never split a word
        self.assertEqual(ingest._cell_text(entry), ingest._itertext(entry))

    def test_an_empty_last_cell_keeps_its_column(self):
        # Arrange / Act
        text = _parse_tables()["Schedule 4"]["text"]

        # Assert -- 'Section 14 | 5,000' read by position as the corporation amount
        self.assertIn("Section 14 | 5,000 |", text)
        self.assertIn("Subsection 5(1) |  | 25,000", text)


class ScheduleTitleTests(unittest.TestCase):
    def test_a_schedule_titled_on_its_table_takes_that_title(self):
        # Arrange / Act
        sched = _parse_tables()["Schedule 1"]

        # Assert
        self.assertEqual(sched["marginal_note"], "Class 1 Aerodromes")
        self.assertIn("Vancouver | CYVR", sched["text"])

    def test_an_originating_reference_keeps_its_cross_references(self):
        # Arrange / Act
        text = _parse_tables()["Schedule 4"]["text"]

        # Assert -- findtext stopped at the first <XRefInternal>: '(Sections'
        self.assertIn("(Sections 797 and 798)", text.splitlines()[0])


class FrenchStructureTests(unittest.TestCase):
    def test_a_french_section_heading_is_the_division(self):
        # Arrange / Act
        chunk = _parse_tables()["78"]

        # Assert -- French divisions are headed 'SECTION n', English ones 'DIVISION n'
        self.assertEqual(chunk["division"], "SECTION 1 - Dispositions générales")

    def test_an_english_heading_naming_a_section_is_not_a_division(self):
        # Arrange -- SOR/97-234 heads its provisions 'Section 5 of the Act — Trafficking'
        for heading in ("Section 5 of the Act — Trafficking", "SECTION 5 OF THE ACT",
                        "Sections 5 and 7"):
            # Act / Assert
            with self.subTest(heading=heading):
                self.assertIsNone(ingest._FR_DIVISION.match(heading))
        for heading in ("SECTION 1 - Dispositions générales", "SECTION 7.1 - Contrôle",
                        "SECTION IV"):
            with self.subTest(heading=heading):
                self.assertIsNotNone(ingest._FR_DIVISION.match(heading))

    def test_a_french_regulation_link_uses_the_french_citation_form(self):
        # Arrange -- Justice Laws serves /fra/reglements/DORS-... and C.R.C., ch.;
        # the SOR- and c. forms return 404 (checked 2026-10-12)
        base = "https://laws-lois.justice.gc.ca/eng/"
        chunks = [
            {"id": "SOR-2011-318-s78", "citation": "CASR 2012, s. 78",
             "source_url": base + "regulations/SOR-2011-318/section-78.html"},
            {"id": "C.R.C.,_c._870-sA", "citation": "FDR, s. A.01.012",
             "source_url": base + "regulations/C.R.C.%2C_c._870/section-A.01.012.html"},
            {"id": "A-2-s4.75", "citation": "Aeronautics Act, s. 4.75",
             "source_url": base + "acts/a-2/section-4.75.html"},
        ]

        # Act
        urls = [c["source_url"] for c in ingest._frenchify(chunks, "X")]

        # Assert
        fra = "https://laws-lois.justice.gc.ca/fra/"
        self.assertEqual(urls, [fra + "reglements/DORS-2011-318/section-78.html",
                                fra + "reglements/C.R.C.%2C_ch._870/section-A.01.012.html",
                                fra + "lois/a-2/section-4.75.html"])


if __name__ == "__main__":
    unittest.main()