Spaces:
Sleeping
Tighten verifier: institutional-context tokens don't count as topical match
Browse filesCloses the "blind spot A" surfaced during the gemma4 Kasus Ibam diagnostic:
the model cited Permen 24/2018 claiming an audit-mechanism topic, when the
actual title is "Pengelolaan Arsip Terjaga di Lingkungan Kementerian
Pendidikan dan Kebudayaan". The previous overlap check passed because claim
and title shared "kementerian / pendidikan / kebudayaan" β institutional
context, not topic.
_topics_overlap now requires at least one shared token to be NON-
institutional. Adds _INSTITUTIONAL_TOKENS frozenset covering generic
government context (kementerian / pemerintah / lingkungan / instansi /
lembaga / wilayah / kantor) and ministry/sector markers (pendidikan /
kebudayaan / kesehatan / keuangan / ketenagakerjaan / pertanian / etc.).
The acronym-expansion path is unaffected β institutional acronyms (MPR /
DPR / KPK) still match their spelled-out forms because their expansion
tokens (majelis / dewan / komisi / pemberantasan) are deliberately kept
out of the institutional list.
Tests: 4 new cases in test_citations.py β institutional-only rejection,
real topical match, acronym-vs-spelled-out, malformed-title fallback.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
- src/legawa/tools/citations.py +35 -1
- tests/test_citations.py +27 -0
|
@@ -101,6 +101,34 @@ _STOP_TOKENS = frozenset(
|
|
| 101 |
}
|
| 102 |
)
|
| 103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
# Common Indonesian institutional acronyms β agents tend to use the acronym
|
| 105 |
# while the regulation's actual title spells the institution out. We expand
|
| 106 |
# claim-side acronyms before topical-overlap so e.g. "MPR" matches a title
|
|
@@ -169,7 +197,13 @@ def _topics_overlap(claimed: str, title: str) -> bool:
|
|
| 169 |
|
| 170 |
claim_tokens = _significant_tokens(claimed)
|
| 171 |
title_tokens = _significant_tokens(title)
|
| 172 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
return True
|
| 174 |
|
| 175 |
# Expand any claim-side acronyms and check for partial title overlap.
|
|
|
|
| 101 |
}
|
| 102 |
)
|
| 103 |
|
| 104 |
+
# Tokens that name an institution, sector, or generic government context.
|
| 105 |
+
# These appear in the *title* of nearly every regulation issued by that
|
| 106 |
+
# institution (e.g. "Kementerian Pendidikan" appears in every Kemendikbud
|
| 107 |
+
# Permen) and in any *claim* that names the institution β so an overlap on
|
| 108 |
+
# these tokens alone does not confirm topical alignment.
|
| 109 |
+
#
|
| 110 |
+
# Used by ``_topics_overlap`` to demand at least one *non-institutional*
|
| 111 |
+
# token in common before declaring a match. Acronyms (MPR/DPR/KPK/...) are
|
| 112 |
+
# kept out of this set because their expansions ARE the topic in regulations
|
| 113 |
+
# whose subject is the institution itself (e.g. UU 17/2014 about MPR/DPR).
|
| 114 |
+
_INSTITUTIONAL_TOKENS = frozenset(
|
| 115 |
+
{
|
| 116 |
+
# Generic government context.
|
| 117 |
+
"kementerian", "pemerintah", "pemerintahan", "negeri", "lingkungan",
|
| 118 |
+
"instansi", "lembaga", "publik", "pusat", "lingkup", "wilayah",
|
| 119 |
+
"kantor",
|
| 120 |
+
# Sector markers β when alone, signal sector not topic. Specific
|
| 121 |
+
# subject matter (audit / pengelolaan / arsip / outsourcing / etc.)
|
| 122 |
+
# is what should drive topical match, not the ministry name.
|
| 123 |
+
"pendidikan", "kebudayaan", "kesehatan", "keuangan",
|
| 124 |
+
"ketenagakerjaan", "pertanian", "perhubungan", "kehutanan",
|
| 125 |
+
"kelautan", "pertahanan", "agama", "sosial", "perindustrian",
|
| 126 |
+
"perdagangan", "ristek", "riset", "teknologi", "informasi",
|
| 127 |
+
"komunikasi", "perekonomian", "perencanaan", "pembangunan",
|
| 128 |
+
}
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
# Common Indonesian institutional acronyms β agents tend to use the acronym
|
| 133 |
# while the regulation's actual title spells the institution out. We expand
|
| 134 |
# claim-side acronyms before topical-overlap so e.g. "MPR" matches a title
|
|
|
|
| 197 |
|
| 198 |
claim_tokens = _significant_tokens(claimed)
|
| 199 |
title_tokens = _significant_tokens(title)
|
| 200 |
+
|
| 201 |
+
# Direct token overlap counts only if at least one shared token is
|
| 202 |
+
# non-institutional. This guards against false positives where claim
|
| 203 |
+
# and title share only ministry/sector context (e.g. "Kementerian
|
| 204 |
+
# Pendidikan") while their actual subjects are different.
|
| 205 |
+
overlap = claim_tokens & title_tokens
|
| 206 |
+
if overlap and (overlap - _INSTITUTIONAL_TOKENS):
|
| 207 |
return True
|
| 208 |
|
| 209 |
# Expand any claim-side acronyms and check for partial title overlap.
|
|
@@ -10,6 +10,7 @@ if str(SRC) not in sys.path:
|
|
| 10 |
|
| 11 |
from legawa.tools.citations import (
|
| 12 |
CitationCheck,
|
|
|
|
| 13 |
extract_citations,
|
| 14 |
format_checks,
|
| 15 |
verify_citation,
|
|
@@ -96,6 +97,32 @@ class CitationTests(unittest.TestCase):
|
|
| 96 |
self.assertEqual(perda.query, "Perda 1/2024")
|
| 97 |
self.assertEqual(perda.frbr_uri, "akn/id/act/perda/2024/1")
|
| 98 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
def test_verify_citations_formats_mixed_results(self) -> None:
|
| 100 |
fake = FakePasalClient(
|
| 101 |
{
|
|
|
|
| 10 |
|
| 11 |
from legawa.tools.citations import (
|
| 12 |
CitationCheck,
|
| 13 |
+
_topics_overlap,
|
| 14 |
extract_citations,
|
| 15 |
format_checks,
|
| 16 |
verify_citation,
|
|
|
|
| 97 |
self.assertEqual(perda.query, "Perda 1/2024")
|
| 98 |
self.assertEqual(perda.frbr_uri, "akn/id/act/perda/2024/1")
|
| 99 |
|
| 100 |
+
def test_topics_overlap_rejects_institutional_only_match(self) -> None:
|
| 101 |
+
# Real false-positive case from gemma4 memo run: model claimed
|
| 102 |
+
# Permen 24/2018 was about audit-mechanism, but it's about Pengelolaan
|
| 103 |
+
# Arsip. They share only ministry-context tokens.
|
| 104 |
+
claim = "Mekanisme Tindak Lanjut Hasil Audit Inspektorat Jenderal Kementerian Pendidikan dan Kebudayaan"
|
| 105 |
+
title = "Pengelolaan Arsip Terjaga di Lingkungan Kementerian Pendidikan dan Kebudayaan"
|
| 106 |
+
self.assertFalse(_topics_overlap(claim, title))
|
| 107 |
+
|
| 108 |
+
def test_topics_overlap_accepts_real_topical_match(self) -> None:
|
| 109 |
+
claim = "Pengadaan Barang/Jasa Pemerintah"
|
| 110 |
+
title = "Peraturan Presiden Nomor 12 Tahun 2021 tentang Perubahan Atas Pengadaan Barang/Jasa Pemerintah"
|
| 111 |
+
self.assertTrue(_topics_overlap(claim, title))
|
| 112 |
+
|
| 113 |
+
def test_topics_overlap_acronym_vs_spelled_out_still_works(self) -> None:
|
| 114 |
+
# Ensure the institutional filter doesn't break the acronym path β
|
| 115 |
+
# UU 17/2014 is genuinely about MPR/DPR/DPD/DPRD.
|
| 116 |
+
claim = "MPR, DPR, DPD, dan DPRD"
|
| 117 |
+
title = "Majelis Permusyawaratan Rakyat, Dewan Perwakilan Rakyat, Dewan Perwakilan Daerah, dan Dewan Perwakilan Rakyat Daerah"
|
| 118 |
+
self.assertTrue(_topics_overlap(claim, title))
|
| 119 |
+
|
| 120 |
+
def test_topics_overlap_uninformative_title_falls_back(self) -> None:
|
| 121 |
+
# Echoed pasal.id title β no usable signal β accept by existence-only.
|
| 122 |
+
claim = "Sistem Pendidikan Nasional"
|
| 123 |
+
title = "Undang-Undang Nomor 20 Tahun 2003 tentang Undang-Undang Nomor 20 Tahun 2003"
|
| 124 |
+
self.assertTrue(_topics_overlap(claim, title))
|
| 125 |
+
|
| 126 |
def test_verify_citations_formats_mixed_results(self) -> None:
|
| 127 |
fake = FakePasalClient(
|
| 128 |
{
|