pebaryan Claude Opus 4.7 commited on
Commit
ed43ddd
Β·
1 Parent(s): 6dd4102

Tighten verifier: institutional-context tokens don't count as topical match

Browse files

Closes the "blind spot A" surfaced during the gemma4 Kasus Ibam diagnostic:
the model cited Permen 24/2018 claiming an audit-mechanism topic, when the
actual title is "Pengelolaan Arsip Terjaga di Lingkungan Kementerian
Pendidikan dan Kebudayaan". The previous overlap check passed because claim
and title shared "kementerian / pendidikan / kebudayaan" β€” institutional
context, not topic.

_topics_overlap now requires at least one shared token to be NON-
institutional. Adds _INSTITUTIONAL_TOKENS frozenset covering generic
government context (kementerian / pemerintah / lingkungan / instansi /
lembaga / wilayah / kantor) and ministry/sector markers (pendidikan /
kebudayaan / kesehatan / keuangan / ketenagakerjaan / pertanian / etc.).

The acronym-expansion path is unaffected β€” institutional acronyms (MPR /
DPR / KPK) still match their spelled-out forms because their expansion
tokens (majelis / dewan / komisi / pemberantasan) are deliberately kept
out of the institutional list.

Tests: 4 new cases in test_citations.py β€” institutional-only rejection,
real topical match, acronym-vs-spelled-out, malformed-title fallback.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/legawa/tools/citations.py CHANGED
@@ -101,6 +101,34 @@ _STOP_TOKENS = frozenset(
101
  }
102
  )
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  # Common Indonesian institutional acronyms β€” agents tend to use the acronym
105
  # while the regulation's actual title spells the institution out. We expand
106
  # claim-side acronyms before topical-overlap so e.g. "MPR" matches a title
@@ -169,7 +197,13 @@ def _topics_overlap(claimed: str, title: str) -> bool:
169
 
170
  claim_tokens = _significant_tokens(claimed)
171
  title_tokens = _significant_tokens(title)
172
- if claim_tokens & title_tokens:
 
 
 
 
 
 
173
  return True
174
 
175
  # Expand any claim-side acronyms and check for partial title overlap.
 
101
  }
102
  )
103
 
104
+ # Tokens that name an institution, sector, or generic government context.
105
+ # These appear in the *title* of nearly every regulation issued by that
106
+ # institution (e.g. "Kementerian Pendidikan" appears in every Kemendikbud
107
+ # Permen) and in any *claim* that names the institution β€” so an overlap on
108
+ # these tokens alone does not confirm topical alignment.
109
+ #
110
+ # Used by ``_topics_overlap`` to demand at least one *non-institutional*
111
+ # token in common before declaring a match. Acronyms (MPR/DPR/KPK/...) are
112
+ # kept out of this set because their expansions ARE the topic in regulations
113
+ # whose subject is the institution itself (e.g. UU 17/2014 about MPR/DPR).
114
+ _INSTITUTIONAL_TOKENS = frozenset(
115
+ {
116
+ # Generic government context.
117
+ "kementerian", "pemerintah", "pemerintahan", "negeri", "lingkungan",
118
+ "instansi", "lembaga", "publik", "pusat", "lingkup", "wilayah",
119
+ "kantor",
120
+ # Sector markers β€” when alone, signal sector not topic. Specific
121
+ # subject matter (audit / pengelolaan / arsip / outsourcing / etc.)
122
+ # is what should drive topical match, not the ministry name.
123
+ "pendidikan", "kebudayaan", "kesehatan", "keuangan",
124
+ "ketenagakerjaan", "pertanian", "perhubungan", "kehutanan",
125
+ "kelautan", "pertahanan", "agama", "sosial", "perindustrian",
126
+ "perdagangan", "ristek", "riset", "teknologi", "informasi",
127
+ "komunikasi", "perekonomian", "perencanaan", "pembangunan",
128
+ }
129
+ )
130
+
131
+
132
  # Common Indonesian institutional acronyms β€” agents tend to use the acronym
133
  # while the regulation's actual title spells the institution out. We expand
134
  # claim-side acronyms before topical-overlap so e.g. "MPR" matches a title
 
197
 
198
  claim_tokens = _significant_tokens(claimed)
199
  title_tokens = _significant_tokens(title)
200
+
201
+ # Direct token overlap counts only if at least one shared token is
202
+ # non-institutional. This guards against false positives where claim
203
+ # and title share only ministry/sector context (e.g. "Kementerian
204
+ # Pendidikan") while their actual subjects are different.
205
+ overlap = claim_tokens & title_tokens
206
+ if overlap and (overlap - _INSTITUTIONAL_TOKENS):
207
  return True
208
 
209
  # Expand any claim-side acronyms and check for partial title overlap.
tests/test_citations.py CHANGED
@@ -10,6 +10,7 @@ if str(SRC) not in sys.path:
10
 
11
  from legawa.tools.citations import (
12
  CitationCheck,
 
13
  extract_citations,
14
  format_checks,
15
  verify_citation,
@@ -96,6 +97,32 @@ class CitationTests(unittest.TestCase):
96
  self.assertEqual(perda.query, "Perda 1/2024")
97
  self.assertEqual(perda.frbr_uri, "akn/id/act/perda/2024/1")
98
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
  def test_verify_citations_formats_mixed_results(self) -> None:
100
  fake = FakePasalClient(
101
  {
 
10
 
11
  from legawa.tools.citations import (
12
  CitationCheck,
13
+ _topics_overlap,
14
  extract_citations,
15
  format_checks,
16
  verify_citation,
 
97
  self.assertEqual(perda.query, "Perda 1/2024")
98
  self.assertEqual(perda.frbr_uri, "akn/id/act/perda/2024/1")
99
 
100
+ def test_topics_overlap_rejects_institutional_only_match(self) -> None:
101
+ # Real false-positive case from gemma4 memo run: model claimed
102
+ # Permen 24/2018 was about audit-mechanism, but it's about Pengelolaan
103
+ # Arsip. They share only ministry-context tokens.
104
+ claim = "Mekanisme Tindak Lanjut Hasil Audit Inspektorat Jenderal Kementerian Pendidikan dan Kebudayaan"
105
+ title = "Pengelolaan Arsip Terjaga di Lingkungan Kementerian Pendidikan dan Kebudayaan"
106
+ self.assertFalse(_topics_overlap(claim, title))
107
+
108
+ def test_topics_overlap_accepts_real_topical_match(self) -> None:
109
+ claim = "Pengadaan Barang/Jasa Pemerintah"
110
+ title = "Peraturan Presiden Nomor 12 Tahun 2021 tentang Perubahan Atas Pengadaan Barang/Jasa Pemerintah"
111
+ self.assertTrue(_topics_overlap(claim, title))
112
+
113
+ def test_topics_overlap_acronym_vs_spelled_out_still_works(self) -> None:
114
+ # Ensure the institutional filter doesn't break the acronym path β€”
115
+ # UU 17/2014 is genuinely about MPR/DPR/DPD/DPRD.
116
+ claim = "MPR, DPR, DPD, dan DPRD"
117
+ title = "Majelis Permusyawaratan Rakyat, Dewan Perwakilan Rakyat, Dewan Perwakilan Daerah, dan Dewan Perwakilan Rakyat Daerah"
118
+ self.assertTrue(_topics_overlap(claim, title))
119
+
120
+ def test_topics_overlap_uninformative_title_falls_back(self) -> None:
121
+ # Echoed pasal.id title β€” no usable signal β€” accept by existence-only.
122
+ claim = "Sistem Pendidikan Nasional"
123
+ title = "Undang-Undang Nomor 20 Tahun 2003 tentang Undang-Undang Nomor 20 Tahun 2003"
124
+ self.assertTrue(_topics_overlap(claim, title))
125
+
126
  def test_verify_citations_formats_mixed_results(self) -> None:
127
  fake = FakePasalClient(
128
  {