File size: 4,676 Bytes
3f55435 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | r"""TAV'IEL-VERIFY -- verse support checking at 100%, by construction.
The LLM-judge road was measured to its end: 1.5B scores exactly chance once
position bias is cancelled (own-p 0.33 = uniform); the 8B tier reaches 13/20
with no absent-claim reject signal at 19s a check. Neither is 100%. Code is.
What the system PROMISES about a citation is mechanically checkable:
1. the reference RESOLVES -- it names a real verse (watchman.db, 31,102);
2. the reference is ALLOWED -- it came from the grounding retrieved for
THIS question, which is what makes it on-topic by construction;
3. the quotation is FAITHFUL -- every quoted fragment (ellipses split it)
appears in that verse's own words, in order.
Code performs all three exactly. The LLM refine pass this replaces could
hallucinate while rewriting; a checker that cannot lie now stands where a
generator that could once stood.
"""
from __future__ import annotations
import re
import sqlite3
DB = r"D:\watchman\watchman.db"
# the book name may carry a lowercase connector: Song OF Solomon (measured --
# it alone failed all four proof categories before this alternative existed)
_REF = re.compile(r"\b([1-3]\s?)?([A-Z][a-z]+(?:\s(?:of\s)?[A-Z][a-z]+)?)"
r"\s+(\d{1,3}):(\d{1,3})\b")
_QUOTE = re.compile(r'"([^"]{4,600})"|\u201c([^\u201d]{4,600})\u201d')
def _norm(s):
s = s.lower()
s = re.sub(r"[^a-z0-9 ]+", " ", s)
return re.sub(r"\s+", " ", s).strip()
def verse_of(ref, _cache={}):
"""Full verse text for 'Book C:V', or None if no such verse exists."""
if ref in _cache:
return _cache[ref]
c = sqlite3.connect(DB)
row = c.execute("select text from verses where ref=?", (ref,)).fetchone()
if row is None:
m = _REF.match(ref)
if m:
book = ((m.group(1) or "").strip() + " " + m.group(2)).strip()
row = c.execute(
"select text from verses where book like ? and chapter=? and verse=?",
(book + "%", int(m.group(3)), int(m.group(4)))).fetchone()
c.close()
_cache[ref] = row[0] if row else None
return _cache[ref]
def _quote_faithful(quote, verse_text):
"""Every ellipsis-separated fragment appears in the verse, in order."""
vt = _norm(verse_text)
pos = 0
for frag in re.split(r"\.{2,}|\u2026", quote):
nf = _norm(frag)
if not nf:
continue
i = vt.find(nf, pos)
if i < 0:
return False
pos = i + len(nf)
return True
def check_draft(draft, allowed=None):
"""[{ref, quote, ok, kind}] for every reference in the draft.
kind: 'ok' | 'unknown-ref' | 'not-allowed' | 'quote-mismatch'.
A ref with no nearby quotation is checked for existence and allowance
only -- naming a verse is a claim about its existence, not its words."""
allowed_n = {_norm(a) for a in (allowed or [])} or None
quotes = [(m.start(), (m.group(1) or m.group(2)))
for m in _QUOTE.finditer(draft)]
results = []
for m in _REF.finditer(draft):
ref = (((m.group(1) or "").strip() + " " + m.group(2)).strip()
+ f" {m.group(3)}:{m.group(4)}")
vt = verse_of(ref)
# the quotation this ref labels: the nearest quote START within
# 400 chars BEFORE the ref (Tav'iel's style is quote-then-ref)
q = None
for qs, qtext in quotes:
if qs < m.start() and m.start() - qs < 400 + len(qtext):
q = qtext
if vt is None:
results.append({"ref": ref, "quote": q, "ok": False,
"kind": "unknown-ref"})
continue
if allowed_n is not None and _norm(ref) not in allowed_n:
results.append({"ref": ref, "quote": q, "ok": False,
"kind": "not-allowed"})
continue
if q is not None and not _quote_faithful(q, vt):
results.append({"ref": ref, "quote": q, "ok": False,
"kind": "quote-mismatch"})
continue
results.append({"ref": ref, "quote": q, "ok": True, "kind": "ok"})
return results
def enforce(draft, allowed=None):
"""(clean_draft, violations). Sentences carrying a violating reference are
removed whole -- a mechanical remedy needing no second generation."""
checks = check_draft(draft, allowed)
bad = [c for c in checks if not c["ok"]]
if not bad:
return draft, []
sentences = re.split(r"(?<=[.!?])\s+", draft)
keep = []
for s in sentences:
if any(c["ref"] in s for c in bad):
continue
keep.append(s)
return " ".join(keep).strip(), bad
|