File size: 16,796 Bytes
041950a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
"""CBX RAG evaluator v0.1: four Ettin-68M judges for retrieval-augmented answers.

    from cbx_rag_eval import Faithfulness, ResponseRelevance, ContextRelevance, ContextUtilization
    judge = ContextRelevance(".")                       # folder with model.safetensors + cbx_rag_config.json
    out = judge(question, passages)                     # passages: str or list of str (or list of sentence lists)

Every judge works on sentences. Raw text is split per profile (`split_sentences`, chosen on validation): "contract"
splits on line breaks and then with NLTK punkt; "financial" splits on blank lines and then with pysbd. Pass pre-split
sentences (a list of strings per passage, `answer_sentences=`) to skip the splitter.

Profiles (settings fixed on validation, see the model card):
    "contract"   long single documents (CUAD-like)
    "financial"  several short passages, numbers and tables (FinQA-like); faithfulness adds a numeric check
    "auto"       (default) "contract" for a single passage, "financial" for two or more
"""
import json
import math
import os
import re
from collections import Counter

import numpy as np
import torch
import torch.nn as nn
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModel, AutoTokenizer

try:
    from cbx_rag_numcheck import Checker
except ImportError:                                                      # development tree
    from numcheck import Checker

VERSION = "v0.1"

# ---------------------------------------------------------------- sentences

_WS = re.compile(r"\s+")
_LINE = re.compile(r"\n+")


def _punkt():
    import nltk
    try:
        nltk.data.find("tokenizers/punkt_tab")
    except LookupError:
        nltk.download("punkt_tab", quiet=True)
    from nltk.tokenize import sent_tokenize
    return sent_tokenize


_PARA = re.compile(r"\n\s*\n")
_SEG = None


def _pysbd():
    global _SEG
    if _SEG is None:
        import warnings
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            import pysbd
        _SEG = pysbd.Segmenter(language="en", clean=False)
    return _SEG


def split_sentences(text, profile="contract"):
    """Whitespace-collapsed sentences. "contract": line breaks, then NLTK punkt inside each line. "financial": blank
    lines, then pysbd inside each paragraph. A JSON table ('[[' ...) stays one sentence."""
    text = str(text)
    if text.lstrip().startswith("[["):
        return [text.strip()]
    if profile == "financial":
        parts = [x for p in _PARA.split(text) for x in _pysbd().segment(p)]
    else:
        tok = _punkt()
        parts = [x for line in _LINE.split(text) for x in tok(line)]
    out = [_WS.sub(" ", x).strip() for x in parts]
    return [x for x in out if x]


def _letters(n):
    s = ""
    n += 1
    while n:
        n, r = divmod(n - 1, 26)
        s = chr(97 + r) + s
    return s


def _passages(passages):
    return [passages] if isinstance(passages, str) else list(passages)


def _context(passages, profile):
    """passages: str | list of str | list of list of str -> list of {key, doc, text}."""
    ctx = []
    for di, p in enumerate(_passages(passages)):
        sents = list(p) if isinstance(p, (list, tuple)) else split_sentences(p, profile)
        ctx += [{"key": f"{di}{_letters(j)}", "doc": di, "text": str(s)} for j, s in enumerate(sents)]
    return ctx


def _answer(answer, answer_sentences, profile):
    sents = list(answer_sentences) if answer_sentences is not None else split_sentences(answer, profile)
    return [{"key": _letters(j), "text": str(s)} for j, s in enumerate(sents)]


def _profile(profile, passages):
    """"auto": one passage -> "contract", two or more -> "financial" (fixed on validation)."""
    if profile in ("contract", "financial"):
        return profile
    if profile not in (None, "auto"):
        raise ValueError("profile must be 'auto', 'contract' or 'financial'")
    return "contract" if len(_passages(passages)) <= 1 else "financial"


# ---------------------------------------------------------------- BM25 (as in training)

_TOKEN = re.compile(r"[a-z0-9]+")
_STOP = set("the a an of to and or in on for by with is are be this that any as at from it its such shall may "
            "will which their there other than not no does do".split())


def _toks(text):
    return [t for t in _TOKEN.findall(text.lower()) if t not in _STOP]


class BM25:
    def __init__(self, docs, k1=1.2, b=0.75):
        self.docs = [Counter(_toks(d)) for d in docs]
        self.len = [sum(d.values()) for d in self.docs]
        self.avg = (sum(self.len) / len(self.len)) if self.len else 1
        df = Counter()
        for d in self.docs:
            df.update(d.keys())
        n = len(docs)
        self.idf = {t: math.log(1 + (n - c + 0.5) / (c + 0.5)) for t, c in df.items()}
        self.k1, self.b = k1, b

    def scores(self, query):
        q = Counter(_toks(query))
        out = []
        for d, ln in zip(self.docs, self.len):
            s = 0.0
            for t in q:
                if t in d:
                    tf = d[t]
                    s += self.idf[t] * tf * (self.k1 + 1) / (tf + self.k1 * (1 - self.b + self.b * ln / self.avg))
            out.append(s)
        return out


# ---------------------------------------------------------------- model

class _Classifier(nn.Module):
    def __init__(self, config, n_classes):
        super().__init__()
        self.backbone = AutoModel.from_config(config, attn_implementation="sdpa")
        self.head = nn.Linear(config.hidden_size, n_classes)

    def forward(self, input_ids, attention_mask):
        h = self.backbone(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
        m = attention_mask.unsqueeze(-1).to(h.dtype)
        return self.head((h * m).sum(1) / m.sum(1).clamp(min=1))


class _Judge:
    TASK = None

    def __init__(self, path=".", device=None, batch_tokens=None):
        with open(os.path.join(path, "cbx_rag_config.json")) as f:
            self.cfg = json.load(f)
        if self.cfg["task"] != self.TASK:
            raise ValueError(f"{path} holds the {self.cfg['task']} model, not {self.TASK}")
        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
        self.tok = AutoTokenizer.from_pretrained(path)
        self.model = _Classifier(AutoConfig.from_pretrained(path), self.cfg["n_classes"])
        self.model.load_state_dict(load_file(os.path.join(path, "model.safetensors")))
        self.model.to(self.device).eval()
        self.batch_tokens = batch_tokens or self.cfg["batch_tokens"]
        self._len = {}

    def _ids(self, text):
        return self.tok(text, add_special_tokens=False)["input_ids"]

    def _encode(self, rows):
        cls, sep = self.tok.cls_token_id, self.tok.sep_token_id
        out = []
        for q, ev, s in rows:
            qi = self._ids(q)[:self.cfg["q_max"]]
            si = self._ids(s)[:384]
            ei = self._ids(ev)
            room = self.cfg["max_len"] - len(qi) - len(si) - 4
            out.append([cls] + qi + [sep] + ei[:max(0, room)] + [sep] + si + [sep])
        return out

    @torch.no_grad()
    def _probs(self, rows):
        """rows: (question, evidence, sentence) -> class probabilities [n, n_classes]."""
        enc = self._encode(rows)
        probs = np.zeros((len(enc), self.cfg["n_classes"]), dtype=np.float32)
        if not enc:
            return probs
        lengths = [len(x) for x in enc]
        order = sorted(range(len(enc)), key=lambda i: lengths[i])
        pad = self.tok.pad_token_id
        cur = []

        def flush():
            n = max(lengths[i] for i in cur)
            ids = torch.full((len(cur), n), pad, dtype=torch.long)
            mask = torch.zeros((len(cur), n), dtype=torch.long)
            for k, i in enumerate(cur):
                ids[k, :lengths[i]] = torch.tensor(enc[i])
                mask[k, :lengths[i]] = 1
            ids, mask = ids.to(self.device), mask.to(self.device)
            if self.device.startswith("cuda"):
                with torch.autocast("cuda", dtype=torch.float16):
                    logits = self.model(ids, mask).float()
            else:
                logits = self.model(ids, mask).float()
            probs[cur] = torch.softmax(logits, -1).cpu().numpy()
        for i in order:
            if cur and lengths[i] * (len(cur) + 1) > self.batch_tokens * 2:
                flush()
                cur = []
            cur.append(i)
        flush()
        return probs


# ---------------------------------------------------------------- faithfulness

class Faithfulness(_Judge):
    """Is each answer sentence supported by the passages? Returns sentence scores, an answer score and the
    hallucination flag (an answer is flagged when any sentence is)."""
    TASK = "faithfulness"
    VERDICTS = ("all_verified", "some_unverified")

    def _ntok(self, text):
        n = self._len.get(text)
        if n is None:
            n = len(self._ids(text))
            if len(self._len) < 200_000:
                self._len[text] = n
        return n

    def _line(self, c):
        text = c["text"]
        cap = self.cfg["sentence_cap_tokens"]
        if self._ntok(text) > cap:
            text = self.tok.decode(self._ids(text)[:cap]) + " ..."
        return f"[{c['key']}] {text}"

    def _evidence(self, question, ctx, sents, si, bm25):
        budget = self.cfg["evidence_budget"]
        overhead = self._ntok(question) + self._ntok(sents[si]["text"]) + 8
        room = max(64, budget - overhead)
        lines = [self._line(c) for c in ctx]
        lens = [self._ntok(x) for x in lines]
        if sum(lens) <= room:
            return "\n".join(lines)
        chosen, used = set(), 0

        def take(i):
            nonlocal used
            if i in chosen or i < 0 or i >= len(ctx) or used + lens[i] > room:
                return False
            chosen.add(i)
            used += lens[i]
            return True
        query = question + " " + sents[si]["text"]
        if si > 0:
            query += " " + sents[si - 1]["text"]
        sc = bm25.scores(query)
        for i in sorted(range(len(ctx)), key=lambda i: -sc[i]):
            if used >= room - 16 or sc[i] <= 0:
                break
            for j in (i, i - 1, i + 1):
                take(j)
        if used < room:
            for i in range(len(ctx)):
                if not take(i) and used + min(lens) > room:
                    break
        return "\n".join(lines[i] for i in sorted(chosen))

    def __call__(self, question, passages, answer, answer_sentences=None, profile="auto", operating_point="default"):
        """operating_point: "default" (flags about 4.5% of sentences on validation) or "high_recall"."""
        prof = _profile(profile, passages)
        ctx = _context(passages, prof)
        sents = _answer(answer, answer_sentences, prof)
        pc = self.cfg["profiles"][prof]
        bm25 = BM25([c["text"] for c in ctx])
        rows = [(question, self._evidence(question, ctx, sents, si, bm25), s["text"]) for si, s in enumerate(sents)]
        p = 1 - self._probs(rows)[:, 0]                                  # probability "unsupported"
        score, checks = p.astype(float), [None] * len(sents)
        thr = pc["thresholds"][operating_point]
        if pc.get("numeric_check") and operating_point == "default":
            ck = Checker(" ".join(c["text"] for c in ctx))
            checks = ck.check_answer([s["text"] for s in sents])
            c = pc["combiner"]
            X = []
            for pi, nc in zip(p, checks):
                q = min(max(float(pi), 1e-6), 1 - 1e-6)
                X.append([math.log(q / (1 - q))] + [float(nc["verdict"] == v) for v in self.VERDICTS]
                         + [min(nc["n_unverified"], 3), float(nc["eq_bad"] > 0), float(nc["eq_ok"] > 0)])
            score = 1 / (1 + np.exp(-(np.array(X) @ np.array(c["coef"]) + c["intercept"])))
        out = [{"key": s["key"], "text": s["text"], "unsupported_prob": round(float(pi), 6),
                "score": round(float(si), 6), "flag": bool(si >= thr),
                **({"numbers": checks[k]["verdict"]} if checks[k] else {})}
               for k, (s, pi, si) in enumerate(zip(sents, p, score))]
        return {"profile": prof, "operating_point": operating_point, "sentences": out,
                "answer_score": round(float(max(score)) if len(out) else 0.0, 6),
                "hallucination": any(x["flag"] for x in out)}


# ---------------------------------------------------------------- response relevance

class ResponseRelevance(_Judge):
    """Does the answer address the question? direct / partial / irrelevant (question and answer only)."""
    TASK = "response_relevance"

    def __call__(self, question, answer):
        return self.batch([(question, answer)])[0]

    def batch(self, pairs):
        pr = self._probs([(q, "", a) for q, a in pairs])
        labels = self.cfg["labels"]
        return [{"label": labels[int(x.argmax())], "probs": {l: round(float(v), 6) for l, v in zip(labels, x)}}
                for x in pr]


# ---------------------------------------------------------------- context relevance / utilization

def _answer_score(p, spec):
    if spec["agg"] == "count":
        return float((p >= spec["thr"]).mean())
    if spec["agg"] == "mean_iso":
        return float(np.interp(p, spec["iso_x"], spec["iso_y"]).mean())
    return float(p.mean())


def _ap_at_k(v):
    v = np.asarray(v, float)
    if v.sum() == 0:
        return 0.0
    return float(((np.cumsum(v) / np.arange(1, len(v) + 1)) * v).sum() / v.sum())


class _ContextJudge(_Judge):
    def _rows(self, q, ctx, idx):
        w, ch = self.cfg["window"], self.cfg["neighbour_chars"]
        rows = []
        for i in idx:
            c = ctx[i]
            js = [j for j in range(i - w, i + w + 1) if j != i and 0 <= j < len(ctx) and ctx[j]["doc"] == c["doc"]]
            rows.append((q, " ".join(ctx[j]["text"][:ch] for j in js), c["text"]))
        return rows

    def _run(self, q, ctx, pc, prefilter_query=None):
        idx = list(range(len(ctx)))
        top = pc.get("prefilter_top_n")
        if top and prefilter_query is not None and len(ctx) > top:
            sc = np.asarray(BM25([c["text"] for c in ctx]).scores(prefilter_query))
            m = np.zeros(len(ctx), bool)
            for i in np.argsort(-sc)[:top]:
                m[max(0, i - 1):i + 2] = True
            idx = [i for i in idx if m[i]]
        p = np.zeros(len(ctx))
        if idx:
            p[idx] = 1 - self._probs(self._rows(q, ctx, idx))[:, 0]
        return p, len(idx)


class ContextRelevance(_ContextJudge):
    """Is each context sentence relevant to the question? Returns sentence scores, the answer-level relevance
    fraction and, with two or more passages, context precision (AP over passages in the given order)."""
    TASK = "context_relevance"

    def __call__(self, question, passages, profile="auto"):
        prof = _profile(profile, passages)
        ctx = _context(passages, prof)
        pc = self.cfg["profiles"][prof]
        p, _ = self._run(question, ctx, pc)
        thr = pc["sentence_threshold"]
        out = {"profile": prof,
               "sentences": [{"key": c["key"], "passage": c["doc"], "text": c["text"], "score": round(float(x), 6),
                              "relevant": bool(x >= thr)} for c, x in zip(ctx, p)],
               "relevance": round(_answer_score(p, pc["answer_score"]), 6)}
        docs = sorted({c["doc"] for c in ctx})
        if len(docs) > 1:
            d = np.array([c["doc"] for c in ctx])
            out["passage_relevant"] = [bool((p[d == k] >= thr).any()) for k in docs]
            out["context_precision"] = round(_ap_at_k(out["passage_relevant"]), 6)
        return out


class ContextUtilization(_ContextJudge):
    """Did the answer use each context sentence? Returns sentence scores and the utilization fraction."""
    TASK = "context_utilization"

    def __call__(self, question, passages, answer, profile="auto", prefilter=True):
        prof = _profile(profile, passages)
        ctx = _context(passages, prof)
        pc = dict(self.cfg["profiles"][prof])
        if not prefilter:
            pc["prefilter_top_n"] = None
        q = question + " || " + str(answer)[:1200]
        p, n_scored = self._run(q, ctx, pc, prefilter_query=question + " " + str(answer))
        thr = pc["sentence_threshold"]
        return {"profile": prof, "sentences_scored": n_scored,
                "sentences": [{"key": c["key"], "passage": c["doc"], "text": c["text"], "score": round(float(x), 6),
                               "utilized": bool(x >= thr)} for c, x in zip(ctx, p)],
                "utilization": round(_answer_score(p, pc["answer_score"]), 6)}