Download cbx_rag_numcheck.py from CobuildX-AI/cbx-rag-context-utilization-68M: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/CobuildX-AI/cbx-rag-context-utilization-68M/resolve/main/cbx_rag_numcheck.py
- Command line
-
hf download hf://CobuildX-AI/cbx-rag-context-utilization-68M/cbx_rag_numcheck.py
-
curl -L -o cbx_rag_numcheck.py https://huggingface.co/CobuildX-AI/cbx-rag-context-utilization-68M/resolve/main/cbx_rag_numcheck.py
11.5 kB
| """plan_v0.3 U3: deterministic FinQA numeric check (no learning). | |
| .venv/bin/python src/eval/numcheck.py test # unit tests on hand-made cases | |
| .venv/bin/python src/eval/numcheck.py run --split validation # en/cache/v0.3/numcheck/<split>.jsonl | |
| For each answer sentence, every number (years and the constant 100 excluded) is classified: | |
| direct it appears in the context (scale variants x1e3 / x1e6 / x1e-3 / x1e-6 allowed), within the rounding | |
| of the sentence's printed precision; | |
| derived it equals one operation on two context numbers: a+b, a-b, |a-b|, a*b, a/b, a/b*100, | |
| (a-b)/b*100 (percent change), or the sum / mean of 2-5 consecutive context numbers; | |
| chain same as derived, with one or both operands taken from numbers earlier in the same answer (incl. the | |
| same sentence) that were themselves verified; | |
| unverified otherwise. | |
| Sentence verdict: all_verified / some_unverified / no_numbers. Tolerance: half a unit of the last printed digit | |
| (+0.1% for the rounding of intermediate values), or 1e-4 relative. | |
| """ | |
| import argparse | |
| import bisect | |
| import json | |
| import math | |
| import re | |
| import sys | |
| import warnings | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| NUM = re.compile(r"(?<![\w.])(\()?\s*(-|−)?\s*\$?\s*(\d[\d,]*(?:\.\d+)?|\.\d+)\s*(\))?\s*(%|percent\b|pct\b)?" | |
| r"(?:\s*(thousand|million|billion|mn|bn|k)\b)?", re.I) | |
| SCALE = {"thousand": 1e3, "k": 1e3, "million": 1e6, "mn": 1e6, "billion": 1e9, "bn": 1e9} | |
| def numbers(text): | |
| """-> list of dicts {v, tol, pct, year}. v includes sign and the scale word; tol = half a unit of the last digit.""" | |
| out = [] | |
| for m in NUM.finditer(text): | |
| lp, neg, raw, rp, pct, scale = m.groups() | |
| raw = raw.rstrip(",") | |
| s = raw.replace(",", "") | |
| if s.count(".") > 1: | |
| continue | |
| try: | |
| v = float(s) | |
| except ValueError: | |
| continue | |
| dec = len(s.split(".")[1]) if "." in s else 0 | |
| tol = 0.5 * 10 ** (-dec) | |
| is_year = (dec == 0 and "," not in raw and 1900 <= v <= 2100 and not pct and not scale) | |
| if neg or (lp and rp): | |
| v = -v | |
| mult = SCALE.get((scale or "").lower(), 1.0) | |
| out.append({"v": v * mult, "tol": tol * mult, "pct": bool(pct), "year": is_year, "raw": m.group(0).strip(), | |
| "mult": mult}) | |
| return out | |
| EQ_EXPR = re.compile(r"([\d$.,()%\s+\-−*/×÷x]+?)\s*(=|≈)\s*(\(?-?\$?\s*\d[\d,]*(?:\.\d+)?\)?\s*%?)") | |
| def _eval(expr): | |
| e = expr.replace("$", "").replace("%", "").replace("−", "-").replace("×", "*").replace("÷", "/") | |
| e = re.sub(r"(?<=\d)\s*x\s*(?=[\d(])", "*", e) | |
| e = re.sub(r"(?<=\d),(?=\d{3})", "", e).strip() | |
| if not re.fullmatch(r"[\d.\s+\-*/()]+", e) or not re.search(r"\d\s*[-+*/]\s*[\d(]|\)\s*[-+*/]", e): | |
| return None | |
| if e.count("(") != e.count(")"): | |
| e = e.strip("()") if e.count("(") + e.count(")") == 1 else e | |
| try: | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| return float(eval(e, {"__builtins__": {}}, {})) # digits, operators and brackets only | |
| except Exception: # noqa: BLE001 | |
| return None | |
| def equations(text): | |
| """Explicit 'expression = number' statements in a sentence (chains 'a - b = c / d = e' included) -> list of | |
| 1 (holds) / 0 (does not). Scale words are dropped on both sides (they cancel in these FinQA answers); a percent | |
| result may be a ratio x100. Tolerance: half a unit of the result's last digit + 0.2% (0.4% after an | |
| approximate sign) for the rounding of intermediate values.""" | |
| t = re.sub(r"\b(thousand|million|billion|mn|bn)\b", "", text, flags=re.I) | |
| parts = re.split(r"(=|≈|approximately equals)", t) | |
| out = [] | |
| for i in range(0, len(parts) - 2, 2): | |
| m = re.search(r"[\d$.,()%\s+\-−*/×÷x]+$", parts[i]) | |
| r = re.match(r"\s*(\(?\s*-?\s*\$?\s*\d[\d,]*(?:\.\d+)?\s*\)?\s*%?)", parts[i + 2]) | |
| if not m or not r: | |
| continue | |
| val = _eval(m.group(0)) | |
| if val is None: | |
| continue | |
| rr = r.group(1).strip() | |
| neg = (rr.startswith("(") and rr.endswith(")")) or "-" in rr | |
| rs = re.sub(r"[^\d.]", "", rr.replace(",", "")) | |
| try: | |
| rv = float(rs) * (-1 if neg else 1) | |
| except ValueError: | |
| continue | |
| dec = len(rs.split(".")[1]) if "." in rs else 0 | |
| tol = 0.5 * 10 ** (-dec) + 0.002 * abs(rv) * (2 if parts[i + 1] != "=" else 1) | |
| out.append(int(any(abs(abs(c) - abs(rv)) <= tol for c in (val, val * 100, val / 100)))) | |
| return out | |
| def _close(x, tol, sorted_vals): | |
| """Is any value in sorted_vals within tol of x?""" | |
| i = bisect.bisect_left(sorted_vals, x - tol) | |
| return i < len(sorted_vals) and sorted_vals[i] <= x + tol | |
| def candidates(ops): | |
| """All one-step results from operand list (context order kept for the consecutive sums / means).""" | |
| vals = [o for o in ops if o != 0 or True] | |
| out = [] | |
| n = len(vals) | |
| for i in range(n): | |
| a = vals[i] | |
| for j in range(n): | |
| if i == j: | |
| continue | |
| b = vals[j] | |
| out += [a + b, a - b, a * b] | |
| if b != 0: | |
| out += [a / b, a / b * 100, (a - b) / b * 100, (a - b) / b] | |
| for k in range(2, 6): | |
| for i in range(0, n - k + 1): | |
| s = sum(vals[i:i + k]) | |
| out += [s, s / k] | |
| return out | |
| class Checker: | |
| def __init__(self, context_text): | |
| cn = [x for x in numbers(context_text) if not x["year"]] | |
| self.ctx = [x["v"] for x in cn] | |
| base = set() | |
| for v in self.ctx: | |
| for m in (1, 1e3, 1e6, 1e-3, 1e-6): | |
| base.add(v * m) | |
| base |= {abs(v) for v in list(base)} | |
| self.direct = sorted(base) | |
| cand = candidates(self.ctx[:220]) | |
| self.derived = sorted({c for c in cand if math.isfinite(c)} | {abs(c) for c in cand if math.isfinite(c)}) | |
| def tol(x): | |
| return max(x["tol"] * 1.0 + 0.001 * abs(x["v"]) * 0 + 1e-9, 1e-4 * abs(x["v"])) | |
| def check_answer(self, sentences): | |
| """sentences: list of texts in answer order. Returns per sentence {verdict, n, n_unverified, kinds}.""" | |
| verified_answer = [] | |
| res = [] | |
| for text in sentences: | |
| nums = [x for x in numbers(text) if not x["year"] and abs(x["v"]) != 100] | |
| kinds = [] | |
| for x in nums: | |
| t = self.tol(x) | |
| v = x["v"] | |
| k = None | |
| forms = [(v, t)] + ([(v / x["mult"], t / x["mult"])] if x["mult"] != 1 else []) | |
| if any(_close(abs(fv), ft, self.direct) for fv, ft in forms): | |
| k = "direct" | |
| elif any(_close(abs(fv), ft, self.derived) for fv, ft in forms): | |
| k = "derived" | |
| else: | |
| pool = verified_answer + [y["v"] for y in nums if y is not x] | |
| if pool: | |
| mixed = [] | |
| for a in pool: | |
| for b in self.ctx + pool: | |
| if a is b: | |
| continue | |
| mixed += [a + b, a - b, b - a, a * b] | |
| if b: | |
| mixed += [a / b, a / b * 100, (a - b) / b * 100] | |
| if a: | |
| mixed += [b / a, b / a * 100, (b - a) / a * 100] | |
| if any(abs(abs(fv) - abs(c)) <= ft for c in mixed if math.isfinite(c) for fv, ft in forms): | |
| k = "chain" | |
| kinds.append(k or "unverified") | |
| if k: | |
| verified_answer.append(v) | |
| nu = kinds.count("unverified") | |
| verdict = "no_numbers" if not nums else ("all_verified" if nu == 0 else "some_unverified") | |
| eq = equations(text) | |
| res.append({"verdict": verdict, "n": len(nums), "n_unverified": nu, "kinds": kinds, | |
| "nums": [x["raw"] for x in nums], "eq_ok": sum(e for e in eq), "eq_bad": sum(1 - e for e in eq)}) | |
| return res | |
| def run(a): | |
| from paths import P | |
| _, files = P.dataset("v0.2") | |
| d = P.cache / "v0.3" / "numcheck" | |
| d.mkdir(parents=True, exist_ok=True) | |
| from collections import Counter | |
| st = Counter() | |
| with open(d / f"{a.split}.jsonl", "w") as f: | |
| for line in open(files[a.split]): | |
| r = json.loads(line) | |
| if r["subset"] != "finqa": | |
| continue | |
| ck = Checker(" ".join(c["text"] for c in r["context_sentences"])) | |
| out = ck.check_answer([s["text"] for s in r["response_sentences"]]) | |
| for s, o in zip(r["response_sentences"], out): | |
| y = int(not s["reviewed_supported"]) | |
| f.write(json.dumps({"id": f"{r['example_id']}:{s['key']}", "label_reviewed": y, **o}) + "\n") | |
| st[f"{o['verdict']}:n"] += 1 | |
| st[f"{o['verdict']}:unsup"] += y | |
| for v in ("all_verified", "some_unverified", "no_numbers"): | |
| n = st[f"{v}:n"] | |
| print(f"{a.split} {v:16s} n {n:6d} reviewed-unsupported rate {st[f'{v}:unsup'] / max(n, 1):.3f}") | |
| def test(_): | |
| ctx = ("rental expense was approximately $ 66.9 million , $ 57.2 million and $ 49.0 million during 2010 , 2009 and " | |
| "2008 . [[\"2011\", \"$ 62465\"], [\"2012\", \"54236\"]] net loss ( 1,234 ) . total 22176 vs 27885 .") | |
| ck = Checker(ctx) | |
| cases = [ | |
| ("Rental expense in 2010: $66.9 million", "all_verified"), | |
| ("Absolute change = $66.9 million - $57.2 million = $9.7 million", "all_verified"), | |
| ("Percentage change = ($9.7 million / $57.2 million) * 100 ≈ 16.96%", "all_verified"), | |
| ("The change was (22176 - 27885) / 27885 = -20.47%", "all_verified"), | |
| ("The net loss was $1,234.", "all_verified"), | |
| ("Rental expense in 2010 was $71.3 million", "some_unverified"), | |
| ("The change was 25.3%", "some_unverified"), | |
| ("The leases are described below.", "no_numbers"), | |
| ("Payments in 2011 and 2012 total 116701", "all_verified"), | |
| ("In 2012, the net trading assets were $66.9 million.", "all_verified"), | |
| ] | |
| out = ck.check_answer([c for c, _ in cases]) | |
| bad = 0 | |
| for (c, want), o in zip(cases, out): | |
| ok = o["verdict"] == want | |
| bad += not ok | |
| print("ok " if ok else "BAD", want, o["verdict"], o["kinds"], "|", c) | |
| for txt, want in (("(226 + 209 + 221) / 3 = 220", 0), ("(22176 - 27885) / 27885 = -20.47%", 1), | |
| ("$66.9 million - $57.2 million = $9.7 million", 1), ("908.4 / 3 = 302.8 million", 1), | |
| ("$19686 * 1.05 = $20670.30", 1), ("x = 5 + 5 = 11", 0), ("Margin = 473 / 7170 ≈ 6.6%", 1)): | |
| e = equations(txt) | |
| ok = bool(e) and e[-1] == want | |
| bad += not ok | |
| print("ok " if ok else "BAD", "eq", want, e, "|", txt) | |
| print("failures", bad) | |
| return bad | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("cmd", choices=["test", "run"]) | |
| ap.add_argument("--split", default="validation") | |
| a = ap.parse_args() | |
| sys.exit(test(a) if a.cmd == "test" else run(a)) | |
| if __name__ == "__main__": | |
| main() | |