cbx-rag-context-utilization-68M / cbx_rag_numcheck.py
SRK-CobuildX's picture
Release v0.1
041950a verified
Raw History Blame Contribute Delete
11.5 kB
"""plan_v0.3 U3: deterministic FinQA numeric check (no learning).
.venv/bin/python src/eval/numcheck.py test # unit tests on hand-made cases
.venv/bin/python src/eval/numcheck.py run --split validation # en/cache/v0.3/numcheck/<split>.jsonl
For each answer sentence, every number (years and the constant 100 excluded) is classified:
direct it appears in the context (scale variants x1e3 / x1e6 / x1e-3 / x1e-6 allowed), within the rounding
of the sentence's printed precision;
derived it equals one operation on two context numbers: a+b, a-b, |a-b|, a*b, a/b, a/b*100,
(a-b)/b*100 (percent change), or the sum / mean of 2-5 consecutive context numbers;
chain same as derived, with one or both operands taken from numbers earlier in the same answer (incl. the
same sentence) that were themselves verified;
unverified otherwise.
Sentence verdict: all_verified / some_unverified / no_numbers. Tolerance: half a unit of the last printed digit
(+0.1% for the rounding of intermediate values), or 1e-4 relative.
"""
import argparse
import bisect
import json
import math
import re
import sys
import warnings
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
NUM = re.compile(r"(?<![\w.])(\()?\s*(-|−)?\s*\$?\s*(\d[\d,]*(?:\.\d+)?|\.\d+)\s*(\))?\s*(%|percent\b|pct\b)?"
r"(?:\s*(thousand|million|billion|mn|bn|k)\b)?", re.I)
SCALE = {"thousand": 1e3, "k": 1e3, "million": 1e6, "mn": 1e6, "billion": 1e9, "bn": 1e9}
def numbers(text):
"""-> list of dicts {v, tol, pct, year}. v includes sign and the scale word; tol = half a unit of the last digit."""
out = []
for m in NUM.finditer(text):
lp, neg, raw, rp, pct, scale = m.groups()
raw = raw.rstrip(",")
s = raw.replace(",", "")
if s.count(".") > 1:
continue
try:
v = float(s)
except ValueError:
continue
dec = len(s.split(".")[1]) if "." in s else 0
tol = 0.5 * 10 ** (-dec)
is_year = (dec == 0 and "," not in raw and 1900 <= v <= 2100 and not pct and not scale)
if neg or (lp and rp):
v = -v
mult = SCALE.get((scale or "").lower(), 1.0)
out.append({"v": v * mult, "tol": tol * mult, "pct": bool(pct), "year": is_year, "raw": m.group(0).strip(),
"mult": mult})
return out
EQ_EXPR = re.compile(r"([\d$.,()%\s+\-−*/×÷x]+?)\s*(=|≈)\s*(\(?-?\$?\s*\d[\d,]*(?:\.\d+)?\)?\s*%?)")
def _eval(expr):
e = expr.replace("$", "").replace("%", "").replace("−", "-").replace("×", "*").replace("÷", "/")
e = re.sub(r"(?<=\d)\s*x\s*(?=[\d(])", "*", e)
e = re.sub(r"(?<=\d),(?=\d{3})", "", e).strip()
if not re.fullmatch(r"[\d.\s+\-*/()]+", e) or not re.search(r"\d\s*[-+*/]\s*[\d(]|\)\s*[-+*/]", e):
return None
if e.count("(") != e.count(")"):
e = e.strip("()") if e.count("(") + e.count(")") == 1 else e
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return float(eval(e, {"__builtins__": {}}, {})) # digits, operators and brackets only
except Exception: # noqa: BLE001
return None
def equations(text):
"""Explicit 'expression = number' statements in a sentence (chains 'a - b = c / d = e' included) -> list of
1 (holds) / 0 (does not). Scale words are dropped on both sides (they cancel in these FinQA answers); a percent
result may be a ratio x100. Tolerance: half a unit of the result's last digit + 0.2% (0.4% after an
approximate sign) for the rounding of intermediate values."""
t = re.sub(r"\b(thousand|million|billion|mn|bn)\b", "", text, flags=re.I)
parts = re.split(r"(=|≈|approximately equals)", t)
out = []
for i in range(0, len(parts) - 2, 2):
m = re.search(r"[\d$.,()%\s+\-−*/×÷x]+$", parts[i])
r = re.match(r"\s*(\(?\s*-?\s*\$?\s*\d[\d,]*(?:\.\d+)?\s*\)?\s*%?)", parts[i + 2])
if not m or not r:
continue
val = _eval(m.group(0))
if val is None:
continue
rr = r.group(1).strip()
neg = (rr.startswith("(") and rr.endswith(")")) or "-" in rr
rs = re.sub(r"[^\d.]", "", rr.replace(",", ""))
try:
rv = float(rs) * (-1 if neg else 1)
except ValueError:
continue
dec = len(rs.split(".")[1]) if "." in rs else 0
tol = 0.5 * 10 ** (-dec) + 0.002 * abs(rv) * (2 if parts[i + 1] != "=" else 1)
out.append(int(any(abs(abs(c) - abs(rv)) <= tol for c in (val, val * 100, val / 100))))
return out
def _close(x, tol, sorted_vals):
"""Is any value in sorted_vals within tol of x?"""
i = bisect.bisect_left(sorted_vals, x - tol)
return i < len(sorted_vals) and sorted_vals[i] <= x + tol
def candidates(ops):
"""All one-step results from operand list (context order kept for the consecutive sums / means)."""
vals = [o for o in ops if o != 0 or True]
out = []
n = len(vals)
for i in range(n):
a = vals[i]
for j in range(n):
if i == j:
continue
b = vals[j]
out += [a + b, a - b, a * b]
if b != 0:
out += [a / b, a / b * 100, (a - b) / b * 100, (a - b) / b]
for k in range(2, 6):
for i in range(0, n - k + 1):
s = sum(vals[i:i + k])
out += [s, s / k]
return out
class Checker:
def __init__(self, context_text):
cn = [x for x in numbers(context_text) if not x["year"]]
self.ctx = [x["v"] for x in cn]
base = set()
for v in self.ctx:
for m in (1, 1e3, 1e6, 1e-3, 1e-6):
base.add(v * m)
base |= {abs(v) for v in list(base)}
self.direct = sorted(base)
cand = candidates(self.ctx[:220])
self.derived = sorted({c for c in cand if math.isfinite(c)} | {abs(c) for c in cand if math.isfinite(c)})
@staticmethod
def tol(x):
return max(x["tol"] * 1.0 + 0.001 * abs(x["v"]) * 0 + 1e-9, 1e-4 * abs(x["v"]))
def check_answer(self, sentences):
"""sentences: list of texts in answer order. Returns per sentence {verdict, n, n_unverified, kinds}."""
verified_answer = []
res = []
for text in sentences:
nums = [x for x in numbers(text) if not x["year"] and abs(x["v"]) != 100]
kinds = []
for x in nums:
t = self.tol(x)
v = x["v"]
k = None
forms = [(v, t)] + ([(v / x["mult"], t / x["mult"])] if x["mult"] != 1 else [])
if any(_close(abs(fv), ft, self.direct) for fv, ft in forms):
k = "direct"
elif any(_close(abs(fv), ft, self.derived) for fv, ft in forms):
k = "derived"
else:
pool = verified_answer + [y["v"] for y in nums if y is not x]
if pool:
mixed = []
for a in pool:
for b in self.ctx + pool:
if a is b:
continue
mixed += [a + b, a - b, b - a, a * b]
if b:
mixed += [a / b, a / b * 100, (a - b) / b * 100]
if a:
mixed += [b / a, b / a * 100, (b - a) / a * 100]
if any(abs(abs(fv) - abs(c)) <= ft for c in mixed if math.isfinite(c) for fv, ft in forms):
k = "chain"
kinds.append(k or "unverified")
if k:
verified_answer.append(v)
nu = kinds.count("unverified")
verdict = "no_numbers" if not nums else ("all_verified" if nu == 0 else "some_unverified")
eq = equations(text)
res.append({"verdict": verdict, "n": len(nums), "n_unverified": nu, "kinds": kinds,
"nums": [x["raw"] for x in nums], "eq_ok": sum(e for e in eq), "eq_bad": sum(1 - e for e in eq)})
return res
def run(a):
from paths import P
_, files = P.dataset("v0.2")
d = P.cache / "v0.3" / "numcheck"
d.mkdir(parents=True, exist_ok=True)
from collections import Counter
st = Counter()
with open(d / f"{a.split}.jsonl", "w") as f:
for line in open(files[a.split]):
r = json.loads(line)
if r["subset"] != "finqa":
continue
ck = Checker(" ".join(c["text"] for c in r["context_sentences"]))
out = ck.check_answer([s["text"] for s in r["response_sentences"]])
for s, o in zip(r["response_sentences"], out):
y = int(not s["reviewed_supported"])
f.write(json.dumps({"id": f"{r['example_id']}:{s['key']}", "label_reviewed": y, **o}) + "\n")
st[f"{o['verdict']}:n"] += 1
st[f"{o['verdict']}:unsup"] += y
for v in ("all_verified", "some_unverified", "no_numbers"):
n = st[f"{v}:n"]
print(f"{a.split} {v:16s} n {n:6d} reviewed-unsupported rate {st[f'{v}:unsup'] / max(n, 1):.3f}")
def test(_):
ctx = ("rental expense was approximately $ 66.9 million , $ 57.2 million and $ 49.0 million during 2010 , 2009 and "
"2008 . [[\"2011\", \"$ 62465\"], [\"2012\", \"54236\"]] net loss ( 1,234 ) . total 22176 vs 27885 .")
ck = Checker(ctx)
cases = [
("Rental expense in 2010: $66.9 million", "all_verified"),
("Absolute change = $66.9 million - $57.2 million = $9.7 million", "all_verified"),
("Percentage change = ($9.7 million / $57.2 million) * 100 ≈ 16.96%", "all_verified"),
("The change was (22176 - 27885) / 27885 = -20.47%", "all_verified"),
("The net loss was $1,234.", "all_verified"),
("Rental expense in 2010 was $71.3 million", "some_unverified"),
("The change was 25.3%", "some_unverified"),
("The leases are described below.", "no_numbers"),
("Payments in 2011 and 2012 total 116701", "all_verified"),
("In 2012, the net trading assets were $66.9 million.", "all_verified"),
]
out = ck.check_answer([c for c, _ in cases])
bad = 0
for (c, want), o in zip(cases, out):
ok = o["verdict"] == want
bad += not ok
print("ok " if ok else "BAD", want, o["verdict"], o["kinds"], "|", c)
for txt, want in (("(226 + 209 + 221) / 3 = 220", 0), ("(22176 - 27885) / 27885 = -20.47%", 1),
("$66.9 million - $57.2 million = $9.7 million", 1), ("908.4 / 3 = 302.8 million", 1),
("$19686 * 1.05 = $20670.30", 1), ("x = 5 + 5 = 11", 0), ("Margin = 473 / 7170 ≈ 6.6%", 1)):
e = equations(txt)
ok = bool(e) and e[-1] == want
bad += not ok
print("ok " if ok else "BAD", "eq", want, e, "|", txt)
print("failures", bad)
return bad
def main():
ap = argparse.ArgumentParser()
ap.add_argument("cmd", choices=["test", "run"])
ap.add_argument("--split", default="validation")
a = ap.parse_args()
sys.exit(test(a) if a.cmd == "test" else run(a))
if __name__ == "__main__":
main()