"""plan_v0.3 U3: deterministic FinQA numeric check (no learning). .venv/bin/python src/eval/numcheck.py test # unit tests on hand-made cases .venv/bin/python src/eval/numcheck.py run --split validation # en/cache/v0.3/numcheck/.jsonl For each answer sentence, every number (years and the constant 100 excluded) is classified: direct it appears in the context (scale variants x1e3 / x1e6 / x1e-3 / x1e-6 allowed), within the rounding of the sentence's printed precision; derived it equals one operation on two context numbers: a+b, a-b, |a-b|, a*b, a/b, a/b*100, (a-b)/b*100 (percent change), or the sum / mean of 2-5 consecutive context numbers; chain same as derived, with one or both operands taken from numbers earlier in the same answer (incl. the same sentence) that were themselves verified; unverified otherwise. Sentence verdict: all_verified / some_unverified / no_numbers. Tolerance: half a unit of the last printed digit (+0.1% for the rounding of intermediate values), or 1e-4 relative. """ import argparse import bisect import json import math import re import sys import warnings from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) NUM = re.compile(r"(? list of dicts {v, tol, pct, year}. v includes sign and the scale word; tol = half a unit of the last digit.""" out = [] for m in NUM.finditer(text): lp, neg, raw, rp, pct, scale = m.groups() raw = raw.rstrip(",") s = raw.replace(",", "") if s.count(".") > 1: continue try: v = float(s) except ValueError: continue dec = len(s.split(".")[1]) if "." in s else 0 tol = 0.5 * 10 ** (-dec) is_year = (dec == 0 and "," not in raw and 1900 <= v <= 2100 and not pct and not scale) if neg or (lp and rp): v = -v mult = SCALE.get((scale or "").lower(), 1.0) out.append({"v": v * mult, "tol": tol * mult, "pct": bool(pct), "year": is_year, "raw": m.group(0).strip(), "mult": mult}) return out EQ_EXPR = re.compile(r"([\d$.,()%\s+\-−*/×÷x]+?)\s*(=|≈)\s*(\(?-?\$?\s*\d[\d,]*(?:\.\d+)?\)?\s*%?)") def _eval(expr): e = expr.replace("$", "").replace("%", "").replace("−", "-").replace("×", "*").replace("÷", "/") e = re.sub(r"(?<=\d)\s*x\s*(?=[\d(])", "*", e) e = re.sub(r"(?<=\d),(?=\d{3})", "", e).strip() if not re.fullmatch(r"[\d.\s+\-*/()]+", e) or not re.search(r"\d\s*[-+*/]\s*[\d(]|\)\s*[-+*/]", e): return None if e.count("(") != e.count(")"): e = e.strip("()") if e.count("(") + e.count(")") == 1 else e try: with warnings.catch_warnings(): warnings.simplefilter("ignore") return float(eval(e, {"__builtins__": {}}, {})) # digits, operators and brackets only except Exception: # noqa: BLE001 return None def equations(text): """Explicit 'expression = number' statements in a sentence (chains 'a - b = c / d = e' included) -> list of 1 (holds) / 0 (does not). Scale words are dropped on both sides (they cancel in these FinQA answers); a percent result may be a ratio x100. Tolerance: half a unit of the result's last digit + 0.2% (0.4% after an approximate sign) for the rounding of intermediate values.""" t = re.sub(r"\b(thousand|million|billion|mn|bn)\b", "", text, flags=re.I) parts = re.split(r"(=|≈|approximately equals)", t) out = [] for i in range(0, len(parts) - 2, 2): m = re.search(r"[\d$.,()%\s+\-−*/×÷x]+$", parts[i]) r = re.match(r"\s*(\(?\s*-?\s*\$?\s*\d[\d,]*(?:\.\d+)?\s*\)?\s*%?)", parts[i + 2]) if not m or not r: continue val = _eval(m.group(0)) if val is None: continue rr = r.group(1).strip() neg = (rr.startswith("(") and rr.endswith(")")) or "-" in rr rs = re.sub(r"[^\d.]", "", rr.replace(",", "")) try: rv = float(rs) * (-1 if neg else 1) except ValueError: continue dec = len(rs.split(".")[1]) if "." in rs else 0 tol = 0.5 * 10 ** (-dec) + 0.002 * abs(rv) * (2 if parts[i + 1] != "=" else 1) out.append(int(any(abs(abs(c) - abs(rv)) <= tol for c in (val, val * 100, val / 100)))) return out def _close(x, tol, sorted_vals): """Is any value in sorted_vals within tol of x?""" i = bisect.bisect_left(sorted_vals, x - tol) return i < len(sorted_vals) and sorted_vals[i] <= x + tol def candidates(ops): """All one-step results from operand list (context order kept for the consecutive sums / means).""" vals = [o for o in ops if o != 0 or True] out = [] n = len(vals) for i in range(n): a = vals[i] for j in range(n): if i == j: continue b = vals[j] out += [a + b, a - b, a * b] if b != 0: out += [a / b, a / b * 100, (a - b) / b * 100, (a - b) / b] for k in range(2, 6): for i in range(0, n - k + 1): s = sum(vals[i:i + k]) out += [s, s / k] return out class Checker: def __init__(self, context_text): cn = [x for x in numbers(context_text) if not x["year"]] self.ctx = [x["v"] for x in cn] base = set() for v in self.ctx: for m in (1, 1e3, 1e6, 1e-3, 1e-6): base.add(v * m) base |= {abs(v) for v in list(base)} self.direct = sorted(base) cand = candidates(self.ctx[:220]) self.derived = sorted({c for c in cand if math.isfinite(c)} | {abs(c) for c in cand if math.isfinite(c)}) @staticmethod def tol(x): return max(x["tol"] * 1.0 + 0.001 * abs(x["v"]) * 0 + 1e-9, 1e-4 * abs(x["v"])) def check_answer(self, sentences): """sentences: list of texts in answer order. Returns per sentence {verdict, n, n_unverified, kinds}.""" verified_answer = [] res = [] for text in sentences: nums = [x for x in numbers(text) if not x["year"] and abs(x["v"]) != 100] kinds = [] for x in nums: t = self.tol(x) v = x["v"] k = None forms = [(v, t)] + ([(v / x["mult"], t / x["mult"])] if x["mult"] != 1 else []) if any(_close(abs(fv), ft, self.direct) for fv, ft in forms): k = "direct" elif any(_close(abs(fv), ft, self.derived) for fv, ft in forms): k = "derived" else: pool = verified_answer + [y["v"] for y in nums if y is not x] if pool: mixed = [] for a in pool: for b in self.ctx + pool: if a is b: continue mixed += [a + b, a - b, b - a, a * b] if b: mixed += [a / b, a / b * 100, (a - b) / b * 100] if a: mixed += [b / a, b / a * 100, (b - a) / a * 100] if any(abs(abs(fv) - abs(c)) <= ft for c in mixed if math.isfinite(c) for fv, ft in forms): k = "chain" kinds.append(k or "unverified") if k: verified_answer.append(v) nu = kinds.count("unverified") verdict = "no_numbers" if not nums else ("all_verified" if nu == 0 else "some_unverified") eq = equations(text) res.append({"verdict": verdict, "n": len(nums), "n_unverified": nu, "kinds": kinds, "nums": [x["raw"] for x in nums], "eq_ok": sum(e for e in eq), "eq_bad": sum(1 - e for e in eq)}) return res def run(a): from paths import P _, files = P.dataset("v0.2") d = P.cache / "v0.3" / "numcheck" d.mkdir(parents=True, exist_ok=True) from collections import Counter st = Counter() with open(d / f"{a.split}.jsonl", "w") as f: for line in open(files[a.split]): r = json.loads(line) if r["subset"] != "finqa": continue ck = Checker(" ".join(c["text"] for c in r["context_sentences"])) out = ck.check_answer([s["text"] for s in r["response_sentences"]]) for s, o in zip(r["response_sentences"], out): y = int(not s["reviewed_supported"]) f.write(json.dumps({"id": f"{r['example_id']}:{s['key']}", "label_reviewed": y, **o}) + "\n") st[f"{o['verdict']}:n"] += 1 st[f"{o['verdict']}:unsup"] += y for v in ("all_verified", "some_unverified", "no_numbers"): n = st[f"{v}:n"] print(f"{a.split} {v:16s} n {n:6d} reviewed-unsupported rate {st[f'{v}:unsup'] / max(n, 1):.3f}") def test(_): ctx = ("rental expense was approximately $ 66.9 million , $ 57.2 million and $ 49.0 million during 2010 , 2009 and " "2008 . [[\"2011\", \"$ 62465\"], [\"2012\", \"54236\"]] net loss ( 1,234 ) . total 22176 vs 27885 .") ck = Checker(ctx) cases = [ ("Rental expense in 2010: $66.9 million", "all_verified"), ("Absolute change = $66.9 million - $57.2 million = $9.7 million", "all_verified"), ("Percentage change = ($9.7 million / $57.2 million) * 100 ≈ 16.96%", "all_verified"), ("The change was (22176 - 27885) / 27885 = -20.47%", "all_verified"), ("The net loss was $1,234.", "all_verified"), ("Rental expense in 2010 was $71.3 million", "some_unverified"), ("The change was 25.3%", "some_unverified"), ("The leases are described below.", "no_numbers"), ("Payments in 2011 and 2012 total 116701", "all_verified"), ("In 2012, the net trading assets were $66.9 million.", "all_verified"), ] out = ck.check_answer([c for c, _ in cases]) bad = 0 for (c, want), o in zip(cases, out): ok = o["verdict"] == want bad += not ok print("ok " if ok else "BAD", want, o["verdict"], o["kinds"], "|", c) for txt, want in (("(226 + 209 + 221) / 3 = 220", 0), ("(22176 - 27885) / 27885 = -20.47%", 1), ("$66.9 million - $57.2 million = $9.7 million", 1), ("908.4 / 3 = 302.8 million", 1), ("$19686 * 1.05 = $20670.30", 1), ("x = 5 + 5 = 11", 0), ("Margin = 473 / 7170 ≈ 6.6%", 1)): e = equations(txt) ok = bool(e) and e[-1] == want bad += not ok print("ok " if ok else "BAD", "eq", want, e, "|", txt) print("failures", bad) return bad def main(): ap = argparse.ArgumentParser() ap.add_argument("cmd", choices=["test", "run"]) ap.add_argument("--split", default="validation") a = ap.parse_args() sys.exit(test(a) if a.cmd == "test" else run(a)) if __name__ == "__main__": main()