File size: 11,451 Bytes
041950a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 | """plan_v0.3 U3: deterministic FinQA numeric check (no learning).
.venv/bin/python src/eval/numcheck.py test # unit tests on hand-made cases
.venv/bin/python src/eval/numcheck.py run --split validation # en/cache/v0.3/numcheck/<split>.jsonl
For each answer sentence, every number (years and the constant 100 excluded) is classified:
direct it appears in the context (scale variants x1e3 / x1e6 / x1e-3 / x1e-6 allowed), within the rounding
of the sentence's printed precision;
derived it equals one operation on two context numbers: a+b, a-b, |a-b|, a*b, a/b, a/b*100,
(a-b)/b*100 (percent change), or the sum / mean of 2-5 consecutive context numbers;
chain same as derived, with one or both operands taken from numbers earlier in the same answer (incl. the
same sentence) that were themselves verified;
unverified otherwise.
Sentence verdict: all_verified / some_unverified / no_numbers. Tolerance: half a unit of the last printed digit
(+0.1% for the rounding of intermediate values), or 1e-4 relative.
"""
import argparse
import bisect
import json
import math
import re
import sys
import warnings
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
NUM = re.compile(r"(?<![\w.])(\()?\s*(-|−)?\s*\$?\s*(\d[\d,]*(?:\.\d+)?|\.\d+)\s*(\))?\s*(%|percent\b|pct\b)?"
r"(?:\s*(thousand|million|billion|mn|bn|k)\b)?", re.I)
SCALE = {"thousand": 1e3, "k": 1e3, "million": 1e6, "mn": 1e6, "billion": 1e9, "bn": 1e9}
def numbers(text):
"""-> list of dicts {v, tol, pct, year}. v includes sign and the scale word; tol = half a unit of the last digit."""
out = []
for m in NUM.finditer(text):
lp, neg, raw, rp, pct, scale = m.groups()
raw = raw.rstrip(",")
s = raw.replace(",", "")
if s.count(".") > 1:
continue
try:
v = float(s)
except ValueError:
continue
dec = len(s.split(".")[1]) if "." in s else 0
tol = 0.5 * 10 ** (-dec)
is_year = (dec == 0 and "," not in raw and 1900 <= v <= 2100 and not pct and not scale)
if neg or (lp and rp):
v = -v
mult = SCALE.get((scale or "").lower(), 1.0)
out.append({"v": v * mult, "tol": tol * mult, "pct": bool(pct), "year": is_year, "raw": m.group(0).strip(),
"mult": mult})
return out
EQ_EXPR = re.compile(r"([\d$.,()%\s+\-−*/×÷x]+?)\s*(=|≈)\s*(\(?-?\$?\s*\d[\d,]*(?:\.\d+)?\)?\s*%?)")
def _eval(expr):
e = expr.replace("$", "").replace("%", "").replace("−", "-").replace("×", "*").replace("÷", "/")
e = re.sub(r"(?<=\d)\s*x\s*(?=[\d(])", "*", e)
e = re.sub(r"(?<=\d),(?=\d{3})", "", e).strip()
if not re.fullmatch(r"[\d.\s+\-*/()]+", e) or not re.search(r"\d\s*[-+*/]\s*[\d(]|\)\s*[-+*/]", e):
return None
if e.count("(") != e.count(")"):
e = e.strip("()") if e.count("(") + e.count(")") == 1 else e
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return float(eval(e, {"__builtins__": {}}, {})) # digits, operators and brackets only
except Exception: # noqa: BLE001
return None
def equations(text):
"""Explicit 'expression = number' statements in a sentence (chains 'a - b = c / d = e' included) -> list of
1 (holds) / 0 (does not). Scale words are dropped on both sides (they cancel in these FinQA answers); a percent
result may be a ratio x100. Tolerance: half a unit of the result's last digit + 0.2% (0.4% after an
approximate sign) for the rounding of intermediate values."""
t = re.sub(r"\b(thousand|million|billion|mn|bn)\b", "", text, flags=re.I)
parts = re.split(r"(=|≈|approximately equals)", t)
out = []
for i in range(0, len(parts) - 2, 2):
m = re.search(r"[\d$.,()%\s+\-−*/×÷x]+$", parts[i])
r = re.match(r"\s*(\(?\s*-?\s*\$?\s*\d[\d,]*(?:\.\d+)?\s*\)?\s*%?)", parts[i + 2])
if not m or not r:
continue
val = _eval(m.group(0))
if val is None:
continue
rr = r.group(1).strip()
neg = (rr.startswith("(") and rr.endswith(")")) or "-" in rr
rs = re.sub(r"[^\d.]", "", rr.replace(",", ""))
try:
rv = float(rs) * (-1 if neg else 1)
except ValueError:
continue
dec = len(rs.split(".")[1]) if "." in rs else 0
tol = 0.5 * 10 ** (-dec) + 0.002 * abs(rv) * (2 if parts[i + 1] != "=" else 1)
out.append(int(any(abs(abs(c) - abs(rv)) <= tol for c in (val, val * 100, val / 100))))
return out
def _close(x, tol, sorted_vals):
"""Is any value in sorted_vals within tol of x?"""
i = bisect.bisect_left(sorted_vals, x - tol)
return i < len(sorted_vals) and sorted_vals[i] <= x + tol
def candidates(ops):
"""All one-step results from operand list (context order kept for the consecutive sums / means)."""
vals = [o for o in ops if o != 0 or True]
out = []
n = len(vals)
for i in range(n):
a = vals[i]
for j in range(n):
if i == j:
continue
b = vals[j]
out += [a + b, a - b, a * b]
if b != 0:
out += [a / b, a / b * 100, (a - b) / b * 100, (a - b) / b]
for k in range(2, 6):
for i in range(0, n - k + 1):
s = sum(vals[i:i + k])
out += [s, s / k]
return out
class Checker:
def __init__(self, context_text):
cn = [x for x in numbers(context_text) if not x["year"]]
self.ctx = [x["v"] for x in cn]
base = set()
for v in self.ctx:
for m in (1, 1e3, 1e6, 1e-3, 1e-6):
base.add(v * m)
base |= {abs(v) for v in list(base)}
self.direct = sorted(base)
cand = candidates(self.ctx[:220])
self.derived = sorted({c for c in cand if math.isfinite(c)} | {abs(c) for c in cand if math.isfinite(c)})
@staticmethod
def tol(x):
return max(x["tol"] * 1.0 + 0.001 * abs(x["v"]) * 0 + 1e-9, 1e-4 * abs(x["v"]))
def check_answer(self, sentences):
"""sentences: list of texts in answer order. Returns per sentence {verdict, n, n_unverified, kinds}."""
verified_answer = []
res = []
for text in sentences:
nums = [x for x in numbers(text) if not x["year"] and abs(x["v"]) != 100]
kinds = []
for x in nums:
t = self.tol(x)
v = x["v"]
k = None
forms = [(v, t)] + ([(v / x["mult"], t / x["mult"])] if x["mult"] != 1 else [])
if any(_close(abs(fv), ft, self.direct) for fv, ft in forms):
k = "direct"
elif any(_close(abs(fv), ft, self.derived) for fv, ft in forms):
k = "derived"
else:
pool = verified_answer + [y["v"] for y in nums if y is not x]
if pool:
mixed = []
for a in pool:
for b in self.ctx + pool:
if a is b:
continue
mixed += [a + b, a - b, b - a, a * b]
if b:
mixed += [a / b, a / b * 100, (a - b) / b * 100]
if a:
mixed += [b / a, b / a * 100, (b - a) / a * 100]
if any(abs(abs(fv) - abs(c)) <= ft for c in mixed if math.isfinite(c) for fv, ft in forms):
k = "chain"
kinds.append(k or "unverified")
if k:
verified_answer.append(v)
nu = kinds.count("unverified")
verdict = "no_numbers" if not nums else ("all_verified" if nu == 0 else "some_unverified")
eq = equations(text)
res.append({"verdict": verdict, "n": len(nums), "n_unverified": nu, "kinds": kinds,
"nums": [x["raw"] for x in nums], "eq_ok": sum(e for e in eq), "eq_bad": sum(1 - e for e in eq)})
return res
def run(a):
from paths import P
_, files = P.dataset("v0.2")
d = P.cache / "v0.3" / "numcheck"
d.mkdir(parents=True, exist_ok=True)
from collections import Counter
st = Counter()
with open(d / f"{a.split}.jsonl", "w") as f:
for line in open(files[a.split]):
r = json.loads(line)
if r["subset"] != "finqa":
continue
ck = Checker(" ".join(c["text"] for c in r["context_sentences"]))
out = ck.check_answer([s["text"] for s in r["response_sentences"]])
for s, o in zip(r["response_sentences"], out):
y = int(not s["reviewed_supported"])
f.write(json.dumps({"id": f"{r['example_id']}:{s['key']}", "label_reviewed": y, **o}) + "\n")
st[f"{o['verdict']}:n"] += 1
st[f"{o['verdict']}:unsup"] += y
for v in ("all_verified", "some_unverified", "no_numbers"):
n = st[f"{v}:n"]
print(f"{a.split} {v:16s} n {n:6d} reviewed-unsupported rate {st[f'{v}:unsup'] / max(n, 1):.3f}")
def test(_):
ctx = ("rental expense was approximately $ 66.9 million , $ 57.2 million and $ 49.0 million during 2010 , 2009 and "
"2008 . [[\"2011\", \"$ 62465\"], [\"2012\", \"54236\"]] net loss ( 1,234 ) . total 22176 vs 27885 .")
ck = Checker(ctx)
cases = [
("Rental expense in 2010: $66.9 million", "all_verified"),
("Absolute change = $66.9 million - $57.2 million = $9.7 million", "all_verified"),
("Percentage change = ($9.7 million / $57.2 million) * 100 ≈ 16.96%", "all_verified"),
("The change was (22176 - 27885) / 27885 = -20.47%", "all_verified"),
("The net loss was $1,234.", "all_verified"),
("Rental expense in 2010 was $71.3 million", "some_unverified"),
("The change was 25.3%", "some_unverified"),
("The leases are described below.", "no_numbers"),
("Payments in 2011 and 2012 total 116701", "all_verified"),
("In 2012, the net trading assets were $66.9 million.", "all_verified"),
]
out = ck.check_answer([c for c, _ in cases])
bad = 0
for (c, want), o in zip(cases, out):
ok = o["verdict"] == want
bad += not ok
print("ok " if ok else "BAD", want, o["verdict"], o["kinds"], "|", c)
for txt, want in (("(226 + 209 + 221) / 3 = 220", 0), ("(22176 - 27885) / 27885 = -20.47%", 1),
("$66.9 million - $57.2 million = $9.7 million", 1), ("908.4 / 3 = 302.8 million", 1),
("$19686 * 1.05 = $20670.30", 1), ("x = 5 + 5 = 11", 0), ("Margin = 473 / 7170 ≈ 6.6%", 1)):
e = equations(txt)
ok = bool(e) and e[-1] == want
bad += not ok
print("ok " if ok else "BAD", "eq", want, e, "|", txt)
print("failures", bad)
return bad
def main():
ap = argparse.ArgumentParser()
ap.add_argument("cmd", choices=["test", "run"])
ap.add_argument("--split", default="validation")
a = ap.parse_args()
sys.exit(test(a) if a.cmd == "test" else run(a))
if __name__ == "__main__":
main()
|