File size: 11,451 Bytes
041950a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
"""plan_v0.3 U3: deterministic FinQA numeric check (no learning).

    .venv/bin/python src/eval/numcheck.py test                        # unit tests on hand-made cases
    .venv/bin/python src/eval/numcheck.py run --split validation      # en/cache/v0.3/numcheck/<split>.jsonl

For each answer sentence, every number (years and the constant 100 excluded) is classified:
    direct    it appears in the context (scale variants x1e3 / x1e6 / x1e-3 / x1e-6 allowed), within the rounding
              of the sentence's printed precision;
    derived   it equals one operation on two context numbers: a+b, a-b, |a-b|, a*b, a/b, a/b*100,
              (a-b)/b*100 (percent change), or the sum / mean of 2-5 consecutive context numbers;
    chain     same as derived, with one or both operands taken from numbers earlier in the same answer (incl. the
              same sentence) that were themselves verified;
    unverified otherwise.
Sentence verdict: all_verified / some_unverified / no_numbers. Tolerance: half a unit of the last printed digit
(+0.1% for the rounding of intermediate values), or 1e-4 relative.
"""
import argparse
import bisect
import json
import math
import re
import sys
import warnings
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

NUM = re.compile(r"(?<![\w.])(\()?\s*(-|−)?\s*\$?\s*(\d[\d,]*(?:\.\d+)?|\.\d+)\s*(\))?\s*(%|percent\b|pct\b)?"
                 r"(?:\s*(thousand|million|billion|mn|bn|k)\b)?", re.I)
SCALE = {"thousand": 1e3, "k": 1e3, "million": 1e6, "mn": 1e6, "billion": 1e9, "bn": 1e9}


def numbers(text):
    """-> list of dicts {v, tol, pct, year}. v includes sign and the scale word; tol = half a unit of the last digit."""
    out = []
    for m in NUM.finditer(text):
        lp, neg, raw, rp, pct, scale = m.groups()
        raw = raw.rstrip(",")
        s = raw.replace(",", "")
        if s.count(".") > 1:
            continue
        try:
            v = float(s)
        except ValueError:
            continue
        dec = len(s.split(".")[1]) if "." in s else 0
        tol = 0.5 * 10 ** (-dec)
        is_year = (dec == 0 and "," not in raw and 1900 <= v <= 2100 and not pct and not scale)
        if neg or (lp and rp):
            v = -v
        mult = SCALE.get((scale or "").lower(), 1.0)
        out.append({"v": v * mult, "tol": tol * mult, "pct": bool(pct), "year": is_year, "raw": m.group(0).strip(),
                    "mult": mult})
    return out


EQ_EXPR = re.compile(r"([\d$.,()%\s+\-−*/×÷x]+?)\s*(=|≈)\s*(\(?-?\$?\s*\d[\d,]*(?:\.\d+)?\)?\s*%?)")


def _eval(expr):
    e = expr.replace("$", "").replace("%", "").replace("−", "-").replace("×", "*").replace("÷", "/")
    e = re.sub(r"(?<=\d)\s*x\s*(?=[\d(])", "*", e)
    e = re.sub(r"(?<=\d),(?=\d{3})", "", e).strip()
    if not re.fullmatch(r"[\d.\s+\-*/()]+", e) or not re.search(r"\d\s*[-+*/]\s*[\d(]|\)\s*[-+*/]", e):
        return None
    if e.count("(") != e.count(")"):
        e = e.strip("()") if e.count("(") + e.count(")") == 1 else e
    try:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            return float(eval(e, {"__builtins__": {}}, {}))             # digits, operators and brackets only
    except Exception:                                                    # noqa: BLE001
        return None


def equations(text):
    """Explicit 'expression = number' statements in a sentence (chains 'a - b = c / d = e' included) -> list of
    1 (holds) / 0 (does not). Scale words are dropped on both sides (they cancel in these FinQA answers); a percent
    result may be a ratio x100. Tolerance: half a unit of the result's last digit + 0.2% (0.4% after an
    approximate sign) for the rounding of intermediate values."""
    t = re.sub(r"\b(thousand|million|billion|mn|bn)\b", "", text, flags=re.I)
    parts = re.split(r"(=|≈|approximately equals)", t)
    out = []
    for i in range(0, len(parts) - 2, 2):
        m = re.search(r"[\d$.,()%\s+\-−*/×÷x]+$", parts[i])
        r = re.match(r"\s*(\(?\s*-?\s*\$?\s*\d[\d,]*(?:\.\d+)?\s*\)?\s*%?)", parts[i + 2])
        if not m or not r:
            continue
        val = _eval(m.group(0))
        if val is None:
            continue
        rr = r.group(1).strip()
        neg = (rr.startswith("(") and rr.endswith(")")) or "-" in rr
        rs = re.sub(r"[^\d.]", "", rr.replace(",", ""))
        try:
            rv = float(rs) * (-1 if neg else 1)
        except ValueError:
            continue
        dec = len(rs.split(".")[1]) if "." in rs else 0
        tol = 0.5 * 10 ** (-dec) + 0.002 * abs(rv) * (2 if parts[i + 1] != "=" else 1)
        out.append(int(any(abs(abs(c) - abs(rv)) <= tol for c in (val, val * 100, val / 100))))
    return out


def _close(x, tol, sorted_vals):
    """Is any value in sorted_vals within tol of x?"""
    i = bisect.bisect_left(sorted_vals, x - tol)
    return i < len(sorted_vals) and sorted_vals[i] <= x + tol


def candidates(ops):
    """All one-step results from operand list (context order kept for the consecutive sums / means)."""
    vals = [o for o in ops if o != 0 or True]
    out = []
    n = len(vals)
    for i in range(n):
        a = vals[i]
        for j in range(n):
            if i == j:
                continue
            b = vals[j]
            out += [a + b, a - b, a * b]
            if b != 0:
                out += [a / b, a / b * 100, (a - b) / b * 100, (a - b) / b]
    for k in range(2, 6):
        for i in range(0, n - k + 1):
            s = sum(vals[i:i + k])
            out += [s, s / k]
    return out


class Checker:
    def __init__(self, context_text):
        cn = [x for x in numbers(context_text) if not x["year"]]
        self.ctx = [x["v"] for x in cn]
        base = set()
        for v in self.ctx:
            for m in (1, 1e3, 1e6, 1e-3, 1e-6):
                base.add(v * m)
        base |= {abs(v) for v in list(base)}
        self.direct = sorted(base)
        cand = candidates(self.ctx[:220])
        self.derived = sorted({c for c in cand if math.isfinite(c)} | {abs(c) for c in cand if math.isfinite(c)})

    @staticmethod
    def tol(x):
        return max(x["tol"] * 1.0 + 0.001 * abs(x["v"]) * 0 + 1e-9, 1e-4 * abs(x["v"]))

    def check_answer(self, sentences):
        """sentences: list of texts in answer order. Returns per sentence {verdict, n, n_unverified, kinds}."""
        verified_answer = []
        res = []
        for text in sentences:
            nums = [x for x in numbers(text) if not x["year"] and abs(x["v"]) != 100]
            kinds = []
            for x in nums:
                t = self.tol(x)
                v = x["v"]
                k = None
                forms = [(v, t)] + ([(v / x["mult"], t / x["mult"])] if x["mult"] != 1 else [])
                if any(_close(abs(fv), ft, self.direct) for fv, ft in forms):
                    k = "direct"
                elif any(_close(abs(fv), ft, self.derived) for fv, ft in forms):
                    k = "derived"
                else:
                    pool = verified_answer + [y["v"] for y in nums if y is not x]
                    if pool:
                        mixed = []
                        for a in pool:
                            for b in self.ctx + pool:
                                if a is b:
                                    continue
                                mixed += [a + b, a - b, b - a, a * b]
                                if b:
                                    mixed += [a / b, a / b * 100, (a - b) / b * 100]
                                if a:
                                    mixed += [b / a, b / a * 100, (b - a) / a * 100]
                        if any(abs(abs(fv) - abs(c)) <= ft for c in mixed if math.isfinite(c) for fv, ft in forms):
                            k = "chain"
                kinds.append(k or "unverified")
                if k:
                    verified_answer.append(v)
            nu = kinds.count("unverified")
            verdict = "no_numbers" if not nums else ("all_verified" if nu == 0 else "some_unverified")
            eq = equations(text)
            res.append({"verdict": verdict, "n": len(nums), "n_unverified": nu, "kinds": kinds,
                        "nums": [x["raw"] for x in nums], "eq_ok": sum(e for e in eq), "eq_bad": sum(1 - e for e in eq)})
        return res


def run(a):
    from paths import P
    _, files = P.dataset("v0.2")
    d = P.cache / "v0.3" / "numcheck"
    d.mkdir(parents=True, exist_ok=True)
    from collections import Counter
    st = Counter()
    with open(d / f"{a.split}.jsonl", "w") as f:
        for line in open(files[a.split]):
            r = json.loads(line)
            if r["subset"] != "finqa":
                continue
            ck = Checker(" ".join(c["text"] for c in r["context_sentences"]))
            out = ck.check_answer([s["text"] for s in r["response_sentences"]])
            for s, o in zip(r["response_sentences"], out):
                y = int(not s["reviewed_supported"])
                f.write(json.dumps({"id": f"{r['example_id']}:{s['key']}", "label_reviewed": y, **o}) + "\n")
                st[f"{o['verdict']}:n"] += 1
                st[f"{o['verdict']}:unsup"] += y
    for v in ("all_verified", "some_unverified", "no_numbers"):
        n = st[f"{v}:n"]
        print(f"{a.split} {v:16s} n {n:6d}  reviewed-unsupported rate {st[f'{v}:unsup'] / max(n, 1):.3f}")


def test(_):
    ctx = ("rental expense was approximately $ 66.9 million , $ 57.2 million and $ 49.0 million during 2010 , 2009 and "
           "2008 . [[\"2011\", \"$ 62465\"], [\"2012\", \"54236\"]] net loss ( 1,234 ) . total 22176 vs 27885 .")
    ck = Checker(ctx)
    cases = [
        ("Rental expense in 2010: $66.9 million", "all_verified"),
        ("Absolute change = $66.9 million - $57.2 million = $9.7 million", "all_verified"),
        ("Percentage change = ($9.7 million / $57.2 million) * 100 ≈ 16.96%", "all_verified"),
        ("The change was (22176 - 27885) / 27885 = -20.47%", "all_verified"),
        ("The net loss was $1,234.", "all_verified"),
        ("Rental expense in 2010 was $71.3 million", "some_unverified"),
        ("The change was 25.3%", "some_unverified"),
        ("The leases are described below.", "no_numbers"),
        ("Payments in 2011 and 2012 total 116701", "all_verified"),
        ("In 2012, the net trading assets were $66.9 million.", "all_verified"),
    ]
    out = ck.check_answer([c for c, _ in cases])
    bad = 0
    for (c, want), o in zip(cases, out):
        ok = o["verdict"] == want
        bad += not ok
        print("ok " if ok else "BAD", want, o["verdict"], o["kinds"], "|", c)
    for txt, want in (("(226 + 209 + 221) / 3 = 220", 0), ("(22176 - 27885) / 27885 = -20.47%", 1),
                      ("$66.9 million - $57.2 million = $9.7 million", 1), ("908.4 / 3 = 302.8 million", 1),
                      ("$19686 * 1.05 = $20670.30", 1), ("x = 5 + 5 = 11", 0), ("Margin = 473 / 7170 ≈ 6.6%", 1)):
        e = equations(txt)
        ok = bool(e) and e[-1] == want
        bad += not ok
        print("ok " if ok else "BAD", "eq", want, e, "|", txt)
    print("failures", bad)
    return bad


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=["test", "run"])
    ap.add_argument("--split", default="validation")
    a = ap.parse_args()
    sys.exit(test(a) if a.cmd == "test" else run(a))


if __name__ == "__main__":
    main()