File size: 1,891 Bytes
4e09985
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""Compare two scored runs message by message, with the exact McNemar test.

Two models scored on the same test can be compared more sharply than their two accuracies suggest: only the
messages where exactly one of them is right carry information about which is better.

Run it with `python -m banking77.compare zero-shot seed0`.
"""

from __future__ import annotations

import argparse
import json
from typing import Dict, List

from banking77.data import home
from banking77.metrics import mcnemar_exact


def load(name: str) -> List[Dict]:
    with open(home() / "runs" / name / "predictions.jsonl", encoding="utf-8") as f:
        return [json.loads(line) for line in f if line.strip()]


def compare(a: List[Dict], b: List[Dict]) -> Dict[str, float]:
    if len(a) != len(b) or any(x["gold"] != y["gold"] for x, y in zip(a, b)):
        raise ValueError("the two runs were not scored on the same test")
    ra = [x["pred"] == x["gold"] for x in a]
    rb = [y["pred"] == y["gold"] for y in b]
    only_a = sum(1 for x, y in zip(ra, rb) if x and not y)
    only_b = sum(1 for x, y in zip(ra, rb) if y and not x)
    return {"n": len(a), "a_right": sum(ra), "b_right": sum(rb), "only_a": only_a, "only_b": only_b,
            "p": mcnemar_exact(only_a, only_b)}


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("a")
    parser.add_argument("b")
    args = parser.parse_args()
    r = compare(load(args.a), load(args.b))
    # Past about 1,100 disagreements the exact p-value is smaller than a float can hold.
    p = "p = %.3g" % r["p"] if r["p"] > 0 else "p < 1e-300"
    print("%s %d right, %s %d right of %d; only %s right %d, only %s right %d; exact McNemar %s"
          % (args.a, r["a_right"], args.b, r["b_right"], r["n"], args.a, r["only_a"], args.b, r["only_b"], p))


if __name__ == "__main__":
    main()