laya-banking77 / code /banking77 /compare.py
VibeHash's picture
Add laya-banking77: Laya fine-tuned on Banking77, its scores and the code that made it
4e09985 verified
Raw History Blame Contribute Delete
1.89 kB
"""Compare two scored runs message by message, with the exact McNemar test.
Two models scored on the same test can be compared more sharply than their two accuracies suggest: only the
messages where exactly one of them is right carry information about which is better.
Run it with `python -m banking77.compare zero-shot seed0`.
"""
from __future__ import annotations
import argparse
import json
from typing import Dict, List
from banking77.data import home
from banking77.metrics import mcnemar_exact
def load(name: str) -> List[Dict]:
with open(home() / "runs" / name / "predictions.jsonl", encoding="utf-8") as f:
return [json.loads(line) for line in f if line.strip()]
def compare(a: List[Dict], b: List[Dict]) -> Dict[str, float]:
if len(a) != len(b) or any(x["gold"] != y["gold"] for x, y in zip(a, b)):
raise ValueError("the two runs were not scored on the same test")
ra = [x["pred"] == x["gold"] for x in a]
rb = [y["pred"] == y["gold"] for y in b]
only_a = sum(1 for x, y in zip(ra, rb) if x and not y)
only_b = sum(1 for x, y in zip(ra, rb) if y and not x)
return {"n": len(a), "a_right": sum(ra), "b_right": sum(rb), "only_a": only_a, "only_b": only_b,
"p": mcnemar_exact(only_a, only_b)}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("a")
parser.add_argument("b")
args = parser.parse_args()
r = compare(load(args.a), load(args.b))
# Past about 1,100 disagreements the exact p-value is smaller than a float can hold.
p = "p = %.3g" % r["p"] if r["p"] > 0 else "p < 1e-300"
print("%s %d right, %s %d right of %d; only %s right %d, only %s right %d; exact McNemar %s"
% (args.a, r["a_right"], args.b, r["b_right"], r["n"], args.a, r["only_a"], args.b, r["only_b"], p))
if __name__ == "__main__":
main()