"""Compare two scored runs message by message, with the exact McNemar test. Two models scored on the same test can be compared more sharply than their two accuracies suggest: only the messages where exactly one of them is right carry information about which is better. Run it with `python -m banking77.compare zero-shot seed0`. """ from __future__ import annotations import argparse import json from typing import Dict, List from banking77.data import home from banking77.metrics import mcnemar_exact def load(name: str) -> List[Dict]: with open(home() / "runs" / name / "predictions.jsonl", encoding="utf-8") as f: return [json.loads(line) for line in f if line.strip()] def compare(a: List[Dict], b: List[Dict]) -> Dict[str, float]: if len(a) != len(b) or any(x["gold"] != y["gold"] for x, y in zip(a, b)): raise ValueError("the two runs were not scored on the same test") ra = [x["pred"] == x["gold"] for x in a] rb = [y["pred"] == y["gold"] for y in b] only_a = sum(1 for x, y in zip(ra, rb) if x and not y) only_b = sum(1 for x, y in zip(ra, rb) if y and not x) return {"n": len(a), "a_right": sum(ra), "b_right": sum(rb), "only_a": only_a, "only_b": only_b, "p": mcnemar_exact(only_a, only_b)} def main() -> None: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("a") parser.add_argument("b") args = parser.parse_args() r = compare(load(args.a), load(args.b)) # Past about 1,100 disagreements the exact p-value is smaller than a float can hold. p = "p = %.3g" % r["p"] if r["p"] > 0 else "p < 1e-300" print("%s %d right, %s %d right of %d; only %s right %d, only %s right %d; exact McNemar %s" % (args.a, r["a_right"], args.b, r["b_right"], r["n"], args.a, r["only_a"], args.b, r["only_b"], p)) if __name__ == "__main__": main()