File size: 1,278 Bytes
e79d917 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | """Macro F0.5 exactly as defined in the problem statement."""
import numpy as np
def entity_scores(pred, truth, beta=0.5):
"""pred, truth: iterables of IDs for ONE Source-1 entity. Returns (f, precision, recall)."""
p, t = set(pred), set(truth)
if not t:
return (1.0, 1.0, 1.0) if not p else (0.0, 0.0, 1.0)
if not p:
return 0.0, 0.0, 0.0
tp = len(p & t)
prec, rec = tp / len(p), tp / len(t)
if tp == 0:
return 0.0, prec, rec
b2 = beta * beta
return (1 + b2) * prec * rec / (b2 * prec + rec), prec, rec
def macro_f05(preds, truths, ids):
"""preds / truths: dict id -> list. Every id in `ids` is scored (missing pred = empty)."""
rows = [entity_scores(preds.get(i, []), truths.get(i, [])) for i in ids]
a = np.array(rows) if rows else np.zeros((0, 3))
return {"f05": float(a[:, 0].mean()), "precision": float(a[:, 1].mean()),
"recall": float(a[:, 2].mean()), "n": len(ids)}, a[:, 0]
def _self_test():
f, p, r = entity_scores(["S2-00047", "S2-00193", "S3-00812"], ["S2-00047", "S3-00812"])
assert abs(f - 0.714) < 1e-3, f
assert entity_scores([], [])[0] == 1.0
assert entity_scores(["S2-1"], [])[0] == 0.0
assert entity_scores([], ["S2-1"])[0] == 0.0
_self_test()
|