Sentinel / eval /label_summary.py
Anbu-00001's picture
Itemator runner: token-guarded evaluation jobs (replaces the probe)
f10d14f verified
Raw History Blame Contribute Delete
7.1 kB
"""Summary of the two labellers' readings of the public replay's extras, by the rubric written before any label existed
(eval/public_replay_label_rubric.json). The labels themselves hold other projects' text, so they stay outside the
repository; this writes counts (an extra the card listed twice, same path and kind, counts once), Cohen's kappa, the disputed extras and, for GitLab's own projects, references and paths.
usage: python3 eval/label_summary.py --labels <dir with L1_labels.json and L2_labels.json> [--raw results/public_replay_a1_raw.json]"""
from __future__ import annotations
import argparse, collections, json, pathlib, sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import replay_gitlab as G
LABELS = ("real", "borderline", "not_an_extra")
OUT = G.ROOT / "results" / "public_replay_labels.json"
def ref_of(packet: str) -> tuple[str, str]:
"""'G_flagged__gitlab-org_gitlab_242496.json' -> ('G_flagged', 'gitlab-org/gitlab!242496'); a field project's hash
has no underscore, so 'F_flagged__1566bdff_1.json' -> ('F_flagged', '1566bdff!1')."""
group, rest = packet.split("__", 1)
project, iid = rest.removesuffix(".json").rsplit("_", 1)
return group, f"{project.replace('_', '/')}!{iid}"
def kappa(pairs: list) -> float | None:
"""Cohen's kappa for two labellers over the same items."""
n = len(pairs)
if not n:
return None
seen = sum(a == b for a, b in pairs) / n
one, two = collections.Counter(a for a, _ in pairs), collections.Counter(b for _, b in pairs)
chance = sum(one[x] * two[x] for x in set(one) | set(two)) / n / n
return None if chance == 1 else round((seen - chance) / (1 - chance), 3)
def per_merge_request(rows: list) -> dict:
"""rows: (packet, l1, l2) per extra. Strict: both call the same extra real. Lenient: either calls any extra real or
borderline."""
out = {}
for packet, a, b in rows:
s = out.setdefault(packet, {"strict": False, "lenient": False})
s["strict"] |= a == b == "real"
s["lenient"] |= a in ("real", "borderline") or b in ("real", "borderline")
return out
def summarize(l1: dict, l2: dict, raw: list, description_of=None) -> dict:
key = lambda e: (e["packet"], e["path"], e["kind"])
one, two = {key(e): e for e in l1["extras"]}, {key(e): e for e in l2["extras"]}
both = sorted(set(one) & set(two))
rows = [(k[0], one[k]["label"], two[k]["label"]) for k in both]
mrs = per_merge_request(rows)
groups = {}
for packet, s in sorted(mrs.items()):
g = groups.setdefault(ref_of(packet)[0], {"merge_requests": 0, "strict": 0, "lenient": 0})
g["merge_requests"] += 1
g["strict"] += s["strict"]
g["lenient"] += s["lenient"]
for g in groups.values():
g["strict_wilson95"], g["lenient_wilson95"] = G.wilson(g["strict"], g["merge_requests"]), G.wilson(g["lenient"], g["merge_requests"])
kinds = collections.defaultdict(collections.Counter)
for k in both:
for lab in (one[k]["label"], two[k]["label"]):
kinds[k[2]][lab] += 1
def shown(k): # field projects: counts only
group, ref = ref_of(k[0])
e = {"group": group, "kind": k[2], "L1": one[k]["label"], "L2": two[k]["label"]}
return e if group.startswith("F") else {"ref": ref, "path": k[1], **e}
disputed = [shown(k) for k in both if one[k]["label"] != two[k]["label"]]
issue_of = {r["ref"]: r for r in raw if r["pop"] == "G" and r["version"] == "agent"}
def named(ref): # found after labelling: does the description name the issue the replay read?
r = issue_of.get(ref)
if not r or not r.get("issue_refs") or description_of is None:
return None
return G.CI.mentions_issue(description_of(ref), int(r["issue_refs"][0].lstrip("#")), "gitlab-org/gitlab")
m2 = {e["packet"]: e["answer"] for e in l2["misses"]}
misses = [{"ref": ref_of(e["packet"])[1], "L1": e["answer"], "L2": m2.get(e["packet"]),
"issue_named_in_description": named(ref_of(e["packet"])[1])} for e in l1["misses"]]
both_yes = [m for m in misses if m["L1"] == m["L2"] == "yes"]
return {
"what": "The public replay's listed extras, read by two labellers by the rubric written before any label existed.",
"rubric": "eval/public_replay_label_rubric.json",
"labellers": {"L1": l1.get("model"), "L2": l2.get("model")},
"agreement": {"entries": {"L1": len(l1["extras"]), "L2": len(l2["extras"])}, "extras": len(both), "only_one_labeller": len(set(one) ^ set(two)), "same_label": sum(a == b for _, a, b in rows),
"kappa": kappa([(a, b) for _, a, b in rows]), "disputed": disputed},
"per_merge_request": groups,
"G_flagged_issue_named_in_description": {ref_of(p)[1]: named(ref_of(p)[1]) for p in sorted(mrs) if p.startswith("G_flagged")},
"labels_by_kind": {k: dict(v) for k, v in sorted(kinds.items())},
"extras": [shown(k) for k in both if not ref_of(k[0])[0].startswith("F")],
"misses": {"sampled": len(misses), "both_yes": len(both_yes), "either_yes": sum("yes" in (m["L1"], m["L2"]) for m in misses),
"both_yes_with_the_issue_not_named_in_the_description": sum(m["issue_named_in_description"] is False for m in both_yes),
"rows": misses},
"found_after_labelling": "Not in the rubric. For the replay's G rows the issue came from the flow's note or the number in its branch name; "
"issue_named_in_description says whether the merge request's description names that issue of gitlab-org/gitlab "
"(as #N or its URL). Where it does not, the labellers may have read the merge request against an unrelated issue, "
"so a 'yes' there is a linking error of the replay more than a missed extra. The CI job reads a branch's issue only "
"when the description names it too (itemator.ci.mentions_issue).",
"limits": "Model readings, not maintainers; one diff each; the rules were not changed after reading these labels.",
}
def main(argv=None) -> int:
ap = argparse.ArgumentParser(); ap.add_argument("--labels", required=True)
ap.add_argument("--raw", default=str(G.ROOT / "results" / "public_replay_a1_raw.json"))
a = ap.parse_args(argv)
d = pathlib.Path(a.labels)
l1, l2 = (json.loads((d / f"L{i}_labels.json").read_text()) for i in (1, 2))
raw = json.loads(pathlib.Path(a.raw).read_text())
describe = lambda ref: (G.get(f"projects/{G.GITLAB}/merge_requests/{ref.split('!')[1]}") or {}).get("description") or ""
s = summarize(l1, l2, raw, describe)
OUT.write_text(json.dumps(s, indent=1) + "\n")
print(json.dumps({k: s[k] for k in ("per_merge_request",)}, indent=1), {k: v for k, v in s["agreement"].items() if k != "disputed"},
{k: v for k, v in s["misses"].items() if k != "rows"})
return 0
if __name__ == "__main__":
sys.exit(main())