"""Summary of the two labellers' readings of the public replay's extras, by the rubric written before any label existed (eval/public_replay_label_rubric.json). The labels themselves hold other projects' text, so they stay outside the repository; this writes counts (an extra the card listed twice, same path and kind, counts once), Cohen's kappa, the disputed extras and, for GitLab's own projects, references and paths. usage: python3 eval/label_summary.py --labels [--raw results/public_replay_a1_raw.json]""" from __future__ import annotations import argparse, collections, json, pathlib, sys sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import replay_gitlab as G LABELS = ("real", "borderline", "not_an_extra") OUT = G.ROOT / "results" / "public_replay_labels.json" def ref_of(packet: str) -> tuple[str, str]: """'G_flagged__gitlab-org_gitlab_242496.json' -> ('G_flagged', 'gitlab-org/gitlab!242496'); a field project's hash has no underscore, so 'F_flagged__1566bdff_1.json' -> ('F_flagged', '1566bdff!1').""" group, rest = packet.split("__", 1) project, iid = rest.removesuffix(".json").rsplit("_", 1) return group, f"{project.replace('_', '/')}!{iid}" def kappa(pairs: list) -> float | None: """Cohen's kappa for two labellers over the same items.""" n = len(pairs) if not n: return None seen = sum(a == b for a, b in pairs) / n one, two = collections.Counter(a for a, _ in pairs), collections.Counter(b for _, b in pairs) chance = sum(one[x] * two[x] for x in set(one) | set(two)) / n / n return None if chance == 1 else round((seen - chance) / (1 - chance), 3) def per_merge_request(rows: list) -> dict: """rows: (packet, l1, l2) per extra. Strict: both call the same extra real. Lenient: either calls any extra real or borderline.""" out = {} for packet, a, b in rows: s = out.setdefault(packet, {"strict": False, "lenient": False}) s["strict"] |= a == b == "real" s["lenient"] |= a in ("real", "borderline") or b in ("real", "borderline") return out def summarize(l1: dict, l2: dict, raw: list, description_of=None) -> dict: key = lambda e: (e["packet"], e["path"], e["kind"]) one, two = {key(e): e for e in l1["extras"]}, {key(e): e for e in l2["extras"]} both = sorted(set(one) & set(two)) rows = [(k[0], one[k]["label"], two[k]["label"]) for k in both] mrs = per_merge_request(rows) groups = {} for packet, s in sorted(mrs.items()): g = groups.setdefault(ref_of(packet)[0], {"merge_requests": 0, "strict": 0, "lenient": 0}) g["merge_requests"] += 1 g["strict"] += s["strict"] g["lenient"] += s["lenient"] for g in groups.values(): g["strict_wilson95"], g["lenient_wilson95"] = G.wilson(g["strict"], g["merge_requests"]), G.wilson(g["lenient"], g["merge_requests"]) kinds = collections.defaultdict(collections.Counter) for k in both: for lab in (one[k]["label"], two[k]["label"]): kinds[k[2]][lab] += 1 def shown(k): # field projects: counts only group, ref = ref_of(k[0]) e = {"group": group, "kind": k[2], "L1": one[k]["label"], "L2": two[k]["label"]} return e if group.startswith("F") else {"ref": ref, "path": k[1], **e} disputed = [shown(k) for k in both if one[k]["label"] != two[k]["label"]] issue_of = {r["ref"]: r for r in raw if r["pop"] == "G" and r["version"] == "agent"} def named(ref): # found after labelling: does the description name the issue the replay read? r = issue_of.get(ref) if not r or not r.get("issue_refs") or description_of is None: return None return G.CI.mentions_issue(description_of(ref), int(r["issue_refs"][0].lstrip("#")), "gitlab-org/gitlab") m2 = {e["packet"]: e["answer"] for e in l2["misses"]} misses = [{"ref": ref_of(e["packet"])[1], "L1": e["answer"], "L2": m2.get(e["packet"]), "issue_named_in_description": named(ref_of(e["packet"])[1])} for e in l1["misses"]] both_yes = [m for m in misses if m["L1"] == m["L2"] == "yes"] return { "what": "The public replay's listed extras, read by two labellers by the rubric written before any label existed.", "rubric": "eval/public_replay_label_rubric.json", "labellers": {"L1": l1.get("model"), "L2": l2.get("model")}, "agreement": {"entries": {"L1": len(l1["extras"]), "L2": len(l2["extras"])}, "extras": len(both), "only_one_labeller": len(set(one) ^ set(two)), "same_label": sum(a == b for _, a, b in rows), "kappa": kappa([(a, b) for _, a, b in rows]), "disputed": disputed}, "per_merge_request": groups, "G_flagged_issue_named_in_description": {ref_of(p)[1]: named(ref_of(p)[1]) for p in sorted(mrs) if p.startswith("G_flagged")}, "labels_by_kind": {k: dict(v) for k, v in sorted(kinds.items())}, "extras": [shown(k) for k in both if not ref_of(k[0])[0].startswith("F")], "misses": {"sampled": len(misses), "both_yes": len(both_yes), "either_yes": sum("yes" in (m["L1"], m["L2"]) for m in misses), "both_yes_with_the_issue_not_named_in_the_description": sum(m["issue_named_in_description"] is False for m in both_yes), "rows": misses}, "found_after_labelling": "Not in the rubric. For the replay's G rows the issue came from the flow's note or the number in its branch name; " "issue_named_in_description says whether the merge request's description names that issue of gitlab-org/gitlab " "(as #N or its URL). Where it does not, the labellers may have read the merge request against an unrelated issue, " "so a 'yes' there is a linking error of the replay more than a missed extra. The CI job reads a branch's issue only " "when the description names it too (itemator.ci.mentions_issue).", "limits": "Model readings, not maintainers; one diff each; the rules were not changed after reading these labels.", } def main(argv=None) -> int: ap = argparse.ArgumentParser(); ap.add_argument("--labels", required=True) ap.add_argument("--raw", default=str(G.ROOT / "results" / "public_replay_a1_raw.json")) a = ap.parse_args(argv) d = pathlib.Path(a.labels) l1, l2 = (json.loads((d / f"L{i}_labels.json").read_text()) for i in (1, 2)) raw = json.loads(pathlib.Path(a.raw).read_text()) describe = lambda ref: (G.get(f"projects/{G.GITLAB}/merge_requests/{ref.split('!')[1]}") or {}).get("description") or "" s = summarize(l1, l2, raw, describe) OUT.write_text(json.dumps(s, indent=1) + "\n") print(json.dumps({k: s[k] for k in ("per_merge_request",)}, indent=1), {k: v for k, v in s["agreement"].items() if k != "disputed"}, {k: v for k, v in s["misses"].items() if k != "rows"}) return 0 if __name__ == "__main__": sys.exit(main())