"""The GitLab-native scope-comment study (eval/gitlab_scope_rubric.json, registered before any note was fetched): on merge requests GitLab Duo's Developer Flow opened in gitlab-org/gitlab, the notes people other than the author wrote, read blind by two labellers, then the rules run on the files those notes point at. GET only, through `glab api`. usage (repository root, glab authenticated): python3 eval/gitlab_scope.py --frame # the duo/ merge requests: results/gitlab_scope_frame.json python3 eval/gitlab_scope.py --notes DIR # people's notes, numbered, in shard files under DIR (not committed) API answers are cached under eval/cache/gitlab/ (gitignored).""" from __future__ import annotations import argparse, concurrent.futures, json, pathlib, re, subprocess, sys ROOT = pathlib.Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "eval")) import replay_gitlab as G # noqa: E402 PROJECT, PATH = 278964, "gitlab-org/gitlab" WINDOW = ("2026-06-01T00:00:00Z", "2026-10-18T00:00:00Z") FRAME = ROOT / "results" / "gitlab_scope_frame.json" SHARD = 60 # notes per shard file NOTE_MAX = 1500 # characters of a note a labeller sees KEYWORDS = ("unrelated", "out of scope", "separate pr", "separate mr", "not needed", "unnecessary") _PAGE = """{ project(fullPath: "%s") { mergeRequests(state: %s, createdAfter: "%s", createdBefore: "%s", first: 100%s) { count pageInfo { endCursor hasNextPage } nodes { iid state sourceBranch createdAt } } } }""" def graphql(query: str) -> dict: r = None for _ in range(5): r = subprocess.run(["glab", "api", "graphql", "-f", f"query={query}"], capture_output=True, text=True) if r.returncode == 0: d = json.loads(r.stdout) if "errors" not in d: return d["data"] raise RuntimeError(f"GraphQL failed: {(r.stderr + r.stdout)[:300] if r else ''}") def listing(state: str) -> tuple[int, list]: rows, after, count = [], "", None while True: m = graphql(_PAGE % (PATH, state, *WINDOW, after))["project"]["mergeRequests"] count = m["count"] if count is None else count rows += m["nodes"] if not m["pageInfo"]["hasNextPage"]: return count, rows after = f', after: "{m["pageInfo"]["endCursor"]}"' def frame() -> dict: out = {"project": PATH, "window": WINDOW, "counts": {}, "rows": []} for state in ("merged", "closed", "opened"): count, rows = listing(state) if state != "opened" else (None, []) if state == "opened": count = graphql((_PAGE % (PATH, state, *WINDOW, "")).replace("first: 100", "first: 1"))["project"]["mergeRequests"]["count"] else: assert len(rows) == count, (state, len(rows), count) # never a silently cut list duo = [r for r in rows if r["sourceBranch"].startswith("duo/")] out["counts"][state] = {"all": count, "duo": len(duo) if state != "opened" else None} out["rows"] += [{"iid": int(r["iid"]), "state": r["state"], "branch": r["sourceBranch"], "created_at": r["createdAt"]} for r in duo] return out def is_flow_account(username: str) -> bool: return (username or "").startswith("duo-") def human_notes(mr: dict, notes: list) -> list: author = (mr.get("author") or {}).get("username") keep = [] for n in notes or []: who = (n.get("author") or {}).get("username") if n.get("system") or who == author or G.is_bot(who) or is_flow_account(who): continue pos = n.get("position") or {} keep.append({"note_id": n["id"], "author": who, "created_at": n["created_at"], "type": n.get("type"), "file": pos.get("new_path") or pos.get("old_path"), "body": n.get("body") or ""}) return keep def changed_paths(iid: int) -> list: rows = G.pages(f"projects/{PROJECT}/merge_requests/{iid}/diffs", limit=30) return sorted({r.get("new_path") or r.get("old_path") for r in rows if isinstance(r, dict)}) def tie(note: dict, paths: list): """A diff note's own file; a general note's file only when it names exactly one changed path.""" if note["file"]: return note["file"] named = [p for p in paths if p in note["body"]] return named[0] if len(named) == 1 else None def keyword(body: str) -> bool: b = body.lower() return any(k in b for k in KEYWORDS) def prefetch(rows: list, workers: int = 4) -> None: """Fill the cache in parallel (GET only); notes() then reads from it in order.""" def one(iid): mr = G.get(f"projects/{PROJECT}/merge_requests/{iid}") if mr and human_notes(mr, G.notes_of(PROJECT, iid)): changed_paths(iid) with concurrent.futures.ThreadPoolExecutor(workers) as ex: list(ex.map(one, [r["iid"] for r in rows])) def notes(out_dir: pathlib.Path) -> dict: fr = json.loads(FRAME.read_text()) prefetch(fr["rows"]) items = [] for row in fr["rows"]: mr = G.get(f"projects/{PROJECT}/merge_requests/{row['iid']}") if not mr: continue hn = human_notes(mr, G.notes_of(PROJECT, row["iid"])) if not hn: continue paths = changed_paths(row["iid"]) for n in hn: items.append({"iid": row["iid"], "state": row["state"], "title": mr.get("title") or "", "branch": row["branch"], "url": f"https://gitlab.com/{PATH}/-/merge_requests/{row['iid']}#note_{n['note_id']}", "changed_paths": paths[:80], "changed_paths_total": len(paths), **n, "tied_file": tie(n, paths), "keyword": keyword(n["body"])}) out_dir.mkdir(parents=True, exist_ok=True) for k in range(0, len(items), SHARD): part = items[k:k + SHARD] lines = [f"# Shard {k // SHARD + 1}: notes {k + 1} to {k + len(part)} of {len(items)}", ""] for pos, it in enumerate(part, start=k + 1): body = it["body"] if len(it["body"]) <= NOTE_MAX else it["body"][:NOTE_MAX] + " [cut]" shown = ", ".join(it["changed_paths"]) + (f" (and {it['changed_paths_total'] - 80} more)" if it["changed_paths_total"] > 80 else "") lines += [f"## Note {pos}", f"- merge request: !{it['iid']} ({it['state']}) {it['title']}", f"- url: {it['url']}", f"- by: {it['author']} on {it['created_at'][:10]}; " + (f"on the line of file `{it['file']}`" if it["file"] else "a general note"), f"- files the merge request changes: {shown}", "", "```text", body.replace("```", "'''"), "```", ""] (out_dir / f"shard-{k // SHARD + 1:02d}.md").write_text("\n".join(lines)) (out_dir / "notes.json").write_text(json.dumps(items, indent=1)) return {"merge_requests_read": len(fr["rows"]), "with_people_notes": len({i["iid"] for i in items}), "notes": len(items), "keyword_tied": sum(1 for i in items if i["keyword"] and i["tied_file"]), "shards": -(-len(items) // SHARD)} SEED, SAMPLE = 20261018, 1500 NOT_FOUND: list = [] GHOST = "ghost1" def flagged_bots(usernames: set) -> set: """Authors GitLab itself marks as bots (the user's `bot` field), plus the Ghost User (amendment 2).""" names, bots, seen = sorted(usernames), set(), set() for k in range(0, len(names), 100): quoted = ", ".join(json.dumps(n) for n in names[k:k + 100]) for u in graphql(f"{{ users(usernames: [{quoted}], first: 100) {{ nodes {{ username bot }} }} }}")["users"]["nodes"]: seen.add(u["username"]) if u["bot"] is True: bots.add(u["username"]) NOT_FOUND[:] = sorted(set(names) - seen) # renamed or blocked accounts: their bot flag cannot be read, so kept return bots | {GHOST} def write_shards(items: list, out_dir: pathlib.Path) -> None: out_dir.mkdir(parents=True, exist_ok=True) for k in range(0, len(items), SHARD): part = items[k:k + SHARD] lines = [f"# Shard {k // SHARD + 1}: notes {k + 1} to {k + len(part)} of {len(items)}", ""] for pos, it in enumerate(part, start=k + 1): body = it["body"] if len(it["body"]) <= NOTE_MAX else it["body"][:NOTE_MAX] + " [cut]" shown = ", ".join(it["changed_paths"]) + (f" (and {it['changed_paths_total'] - 80} more)" if it["changed_paths_total"] > 80 else "") lines += [f"## Note {pos}", f"- merge request: !{it['iid']} ({it['state']}) {it['title']}", f"- url: {it['url']}", f"- by: {it['author']} on {it['created_at'][:10]}; " + (f"on the line of file `{it['file']}`" if it["file"] else "a general note"), f"- files the merge request changes: {shown}", "", "```text", body.replace("```", "\'\'\'"), "```", ""] (out_dir / f"shard-{k // SHARD + 1:02d}.md").write_text("\n".join(lines)) def sample(all_notes: pathlib.Path, out_dir: pathlib.Path) -> dict: """Amendment 2: drop bot-flagged and deleted authors, then a seeded sample plus every keyword note, shuffled.""" import random items = json.loads(all_notes.read_text()) bots = flagged_bots({i["author"] for i in items}) left = [i for i in items if i["author"] not in bots] rng = random.Random(SEED) kw = [i for i in left if i["keyword"]] rest = [i for i in left if not i["keyword"]] drawn = rng.sample(left, SAMPLE) chosen = {id(i) for i in drawn} | {id(i) for i in kw} labelled = [i for i in left if id(i) in chosen] rng.shuffle(labelled) for n, i in enumerate(labelled, start=1): i["position"], i["in_random_draw"] = n, any(i is d for d in drawn) write_shards(labelled, out_dir) (out_dir / "sample.json").write_text(json.dumps(labelled, indent=1)) return {"notes": len(items), "bot_or_ghost_authors": sorted(bots), "authors_not_found_kept": NOT_FOUND, "notes_after_exclusion": len(left), "keyword_notes": len(kw), "random_draw": SAMPLE, "labelled": len(labelled), "shards": -(-len(labelled) // SHARD), "not_keyword_left": len(rest)} _LINE = re.compile(r"^NOTE (\d+) \| (yes|no|unclear) \| (.*?) \| (.*)$") def labels(reports: pathlib.Path, tier: str) -> dict: """The labellers' NOTE lines of one tier (H or S), by note number; a number labelled twice is an error.""" out = {} for f in sorted(reports.glob(f"L-{tier}-*.md")): for line in f.read_text().splitlines(): m = _LINE.match(line.strip().strip("`")) if m: n = int(m.group(1)) assert n not in out, (tier, n, f.name) out[n] = {"answer": m.group(2), "file": m.group(3).strip(), "quote": m.group(4).strip()} return out def adjudication(shards: pathlib.Path, reports: pathlib.Path) -> dict: """Amendment-2 numbering: the notes the two tiers split on, plus every 7th note, for the adjudicator.""" items = {i["position"]: i for i in json.loads((shards / "sample.json").read_text())} h, s = labels(reports, "H"), labels(reports, "S") missing = {"H": sorted(set(items) - set(h)), "S": sorted(set(items) - set(s))} split = sorted(n for n in items if n in h and n in s and h[n]["answer"] != s[n]["answer"]) sample = sorted(n for n in items if n % 7 == 0) todo = sorted(set(split) | set(sample) | set(missing["H"]) | set(missing["S"])) fence = "`" * 3 lines = ["# Notes to adjudicate (split answers, every 7th note, and any note a labeller missed)", ""] for n in todo: it = items[n] body = it["body"] if len(it["body"]) <= NOTE_MAX else it["body"][:NOTE_MAX] + " [cut]" lines += [f"## Note {n}" + (" (sample)" if n % 7 == 0 else ""), f"- merge request: !{it['iid']} ({it['state']}) {it['title']}", f"- by: {it['author']}; " + (f"on the line of file `{it['file']}`" if it["file"] else "a general note"), "- files the merge request changes: " + ", ".join(it["changed_paths"][:40]), "", fence + "text", body.replace(fence, "'" * 3), fence, f"- Haiku: {h.get(n, {}).get('answer', 'MISSING')} | {h.get(n, {}).get('file', '-')}", f"- Sonnet: {s.get(n, {}).get('answer', 'MISSING')} | {s.get(n, {}).get('file', '-')}", ""] both_yes = sorted(n for n in items if h.get(n, {}).get("answer") == "yes" and s.get(n, {}).get("answer") == "yes") agree = sum(1 for n in items if n in h and n in s and h[n]["answer"] == s[n]["answer"]) return {"lines": lines, "missing": missing, "split": split, "sample": sample, "both_yes": both_yes, "agree": agree, "n": len(items)} _ADJ = re.compile(r"^NOTE (\d+) \| (yes|no|unclear) \| (.*?) \| agrees: (\w+) \| (.*)$") def adjudicated(reports: pathlib.Path) -> dict: out = {} for line in (reports / "ADJ.md").read_text().splitlines(): m = _ADJ.match(line.strip().strip("`")) if m: out[int(m.group(1))] = {"answer": m.group(2), "file": m.group(3).strip(), "agrees": m.group(4)} return out def final_labels(shards: pathlib.Path, reports: pathlib.Path) -> dict: """The rubric's rule: a scope comment when both labellers say yes; a split goes to the adjudicator.""" h, s, a = labels(reports, "H"), labels(reports, "S"), adjudicated(reports) out = {} for n in h: if h[n]["answer"] == s[n]["answer"]: out[n] = {"answer": h[n]["answer"], "by": "both", "file": s[n]["file"]} else: assert n in a, f"split note {n} has no adjudication" out[n] = {"answer": a[n]["answer"], "by": "adjudicator", "file": a[n]["file"]} return out def version_at(iid: int, when: str): """The newest diff version created at or before the note, else the oldest (rubric: files).""" vs = sorted(G.get(f"projects/{PROJECT}/merge_requests/{iid}/versions") or [], key=lambda v: v["created_at"]) if not vs: return None before = [v for v in vs if v["created_at"] <= when] v = (before or vs[:1])[-1] return G.rebuilt(PROJECT, G.get(f"projects/{PROJECT}/merge_requests/{iid}/versions/{v['id']}") or {}) def task_of(mr: dict, notes: list) -> tuple[str, str]: n = G.flow_issue(notes, "duo-developer-gitlab-org", mr.get("source_branch")) raw = G.get(f"projects/{PROJECT}/issues/{n}") if n else None if raw: return f"{raw.get('title') or ''}\n{raw.get('description') or ''}", "flow_issue" for k in G.CI.closing_refs(mr.get("description")): raw = G.get(f"projects/{PROJECT}/issues/{k}") if raw: return f"{raw.get('title') or ''}\n{raw.get('description') or ''}", "closing_link" return f"{mr.get('title') or ''}\n{mr.get('description') or ''}", "mr_text" def score_note(it: dict, file: str | None) -> dict: import aidev_recall as A row = {"position": it["position"], "url": it["url"], "iid": it["iid"], "state": it["state"], "path": file, "kind": A.listable_kind(file) if file else None, "in_random_draw": it["in_random_draw"], "keyword": it["keyword"]} if not file: return dict(row, scored=False, why="no file") mr = G.get(f"projects/{PROJECT}/merge_requests/{it['iid']}") or {} v = version_at(it["iid"], it["created_at"]) files = G.to_files((v or {}).get("diffs") or []) if not files: return dict(row, scored=False, why="no readable version") if file not in {f["filename"] for f in files}: return dict(row, scored=False, why="not in the change then") task, src = task_of(mr, G.notes_of(PROJECT, it["iid"])) res = A.X.analyse(task, files, repo=PATH, version=6) return dict(row, scored=True, task_from=src, hit=A.listed(res.extras, file), files=len(files)) def scored_share(rows: list) -> dict: s = [r for r in rows if r.get("scored")] k = sum(1 for r in s if r["hit"]) return {"comments": len(s), "listed": k, "wilson95": G.wilson(k, len(s)) if s else None} def score(shards: pathlib.Path, reports: pathlib.Path) -> dict: items = {i["position"]: i for i in json.loads((shards / "sample.json").read_text())} final = final_labels(shards, reports) read_route = [] for n, lab in sorted(final.items()): if lab["answer"] != "yes": continue it = items[n] file = it["file"] or (lab["file"] if lab["file"] in it["changed_paths"] else None) or it["tied_file"] read_route.append(dict(score_note(it, file), by=lab["by"])) keyword_route = [dict(score_note(it, it["file"] or it["tied_file"]), label=final[n]["answer"]) for n, it in sorted(items.items()) if it["keyword"] and (it["file"] or it["tied_file"])] listable = [r for r in read_route if r["kind"]] h, s = labels(reports, "H"), labels(reports, "S") return { "rubric": "eval/gitlab_scope_rubric.json", "rules_version": 6, "labelled_notes": len(items), "read_route": {"scope_comments": len(read_route), "primary_listable_kind": scored_share(listable), "all": scored_share(read_route), "not_scored": {w: sum(1 for r in read_route if r.get("why") == w) for w in ("no file", "no readable version", "not in the change then")}, "by_state": {st: scored_share([r for r in read_route if r["state"] == st]) for st in ("merged", "closed")}, "in_random_draw": sum(1 for r in read_route if r["in_random_draw"])}, "keyword_route": {"notes": len(keyword_route), "all": scored_share(keyword_route), "listable_kind": scored_share([r for r in keyword_route if r["kind"]]), "labelled_not_scope": sum(1 for r in keyword_route if r["label"] != "yes")}, "agreement": {"both_yes": sum(1 for n in h if h[n]["answer"] == s[n]["answer"] == "yes"), "both_no": sum(1 for n in h if h[n]["answer"] == s[n]["answer"] == "no"), "same_answer": sum(1 for n in h if h[n]["answer"] == s[n]["answer"]), "split": sum(1 for n in h if h[n]["answer"] != s[n]["answer"])}, "rows": read_route, "keyword_rows": keyword_route, } def main(argv=None) -> int: ap = argparse.ArgumentParser() ap.add_argument("--frame", action="store_true") ap.add_argument("--notes") ap.add_argument("--sample", nargs=2, metavar=("ALL_NOTES_JSON", "DIR")) ap.add_argument("--score", nargs=2, metavar=("SHARDS_DIR", "REPORTS_DIR")) a = ap.parse_args(argv) if a.frame: fr = frame() FRAME.write_text(json.dumps(fr, indent=1) + "\n") print(json.dumps(fr["counts"])) if a.notes: print(json.dumps(notes(pathlib.Path(a.notes)))) if a.sample: print(json.dumps(sample(pathlib.Path(a.sample[0]), pathlib.Path(a.sample[1])))) if a.score: out = score(pathlib.Path(a.score[0]), pathlib.Path(a.score[1])) (ROOT / "results" / "gitlab_scope_v6.json").write_text(json.dumps(out, indent=1) + "\n") print(json.dumps({k: v for k, v in out.items() if not k.endswith("rows")})) return 0 if __name__ == "__main__": sys.exit(main())