Blame for eval/asker_study.py

"""Reply study for names.asker_required: six pre-registered cases (eval/asker_study_cases.json) on one throwaway merge request.
The issue is written by the second account (the ASKER); the merge request is opened by the first account (the OTHER person,
also the merge request author). The target branch's .itemator.toml sets names.asker_required = true. The card flow opens one
thread per extra; each case's reply is typed by the account named in the case ('ASKER' = second account with its token from
~/.itemator-second-token, 'OTHER' = the first account); the reaction flow is started once with a mention and every thread is read
afterwards. Raw record: results/asker_study_raw.json, counts: results/asker_study.json.

usage (repository root, glab authenticated; spends Duo credits, about two flow runs):
    python3 eval/asker_study.py --credit-stop 241
    python3 eval/asker_study.py --summarize
The token is read from the file and sent only to gitlab.com; it is never printed or stored.
"""
from __future__ import annotations

import argparse, base64, json, os, pathlib, sys, time, urllib.parse, urllib.request

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import react_study as R                                    # the same helpers and the same platform constants

ROOT = R.ROOT
CASES = ROOT / "eval" / "asker_study_cases.json"
RAW = ROOT / "results" / "asker_study_raw.json"
OUT = ROOT / "results" / "asker_study.json"
BRANCHES = ("probe/as-src", "probe/as-target")
P = R.P
MENTION = R.REACT_MENTION
TOKEN_FILE = pathlib.Path("~/.itemator-second-token").expanduser()


def second(method, path, **fields):
    """One call as the second account. The token never leaves this function."""
    tok = TOKEN_FILE.read_text().strip()
    data = urllib.parse.urlencode(fields).encode() if fields else None
    req = urllib.request.Request(f"https://gitlab.com/api/v4/{path}", data=data, method=method, headers={"PRIVATE-TOKEN": tok})
    with urllib.request.urlopen(req, timeout=60) as r:
        return json.loads(r.read() or b"null")


def run(doc):
    cases = doc["cases"]
    rec = {"started": time.strftime("%Y-%m-%dT%H:%M:%S"), "cases": []}
    issue = second("POST", f"projects/{P}/issues", title="Probe: fix a typo in the README (asker study)",
                   description="Fix one typo in the README heading. Nothing else is asked.")
    asker = issue["author"]["username"]
    rec.update(issue_iid=issue["iid"], asker=asker)
    R.api(f"projects/{P}/repository/branches", "POST", branch=BRANCHES[1], ref="main")
    R.api(f"projects/{P}/repository/branches", "POST", branch=BRANCHES[0], ref=BRANCHES[1])

    def read(path, ref):
        return base64.b64decode(R.api(f"projects/{P}/repository/files/{path.replace('/', '%2F')}?ref={ref}")["content"]).decode()
    pol = read(".itemator.toml", BRANCHES[1])
    assert "other_person_required = false" in pol
    R.commit(BRANCHES[1], "Only the issue's author keeps an extra (test target only)",
             [{"action": "update", "file_path": ".itemator.toml",
               "content": pol.replace("other_person_required = false", "other_person_required = false\nasker_required = true", 1)}])
    rd, ci = read("README.md", BRANCHES[0]), read(".gitlab-ci.yml", BRANCHES[0])
    R.commit(BRANCHES[0], "Fix a typo in the README", [
        {"action": "update", "file_path": "README.md", "content": rd.replace("## Related tools\n", "## Related tools:\n", 1)},
        {"action": "create", "file_path": "docker-compose.yml", "content": "services: {}\n"},
        {"action": "create", "file_path": "Dockerfile", "content": "FROM python:3.12-slim\nRUN true\n"},
        {"action": "create", "file_path": "notes.txt", "content": "scratch notes\n"},
        {"action": "create", "file_path": "setup.cfg", "content": "[metadata]\nname = probe\n"},
        {"action": "create", "file_path": "tox.ini", "content": "[tox]\nenvlist = py312\n"},
        {"action": "update", "file_path": ".gitlab-ci.yml", "content": "# probe: ordering note\n" + ci}])
    mr = R.api(f"projects/{P}/merge_requests", "POST", source_branch=BRANCHES[0], target_branch=BRANCHES[1],
               title="Fix a typo in the README", description=f"Closes #{issue['iid']}\n\nFixes the heading.")
    iid = rec["mr_iid"] = mr["iid"]
    t0 = time.time()
    while time.time() - t0 < 720:
        pl = R.api(f"projects/{P}/merge_requests/{iid}/pipelines")
        if pl and pl[0]["status"] in ("success", "failed"):
            rec["pipeline"] = pl[0]["status"]; break
        time.sleep(15)
    try:
        R.trigger("card", True)
        R.api(f"projects/{P}/merge_requests/{iid}", "PUT", reviewer_ids=R.CARD_USER)
        rec["card_done"] = R.wait_session_done(iid, "card", 1200)
    finally:
        R.trigger("card", False)
    time.sleep(5)
    threads = {}
    for d in R.discussions(iid):
        first = d["notes"][0]
        if "card" in first["author"]["username"] and "reply with its exact name" in first["body"]:
            ticks = first["body"].split("`")
            if len(ticks) > 1:
                threads[ticks[1]] = (d["id"], first["body"])
    rec["threads_found"] = sorted(threads)
    rec["asker_sentence_in_every_thread"] = all(f"(@{asker}) can keep this." in b for _, b in threads.values()) if threads else False
    for c in cases:
        d = threads.get(c["name"], (None, ""))[0]
        c["sent"] = [r["text"].replace("{NAME}", c["name"]) for r in c["replies"]]
        if not d:
            continue
        for text in c["sent"]:
            if c["author"] == "ASKER":
                second("POST", f"projects/{P}/merge_requests/{iid}/discussions/{d}/notes", body=text)
            else:
                R.api(f"projects/{P}/merge_requests/{iid}/discussions/{d}/notes", "POST", body=text)
    try:
        R.trigger("react", True)
        R.api(f"projects/{P}/merge_requests/{iid}/notes", "POST", body=f"{MENTION} check")
        rec["react_done"] = R.wait_session_done(iid, "react", 1200)
    finally:
        R.trigger("react", False)
    time.sleep(10)
    by_id = {d["id"]: d for d in R.discussions(iid)}
    for c in cases:
        d = threads.get(c["name"], (None, ""))[0]
        got = R.classify(by_id[d]) if d in by_id else {"settled": None, "resolved": False, "bot_notes": [], "missing_thread": True}
        outcome = got["settled"] or "open"
        if outcome == "open" and got["resolved"]:
            outcome = "resolved silently"
        rec["cases"].append({"n": c["n"], "name": c["name"], "label": c["label"], "author": c["author"], "sent": c["sent"],
                             "expected": c["expected"], "got": outcome,
                             "match": outcome == c["expected"] and not got.get("missing_thread"),
                             "bot_notes": got["bot_notes"], "thread_resolved": got["resolved"]})
    m = R.api(f"projects/{P}/merge_requests/{iid}")
    rec["mr_after"] = {"state": m["state"], "title": m["title"], "labels": m["labels"], "merge_status": m.get("detailed_merge_status")}
    issue_notes = second("GET", f"projects/{P}/issues/{issue['iid']}/notes?per_page=50")
    rec["receipt_on_issue"] = any("Receipt from Itemator" in (n.get("body") or "") for n in issue_notes)
    for step in (lambda: R.api(f"projects/{P}/merge_requests/{iid}", "PUT", state_event="close"),
                 lambda: R.api(f"projects/{P}/issues/{issue['iid']}", "PUT", state_event="close"),
                 lambda: R.api(f"projects/{P}/repository/branches/{BRANCHES[0].replace('/', '%2F')}", "DELETE"),
                 lambda: R.api(f"projects/{P}/repository/branches/{BRANCHES[1].replace('/', '%2F')}", "DELETE")):
        try: step()
        except Exception: pass
    return rec


def summarize(runs):
    rows = [c for r in runs for c in r["cases"]]
    ok = [c for c in rows if c["match"]]
    return {"what": "Six reply cases for names.asker_required (reaction flow), pre-registered in eval/asker_study_cases.json. Counts only.",
            "limits": "Two runs (on two versions of the flow), one merge request each, the same six cases, six cases written by us, two accounts, one model reading. The cases file names the first account as ASKER; the runner (committed before the run, a86e848) used the second account as ASKER and the first as OTHER, the same swap in every case, so the outcomes stand. It shows whether the model applies the username test, not how people answer; a username that differs only in letter case cannot be produced on GitLab, so that clause is covered by unit tests only.",
            "runs": len(runs), "cases": len(rows), "as_expected": len(ok), "misread": len(rows) - len(ok),
            "misread_cases": [{"n": c["n"], "label": c["label"], "expected": c["expected"], "got": c["got"]} for c in rows if not c["match"]],
            "asker_sentence_in_every_thread": all(r.get("asker_sentence_in_every_thread") for r in runs),
            "record_on_issue_posted": any(r.get("receipt_on_issue") for r in runs)}


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--credit-stop", type=float, default=0.0); ap.add_argument("--summarize", action="store_true")
    a = ap.parse_args()
    runs = json.loads(RAW.read_text()) if RAW.exists() else []
    if a.summarize:
        OUT.write_text(json.dumps(summarize(runs), indent=1) + "\n"); print("wrote", OUT.name); return 0
    m = R.meter(); print("meter", m)
    if a.credit_stop and m is not None and m >= a.credit_stop:
        print("credit stop reached"); return 1
    rec = run(json.loads(CASES.read_text()))
    runs.append(rec); RAW.write_text(json.dumps(runs, indent=1) + "\n")
    for c in rec["cases"]:
        print(f"{c['n']:>2} {c['label'][:52]:52} expected {c['expected']:9} got {c['got']:9} {'OK' if c['match'] else 'MISMATCH'}")
    print("asker sentence in every thread:", rec["asker_sentence_in_every_thread"], "| record on issue:", rec["receipt_on_issue"])
    return 0


if __name__ == "__main__":
    sys.exit(main())