merge.py

#!/usr/bin/env python3
"""Reconcile annotation files into one adjudicated row per key.

    uv run python human_labeling/merge.py [--out annotations/adjudicated.jsonl]

Reads every annotations/rulings*.jsonl + annotations/*-export.json
(export format: {"items": [...]}, same ruling objects), majority-votes
ties -> "REVIEW", and writes adjudicated.jsonl:
{"key", "verdict", "n_keep", "n_drop", "annotators", "origins"?, ...}
Kept inputs are never modified; rerun any time.
"""

import argparse
import glob
import json
from pathlib import Path

HERE = Path(__file__).resolve().parent
ANNOT = HERE / "annotations"


def load() -> list[dict]:
    rulings: list[dict] = []
    for f in sorted(glob.glob(str(ANNOT / "rulings*.jsonl"))):
        for line in open(f):
            line = line.strip()
            if line:
                rulings.append(json.loads(line))
    for f in sorted(glob.glob(str(ANNOT / "*-export.json"))):
        try:
            payload = json.load(open(f))
        except json.JSONDecodeError:
            continue
        items = payload.get("items", payload if isinstance(payload, list) else [])
        rulings.extend(i for i in items if isinstance(i, dict) and i.get("key"))
    return rulings


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=str(ANNOT / "adjudicated.jsonl"))
    a = ap.parse_args()
    by_key: dict[str, list[dict]] = {}
    for r in load():
        by_key.setdefault(r["key"], []).append(r)
    n_tie = 0
    with open(a.out, "w") as f:
        for key in sorted(by_key):
            rs = by_key[key]
            keep = sum(1 for r in rs if r["ruling"] == "DATA_MENTION")
            drop = sum(1 for r in rs if r["ruling"] == "NON_MENTION")
            verdict = "DATA_MENTION" if keep > drop else "NON_MENTION" if drop > keep else "REVIEW"
            n_tie += verdict == "REVIEW"
            f.write(json.dumps({
                "key": key, "verdict": verdict, "n_keep": keep, "n_drop": drop,
                "n": len(rs), "annotators": sorted({r.get("annotator", "?") for r in rs}),
                "surface": rs[0].get("surface"), "origin": rs[0].get("origin"),
                "head_score": rs[0].get("head_score"),
            }) + "\n")
    print(f"keys={len(by_key)} ties={n_tie} -> {a.out}")


if __name__ == "__main__":
    main()