Spaces:
Running
Running
File size: 3,387 Bytes
53ea208 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | #!/usr/bin/env python3
"""Build the gold-controls queue from local human-adjudicated spans.
Joins outputs/gliner_datause_v3_probe_human473.jsonl (key, set, surface)
to analysis/v24_sample_review/gliner2_review.jsonl
(spans[].key -> input passage, start/end, origin). Scores are the
singlepass infer-head H100 predictions
(outputs/gliner-datause-catchall-infer-probe/human473_predictions.jsonl);
truth labels are deliberately excluded (blind controls).
Splits: annotator190 (set=annotator) + jdc283 (set=jdc).
uv run python human_labeling/build_gold_queue.py [--out human_labeling/queue_human473.json]
"""
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from probe_labels import decide # noqa: E402
REPO = Path(__file__).resolve().parent.parent
HUMAN = REPO / "outputs" / "gliner_datause_v3_probe_human473.jsonl"
REVIEW = REPO / "analysis" / "v24_sample_review" / "gliner2_review.jsonl"
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", default=str(REPO / "human_labeling" / "queue_human473.json"))
a = ap.parse_args()
# singlepass (infer-head) rescore, published H100 predictions; 23 of 473
# out-of-grid spans absent here -> head_score None, band "unscored"
SP = REPO / "outputs" / "gliner-datause-catchall-infer-probe" / "human473_predictions.jsonl"
sp = {}
if SP.exists():
for line in SP.read_text().splitlines():
if line.strip():
o = json.loads(line)
sp[o["key"]] = float(o.get("head_score"))
# review spans by key -> (passage, start, end, origin)
ctx: dict[str, tuple] = {}
for line in REVIEW.read_text().splitlines():
if not line.strip():
continue
row = json.loads(line)
for s in row.get("spans", []):
if s.get("key"):
ctx[s["key"]] = (row.get("input", ""), s.get("start"),
s.get("end"), row.get("origin"))
items: list[dict] = []
missing: list[str] = []
from collections import Counter
for line in HUMAN.read_text().splitlines():
if not line.strip():
continue
r = json.loads(line)
key = r["key"]
subset = "annotator190" if r.get("set") == "annotator" else "jdc283"
score = sp.get(key) # singlepass infer-head (H100); None = out-of-grid
if key in ctx:
passage, start, end, origin = ctx[key]
items.append({
"key": key, "surface": r.get("surface"), "ctx": passage,
"ctx_missing": False, "head_score": score,
"start": start, "end": end,
"band": decide(score, origin), "origin": origin,
"specificity": "", "split": "holdout",
"queue": "human473", "subset": subset,
"scored_by": "rafmacalaba/gliner-datause-catchall-infer-probe (H100)",
})
else:
missing.append(key)
Path(a.out).write_text("\n".join(json.dumps(i) for i in items) + "\n")
print(f"queued={len(items)} missing_ctx={len(missing)} "
f"scored={sum(1 for i in items if i['head_score'] is not None)} "
f"{Counter(i['subset'] for i in items)} -> {a.out}")
if missing:
print("missing:", missing[:5])
if __name__ == "__main__":
main()
|