Download code_release/code/construction/pipeline/questions/axis1.py from clinical-agent-memory-bench/meme-bench: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/spaces/clinical-agent-memory-bench/meme-bench/resolve/main/code_release/code/construction/pipeline/questions/axis1.py
- Command line
-
hf download hf://spaces/clinical-agent-memory-bench/meme-bench/code_release/code/construction/pipeline/questions/axis1.py
-
curl -L -o axis1.py https://huggingface.co/spaces/clinical-agent-memory-bench/meme-bench/resolve/main/code_release/code/construction/pipeline/questions/axis1.py
12.8 kB
| #!/usr/bin/env python3 | |
| """Axis 1 (long horizon): seeds from the evidence chain, LLM generation (pipeline.llm step axis1), post-processing. | |
| seeds Form A (range/point): a value held unchanged over a stable span of at least MIN_VIS visits. | |
| Form B (never again): an entity whose last mention lies at least MIN_VIS visits before the end. | |
| -> _seed_axis1.json | |
| post GEN finds each question's seed by (entity, form), anchors gt_date inside the question's own scope (the | |
| earliest evidence date within a range; the shown date for a point), recomputes the appearance | |
| window, checks each range question, deduplicates and assigns qid and grounding | |
| -> questions_axis1.jsonl | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| from collections import Counter, defaultdict | |
| from datetime import date | |
| from pipeline.common import BASE, has_text, norm_value, other_department_segments, patient_id | |
| PATIENT = patient_id() | |
| def seeds() -> None: | |
| """Seed builder over the evidence chain; writes _seed_axis1.json.""" | |
| LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl")) | |
| MIN_VIS = 100 | |
| CAP_A, CAP_B = 3, 4 | |
| CONTENT = ["diagnosis", "patient_self_description", "tests_and_medication", "doctor_notes", "doctor_plans"] | |
| def pdate(s): | |
| y, m, d = map(int, s.split("-")) | |
| return date(y, m, d) | |
| def months(a, b): | |
| return (b.year - a.year) * 12 + (b.month - a.month) | |
| recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")] | |
| vdate = {r["visit_number"]: r["date"] for r in recs} | |
| last_date = max(vdate.values()) | |
| tl = [json.loads(l) for l in open(f"{BASE}/evidence_chain.jsonl")] | |
| TYPE = {"test_result": "test", "medication": "medication", "disease_state": "diagnosis", | |
| "exam_finding": "exam", "care_plan": "treatment", "procedure_or_followup": "treatment", | |
| "symptom": "symptom"} | |
| CORE = {"test", "medication", "diagnosis", "exam", "treatment", "symptom"} | |
| GENERIC = {"unknown", "denied", "reassurance", "lifestyle modification", "none", "nil", "no", "yes", "na"} | |
| def qtype(asp): | |
| return TYPE.get(asp, asp) | |
| def concrete(v): | |
| vl = re.sub(r"\s*\([^)]*\)", "", v).strip().lower() | |
| if vl in GENERIC: | |
| return False | |
| return bool(re.search(r"\d", v)) or len(v.split()) >= 2 | |
| WORD = re.compile(r"[a-z0-9]+") | |
| def _stem(w): | |
| return re.sub(r"(ies|es|ing|ed|s)$", "", w) | |
| def toks(s): | |
| return {_stem(w) for w in WORD.findall(str(s).lower()) if len(w) >= 4} | |
| vis_tokens = {r["visit_number"]: toks(" ".join(sum((r.get(b, []) for b in CONTENT), []))) for r in recs} | |
| def recurs_after(at, last_v, bar=0.6): | |
| for v in range(last_v + 1, LAST_V + 1): | |
| tv = vis_tokens.get(v) | |
| if tv and at and len(at & tv) / len(at) >= bar: | |
| return v | |
| return None | |
| def overlaps(seg, lo, hi): | |
| start = (seg.get("valid_from") or {}).get("date") or "" | |
| vu = seg.get("valid_until") | |
| end = vu.get("date") if isinstance(vu, dict) else last_date | |
| return start <= hi and (end or last_date) >= lo | |
| formA, n_conflict = {}, 0 | |
| for t in tl: | |
| for s in t["segments"]: | |
| supp = sorted(s.get("supporting_visits", [])) | |
| if len(supp) < 3 or not concrete(s["value"]) or qtype(t["aspect"]) not in CORE: | |
| continue | |
| first_v, last_v = supp[0], supp[-1] | |
| hv = last_v - first_v | |
| if hv < MIN_VIS: | |
| continue | |
| margin = max(10, min(40, hv // 8)) | |
| wf_v, wt_v = first_v + margin, last_v - margin | |
| if sum(1 for v in supp if wf_v <= v <= wt_v) < 2: | |
| continue | |
| if any(norm_value(o.get("value")) != norm_value(s["value"]) and overlaps(o, vdate[wf_v], vdate[wt_v]) | |
| for o in other_department_segments(tl, t["entity"], t["department"])): | |
| n_conflict += 1 | |
| continue | |
| seed = {"form": "range", "entity": t["entity"], "aspect": t["aspect"], "qtype": qtype(t["aspect"]), | |
| "department": t["department"], "value": s["value"], | |
| "gt_date": vdate[first_v], "gt_visit": first_v, | |
| "range_from": {"date": vdate[first_v], "visit": first_v}, | |
| "range_to": {"date": vdate[last_v], "visit": last_v}, | |
| "window_from": {"date": vdate[wf_v], "visit": wf_v}, | |
| "window_to": {"date": vdate[wt_v], "visit": wt_v}, | |
| "horizon_visits": hv, "horizon_months": months(pdate(vdate[first_v]), pdate(vdate[last_v])), | |
| "n_mentions": len(supp), "evidence_visits": supp[:12], | |
| "still_current": s["valid_until"] == "current"} | |
| if t["entity"] not in formA or hv > formA[t["entity"]]["horizon_visits"]: | |
| formA[t["entity"]] = seed | |
| formB = {} | |
| for t in tl: | |
| supp = sorted({v for s in t["segments"] for v in s.get("supporting_visits", [])}) | |
| if not supp: | |
| continue | |
| last = supp[-1] | |
| if LAST_V - last < MIN_VIS or qtype(t["aspect"]) not in CORE: | |
| continue | |
| val = t["entity"] | |
| for s in t["segments"]: | |
| if last in s.get("supporting_visits", []): | |
| val = s["value"] | |
| break | |
| if not concrete(val) or len(toks(val) | toks(t["entity"])) < 2: | |
| continue | |
| if recurs_after(toks(t["entity"]) | toks(val), last, 0.6): | |
| continue | |
| formB[t["entity"]] = {"form": "never_again", "entity": t["entity"], "aspect": t["aspect"], | |
| "qtype": qtype(t["aspect"]), "department": t["department"], "value": val, | |
| "gt_date": vdate[last], "gt_visit": last, "horizon_visits": LAST_V - last, | |
| "horizon_months": months(pdate(vdate[last]), pdate(last_date)), | |
| "n_mentions": len(supp), "evidence_visits": supp[-6:], "last_seen_date": vdate[last]} | |
| def stratify(seeds, cap): | |
| by = defaultdict(list) | |
| for s in seeds: | |
| by[s["qtype"]].append(s) | |
| out = [] | |
| for qt, lst in by.items(): | |
| lst.sort(key=lambda x: -x["horizon_visits"]) | |
| out += lst[:cap] | |
| return out | |
| A = stratify(list(formA.values()), CAP_A) | |
| B = stratify(list(formB.values()), CAP_B) | |
| seeds = A + B | |
| json.dump({"axis": "1_long_horizon", "seeds": seeds}, open(f"{BASE}/_seed_axis1.json", "w"), | |
| ensure_ascii=False, indent=1) | |
| print(f"FORM A: {len(formA)} entities -> {len(A)} seeds (cap {CAP_A}/type); {n_conflict} windows left out " | |
| f"because another department records a different value in them") | |
| print(f"FORM B: {len(formB)} entities pass the lexical filter -> {len(B)} seeds (cap {CAP_B}/type)") | |
| print(f"total seeds: {len(seeds)} | min horizon_visits: {min(s['horizon_visits'] for s in seeds)}") | |
| print("FORM A by qtype:", dict(Counter(s["qtype"] for s in A))) | |
| print("FORM B by qtype:", dict(Counter(s["qtype"] for s in B))) | |
| print("\nwrote _seed_axis1.json") | |
| def post(gen_path: str) -> None: | |
| """Post-processing of the generated axis-1 questions; writes questions_axis1.jsonl.""" | |
| LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl")) | |
| MIN_HORIZON_VISITS = 100 | |
| DATE = re.compile(r"\d{4}-\d{2}-\d{2}") | |
| recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")] | |
| vdate = {r["visit_number"]: r["date"] for r in recs} | |
| last_date = max(vdate.values()) | |
| date2v = {} | |
| for r in sorted(recs, key=lambda x: x["visit_number"]): | |
| date2v.setdefault(r["date"], r["visit_number"]) | |
| def gt_visit(d): | |
| if d in date2v: | |
| return date2v[d] | |
| later = [r["visit_number"] for r in recs if r["date"] >= d] | |
| return min(later) if later else 1 | |
| seed_list = json.load(open(f"{BASE}/_seed_axis1.json"))["seeds"] | |
| by_entity = {(str(s["entity"]).strip(), s["form"]): s for s in seed_list} | |
| n_date = Counter(s["gt_date"] for s in seed_list) | |
| by_date = {s["gt_date"]: s for s in seed_list if n_date[s["gt_date"]] == 1} | |
| def seed_of(q): | |
| """The question's seed: by (entity, form), else by gt_date when exactly one seed has that date.""" | |
| sd = by_entity.get((str(q.get("entity") or "").strip(), q.get("form"))) | |
| return sd or by_date.get(q["gt_date"]) | |
| raw = json.load(open(gen_path)) | |
| qs = raw["questions"] if isinstance(raw, dict) else raw | |
| out, seen, warns = [], set(), [] | |
| no_text = sum(1 for q in qs if not has_text(q)) | |
| if no_text: | |
| print(f"skipped {no_text} generated question(s) without question text") | |
| qs = [q for q in qs if has_text(q)] | |
| bad = [q for q in qs if not (isinstance(q.get("gt_date"), str) and DATE.fullmatch(q["gt_date"]) | |
| and isinstance(q.get("evidence_visits"), list))] | |
| bad += [q for q in qs if q not in bad and q.get("template") == "T2" and not DATE.search(q["question"])] | |
| for q in bad: | |
| print(f"skipped a generated question without a gt_date, evidence_visits or, for T2, dates in its text: " | |
| f"{q['question'][:60]!r}") | |
| qs = [q for q in qs if q not in bad] | |
| for q in qs: | |
| key = q["question"].strip().lower() | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| onset = q["gt_date"] | |
| sd = seed_of(q) | |
| if sd is None: | |
| print(f"warning: no seed found for {q.get('entity')!r} ({q.get('form')}, gt_date {onset}); " | |
| f"no entity attached: {q['question'][:55]!r}") | |
| qdates = sorted(DATE.findall(q["question"])) | |
| if q.get("form") == "range": | |
| if len(qdates) >= 2: | |
| lo, hi = qdates[0], qdates[-1] | |
| in_range = sorted(vdate[v] for v in q["evidence_visits"] if lo <= vdate.get(v, "") <= hi) | |
| q["gt_date"] = in_range[0] if in_range else onset | |
| elif len(qdates) == 1: | |
| q["gt_date"] = qdates[0] | |
| gt = q["gt_date"] | |
| gv = gt_visit(gt) | |
| latest_q_v = max((gt_visit(d) for d in qdates), default=0) | |
| fv = min(max(gv + MIN_HORIZON_VISITS, latest_q_v), LAST_V) | |
| q["gt_visit"] = gv | |
| q["question_appearance_window"] = {"from": vdate[fv], "to": last_date} | |
| q["axis"] = "1_long_horizon" | |
| if sd: | |
| q["axis_specific"] = {**(q.get("axis_specific") or {}), "entity": sd["entity"]} | |
| q["qid"] = f"{PATIENT}/axis1/{len(out):03d}" | |
| q["grounding"] = "record" | |
| if q.get("template") == "T2": | |
| if onset in qdates: | |
| warns.append(f"T2 onset {onset} is a shown endpoint: {q['question'][:55]}") | |
| if not (qdates[0] <= gt <= qdates[-1]): | |
| warns.append(f"T2 gt {gt} NOT inside range {qdates}: {q['question'][:55]}") | |
| if sd and "window_from" in sd: | |
| wf, wt = sd["window_from"]["date"], sd["window_to"]["date"] | |
| if not (wf in qdates and wt in qdates): | |
| warns.append(f"T2 dates {qdates} != seed window [{wf},{wt}]: {q['question'][:55]}") | |
| ev_in = [v for v in q["evidence_visits"] if wf <= vdate.get(v, "") <= wt] | |
| if len(ev_in) < 1: | |
| warns.append(f"T2 NO in-range evidence: {q['question'][:55]}") | |
| out.append(q) | |
| with open(f"{BASE}/questions_axis1.jsonl", "w", encoding="utf-8") as fh: | |
| for q in out: | |
| fh.write(json.dumps(q, ensure_ascii=False) + "\n") | |
| print(f"axis-1 questions: {len(out)} ({len(qs) - len(out)} dropped as dup)") | |
| print("by form :", dict(Counter(q["form"] for q in out))) | |
| print("by qtype:", dict(Counter(q["qtype"] for q in out))) | |
| print("by template:", dict(Counter(q.get("template", "?") for q in out))) | |
| hv = [q["horizon_visits"] for q in out] | |
| if hv: | |
| print(f"horizon_visits: min {min(hv)} max {max(hv)} (minimum {MIN_HORIZON_VISITS})") | |
| print("horizon below minimum:", sum(1 for q in out if q["horizon_visits"] < MIN_HORIZON_VISITS)) | |
| print("degenerate appearance window:", | |
| sum(1 for q in out if q["question_appearance_window"]["from"] >= q["question_appearance_window"]["to"])) | |
| print("range warnings:", len(warns)) | |
| print("\nwrote questions_axis1.jsonl") | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) | |
| sp = ap.add_subparsers(dest="cmd", required=True) | |
| sp.add_parser("seeds") | |
| sp.add_parser("post").add_argument("generated") | |
| a = ap.parse_args() | |
| seeds() if a.cmd == "seeds" else post(a.generated) | |
| if __name__ == "__main__": | |
| main() | |