clinical-agent-memory-bench's picture
Upload 99 files
750b88d verified
Raw History Blame Contribute Delete
12.8 kB
#!/usr/bin/env python3
"""Axis 1 (long horizon): seeds from the evidence chain, LLM generation (pipeline.llm step axis1), post-processing.
seeds Form A (range/point): a value held unchanged over a stable span of at least MIN_VIS visits.
Form B (never again): an entity whose last mention lies at least MIN_VIS visits before the end.
-> _seed_axis1.json
post GEN finds each question's seed by (entity, form), anchors gt_date inside the question's own scope (the
earliest evidence date within a range; the shown date for a point), recomputes the appearance
window, checks each range question, deduplicates and assigns qid and grounding
-> questions_axis1.jsonl
"""
from __future__ import annotations
import argparse
import json
import re
from collections import Counter, defaultdict
from datetime import date
from pipeline.common import BASE, has_text, norm_value, other_department_segments, patient_id
PATIENT = patient_id()
def seeds() -> None:
"""Seed builder over the evidence chain; writes _seed_axis1.json."""
LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl"))
MIN_VIS = 100
CAP_A, CAP_B = 3, 4
CONTENT = ["diagnosis", "patient_self_description", "tests_and_medication", "doctor_notes", "doctor_plans"]
def pdate(s):
y, m, d = map(int, s.split("-"))
return date(y, m, d)
def months(a, b):
return (b.year - a.year) * 12 + (b.month - a.month)
recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")]
vdate = {r["visit_number"]: r["date"] for r in recs}
last_date = max(vdate.values())
tl = [json.loads(l) for l in open(f"{BASE}/evidence_chain.jsonl")]
TYPE = {"test_result": "test", "medication": "medication", "disease_state": "diagnosis",
"exam_finding": "exam", "care_plan": "treatment", "procedure_or_followup": "treatment",
"symptom": "symptom"}
CORE = {"test", "medication", "diagnosis", "exam", "treatment", "symptom"}
GENERIC = {"unknown", "denied", "reassurance", "lifestyle modification", "none", "nil", "no", "yes", "na"}
def qtype(asp):
return TYPE.get(asp, asp)
def concrete(v):
vl = re.sub(r"\s*\([^)]*\)", "", v).strip().lower()
if vl in GENERIC:
return False
return bool(re.search(r"\d", v)) or len(v.split()) >= 2
WORD = re.compile(r"[a-z0-9]+")
def _stem(w):
return re.sub(r"(ies|es|ing|ed|s)$", "", w)
def toks(s):
return {_stem(w) for w in WORD.findall(str(s).lower()) if len(w) >= 4}
vis_tokens = {r["visit_number"]: toks(" ".join(sum((r.get(b, []) for b in CONTENT), []))) for r in recs}
def recurs_after(at, last_v, bar=0.6):
for v in range(last_v + 1, LAST_V + 1):
tv = vis_tokens.get(v)
if tv and at and len(at & tv) / len(at) >= bar:
return v
return None
def overlaps(seg, lo, hi):
start = (seg.get("valid_from") or {}).get("date") or ""
vu = seg.get("valid_until")
end = vu.get("date") if isinstance(vu, dict) else last_date
return start <= hi and (end or last_date) >= lo
formA, n_conflict = {}, 0
for t in tl:
for s in t["segments"]:
supp = sorted(s.get("supporting_visits", []))
if len(supp) < 3 or not concrete(s["value"]) or qtype(t["aspect"]) not in CORE:
continue
first_v, last_v = supp[0], supp[-1]
hv = last_v - first_v
if hv < MIN_VIS:
continue
margin = max(10, min(40, hv // 8))
wf_v, wt_v = first_v + margin, last_v - margin
if sum(1 for v in supp if wf_v <= v <= wt_v) < 2:
continue
if any(norm_value(o.get("value")) != norm_value(s["value"]) and overlaps(o, vdate[wf_v], vdate[wt_v])
for o in other_department_segments(tl, t["entity"], t["department"])):
n_conflict += 1
continue
seed = {"form": "range", "entity": t["entity"], "aspect": t["aspect"], "qtype": qtype(t["aspect"]),
"department": t["department"], "value": s["value"],
"gt_date": vdate[first_v], "gt_visit": first_v,
"range_from": {"date": vdate[first_v], "visit": first_v},
"range_to": {"date": vdate[last_v], "visit": last_v},
"window_from": {"date": vdate[wf_v], "visit": wf_v},
"window_to": {"date": vdate[wt_v], "visit": wt_v},
"horizon_visits": hv, "horizon_months": months(pdate(vdate[first_v]), pdate(vdate[last_v])),
"n_mentions": len(supp), "evidence_visits": supp[:12],
"still_current": s["valid_until"] == "current"}
if t["entity"] not in formA or hv > formA[t["entity"]]["horizon_visits"]:
formA[t["entity"]] = seed
formB = {}
for t in tl:
supp = sorted({v for s in t["segments"] for v in s.get("supporting_visits", [])})
if not supp:
continue
last = supp[-1]
if LAST_V - last < MIN_VIS or qtype(t["aspect"]) not in CORE:
continue
val = t["entity"]
for s in t["segments"]:
if last in s.get("supporting_visits", []):
val = s["value"]
break
if not concrete(val) or len(toks(val) | toks(t["entity"])) < 2:
continue
if recurs_after(toks(t["entity"]) | toks(val), last, 0.6):
continue
formB[t["entity"]] = {"form": "never_again", "entity": t["entity"], "aspect": t["aspect"],
"qtype": qtype(t["aspect"]), "department": t["department"], "value": val,
"gt_date": vdate[last], "gt_visit": last, "horizon_visits": LAST_V - last,
"horizon_months": months(pdate(vdate[last]), pdate(last_date)),
"n_mentions": len(supp), "evidence_visits": supp[-6:], "last_seen_date": vdate[last]}
def stratify(seeds, cap):
by = defaultdict(list)
for s in seeds:
by[s["qtype"]].append(s)
out = []
for qt, lst in by.items():
lst.sort(key=lambda x: -x["horizon_visits"])
out += lst[:cap]
return out
A = stratify(list(formA.values()), CAP_A)
B = stratify(list(formB.values()), CAP_B)
seeds = A + B
json.dump({"axis": "1_long_horizon", "seeds": seeds}, open(f"{BASE}/_seed_axis1.json", "w"),
ensure_ascii=False, indent=1)
print(f"FORM A: {len(formA)} entities -> {len(A)} seeds (cap {CAP_A}/type); {n_conflict} windows left out "
f"because another department records a different value in them")
print(f"FORM B: {len(formB)} entities pass the lexical filter -> {len(B)} seeds (cap {CAP_B}/type)")
print(f"total seeds: {len(seeds)} | min horizon_visits: {min(s['horizon_visits'] for s in seeds)}")
print("FORM A by qtype:", dict(Counter(s["qtype"] for s in A)))
print("FORM B by qtype:", dict(Counter(s["qtype"] for s in B)))
print("\nwrote _seed_axis1.json")
def post(gen_path: str) -> None:
"""Post-processing of the generated axis-1 questions; writes questions_axis1.jsonl."""
LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl"))
MIN_HORIZON_VISITS = 100
DATE = re.compile(r"\d{4}-\d{2}-\d{2}")
recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")]
vdate = {r["visit_number"]: r["date"] for r in recs}
last_date = max(vdate.values())
date2v = {}
for r in sorted(recs, key=lambda x: x["visit_number"]):
date2v.setdefault(r["date"], r["visit_number"])
def gt_visit(d):
if d in date2v:
return date2v[d]
later = [r["visit_number"] for r in recs if r["date"] >= d]
return min(later) if later else 1
seed_list = json.load(open(f"{BASE}/_seed_axis1.json"))["seeds"]
by_entity = {(str(s["entity"]).strip(), s["form"]): s for s in seed_list}
n_date = Counter(s["gt_date"] for s in seed_list)
by_date = {s["gt_date"]: s for s in seed_list if n_date[s["gt_date"]] == 1}
def seed_of(q):
"""The question's seed: by (entity, form), else by gt_date when exactly one seed has that date."""
sd = by_entity.get((str(q.get("entity") or "").strip(), q.get("form")))
return sd or by_date.get(q["gt_date"])
raw = json.load(open(gen_path))
qs = raw["questions"] if isinstance(raw, dict) else raw
out, seen, warns = [], set(), []
no_text = sum(1 for q in qs if not has_text(q))
if no_text:
print(f"skipped {no_text} generated question(s) without question text")
qs = [q for q in qs if has_text(q)]
bad = [q for q in qs if not (isinstance(q.get("gt_date"), str) and DATE.fullmatch(q["gt_date"])
and isinstance(q.get("evidence_visits"), list))]
bad += [q for q in qs if q not in bad and q.get("template") == "T2" and not DATE.search(q["question"])]
for q in bad:
print(f"skipped a generated question without a gt_date, evidence_visits or, for T2, dates in its text: "
f"{q['question'][:60]!r}")
qs = [q for q in qs if q not in bad]
for q in qs:
key = q["question"].strip().lower()
if key in seen:
continue
seen.add(key)
onset = q["gt_date"]
sd = seed_of(q)
if sd is None:
print(f"warning: no seed found for {q.get('entity')!r} ({q.get('form')}, gt_date {onset}); "
f"no entity attached: {q['question'][:55]!r}")
qdates = sorted(DATE.findall(q["question"]))
if q.get("form") == "range":
if len(qdates) >= 2:
lo, hi = qdates[0], qdates[-1]
in_range = sorted(vdate[v] for v in q["evidence_visits"] if lo <= vdate.get(v, "") <= hi)
q["gt_date"] = in_range[0] if in_range else onset
elif len(qdates) == 1:
q["gt_date"] = qdates[0]
gt = q["gt_date"]
gv = gt_visit(gt)
latest_q_v = max((gt_visit(d) for d in qdates), default=0)
fv = min(max(gv + MIN_HORIZON_VISITS, latest_q_v), LAST_V)
q["gt_visit"] = gv
q["question_appearance_window"] = {"from": vdate[fv], "to": last_date}
q["axis"] = "1_long_horizon"
if sd:
q["axis_specific"] = {**(q.get("axis_specific") or {}), "entity": sd["entity"]}
q["qid"] = f"{PATIENT}/axis1/{len(out):03d}"
q["grounding"] = "record"
if q.get("template") == "T2":
if onset in qdates:
warns.append(f"T2 onset {onset} is a shown endpoint: {q['question'][:55]}")
if not (qdates[0] <= gt <= qdates[-1]):
warns.append(f"T2 gt {gt} NOT inside range {qdates}: {q['question'][:55]}")
if sd and "window_from" in sd:
wf, wt = sd["window_from"]["date"], sd["window_to"]["date"]
if not (wf in qdates and wt in qdates):
warns.append(f"T2 dates {qdates} != seed window [{wf},{wt}]: {q['question'][:55]}")
ev_in = [v for v in q["evidence_visits"] if wf <= vdate.get(v, "") <= wt]
if len(ev_in) < 1:
warns.append(f"T2 NO in-range evidence: {q['question'][:55]}")
out.append(q)
with open(f"{BASE}/questions_axis1.jsonl", "w", encoding="utf-8") as fh:
for q in out:
fh.write(json.dumps(q, ensure_ascii=False) + "\n")
print(f"axis-1 questions: {len(out)} ({len(qs) - len(out)} dropped as dup)")
print("by form :", dict(Counter(q["form"] for q in out)))
print("by qtype:", dict(Counter(q["qtype"] for q in out)))
print("by template:", dict(Counter(q.get("template", "?") for q in out)))
hv = [q["horizon_visits"] for q in out]
if hv:
print(f"horizon_visits: min {min(hv)} max {max(hv)} (minimum {MIN_HORIZON_VISITS})")
print("horizon below minimum:", sum(1 for q in out if q["horizon_visits"] < MIN_HORIZON_VISITS))
print("degenerate appearance window:",
sum(1 for q in out if q["question_appearance_window"]["from"] >= q["question_appearance_window"]["to"]))
print("range warnings:", len(warns))
print("\nwrote questions_axis1.jsonl")
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
sp = ap.add_subparsers(dest="cmd", required=True)
sp.add_parser("seeds")
sp.add_parser("post").add_argument("generated")
a = ap.parse_args()
seeds() if a.cmd == "seeds" else post(a.generated)
if __name__ == "__main__":
main()