#!/usr/bin/env python3 """Axis 1 (long horizon): seeds from the evidence chain, LLM generation (pipeline.llm step axis1), post-processing. seeds Form A (range/point): a value held unchanged over a stable span of at least MIN_VIS visits. Form B (never again): an entity whose last mention lies at least MIN_VIS visits before the end. -> _seed_axis1.json post GEN finds each question's seed by (entity, form), anchors gt_date inside the question's own scope (the earliest evidence date within a range; the shown date for a point), recomputes the appearance window, checks each range question, deduplicates and assigns qid and grounding -> questions_axis1.jsonl """ from __future__ import annotations import argparse import json import re from collections import Counter, defaultdict from datetime import date from pipeline.common import BASE, has_text, norm_value, other_department_segments, patient_id PATIENT = patient_id() def seeds() -> None: """Seed builder over the evidence chain; writes _seed_axis1.json.""" LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl")) MIN_VIS = 100 CAP_A, CAP_B = 3, 4 CONTENT = ["diagnosis", "patient_self_description", "tests_and_medication", "doctor_notes", "doctor_plans"] def pdate(s): y, m, d = map(int, s.split("-")) return date(y, m, d) def months(a, b): return (b.year - a.year) * 12 + (b.month - a.month) recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")] vdate = {r["visit_number"]: r["date"] for r in recs} last_date = max(vdate.values()) tl = [json.loads(l) for l in open(f"{BASE}/evidence_chain.jsonl")] TYPE = {"test_result": "test", "medication": "medication", "disease_state": "diagnosis", "exam_finding": "exam", "care_plan": "treatment", "procedure_or_followup": "treatment", "symptom": "symptom"} CORE = {"test", "medication", "diagnosis", "exam", "treatment", "symptom"} GENERIC = {"unknown", "denied", "reassurance", "lifestyle modification", "none", "nil", "no", "yes", "na"} def qtype(asp): return TYPE.get(asp, asp) def concrete(v): vl = re.sub(r"\s*\([^)]*\)", "", v).strip().lower() if vl in GENERIC: return False return bool(re.search(r"\d", v)) or len(v.split()) >= 2 WORD = re.compile(r"[a-z0-9]+") def _stem(w): return re.sub(r"(ies|es|ing|ed|s)$", "", w) def toks(s): return {_stem(w) for w in WORD.findall(str(s).lower()) if len(w) >= 4} vis_tokens = {r["visit_number"]: toks(" ".join(sum((r.get(b, []) for b in CONTENT), []))) for r in recs} def recurs_after(at, last_v, bar=0.6): for v in range(last_v + 1, LAST_V + 1): tv = vis_tokens.get(v) if tv and at and len(at & tv) / len(at) >= bar: return v return None def overlaps(seg, lo, hi): start = (seg.get("valid_from") or {}).get("date") or "" vu = seg.get("valid_until") end = vu.get("date") if isinstance(vu, dict) else last_date return start <= hi and (end or last_date) >= lo formA, n_conflict = {}, 0 for t in tl: for s in t["segments"]: supp = sorted(s.get("supporting_visits", [])) if len(supp) < 3 or not concrete(s["value"]) or qtype(t["aspect"]) not in CORE: continue first_v, last_v = supp[0], supp[-1] hv = last_v - first_v if hv < MIN_VIS: continue margin = max(10, min(40, hv // 8)) wf_v, wt_v = first_v + margin, last_v - margin if sum(1 for v in supp if wf_v <= v <= wt_v) < 2: continue if any(norm_value(o.get("value")) != norm_value(s["value"]) and overlaps(o, vdate[wf_v], vdate[wt_v]) for o in other_department_segments(tl, t["entity"], t["department"])): n_conflict += 1 continue seed = {"form": "range", "entity": t["entity"], "aspect": t["aspect"], "qtype": qtype(t["aspect"]), "department": t["department"], "value": s["value"], "gt_date": vdate[first_v], "gt_visit": first_v, "range_from": {"date": vdate[first_v], "visit": first_v}, "range_to": {"date": vdate[last_v], "visit": last_v}, "window_from": {"date": vdate[wf_v], "visit": wf_v}, "window_to": {"date": vdate[wt_v], "visit": wt_v}, "horizon_visits": hv, "horizon_months": months(pdate(vdate[first_v]), pdate(vdate[last_v])), "n_mentions": len(supp), "evidence_visits": supp[:12], "still_current": s["valid_until"] == "current"} if t["entity"] not in formA or hv > formA[t["entity"]]["horizon_visits"]: formA[t["entity"]] = seed formB = {} for t in tl: supp = sorted({v for s in t["segments"] for v in s.get("supporting_visits", [])}) if not supp: continue last = supp[-1] if LAST_V - last < MIN_VIS or qtype(t["aspect"]) not in CORE: continue val = t["entity"] for s in t["segments"]: if last in s.get("supporting_visits", []): val = s["value"] break if not concrete(val) or len(toks(val) | toks(t["entity"])) < 2: continue if recurs_after(toks(t["entity"]) | toks(val), last, 0.6): continue formB[t["entity"]] = {"form": "never_again", "entity": t["entity"], "aspect": t["aspect"], "qtype": qtype(t["aspect"]), "department": t["department"], "value": val, "gt_date": vdate[last], "gt_visit": last, "horizon_visits": LAST_V - last, "horizon_months": months(pdate(vdate[last]), pdate(last_date)), "n_mentions": len(supp), "evidence_visits": supp[-6:], "last_seen_date": vdate[last]} def stratify(seeds, cap): by = defaultdict(list) for s in seeds: by[s["qtype"]].append(s) out = [] for qt, lst in by.items(): lst.sort(key=lambda x: -x["horizon_visits"]) out += lst[:cap] return out A = stratify(list(formA.values()), CAP_A) B = stratify(list(formB.values()), CAP_B) seeds = A + B json.dump({"axis": "1_long_horizon", "seeds": seeds}, open(f"{BASE}/_seed_axis1.json", "w"), ensure_ascii=False, indent=1) print(f"FORM A: {len(formA)} entities -> {len(A)} seeds (cap {CAP_A}/type); {n_conflict} windows left out " f"because another department records a different value in them") print(f"FORM B: {len(formB)} entities pass the lexical filter -> {len(B)} seeds (cap {CAP_B}/type)") print(f"total seeds: {len(seeds)} | min horizon_visits: {min(s['horizon_visits'] for s in seeds)}") print("FORM A by qtype:", dict(Counter(s["qtype"] for s in A))) print("FORM B by qtype:", dict(Counter(s["qtype"] for s in B))) print("\nwrote _seed_axis1.json") def post(gen_path: str) -> None: """Post-processing of the generated axis-1 questions; writes questions_axis1.jsonl.""" LAST_V = max(json.loads(_l)["visit_number"] for _l in open(f"{BASE}/event_chain.jsonl")) MIN_HORIZON_VISITS = 100 DATE = re.compile(r"\d{4}-\d{2}-\d{2}") recs = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl")] vdate = {r["visit_number"]: r["date"] for r in recs} last_date = max(vdate.values()) date2v = {} for r in sorted(recs, key=lambda x: x["visit_number"]): date2v.setdefault(r["date"], r["visit_number"]) def gt_visit(d): if d in date2v: return date2v[d] later = [r["visit_number"] for r in recs if r["date"] >= d] return min(later) if later else 1 seed_list = json.load(open(f"{BASE}/_seed_axis1.json"))["seeds"] by_entity = {(str(s["entity"]).strip(), s["form"]): s for s in seed_list} n_date = Counter(s["gt_date"] for s in seed_list) by_date = {s["gt_date"]: s for s in seed_list if n_date[s["gt_date"]] == 1} def seed_of(q): """The question's seed: by (entity, form), else by gt_date when exactly one seed has that date.""" sd = by_entity.get((str(q.get("entity") or "").strip(), q.get("form"))) return sd or by_date.get(q["gt_date"]) raw = json.load(open(gen_path)) qs = raw["questions"] if isinstance(raw, dict) else raw out, seen, warns = [], set(), [] no_text = sum(1 for q in qs if not has_text(q)) if no_text: print(f"skipped {no_text} generated question(s) without question text") qs = [q for q in qs if has_text(q)] bad = [q for q in qs if not (isinstance(q.get("gt_date"), str) and DATE.fullmatch(q["gt_date"]) and isinstance(q.get("evidence_visits"), list))] bad += [q for q in qs if q not in bad and q.get("template") == "T2" and not DATE.search(q["question"])] for q in bad: print(f"skipped a generated question without a gt_date, evidence_visits or, for T2, dates in its text: " f"{q['question'][:60]!r}") qs = [q for q in qs if q not in bad] for q in qs: key = q["question"].strip().lower() if key in seen: continue seen.add(key) onset = q["gt_date"] sd = seed_of(q) if sd is None: print(f"warning: no seed found for {q.get('entity')!r} ({q.get('form')}, gt_date {onset}); " f"no entity attached: {q['question'][:55]!r}") qdates = sorted(DATE.findall(q["question"])) if q.get("form") == "range": if len(qdates) >= 2: lo, hi = qdates[0], qdates[-1] in_range = sorted(vdate[v] for v in q["evidence_visits"] if lo <= vdate.get(v, "") <= hi) q["gt_date"] = in_range[0] if in_range else onset elif len(qdates) == 1: q["gt_date"] = qdates[0] gt = q["gt_date"] gv = gt_visit(gt) latest_q_v = max((gt_visit(d) for d in qdates), default=0) fv = min(max(gv + MIN_HORIZON_VISITS, latest_q_v), LAST_V) q["gt_visit"] = gv q["question_appearance_window"] = {"from": vdate[fv], "to": last_date} q["axis"] = "1_long_horizon" if sd: q["axis_specific"] = {**(q.get("axis_specific") or {}), "entity": sd["entity"]} q["qid"] = f"{PATIENT}/axis1/{len(out):03d}" q["grounding"] = "record" if q.get("template") == "T2": if onset in qdates: warns.append(f"T2 onset {onset} is a shown endpoint: {q['question'][:55]}") if not (qdates[0] <= gt <= qdates[-1]): warns.append(f"T2 gt {gt} NOT inside range {qdates}: {q['question'][:55]}") if sd and "window_from" in sd: wf, wt = sd["window_from"]["date"], sd["window_to"]["date"] if not (wf in qdates and wt in qdates): warns.append(f"T2 dates {qdates} != seed window [{wf},{wt}]: {q['question'][:55]}") ev_in = [v for v in q["evidence_visits"] if wf <= vdate.get(v, "") <= wt] if len(ev_in) < 1: warns.append(f"T2 NO in-range evidence: {q['question'][:55]}") out.append(q) with open(f"{BASE}/questions_axis1.jsonl", "w", encoding="utf-8") as fh: for q in out: fh.write(json.dumps(q, ensure_ascii=False) + "\n") print(f"axis-1 questions: {len(out)} ({len(qs) - len(out)} dropped as dup)") print("by form :", dict(Counter(q["form"] for q in out))) print("by qtype:", dict(Counter(q["qtype"] for q in out))) print("by template:", dict(Counter(q.get("template", "?") for q in out))) hv = [q["horizon_visits"] for q in out] if hv: print(f"horizon_visits: min {min(hv)} max {max(hv)} (minimum {MIN_HORIZON_VISITS})") print("horizon below minimum:", sum(1 for q in out if q["horizon_visits"] < MIN_HORIZON_VISITS)) print("degenerate appearance window:", sum(1 for q in out if q["question_appearance_window"]["from"] >= q["question_appearance_window"]["to"])) print("range warnings:", len(warns)) print("\nwrote questions_axis1.jsonl") def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) sp = ap.add_subparsers(dest="cmd", required=True) sp.add_parser("seeds") sp.add_parser("post").add_argument("generated") a = ap.parse_args() seeds() if a.cmd == "seeds" else post(a.generated) if __name__ == "__main__": main()