Download code_release/code/construction/pipeline/questions/axis8.py from clinical-agent-memory-bench/meme-bench: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/spaces/clinical-agent-memory-bench/meme-bench/resolve/main/code_release/code/construction/pipeline/questions/axis8.py
- Command line
-
hf download hf://spaces/clinical-agent-memory-bench/meme-bench/code_release/code/construction/pipeline/questions/axis8.py
-
curl -L -o axis8.py https://huggingface.co/spaces/clinical-agent-memory-bench/meme-bench/resolve/main/code_release/code/construction/pipeline/questions/axis8.py
12.2 kB
| #!/usr/bin/env python3 | |
| """Axis 8 (implicit temporal state): the LLM finds time-bounded instructions (deadline, fixed_duration, | |
| event_bounded); code computes the state at a checkpoint after the instruction's visit, alternately before and after | |
| the boundary (start + duration). An event_bounded instruction is asked as not yet ended. A passed deadline and an | |
| event_bounded instruction are used only if no visit to the same department lies between the instruction and the | |
| checkpoint. | |
| python -m pipeline.questions.axis8 discovery -> _discover_axis8.json | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from collections import Counter | |
| from datetime import date, timedelta | |
| from pathlib import Path | |
| from pipeline.common import BASE, Progress, model, patient_id | |
| from pipeline.llm import ChatClient | |
| PATIENT = patient_id() | |
| ENGINE = model("generation") | |
| AXIS = 8 | |
| SCAN_FIELDS = ["doctor_plans", "health_advice", "doctor_extra_notes", "doctor_notes"] | |
| UNIT = {"day": 1, "days": 1, "week": 7, "weeks": 7, "wk": 7, "month": 30, "months": 30, "mo": 30, | |
| "year": 365, "years": 365, "yr": 365} | |
| QUESTION_DEADLINE = ("At my {department_display} visit on {start_date} I was asked to come back for a " | |
| "follow-up. Is it time for me to book that one yet?") | |
| PROSE_DEADLINE = { | |
| "not_yet_due": ("Not yet. At your {department_display} visit on {start_date} you were asked to come back " | |
| "in {duration_phrase}, which puts it around {derived_boundary}. Today is " | |
| "{checkpoint_date}, so the interval has not elapsed and there is nothing to arrange yet."), | |
| "overdue": ("Yes — and it is already late. The {department_display} plan on {start_date} was to come " | |
| "back in {duration_phrase}, so it was due around {derived_boundary}. Today is " | |
| "{checkpoint_date} and no {department_display} visit has been recorded since, so it is " | |
| "{overdue_phrase} overdue — book it now."), | |
| } | |
| QUESTION_FIXED = ("My {department_display} team told me on {start_date} to {restriction_phrase}. " | |
| "Am I clear of that restriction now?") | |
| PROSE_FIXED = { | |
| "active": ("Not yet. On {start_date} your {department_display} team said to {restriction_phrase} for " | |
| "{duration_phrase}, which runs until about {derived_boundary}. Today is {checkpoint_date}, so " | |
| "the restriction is still in force."), | |
| "expired": ("Yes. On {start_date} your {department_display} team said to {restriction_phrase} for " | |
| "{duration_phrase}, which ended around {derived_boundary}. Today is {checkpoint_date}, so the " | |
| "restriction has lapsed."), | |
| } | |
| QUESTION_EVENT = ("On {start_date} my {department_display} team said to {restriction_phrase} until " | |
| "{terminal_condition}. Can I stop now?") | |
| PROSE_EVENT = { | |
| "indeterminate": ("Not yet. On {start_date} {department_display} said to {restriction_phrase} until " | |
| "{terminal_condition}. By {checkpoint_date} no visit has recorded that, so keep to it " | |
| "until {department_display} confirms it."), | |
| } | |
| DISCOVER_SYS = ( | |
| "You read one patient's care plans and find time-bounded instructions: instructions whose current\n" | |
| "state has to be worked out from when they were given and how long they last, not just recalled.\n" | |
| "Return only JSON.") | |
| def _dept_head(dept): | |
| m = re.match(r"[A-Za-z][A-Za-z0-9 &/\-']*", dept) | |
| return m.group(0).strip().lower() if m else None | |
| def same_department(a, b): | |
| """Whether two department names are equal or have the same leading name.""" | |
| return a == b or (_dept_head(a) is not None and _dept_head(a) == _dept_head(b)) | |
| def discover_rules(cli, by_v): | |
| rows = [] | |
| for v in sorted(by_v): | |
| for f in SCAN_FIELDS: | |
| for a in by_v[v].get(f) or []: | |
| rows.append(f"v{v} {by_v[v]['date']} [{by_v[v]['department']}] {f}: {str(a)[:120]}") | |
| chunks = ["\n".join(rows[i:i + 120]) for i in range(0, len(rows), 120)] or [""] | |
| found = [] | |
| for plan_facts in chunks: | |
| usr = ( | |
| "Plan and advice facts, one per line (visit, date, department, field, text):\n" + plan_facts + "\n\n" | |
| "Find every time-bounded instruction and classify it by mechanism:\n" | |
| "- deadline: 'come back / recheck in N <unit>', a follow-up that falls due after an interval.\n" | |
| "- fixed_duration: 'avoid / hold / restrict X for N <unit>', a restriction that lifts after an\n" | |
| " interval. A follow-up appointment is not fixed_duration, nor is a treatment course (take,\n" | |
| " continue or change a drug for N <unit>).\n" | |
| "- event_bounded: 'hold / avoid X until Y happens / resolves / normalizes', which ends on a\n" | |
| " condition, not on a date.\n\n" | |
| "Ignore one-off notes with no time bound. For each instruction give its visit, its mechanism,\n" | |
| "the instruction word for word, and:\n" | |
| "- for deadline and fixed_duration: duration_n (an integer) and duration_unit (day, week, month\n" | |
| " or year); for fixed_duration also a short restriction_phrase, such as 'avoid heavy lifting';\n" | |
| "- for event_bounded: restriction_phrase and terminal_condition, such as 'the corneal defect\n" | |
| " heals'.\n\n" | |
| "Return JSON in this form:\n" | |
| '{"rules": [{\n' | |
| ' "visit": <int>,\n' | |
| ' "mechanism": "<deadline|fixed_duration|event_bounded>",\n' | |
| ' "rule_text": "<word for word>",\n' | |
| ' "entity": "<what it is about>",\n' | |
| ' "duration_n": <int|null>,\n' | |
| ' "duration_unit": "<unit|null>",\n' | |
| ' "restriction_phrase": "<str|null>",\n' | |
| ' "terminal_condition": "<str|null>"\n' | |
| "}]}") | |
| try: | |
| found += cli.complete_json(DISCOVER_SYS, usr).get("rules") or [] | |
| except Exception as e: | |
| print(f" discover chunk FAILED: {str(e)[:100]}") | |
| return found | |
| def discover() -> None: | |
| """Discover the axis-8 candidates of this patient -> _discover_axis8.json.""" | |
| cli = ChatClient(model=ENGINE, reasoning_effort="medium", max_completion_tokens=6000) | |
| chain = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl") if l.strip()] | |
| by_v = {r["visit_number"]: r for r in chain} | |
| vs = sorted(by_v) | |
| plast = max(r["date"] for r in chain) | |
| def dt(v): | |
| return date.fromisoformat(by_v[v]["date"]) | |
| with Progress("axis8 discover (LLM)", total=1, out=f"{BASE}/progress_axis8_discover.json") as pg: | |
| rules = discover_rules(cli, by_v) | |
| pg.tick(ok=bool(rules)) | |
| candidates, revisited, no_checkpoint = [], [], [] | |
| by_mech = {"deadline": 0, "fixed_duration": 0, "event_bounded": 0} | |
| n_bad = sum(1 for r in rules if not isinstance(r, dict)) | |
| if n_bad: | |
| print(f" skipped {n_bad} rule(s) that are not objects") | |
| for r in (r for r in rules if isinstance(r, dict)): | |
| mech = r.get("mechanism") | |
| rv = r.get("visit") | |
| if mech not in by_mech or rv not in by_v: | |
| continue | |
| dept = by_v[rv]["department"] | |
| disp = dept.split(" (")[0] | |
| start = dt(rv) | |
| base = {"department_display": disp, "start_date": start.isoformat(), | |
| "rule_text": r.get("rule_text"), "temporal_mechanism": mech, "rule_visit": rv} | |
| if mech in ("deadline", "fixed_duration"): | |
| n, unit = r.get("duration_n"), str(r.get("duration_unit") or "").rstrip("s") | |
| if not isinstance(n, int) or unit not in UNIT: | |
| continue | |
| dur_days = n * UNIT[unit] | |
| boundary = start + timedelta(days=dur_days) | |
| want_after = by_mech[mech] % 2 == 1 | |
| target = boundary + timedelta(days=max(20, dur_days // 2)) if want_after \ | |
| else boundary - timedelta(days=max(2, dur_days // 3)) | |
| ck = next((v for v in vs if v > rv and dt(v) >= target), vs[-1]) | |
| ck_date = dt(ck) | |
| ck_ok = ck > rv and (want_after or ck_date < boundary) | |
| dur_phrase = f"{n} {unit}" + ("s" if n > 1 else "") | |
| ops = {**base, "duration_phrase": dur_phrase, "derived_boundary": boundary.isoformat(), | |
| "checkpoint_date": ck_date.isoformat(), "duration_n": n, "duration_unit": unit, | |
| "duration_days": dur_days, "boundary_delta_days": (ck_date - boundary).days, | |
| "checkpoint_visit": ck, "overdue_phrase": f"{(ck_date - boundary).days} days"} | |
| if mech == "deadline": | |
| state = "overdue" if ck_date >= boundary else "not_yet_due" | |
| q, gold = QUESTION_DEADLINE.format(**ops), PROSE_DEADLINE[state].format(**ops) | |
| else: | |
| ops["restriction_phrase"] = r.get("restriction_phrase") or "follow the restriction" | |
| state = "expired" if ck_date >= boundary else "active" | |
| q, gold = QUESTION_FIXED.format(**ops), PROSE_FIXED[state].format(**ops) | |
| else: | |
| cond = r.get("terminal_condition") | |
| if not cond: | |
| continue | |
| ck = vs[min(len(vs) - 1, vs.index(rv) + 8)] | |
| ck_date = dt(ck) | |
| ck_ok = ck > rv | |
| state = "indeterminate" | |
| ops = {**base, "restriction_phrase": r.get("restriction_phrase") or "hold it", | |
| "terminal_condition": cond, "checkpoint_date": ck_date.isoformat(), | |
| "checkpoint_visit": ck, "resolved_date": None, "duration_days": None, | |
| "boundary_delta_days": None} | |
| q, gold = QUESTION_EVENT.format(**ops), PROSE_EVENT[state].format(**ops) | |
| qid = f"{PATIENT}/axis{AXIS}/{sum(by_mech.values()):03d}" | |
| by_mech[mech] += 1 | |
| if not ck_ok: | |
| no_checkpoint.append(qid) | |
| continue | |
| if state in ("overdue", "indeterminate"): | |
| asked_at = ck if ck > rv else vs[-1] | |
| if any(same_department(by_v[v]["department"], dept) for v in vs if rv < v <= asked_at): | |
| revisited.append(qid) | |
| continue | |
| candidates.append({ | |
| "qid": qid, "patient_id": PATIENT, | |
| "axis": "8_implicit_temporal_state", "question": q, "answer": gold, | |
| "answer_type": "state", "grounding": "record", | |
| "evidence_visits": [rv], "gt_visit": rv, "gt_date": by_v[rv]["date"], | |
| "question_appearance_window": {"from": by_v[ck]["date"], "to": plast}, | |
| "derivation": { | |
| "evidence": [{"visit": rv, "date": by_v[rv]["date"], "field": "doctor_plans", | |
| "atom": r.get("rule_text")}], | |
| "steps": [f"{mech} rule at v{rv} ({start}): {r.get('rule_text')!r}.", | |
| (f"boundary = {start} + {ops.get('duration_phrase','(condition)')} = " | |
| f"{ops.get('derived_boundary','n/a')}."), | |
| f"checkpoint {ops['checkpoint_date']} -> {state}."], | |
| "gold": gold, "rivals": [], "operands": ops, "derived": True}, | |
| "provenance": f"LLM-discovered {mech} ({ENGINE}); state COMPUTED, not recalled", | |
| "axis_specific": {"state": state, "mechanism": mech, "checkpoint_visit": ck}, | |
| }) | |
| out = {"axis": AXIS, "patient_id": PATIENT, "n_record": len(candidates), "candidates": candidates} | |
| Path(f"{BASE}/_discover_axis{AXIS}.json").write_text(json.dumps(out, ensure_ascii=False, indent=1)) | |
| print(f"\npatient {PATIENT} engine={ENGINE}") | |
| print(f" time-bounded rules discovered (LLM): {len(rules)}") | |
| print(f" valid rules by mechanism : {by_mech}") | |
| print(f" overdue / event-bounded with a same-dept revisit : {len(revisited)} (skipped)") | |
| print(f" no checkpoint after the rule (and before the boundary, when asked before it): " | |
| f"{len(no_checkpoint)} (skipped)") | |
| print(f" candidates by mechanism : " | |
| f"{dict(Counter(c['axis_specific']['mechanism'] for c in candidates))}") | |
| print(f" CANDIDATES: {len(candidates)} -> {BASE}/_discover_axis{AXIS}.json") | |
| if __name__ == "__main__": | |
| discover() | |