clinical-agent-memory-bench's picture
Upload 99 files
750b88d verified
Raw History Blame Contribute Delete
12.2 kB
#!/usr/bin/env python3
"""Axis 8 (implicit temporal state): the LLM finds time-bounded instructions (deadline, fixed_duration,
event_bounded); code computes the state at a checkpoint after the instruction's visit, alternately before and after
the boundary (start + duration). An event_bounded instruction is asked as not yet ended. A passed deadline and an
event_bounded instruction are used only if no visit to the same department lies between the instruction and the
checkpoint.
python -m pipeline.questions.axis8 discovery -> _discover_axis8.json
"""
from __future__ import annotations
import json
import re
from collections import Counter
from datetime import date, timedelta
from pathlib import Path
from pipeline.common import BASE, Progress, model, patient_id
from pipeline.llm import ChatClient
PATIENT = patient_id()
ENGINE = model("generation")
AXIS = 8
SCAN_FIELDS = ["doctor_plans", "health_advice", "doctor_extra_notes", "doctor_notes"]
UNIT = {"day": 1, "days": 1, "week": 7, "weeks": 7, "wk": 7, "month": 30, "months": 30, "mo": 30,
"year": 365, "years": 365, "yr": 365}
QUESTION_DEADLINE = ("At my {department_display} visit on {start_date} I was asked to come back for a "
"follow-up. Is it time for me to book that one yet?")
PROSE_DEADLINE = {
"not_yet_due": ("Not yet. At your {department_display} visit on {start_date} you were asked to come back "
"in {duration_phrase}, which puts it around {derived_boundary}. Today is "
"{checkpoint_date}, so the interval has not elapsed and there is nothing to arrange yet."),
"overdue": ("Yes — and it is already late. The {department_display} plan on {start_date} was to come "
"back in {duration_phrase}, so it was due around {derived_boundary}. Today is "
"{checkpoint_date} and no {department_display} visit has been recorded since, so it is "
"{overdue_phrase} overdue — book it now."),
}
QUESTION_FIXED = ("My {department_display} team told me on {start_date} to {restriction_phrase}. "
"Am I clear of that restriction now?")
PROSE_FIXED = {
"active": ("Not yet. On {start_date} your {department_display} team said to {restriction_phrase} for "
"{duration_phrase}, which runs until about {derived_boundary}. Today is {checkpoint_date}, so "
"the restriction is still in force."),
"expired": ("Yes. On {start_date} your {department_display} team said to {restriction_phrase} for "
"{duration_phrase}, which ended around {derived_boundary}. Today is {checkpoint_date}, so the "
"restriction has lapsed."),
}
QUESTION_EVENT = ("On {start_date} my {department_display} team said to {restriction_phrase} until "
"{terminal_condition}. Can I stop now?")
PROSE_EVENT = {
"indeterminate": ("Not yet. On {start_date} {department_display} said to {restriction_phrase} until "
"{terminal_condition}. By {checkpoint_date} no visit has recorded that, so keep to it "
"until {department_display} confirms it."),
}
DISCOVER_SYS = (
"You read one patient's care plans and find time-bounded instructions: instructions whose current\n"
"state has to be worked out from when they were given and how long they last, not just recalled.\n"
"Return only JSON.")
def _dept_head(dept):
m = re.match(r"[A-Za-z][A-Za-z0-9 &/\-']*", dept)
return m.group(0).strip().lower() if m else None
def same_department(a, b):
"""Whether two department names are equal or have the same leading name."""
return a == b or (_dept_head(a) is not None and _dept_head(a) == _dept_head(b))
def discover_rules(cli, by_v):
rows = []
for v in sorted(by_v):
for f in SCAN_FIELDS:
for a in by_v[v].get(f) or []:
rows.append(f"v{v} {by_v[v]['date']} [{by_v[v]['department']}] {f}: {str(a)[:120]}")
chunks = ["\n".join(rows[i:i + 120]) for i in range(0, len(rows), 120)] or [""]
found = []
for plan_facts in chunks:
usr = (
"Plan and advice facts, one per line (visit, date, department, field, text):\n" + plan_facts + "\n\n"
"Find every time-bounded instruction and classify it by mechanism:\n"
"- deadline: 'come back / recheck in N <unit>', a follow-up that falls due after an interval.\n"
"- fixed_duration: 'avoid / hold / restrict X for N <unit>', a restriction that lifts after an\n"
" interval. A follow-up appointment is not fixed_duration, nor is a treatment course (take,\n"
" continue or change a drug for N <unit>).\n"
"- event_bounded: 'hold / avoid X until Y happens / resolves / normalizes', which ends on a\n"
" condition, not on a date.\n\n"
"Ignore one-off notes with no time bound. For each instruction give its visit, its mechanism,\n"
"the instruction word for word, and:\n"
"- for deadline and fixed_duration: duration_n (an integer) and duration_unit (day, week, month\n"
" or year); for fixed_duration also a short restriction_phrase, such as 'avoid heavy lifting';\n"
"- for event_bounded: restriction_phrase and terminal_condition, such as 'the corneal defect\n"
" heals'.\n\n"
"Return JSON in this form:\n"
'{"rules": [{\n'
' "visit": <int>,\n'
' "mechanism": "<deadline|fixed_duration|event_bounded>",\n'
' "rule_text": "<word for word>",\n'
' "entity": "<what it is about>",\n'
' "duration_n": <int|null>,\n'
' "duration_unit": "<unit|null>",\n'
' "restriction_phrase": "<str|null>",\n'
' "terminal_condition": "<str|null>"\n'
"}]}")
try:
found += cli.complete_json(DISCOVER_SYS, usr).get("rules") or []
except Exception as e:
print(f" discover chunk FAILED: {str(e)[:100]}")
return found
def discover() -> None:
"""Discover the axis-8 candidates of this patient -> _discover_axis8.json."""
cli = ChatClient(model=ENGINE, reasoning_effort="medium", max_completion_tokens=6000)
chain = [json.loads(l) for l in open(f"{BASE}/event_chain.jsonl") if l.strip()]
by_v = {r["visit_number"]: r for r in chain}
vs = sorted(by_v)
plast = max(r["date"] for r in chain)
def dt(v):
return date.fromisoformat(by_v[v]["date"])
with Progress("axis8 discover (LLM)", total=1, out=f"{BASE}/progress_axis8_discover.json") as pg:
rules = discover_rules(cli, by_v)
pg.tick(ok=bool(rules))
candidates, revisited, no_checkpoint = [], [], []
by_mech = {"deadline": 0, "fixed_duration": 0, "event_bounded": 0}
n_bad = sum(1 for r in rules if not isinstance(r, dict))
if n_bad:
print(f" skipped {n_bad} rule(s) that are not objects")
for r in (r for r in rules if isinstance(r, dict)):
mech = r.get("mechanism")
rv = r.get("visit")
if mech not in by_mech or rv not in by_v:
continue
dept = by_v[rv]["department"]
disp = dept.split(" (")[0]
start = dt(rv)
base = {"department_display": disp, "start_date": start.isoformat(),
"rule_text": r.get("rule_text"), "temporal_mechanism": mech, "rule_visit": rv}
if mech in ("deadline", "fixed_duration"):
n, unit = r.get("duration_n"), str(r.get("duration_unit") or "").rstrip("s")
if not isinstance(n, int) or unit not in UNIT:
continue
dur_days = n * UNIT[unit]
boundary = start + timedelta(days=dur_days)
want_after = by_mech[mech] % 2 == 1
target = boundary + timedelta(days=max(20, dur_days // 2)) if want_after \
else boundary - timedelta(days=max(2, dur_days // 3))
ck = next((v for v in vs if v > rv and dt(v) >= target), vs[-1])
ck_date = dt(ck)
ck_ok = ck > rv and (want_after or ck_date < boundary)
dur_phrase = f"{n} {unit}" + ("s" if n > 1 else "")
ops = {**base, "duration_phrase": dur_phrase, "derived_boundary": boundary.isoformat(),
"checkpoint_date": ck_date.isoformat(), "duration_n": n, "duration_unit": unit,
"duration_days": dur_days, "boundary_delta_days": (ck_date - boundary).days,
"checkpoint_visit": ck, "overdue_phrase": f"{(ck_date - boundary).days} days"}
if mech == "deadline":
state = "overdue" if ck_date >= boundary else "not_yet_due"
q, gold = QUESTION_DEADLINE.format(**ops), PROSE_DEADLINE[state].format(**ops)
else:
ops["restriction_phrase"] = r.get("restriction_phrase") or "follow the restriction"
state = "expired" if ck_date >= boundary else "active"
q, gold = QUESTION_FIXED.format(**ops), PROSE_FIXED[state].format(**ops)
else:
cond = r.get("terminal_condition")
if not cond:
continue
ck = vs[min(len(vs) - 1, vs.index(rv) + 8)]
ck_date = dt(ck)
ck_ok = ck > rv
state = "indeterminate"
ops = {**base, "restriction_phrase": r.get("restriction_phrase") or "hold it",
"terminal_condition": cond, "checkpoint_date": ck_date.isoformat(),
"checkpoint_visit": ck, "resolved_date": None, "duration_days": None,
"boundary_delta_days": None}
q, gold = QUESTION_EVENT.format(**ops), PROSE_EVENT[state].format(**ops)
qid = f"{PATIENT}/axis{AXIS}/{sum(by_mech.values()):03d}"
by_mech[mech] += 1
if not ck_ok:
no_checkpoint.append(qid)
continue
if state in ("overdue", "indeterminate"):
asked_at = ck if ck > rv else vs[-1]
if any(same_department(by_v[v]["department"], dept) for v in vs if rv < v <= asked_at):
revisited.append(qid)
continue
candidates.append({
"qid": qid, "patient_id": PATIENT,
"axis": "8_implicit_temporal_state", "question": q, "answer": gold,
"answer_type": "state", "grounding": "record",
"evidence_visits": [rv], "gt_visit": rv, "gt_date": by_v[rv]["date"],
"question_appearance_window": {"from": by_v[ck]["date"], "to": plast},
"derivation": {
"evidence": [{"visit": rv, "date": by_v[rv]["date"], "field": "doctor_plans",
"atom": r.get("rule_text")}],
"steps": [f"{mech} rule at v{rv} ({start}): {r.get('rule_text')!r}.",
(f"boundary = {start} + {ops.get('duration_phrase','(condition)')} = "
f"{ops.get('derived_boundary','n/a')}."),
f"checkpoint {ops['checkpoint_date']} -> {state}."],
"gold": gold, "rivals": [], "operands": ops, "derived": True},
"provenance": f"LLM-discovered {mech} ({ENGINE}); state COMPUTED, not recalled",
"axis_specific": {"state": state, "mechanism": mech, "checkpoint_visit": ck},
})
out = {"axis": AXIS, "patient_id": PATIENT, "n_record": len(candidates), "candidates": candidates}
Path(f"{BASE}/_discover_axis{AXIS}.json").write_text(json.dumps(out, ensure_ascii=False, indent=1))
print(f"\npatient {PATIENT} engine={ENGINE}")
print(f" time-bounded rules discovered (LLM): {len(rules)}")
print(f" valid rules by mechanism : {by_mech}")
print(f" overdue / event-bounded with a same-dept revisit : {len(revisited)} (skipped)")
print(f" no checkpoint after the rule (and before the boundary, when asked before it): "
f"{len(no_checkpoint)} (skipped)")
print(f" candidates by mechanism : "
f"{dict(Counter(c['axis_specific']['mechanism'] for c in candidates))}")
print(f" CANDIDATES: {len(candidates)} -> {BASE}/_discover_axis{AXIS}.json")
if __name__ == "__main__":
discover()