File size: 10,171 Bytes
750b88d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#!/usr/bin/env bash
# Build one patient of the benchmark end to end; all outputs go to build/<patient_id>/.
#
#   bash build_patient.sh <patient_id> ["<clinical descriptor>"]
#
# Stages: A preprocessing (deterministic) -> B event chain (extract), evidence chain (evidence), change tags (tags)
# -> C questions (axes 1-2: seeds, LLM, post; axes 3-8: LLM discovery; golds written by the model for axes 3-4, from
#    templates for axes 5-8)
# -> D claim registry and synthetic-question targets -> E synthetic questions, gates, injection
# -> F dialogue simulation -> G checkpoints, validation, manifest.
# Requires the visit table and the term glossary (set at the top; neither distributed) and OPENAI_API_KEY.
# Set MEMEBENCH_ENV to a script that activates the Python environment, if needed, or PYTHON to the interpreter.
# MEMEBENCH_FROM=<step> resumes from a step (B1 ... G), keeping earlier outputs. Once stage E has written synthetic
# facts into the chains (event_chain.preinject.jsonl holds the chain before them), only F and G can be resumed;
# B1 rebuilds the event chain from the extraction.
# MEMEBENCH_WORKERS (default 64) and MEMEBENCH_WORKERS_SMALL (default 32) set the parallel calls of the LLM steps.
set -euo pipefail
export PYTHONHASHSEED=0

# Inputs. The visit table holds the synthetic timeline (SOAP-structured), the output of the abstraction and
# regeneration steps described in the paper; those steps are not part of this code.
# Fill in before running.
VISIT_TABLE="${MEMEBENCH_VISIT_TABLE:-<VISIT_TABLE>}"              # parquet, one row per visit
GLOSSARY="${MEMEBENCH_GLOSSARY:-<TERM_GLOSSARY>}"                  # term <TAB> normalized form, one pair per line
for f in "$VISIT_TABLE" "$GLOSSARY"; do
  [[ $f == \<* ]] && { echo "Set the visit table and the term glossary at the top of build_patient.sh"; exit 1; }
done
export MEMEBENCH_VISIT_TABLE="$VISIT_TABLE" MEMEBENCH_GLOSSARY="$GLOSSARY"

# Models (OpenAI-compatible names). Fill in before running.
EXTRACTION_MODEL="${MEMEBENCH_EXTRACTION_MODEL:-<EXTRACTION_MODEL>}"  # B1 event extraction, B3 change tags
GENERATION_MODEL="${MEMEBENCH_GENERATION_MODEL:-<GENERATION_MODEL>}"  # B2, C questions, E synthetic/gates, G validation
DIALOGUE_MODEL="${MEMEBENCH_DIALOGUE_MODEL:-<DIALOGUE_MODEL>}"        # F dialogue simulation
for m in "$EXTRACTION_MODEL" "$GENERATION_MODEL" "$DIALOGUE_MODEL"; do
  [[ $m == \<* ]] && { echo "Set the models at the top of build_patient.sh"; exit 1; }
done
export MEMEBENCH_EXTRACTION_MODEL="$EXTRACTION_MODEL" MEMEBENCH_GENERATION_MODEL="$GENERATION_MODEL" \
       MEMEBENCH_DIALOGUE_MODEL="$DIALOGUE_MODEL"

# Validation (stage G): axes whose questions are not filtered by the no-memory probe (comma-separated axis numbers).
NO_MEMORY_EXEMPT_AXES="${MEMEBENCH_NO_MEMORY_EXEMPT_AXES-5}"
export MEMEBENCH_NO_MEMORY_EXEMPT_AXES="$NO_MEMORY_EXEMPT_AXES"

PID="${1:?usage: bash build_patient.sh <patient_id> [<clinical descriptor>]}"
PDESC="${2:-}"
BASE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
W="$BASE/build/$PID"
cd "$BASE"
if [ -n "${MEMEBENCH_ENV:-}" ]; then
  source "$MEMEBENCH_ENV" >/dev/null || { echo "MEMEBENCH_ENV failed: $MEMEBENCH_ENV"; exit 1; }
fi

mkdir -p "$W"
export MEMEBENCH_WORK="$W" MEMEBENCH_PATIENT="$PID"

say() { printf "\n\033[1m== %s\033[0m\n" "$*"; }
run() { "${PYTHON:-python}" -u -m "$@"; }

say "A. preprocessing"
[ -f "$W/ALL_visits.txt" ] || run pipeline.preprocess export --out "$W/ALL_visits.txt"
[ -f "$W/visits.jsonl" ]      || run pipeline.preprocess parse --inp "$W/ALL_visits.txt" --out "$W/visits.jsonl"
[ -d "$W/visits" ]            || run pipeline.preprocess split              # -> visits/v*.json + visit_manifest.json
run pipeline.preprocess vitals                                               # -> vitals.json, vitals_timeline.json
[ -f "$W/patient.env" ]       || run pipeline.preprocess facts --pdesc "$PDESC"
source "$W/patient.env"

STEPS=(B1 B2 B3 C1 C2 D E F G)
step_i() { local i; for i in "${!STEPS[@]}"; do [ "${STEPS[$i]}" = "$1" ] && { echo "$i"; return 0; }; done; return 1; }
FROM="${MEMEBENCH_FROM:-B1}"
FROM_I=$(step_i "$FROM") || { echo "MEMEBENCH_FROM=$FROM is not one of: ${STEPS[*]}"; exit 1; }
from() { [ "$(step_i "$1")" -ge "$FROM_I" ]; }
if [ "$FROM" != "B1" ]; then
  [ -f "$W/event_records.json" ] && [ -f "$W/event_chain.jsonl" ] \
    || { echo "MEMEBENCH_FROM=$FROM but $W has no finished B1 output"; exit 1; }
  if [ -f "$W/event_chain.preinject.jsonl" ] && [ "$FROM_I" -lt "$(step_i F)" ]; then
    echo "MEMEBENCH_FROM=$FROM: stage E has already written synthetic facts into the event and evidence chains of"
    echo "$W (event_chain.preinject.jsonl is the chain before them); resume from F or G, or rerun from B1"
    exit 1
  fi
  if [ "$FROM_I" -ge "$(step_i F)" ] && [ ! -f "$W/event_chain.preinject.jsonl" ] \
     && grep -qs '"grounding": "synthetic' "$W"/questions_axis*.jsonl; then
    echo "MEMEBENCH_FROM=$FROM: the question banks hold synthetic questions whose facts stage E has not written into"
    echo "the chains; resume from D"
    exit 1
  fi
  say "RESUMING from $FROM (steps before it are skipped)"
fi

if from B1; then
say "B1. event extraction (one call per visit)"
rm -f "$W/failed_extract.json"
run pipeline.llm extract --manifest "$W/visit_manifest.json" --out "$W/event_records.json" \
    --model "$EXTRACTION_MODEL" --workers "${MEMEBENCH_WORKERS:-64}"
[ ! -f "$W/failed_extract.json" ] || { echo "extraction failed for some visits; see $W/failed_extract.json"; exit 1; }
run pipeline.event_chain event-chain "$W/event_records.json"                 # -> event_chain.jsonl
rm -f "${W:?}/event_chain.preinject.jsonl"                                    # the new chain has no synthetic facts
fi

if from B2; then
say "B2. evidence chain"
run pipeline.evidence_chain dept-index                                       # -> _dept_event_index.json
run pipeline.evidence_chain prep                                             # -> _evidence_manifest.json + batches
run pipeline.llm step --step evidence --manifest "$W/_evidence_manifest.json" \
    --out "$W/_evidence_records.json" --model "$GENERATION_MODEL" --workers "${MEMEBENCH_WORKERS_SMALL:-32}"
[ ! -f "$W/failed_evidence.json" ] || { echo "evidence failed for some batches; see $W/failed_evidence.json"; exit 1; }
run pipeline.evidence_chain post "$W/_evidence_records.json"                 # -> evidence_chain.jsonl
fi

if from B3; then
say "B3. change tags"
run pipeline.event_chain windows                                             # -> _tag_manifest.json + batches
run pipeline.llm step --step tags --manifest "$W/_tag_manifest.json" \
    --out "$W/_tag_records.json" --model "$EXTRACTION_MODEL" --workers "${MEMEBENCH_WORKERS_SMALL:-32}"
[ ! -f "$W/failed_tags.json" ] || { echo "tags failed for some windows; see $W/failed_tags.json"; exit 1; }
run pipeline.event_chain tags-post "$W/_tag_records.json"                    # -> tags.jsonl
fi

if from C1; then
say "C1. axes 1-2 (seeds -> LLM -> post)"
for N in 1 2; do
  run "pipeline.questions.axis$N" seeds
  run pipeline.llm step --step "axis$N" --input "$W/_seed_axis${N}.json" \
      --out "$W/_axis${N}_generated.json" --model "$GENERATION_MODEL"
  run "pipeline.questions.axis$N" post "$W/_axis${N}_generated.json"
done
fi

if from C2; then
say "C2. axes 3-8 (LLM discovery; golds from the model for axes 3-4, from templates for axes 5-8)"
for N in 3 4 5 6 7 8; do run "pipeline.questions.axis$N"; done
run pipeline.synthetic banks                                             # -> questions_axis{3..8}.jsonl
fi

if from D; then
say "D. claim registry and synthetic-question targets"
[ -f "$W/injection_registry.json" ] || run pipeline.synthetic registry --init
run pipeline.synthetic claims
run pipeline.synthetic targets                                           # -> synthetic_targets.json
fi

if from E; then
say "E. synthetic questions and injection"
SHORT=$(run pipeline.synthetic short-axes)
if [ -n "$SHORT" ]; then
  run pipeline.synthetic generate --axes "$SHORT" --model "$GENERATION_MODEL" \
    || echo "  synthetic generation reported failures; continuing"
  run pipeline.synthetic wire                                            # bank += synthetic Q; injection_plan.json
  rm -f "$W/injection_registry.json"
  run pipeline.synthetic registry --init
  run pipeline.synthetic batches
  run pipeline.synthetic claims >/dev/null                               # all claims incl. synthetic
  run pipeline.gates lint || { echo "LINT FAILED: not injecting"; exit 1; }
  run pipeline.gates persona || { echo "PERSONA GATE FAILED: the persona check did not complete"; exit 1; }
  run pipeline.gates lint || { echo "LINT FAILED after the persona gate: not injecting"; exit 1; }
  run pipeline.synthetic apply                                           # atoms -> event_chain + evidence_chain
else
  say "E. no synthetic questions needed"
  rm -f "${W:?}/synthetic_pairs.json"
  run pipeline.synthetic wire
fi
fi

if from F; then
say "F. dialogues"
[ "$FROM_I" -ge "$(step_i F)" ] || rm -f "${W:?}/dialogues.jsonl"
run pipeline.dialogues generate --all --workers "${MEMEBENCH_WORKERS:-64}" --out "$W/dialogues.jsonl"
if [ "$(run pipeline.dialogues count-empty)" != "0" ]; then
  run pipeline.dialogues generate --all --workers "${MEMEBENCH_WORKERS:-64}" --out "$W/dialogues.jsonl"
fi
N_EMPTY=$(run pipeline.dialogues count-empty)
[ "$N_EMPTY" = "0" ] || { echo "dialogues: $N_EMPTY visits still empty after a retry pass"; exit 1; }
fi

if from G; then
say "G1. checkpoints"
run pipeline.validation assign                                             # -> checkpoints.jsonl
run pipeline.validation audit                                              # -> checkpoint_audit.json
say "G2. validation (oracle + no-memory probe)"
run pipeline.validation validate --workers "${MEMEBENCH_WORKERS:-64}"      # -> validation.jsonl
N_ERR=$(run pipeline.validation count-errors)
[ "$N_ERR" = "0" ] \
  || { echo "validation: $N_ERR questions have failed probe or judge calls; see validation.jsonl"; exit 1; }
say "G3. package"
run pipeline.validation manifest                                           # -> MANIFEST.json
fi

say "DONE: $W"