parsa-hke's picture
Stroke Decision Copilot: page, bank of 195 cases (held-out by alias), EXP-66 results (part 2)
713f736 verified
Raw History Blame Contribute Delete
4.14 kB
"""The fine-tuning dataset for the small reasoner (Item 4, DEC-30): training scenarios (scenarios.generate_training)
with their gold answers (gold.gold_answer), as chat examples in prompt v3's own format.
Each example is the request the reasoner gets in evaluation (system prompt v3 with the rulebook; the user message
holding the vision JSON and the note) and the gold reply as the assistant's answer, so the model is trained on exactly
the input it will be scored on. Every gold answer must pass gold.check(); one that does not stops the build, because a
training set with a wrong answer in it would teach that error. The validation slice (for early stopping) is every
tenth example by a fixed shuffle. Nothing here uses patient data: the scenarios are synthetic.
"""
import hashlib
import json
import random
from collections import Counter
from pathlib import Path
from strokecopilot.nlp.corpus import ingest
from strokecopilot.nlp.reasoner import gold
from strokecopilot.nlp.reasoner import prompt as rp
from strokecopilot.nlp.verifier import scenarios as sc
PROMPT = "reasoner_v3"
VAL_EVERY = 10
SPLIT_SEED = 20261117
def examples(scenarios, version=PROMPT, corpus=None):
"""[{id, route, user, answer}] for the scenarios, each answer a gold reply as compact JSON. Raises if a gold
answer fails gold.check()."""
corpus = corpus or ingest.load()
out = []
for s in scenarios:
reply = gold.gold_answer(s.vision, s.facts, corpus)
problems = gold.check(s.vision, s.facts, s.note, reply, corpus)
if problems:
raise ValueError(f"The gold answer for {s.id} fails its checks: {problems[:3]}")
out.append({"id": s.id, "route": [s.expected["thrombolysis"]["reason"], s.expected["thrombectomy"]["reason"]],
"user": rp.user_message(s.vision, s.note), "answer": json.dumps(reply, ensure_ascii=False)})
return out
def split(items, every=VAL_EVERY, seed=SPLIT_SEED):
"""(train, validation): a fixed shuffle, every `every`-th item to validation."""
order = list(items)
random.Random(seed).shuffle(order)
return [x for i, x in enumerate(order) if i % every], [x for i, x in enumerate(order) if not i % every]
def build(out_dir, n, seed=sc.TRAIN_SEED, held_out=None, version=PROMPT):
"""Write train.jsonl, val.jsonl, system_prompt.md and meta.json to out_dir; returns meta. The system prompt is
the same for every example, so it is written once and its hash recorded."""
held_out = sc.held_out_routes(seed) if held_out is None else list(held_out)
scenarios = sc.generate_training(n, seed=seed, held_out=held_out)
items = examples(scenarios, version)
train, val = split(items)
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
system = rp.system_prompt(version)
(out / "system_prompt.md").write_text(system, encoding="utf-8")
for name, rows in (("train", train), ("val", val)):
with (out / f"{name}.jsonl").open("w", encoding="utf-8") as f:
for row in rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
chars = [len(system) + len(x["user"]) + len(x["answer"]) for x in items]
answer_chars = [len(x["answer"]) for x in items]
meta = {"n": len(items), "train": len(train), "val": len(val), "seed": seed, "split_seed": SPLIT_SEED,
"prompt_version": version, "system_prompt_sha256": hashlib.sha256(system.encode()).hexdigest(),
"held_out_routes": held_out, "grids": "scenarios.TRAIN_GRIDS",
"test_set_sha256": sc.manifest(sc.generate())["sha256"],
"thrombolysis": dict(Counter(s.expected["thrombolysis"]["value"] for s in scenarios)),
"thrombectomy": dict(Counter(s.expected["thrombectomy"]["value"] for s in scenarios)),
"routes": {r: c for r, c in sc.routes(scenarios).items()},
"chars_per_example": {"mean": round(sum(chars) / len(chars)), "max": max(chars)},
"answer_chars": {"mean": round(sum(answer_chars) / len(answer_chars)), "max": max(answer_chars)}}
(out / "meta.json").write_text(json.dumps(meta, indent=1))
return meta