diet-nutrition-qa / scripts /gen_router_data.py
chenhaodev's picture
diet-nutrition QA harness: code, data, RAG indexes, router LoRA
709d759 verified
Raw History Blame Contribute Delete
11.9 kB
#!/usr/bin/env python3
"""Build router (small model) SFT dataset for the diet/nutrition Q&A harness.
Sources (weak supervision, consistent with the rule engine):
(1) REAL curated questions from the 3 books -> category = book chapter (authoritative),
intent from a keyword classifier, mode/red_flags from rules.
(2) AUGMENTED alerts : chronicity / acuteness / progression / domain red-flag entities
(GI bleed, diabetes hypo/DKA, osteoporosis fall/fracture).
(3) AUGMENTED intents : what-eat / what-avoid / diet-advice / nutrient / meal-plan /
lifestyle / definition / when-seek-care coverage.
(4) AUGMENTED compound: parallel multi-sub-intent queries.
Every example passes through scripts.red_flag_rules.scan so the supervised label is
EXACTLY what the harness will produce at inference (train/serve consistency).
For real curated questions the category is overridden to the authoritative book value.
Output: data/router/router_sft_train.jsonl + router_sft_val.jsonl
"""
import json
import os
import random
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from scripts.red_flag_rules import scan
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "data/router")
os.makedirs(OUT, exist_ok=True)
# intent keyword classifier used to label real QA questions consistently
INTENT_KEYS = [
("what-avoid", ["禁忌", "不能吃", "不宜", "忌", "避免", "远离", "不要吃", "禁", "少吃", "不要喝"]),
("what-eat", ["吃多少", "可以吃", "能吃", "吃什么", "吃合适", "吃好", "吃高", "选择肉类", "挑选", "多吃", "吃些", "吃食", "吃哪些"]),
("meal-plan", ["食谱", "一日", "三餐", "早餐", "午餐", "晚餐", "搭配", "餐次", "加餐", "一周", "烹调", "烹饪", "做法"]),
("nutrient", ["补钙", "钙", "维生素", "营养", "蛋白质", "膳食纤维", "矿物质", "营养素", "脂肪", "盐", "糖", "含钙", "铁", "锌", "磷", "镁", "氨基酸"]),
("lifestyle", ["运动", "锻炼", "晒太阳", "阳光", "作息", "睡眠", "戒烟", "戒酒", "饮酒", "抽烟", "生活习惯", "心情"]),
("when-seek-care", ["就医", "看医生", "医院", "就诊", "医生", "检测", "检查", "诊断", "胃镜", "肠镜"]),
("definition", ["什么是", "是什么", "认识", "了解", "怎么回事", "什么样"]),
("diet-advice", ["饮食", "膳食", "调理", "营养治疗", "怎么吃", "护理"]),
]
def rule_intent(text):
for it, kws in INTENT_KEYS:
if any(k in text for k in kws):
return it
return "other"
def build_example(utterance, plan):
return {
"messages": [
{"role": "user",
"content": f"请把下面这句话拆成结构化的患者饮食意图计划。只输出 JSON。\n患者:{utterance}"},
{"role": "assistant", "content": json.dumps(plan, ensure_ascii=False)},
],
"utterance": utterance,
}
def plan_of(utterance, override_cat=None, override_intent=None):
red, subs, mode = scan(utterance)
plan = {"mode": mode, "red_flags": red if mode == "alert" else [], "sub_intents": subs}
if subs and (override_cat or override_intent):
s = plan["sub_intents"][0]
if override_cat:
s["category"] = override_cat
if override_intent:
s["intent"] = override_intent
return plan
# ========================= 1) REAL curated questions =========================
def real_questions():
recs = []
for key in ("gastro", "diabetes", "osteoporosis"):
p = os.path.join(ROOT, "data", f"qa_{key}.jsonl")
for line in open(p, encoding="utf-8"):
r = json.loads(line)
u = r["question"].rstrip("。").rstrip("?") + "?"
cat = r["category"]
intent = rule_intent(u)
plan = plan_of(u, override_cat=cat, override_intent=intent)
recs.append(build_example(u, plan))
return recs
# ========================= 2) AUGMENTED alerts =========================
CAT_ENTITY = {
"gastritis": ["胃痛", "胃胀", "反酸", "胃病反复", "浅表性胃炎"],
"peptic-ulcer": ["胃溃疡", "十二指肠溃疡", "溃疡又疼了"],
"diarrhea": ["拉肚子", "腹泻", "拉稀"],
"gi-bleeding": ["黑便", "大便带血", "便血", "呕吐物带血"],
"constipation": ["便秘", "排便困难", "大便干结"],
"ibs": ["肚子胀气", "肠易激综合征", "腹痛腹胀"],
"ibd": ["溃疡性结肠炎", "克罗恩病", "肠道炎症"],
"stomach-cancer": ["胃不舒服", "胃炎", "胃病"],
"diabetes": ["血糖高", "血糖低", "头晕乏力", "手脚发麻", "低血糖"],
"osteoporosis": ["补钙", "骨头疼", "腰背痛", "骨质疏松", "摔伤", "骨折"],
}
TIME_CHRONIC = ["两周", "一个月", "三个月", "半年", "一年", "两个星期"]
REC = ["三", "四", "五六", "七八"]
DOMAIN_ENT = {
"gi-bleeding": ["大便发黑", "呕血", "便血", "柏油样便", "上消化道出血"],
"diabetes": ["出冷汗心慌手抖", "低血糖发作", "意识模糊", "酮症酸中毒", "血糖很高"],
"osteoporosis": ["摔了一跤髋部很疼", "跌倒后腰背剧痛", "骨折了", "剧烈骨痛"],
}
def gen_alert_realcat():
"""Red-flag utterances; label by rules but keep the explicit category by keyword."""
recs = []
# chronicity (grounded-entity that becomes alert by duration)
for cat, ents in CAT_SYN.items():
for e in ents:
if "孩子" in e:
continue
for t in ["我已经{e}{time}了,饮食上怎么办", "{e}一直反复{time}了", "{e}越来越严重,怎么调理"]:
time = random.choice(TIME_CHRONIC)
u = t.format(e=e, time=time)
plan = plan_of(u)
if plan["mode"] == "alert":
# force category to match the synthetic entity's domain
if not plan["sub_intents"]:
plan["sub_intents"] = [{"intent": "what-eat", "entity": "", "category": cat,
"level": "basics", "tmpl": "what-eat",
"risk": "high"}]
plan["sub_intents"][0]["category"] = cat
plan["sub_intents"][0]["intent"] = "what-eat" if "饮食" in u or "调理" in u else "diet-advice"
recs.append(build_example(u, plan))
# domain red-flag entities (force alert)
for cat, ents in DOMAIN_ENT.items():
for e in ents:
u = f"我{e},要紧吗,该怎么吃"
plan = plan_of(u)
if plan["mode"] == "alert":
if not plan["sub_intents"]:
plan["sub_intents"] = [{"intent": "diet-advice", "entity": "", "category": cat,
"level": "basics", "tmpl": "diet-advice",
"risk": "high"}]
plan["sub_intents"][0]["category"] = cat
recs.append(build_example(u, plan))
return recs
CAT_SYN = {
"gastritis": ["胃痛", "胃胀", "反酸", "胃炎"],
"peptic-ulcer": ["胃溃疡", "上腹痛"],
"diarrhea": ["拉肚子", "腹泻"],
"constipation": ["便秘", "排便困难"],
"stomach-cancer": ["胃不舒服"],
"diabetes": ["血糖高", "手脚发麻", "头晕乏力"],
"osteoporosis": ["骨头疼", "腰背痛"],
}
# ========================= 3) intent coverage =========================
ADVICE_TMPL = {
"what-eat": ["胃溃疡病人平时能吃些什么食物", "糖尿病人可以吃水果吗", "骨质疏松的人吃什么补钙食物好"],
"what-avoid": ["胃病有没有什么忌口", "糖尿病病人不能吃哪些食物", "骨质疏松要少吃什么"],
"diet-advice": ["腹泻之后饮食上怎么调理", "糖尿病肾病饮食注意什么", "老年人怎么通过饮食养护骨骼"],
"nutrient": ["补钙每天吃多少合适", "糖尿病人需要补充哪些维生素", "骨质疏松该补蛋白质还是钙"],
"meal-plan": ["糖尿病一日三餐怎么搭配", "骨质疏松的食谱怎么安排", "胃病病人一日三餐怎么吃"],
"lifestyle": ["糖尿病人平时该怎么运动", "补钙要配合晒太阳吗", "骨质疏松病人适合做什么运动"],
"definition": ["什么是肠易激综合征", "骨质疏松是怎么回事", "消化性溃疡是什么"],
"when-seek-care": ["胃痛什么情况要去看医生", "血糖多高需要去医院", "摔伤后要不要拍片"],
}
def gen_intent():
recs = []
for intent, us in ADVICE_TMPL.items():
for u in us:
plan = plan_of(u)
if plan["sub_intents"]:
plan["sub_intents"][0]["intent"] = intent
recs.append(build_example(u, plan))
return recs
# ========================= 4) compound =========================
COMPOUND = [
("胃痛和反酸,吃什么好", ["gastritis"], ["what-eat"]),
("糖尿病还便秘,饮食怎么办", ["diabetes", "constipation"], ["diet-advice"]),
("老人骨质疏松又胃不好,怎么补钙", ["osteoporosis", "gastritis"], ["nutrient"]),
("血糖高还经常拉肚子,怎么调理", ["diabetes", "diarrhea"], ["diet-advice"]),
("想了解骨质疏松,也想问怎么补钙", ["osteoporosis"], ["definition", "nutrient"]),
("胃溃疡何十二指肠溃疡都要注意什么", ["peptic-ulcer"], ["definition"]),
]
def gen_compound():
recs = []
for u, cats, ints in COMPOUND:
plan = plan_of(u)
if plan["sub_intents"]:
plan["sub_intents"] = [
{"intent": ints[0], "entity": "", "category": cats[0], "level": "basics",
"tmpl": ints[0], "risk": "high" if plan["mode"] == "alert" else "low"},
]
recs.append(build_example(u, plan))
return recs
def main():
random.seed(42)
recs = []
# 1) real curated
for key in ("gastro", "diabetes", "osteoporosis"):
for line in open(os.path.join(ROOT, "data", f"qa_{key}.jsonl"), encoding="utf-8"):
r = json.loads(line)
u = r["question"]
if not u or "。" not in u:
u = u.rstrip("。") + "怎么办"
plan = plan_of(u, override_cat=r["category"], override_intent=rule_intent(u))
recs.append(build_example(u, plan))
recs += gen_alert_realcat()
recs += gen_intent()
recs += gen_compound()
# dedup by utterance
seen = set()
uniq = []
for r in recs:
if r["utterance"] in seen:
continue
seen.add(r["utterance"])
uniq.append(r)
random.shuffle(uniq)
val_n = max(30, int(len(uniq) * 0.12))
val = uniq[:val_n]
train = uniq[val_n:]
with open(os.path.join(OUT, "router_sft_train.jsonl"), "w", encoding="utf-8") as f:
for r in train:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
with open(os.path.join(OUT, "router_sft_val.jsonl"), "w", encoding="utf-8") as f:
for r in val:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
from collections import Counter
modes = Counter(json.loads(r["messages"][1]["content"])["mode"] for r in uniq)
cats = Counter()
intents = Counter()
for r in uniq:
p = json.loads(r["messages"][1]["content"])
for s in p["sub_intents"]:
cats[s["category"]] += 1
intents[s["intent"]] += 1
print(f"total unique: {len(uniq)} train={len(train)} val={len(val)}")
print("modes:", dict(modes))
print("categories:", dict(cats.most_common()))
print("intents:", dict(intents.most_common()))
if __name__ == "__main__":
main()