#!/usr/bin/env python3 """Build router (small model) SFT dataset for the diet/nutrition Q&A harness. Sources (weak supervision, consistent with the rule engine): (1) REAL curated questions from the 3 books -> category = book chapter (authoritative), intent from a keyword classifier, mode/red_flags from rules. (2) AUGMENTED alerts : chronicity / acuteness / progression / domain red-flag entities (GI bleed, diabetes hypo/DKA, osteoporosis fall/fracture). (3) AUGMENTED intents : what-eat / what-avoid / diet-advice / nutrient / meal-plan / lifestyle / definition / when-seek-care coverage. (4) AUGMENTED compound: parallel multi-sub-intent queries. Every example passes through scripts.red_flag_rules.scan so the supervised label is EXACTLY what the harness will produce at inference (train/serve consistency). For real curated questions the category is overridden to the authoritative book value. Output: data/router/router_sft_train.jsonl + router_sft_val.jsonl """ import json import os import random import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from scripts.red_flag_rules import scan ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUT = os.path.join(ROOT, "data/router") os.makedirs(OUT, exist_ok=True) # intent keyword classifier used to label real QA questions consistently INTENT_KEYS = [ ("what-avoid", ["禁忌", "不能吃", "不宜", "忌", "避免", "远离", "不要吃", "禁", "少吃", "不要喝"]), ("what-eat", ["吃多少", "可以吃", "能吃", "吃什么", "吃合适", "吃好", "吃高", "选择肉类", "挑选", "多吃", "吃些", "吃食", "吃哪些"]), ("meal-plan", ["食谱", "一日", "三餐", "早餐", "午餐", "晚餐", "搭配", "餐次", "加餐", "一周", "烹调", "烹饪", "做法"]), ("nutrient", ["补钙", "钙", "维生素", "营养", "蛋白质", "膳食纤维", "矿物质", "营养素", "脂肪", "盐", "糖", "含钙", "铁", "锌", "磷", "镁", "氨基酸"]), ("lifestyle", ["运动", "锻炼", "晒太阳", "阳光", "作息", "睡眠", "戒烟", "戒酒", "饮酒", "抽烟", "生活习惯", "心情"]), ("when-seek-care", ["就医", "看医生", "医院", "就诊", "医生", "检测", "检查", "诊断", "胃镜", "肠镜"]), ("definition", ["什么是", "是什么", "认识", "了解", "怎么回事", "什么样"]), ("diet-advice", ["饮食", "膳食", "调理", "营养治疗", "怎么吃", "护理"]), ] def rule_intent(text): for it, kws in INTENT_KEYS: if any(k in text for k in kws): return it return "other" def build_example(utterance, plan): return { "messages": [ {"role": "user", "content": f"请把下面这句话拆成结构化的患者饮食意图计划。只输出 JSON。\n患者:{utterance}"}, {"role": "assistant", "content": json.dumps(plan, ensure_ascii=False)}, ], "utterance": utterance, } def plan_of(utterance, override_cat=None, override_intent=None): red, subs, mode = scan(utterance) plan = {"mode": mode, "red_flags": red if mode == "alert" else [], "sub_intents": subs} if subs and (override_cat or override_intent): s = plan["sub_intents"][0] if override_cat: s["category"] = override_cat if override_intent: s["intent"] = override_intent return plan # ========================= 1) REAL curated questions ========================= def real_questions(): recs = [] for key in ("gastro", "diabetes", "osteoporosis"): p = os.path.join(ROOT, "data", f"qa_{key}.jsonl") for line in open(p, encoding="utf-8"): r = json.loads(line) u = r["question"].rstrip("。").rstrip("?") + "?" cat = r["category"] intent = rule_intent(u) plan = plan_of(u, override_cat=cat, override_intent=intent) recs.append(build_example(u, plan)) return recs # ========================= 2) AUGMENTED alerts ========================= CAT_ENTITY = { "gastritis": ["胃痛", "胃胀", "反酸", "胃病反复", "浅表性胃炎"], "peptic-ulcer": ["胃溃疡", "十二指肠溃疡", "溃疡又疼了"], "diarrhea": ["拉肚子", "腹泻", "拉稀"], "gi-bleeding": ["黑便", "大便带血", "便血", "呕吐物带血"], "constipation": ["便秘", "排便困难", "大便干结"], "ibs": ["肚子胀气", "肠易激综合征", "腹痛腹胀"], "ibd": ["溃疡性结肠炎", "克罗恩病", "肠道炎症"], "stomach-cancer": ["胃不舒服", "胃炎", "胃病"], "diabetes": ["血糖高", "血糖低", "头晕乏力", "手脚发麻", "低血糖"], "osteoporosis": ["补钙", "骨头疼", "腰背痛", "骨质疏松", "摔伤", "骨折"], } TIME_CHRONIC = ["两周", "一个月", "三个月", "半年", "一年", "两个星期"] REC = ["三", "四", "五六", "七八"] DOMAIN_ENT = { "gi-bleeding": ["大便发黑", "呕血", "便血", "柏油样便", "上消化道出血"], "diabetes": ["出冷汗心慌手抖", "低血糖发作", "意识模糊", "酮症酸中毒", "血糖很高"], "osteoporosis": ["摔了一跤髋部很疼", "跌倒后腰背剧痛", "骨折了", "剧烈骨痛"], } def gen_alert_realcat(): """Red-flag utterances; label by rules but keep the explicit category by keyword.""" recs = [] # chronicity (grounded-entity that becomes alert by duration) for cat, ents in CAT_SYN.items(): for e in ents: if "孩子" in e: continue for t in ["我已经{e}{time}了,饮食上怎么办", "{e}一直反复{time}了", "{e}越来越严重,怎么调理"]: time = random.choice(TIME_CHRONIC) u = t.format(e=e, time=time) plan = plan_of(u) if plan["mode"] == "alert": # force category to match the synthetic entity's domain if not plan["sub_intents"]: plan["sub_intents"] = [{"intent": "what-eat", "entity": "", "category": cat, "level": "basics", "tmpl": "what-eat", "risk": "high"}] plan["sub_intents"][0]["category"] = cat plan["sub_intents"][0]["intent"] = "what-eat" if "饮食" in u or "调理" in u else "diet-advice" recs.append(build_example(u, plan)) # domain red-flag entities (force alert) for cat, ents in DOMAIN_ENT.items(): for e in ents: u = f"我{e},要紧吗,该怎么吃" plan = plan_of(u) if plan["mode"] == "alert": if not plan["sub_intents"]: plan["sub_intents"] = [{"intent": "diet-advice", "entity": "", "category": cat, "level": "basics", "tmpl": "diet-advice", "risk": "high"}] plan["sub_intents"][0]["category"] = cat recs.append(build_example(u, plan)) return recs CAT_SYN = { "gastritis": ["胃痛", "胃胀", "反酸", "胃炎"], "peptic-ulcer": ["胃溃疡", "上腹痛"], "diarrhea": ["拉肚子", "腹泻"], "constipation": ["便秘", "排便困难"], "stomach-cancer": ["胃不舒服"], "diabetes": ["血糖高", "手脚发麻", "头晕乏力"], "osteoporosis": ["骨头疼", "腰背痛"], } # ========================= 3) intent coverage ========================= ADVICE_TMPL = { "what-eat": ["胃溃疡病人平时能吃些什么食物", "糖尿病人可以吃水果吗", "骨质疏松的人吃什么补钙食物好"], "what-avoid": ["胃病有没有什么忌口", "糖尿病病人不能吃哪些食物", "骨质疏松要少吃什么"], "diet-advice": ["腹泻之后饮食上怎么调理", "糖尿病肾病饮食注意什么", "老年人怎么通过饮食养护骨骼"], "nutrient": ["补钙每天吃多少合适", "糖尿病人需要补充哪些维生素", "骨质疏松该补蛋白质还是钙"], "meal-plan": ["糖尿病一日三餐怎么搭配", "骨质疏松的食谱怎么安排", "胃病病人一日三餐怎么吃"], "lifestyle": ["糖尿病人平时该怎么运动", "补钙要配合晒太阳吗", "骨质疏松病人适合做什么运动"], "definition": ["什么是肠易激综合征", "骨质疏松是怎么回事", "消化性溃疡是什么"], "when-seek-care": ["胃痛什么情况要去看医生", "血糖多高需要去医院", "摔伤后要不要拍片"], } def gen_intent(): recs = [] for intent, us in ADVICE_TMPL.items(): for u in us: plan = plan_of(u) if plan["sub_intents"]: plan["sub_intents"][0]["intent"] = intent recs.append(build_example(u, plan)) return recs # ========================= 4) compound ========================= COMPOUND = [ ("胃痛和反酸,吃什么好", ["gastritis"], ["what-eat"]), ("糖尿病还便秘,饮食怎么办", ["diabetes", "constipation"], ["diet-advice"]), ("老人骨质疏松又胃不好,怎么补钙", ["osteoporosis", "gastritis"], ["nutrient"]), ("血糖高还经常拉肚子,怎么调理", ["diabetes", "diarrhea"], ["diet-advice"]), ("想了解骨质疏松,也想问怎么补钙", ["osteoporosis"], ["definition", "nutrient"]), ("胃溃疡何十二指肠溃疡都要注意什么", ["peptic-ulcer"], ["definition"]), ] def gen_compound(): recs = [] for u, cats, ints in COMPOUND: plan = plan_of(u) if plan["sub_intents"]: plan["sub_intents"] = [ {"intent": ints[0], "entity": "", "category": cats[0], "level": "basics", "tmpl": ints[0], "risk": "high" if plan["mode"] == "alert" else "low"}, ] recs.append(build_example(u, plan)) return recs def main(): random.seed(42) recs = [] # 1) real curated for key in ("gastro", "diabetes", "osteoporosis"): for line in open(os.path.join(ROOT, "data", f"qa_{key}.jsonl"), encoding="utf-8"): r = json.loads(line) u = r["question"] if not u or "。" not in u: u = u.rstrip("。") + "怎么办" plan = plan_of(u, override_cat=r["category"], override_intent=rule_intent(u)) recs.append(build_example(u, plan)) recs += gen_alert_realcat() recs += gen_intent() recs += gen_compound() # dedup by utterance seen = set() uniq = [] for r in recs: if r["utterance"] in seen: continue seen.add(r["utterance"]) uniq.append(r) random.shuffle(uniq) val_n = max(30, int(len(uniq) * 0.12)) val = uniq[:val_n] train = uniq[val_n:] with open(os.path.join(OUT, "router_sft_train.jsonl"), "w", encoding="utf-8") as f: for r in train: f.write(json.dumps(r, ensure_ascii=False) + "\n") with open(os.path.join(OUT, "router_sft_val.jsonl"), "w", encoding="utf-8") as f: for r in val: f.write(json.dumps(r, ensure_ascii=False) + "\n") from collections import Counter modes = Counter(json.loads(r["messages"][1]["content"])["mode"] for r in uniq) cats = Counter() intents = Counter() for r in uniq: p = json.loads(r["messages"][1]["content"]) for s in p["sub_intents"]: cats[s["category"]] += 1 intents[s["intent"]] += 1 print(f"total unique: {len(uniq)} train={len(train)} val={len(val)}") print("modes:", dict(modes)) print("categories:", dict(cats.most_common())) print("intents:", dict(intents.most_common())) if __name__ == "__main__": main()