File size: 11,892 Bytes
709d759
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
#!/usr/bin/env python3
"""Build router (small model) SFT dataset for the diet/nutrition Q&A harness.

Sources (weak supervision, consistent with the rule engine):
  (1) REAL curated questions from the 3 books -> category = book chapter (authoritative),
      intent from a keyword classifier, mode/red_flags from rules.
  (2) AUGMENTED alerts  : chronicity / acuteness / progression / domain red-flag entities
                          (GI bleed, diabetes hypo/DKA, osteoporosis fall/fracture).
  (3) AUGMENTED intents : what-eat / what-avoid / diet-advice / nutrient / meal-plan /
                          lifestyle / definition / when-seek-care coverage.
  (4) AUGMENTED compound: parallel multi-sub-intent queries.

Every example passes through scripts.red_flag_rules.scan so the supervised label is
EXACTLY what the harness will produce at inference (train/serve consistency).
For real curated questions the category is overridden to the authoritative book value.

Output: data/router/router_sft_train.jsonl + router_sft_val.jsonl
"""
import json
import os
import random
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from scripts.red_flag_rules import scan

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "data/router")
os.makedirs(OUT, exist_ok=True)

# intent keyword classifier used to label real QA questions consistently
INTENT_KEYS = [
    ("what-avoid", ["禁忌", "不能吃", "不宜", "忌", "避免", "远离", "不要吃", "禁", "少吃", "不要喝"]),
    ("what-eat", ["吃多少", "可以吃", "能吃", "吃什么", "吃合适", "吃好", "吃高", "选择肉类", "挑选", "多吃", "吃些", "吃食", "吃哪些"]),
    ("meal-plan", ["食谱", "一日", "三餐", "早餐", "午餐", "晚餐", "搭配", "餐次", "加餐", "一周", "烹调", "烹饪", "做法"]),
    ("nutrient", ["补钙", "钙", "维生素", "营养", "蛋白质", "膳食纤维", "矿物质", "营养素", "脂肪", "盐", "糖", "含钙", "铁", "锌", "磷", "镁", "氨基酸"]),
    ("lifestyle", ["运动", "锻炼", "晒太阳", "阳光", "作息", "睡眠", "戒烟", "戒酒", "饮酒", "抽烟", "生活习惯", "心情"]),
    ("when-seek-care", ["就医", "看医生", "医院", "就诊", "医生", "检测", "检查", "诊断", "胃镜", "肠镜"]),
    ("definition", ["什么是", "是什么", "认识", "了解", "怎么回事", "什么样"]),
    ("diet-advice", ["饮食", "膳食", "调理", "营养治疗", "怎么吃", "护理"]),
]


def rule_intent(text):
    for it, kws in INTENT_KEYS:
        if any(k in text for k in kws):
            return it
    return "other"


def build_example(utterance, plan):
    return {
        "messages": [
            {"role": "user",
             "content": f"请把下面这句话拆成结构化的患者饮食意图计划。只输出 JSON。\n患者:{utterance}"},
            {"role": "assistant", "content": json.dumps(plan, ensure_ascii=False)},
        ],
        "utterance": utterance,
    }


def plan_of(utterance, override_cat=None, override_intent=None):
    red, subs, mode = scan(utterance)
    plan = {"mode": mode, "red_flags": red if mode == "alert" else [], "sub_intents": subs}
    if subs and (override_cat or override_intent):
        s = plan["sub_intents"][0]
        if override_cat:
            s["category"] = override_cat
        if override_intent:
            s["intent"] = override_intent
    return plan


# ========================= 1) REAL curated questions =========================
def real_questions():
    recs = []
    for key in ("gastro", "diabetes", "osteoporosis"):
        p = os.path.join(ROOT, "data", f"qa_{key}.jsonl")
        for line in open(p, encoding="utf-8"):
            r = json.loads(line)
            u = r["question"].rstrip("。").rstrip("?") + "?"
            cat = r["category"]
            intent = rule_intent(u)
            plan = plan_of(u, override_cat=cat, override_intent=intent)
            recs.append(build_example(u, plan))
    return recs


# ========================= 2) AUGMENTED alerts =========================
CAT_ENTITY = {
    "gastritis": ["胃痛", "胃胀", "反酸", "胃病反复", "浅表性胃炎"],
    "peptic-ulcer": ["胃溃疡", "十二指肠溃疡", "溃疡又疼了"],
    "diarrhea": ["拉肚子", "腹泻", "拉稀"],
    "gi-bleeding": ["黑便", "大便带血", "便血", "呕吐物带血"],
    "constipation": ["便秘", "排便困难", "大便干结"],
    "ibs": ["肚子胀气", "肠易激综合征", "腹痛腹胀"],
    "ibd": ["溃疡性结肠炎", "克罗恩病", "肠道炎症"],
    "stomach-cancer": ["胃不舒服", "胃炎", "胃病"],
    "diabetes": ["血糖高", "血糖低", "头晕乏力", "手脚发麻", "低血糖"],
    "osteoporosis": ["补钙", "骨头疼", "腰背痛", "骨质疏松", "摔伤", "骨折"],
}
TIME_CHRONIC = ["两周", "一个月", "三个月", "半年", "一年", "两个星期"]
REC = ["三", "四", "五六", "七八"]
DOMAIN_ENT = {
    "gi-bleeding": ["大便发黑", "呕血", "便血", "柏油样便", "上消化道出血"],
    "diabetes": ["出冷汗心慌手抖", "低血糖发作", "意识模糊", "酮症酸中毒", "血糖很高"],
    "osteoporosis": ["摔了一跤髋部很疼", "跌倒后腰背剧痛", "骨折了", "剧烈骨痛"],
}


def gen_alert_realcat():
    """Red-flag utterances; label by rules but keep the explicit category by keyword."""
    recs = []
    # chronicity (grounded-entity that becomes alert by duration)
    for cat, ents in CAT_SYN.items():
        for e in ents:
            if "孩子" in e:
                continue
            for t in ["我已经{e}{time}了,饮食上怎么办", "{e}一直反复{time}了", "{e}越来越严重,怎么调理"]:
                time = random.choice(TIME_CHRONIC)
                u = t.format(e=e, time=time)
                plan = plan_of(u)
                if plan["mode"] == "alert":
                    # force category to match the synthetic entity's domain
                    if not plan["sub_intents"]:
                        plan["sub_intents"] = [{"intent": "what-eat", "entity": "", "category": cat,
                                                "level": "basics", "tmpl": "what-eat",
                                                "risk": "high"}]
                    plan["sub_intents"][0]["category"] = cat
                    plan["sub_intents"][0]["intent"] = "what-eat" if "饮食" in u or "调理" in u else "diet-advice"
                    recs.append(build_example(u, plan))
    # domain red-flag entities (force alert)
    for cat, ents in DOMAIN_ENT.items():
        for e in ents:
            u = f"我{e},要紧吗,该怎么吃"
            plan = plan_of(u)
            if plan["mode"] == "alert":
                if not plan["sub_intents"]:
                    plan["sub_intents"] = [{"intent": "diet-advice", "entity": "", "category": cat,
                                            "level": "basics", "tmpl": "diet-advice",
                                            "risk": "high"}]
                plan["sub_intents"][0]["category"] = cat
                recs.append(build_example(u, plan))
    return recs


CAT_SYN = {
    "gastritis": ["胃痛", "胃胀", "反酸", "胃炎"],
    "peptic-ulcer": ["胃溃疡", "上腹痛"],
    "diarrhea": ["拉肚子", "腹泻"],
    "constipation": ["便秘", "排便困难"],
    "stomach-cancer": ["胃不舒服"],
    "diabetes": ["血糖高", "手脚发麻", "头晕乏力"],
    "osteoporosis": ["骨头疼", "腰背痛"],
}


# ========================= 3) intent coverage =========================
ADVICE_TMPL = {
    "what-eat": ["胃溃疡病人平时能吃些什么食物", "糖尿病人可以吃水果吗", "骨质疏松的人吃什么补钙食物好"],
    "what-avoid": ["胃病有没有什么忌口", "糖尿病病人不能吃哪些食物", "骨质疏松要少吃什么"],
    "diet-advice": ["腹泻之后饮食上怎么调理", "糖尿病肾病饮食注意什么", "老年人怎么通过饮食养护骨骼"],
    "nutrient": ["补钙每天吃多少合适", "糖尿病人需要补充哪些维生素", "骨质疏松该补蛋白质还是钙"],
    "meal-plan": ["糖尿病一日三餐怎么搭配", "骨质疏松的食谱怎么安排", "胃病病人一日三餐怎么吃"],
    "lifestyle": ["糖尿病人平时该怎么运动", "补钙要配合晒太阳吗", "骨质疏松病人适合做什么运动"],
    "definition": ["什么是肠易激综合征", "骨质疏松是怎么回事", "消化性溃疡是什么"],
    "when-seek-care": ["胃痛什么情况要去看医生", "血糖多高需要去医院", "摔伤后要不要拍片"],
}


def gen_intent():
    recs = []
    for intent, us in ADVICE_TMPL.items():
        for u in us:
            plan = plan_of(u)
            if plan["sub_intents"]:
                plan["sub_intents"][0]["intent"] = intent
                recs.append(build_example(u, plan))
    return recs


# ========================= 4) compound =========================
COMPOUND = [
    ("胃痛和反酸,吃什么好", ["gastritis"], ["what-eat"]),
    ("糖尿病还便秘,饮食怎么办", ["diabetes", "constipation"], ["diet-advice"]),
    ("老人骨质疏松又胃不好,怎么补钙", ["osteoporosis", "gastritis"], ["nutrient"]),
    ("血糖高还经常拉肚子,怎么调理", ["diabetes", "diarrhea"], ["diet-advice"]),
    ("想了解骨质疏松,也想问怎么补钙", ["osteoporosis"], ["definition", "nutrient"]),
    ("胃溃疡何十二指肠溃疡都要注意什么", ["peptic-ulcer"], ["definition"]),
]


def gen_compound():
    recs = []
    for u, cats, ints in COMPOUND:
        plan = plan_of(u)
        if plan["sub_intents"]:
            plan["sub_intents"] = [
                {"intent": ints[0], "entity": "", "category": cats[0], "level": "basics",
                 "tmpl": ints[0], "risk": "high" if plan["mode"] == "alert" else "low"},
            ]
            recs.append(build_example(u, plan))
    return recs


def main():
    random.seed(42)
    recs = []
    # 1) real curated
    for key in ("gastro", "diabetes", "osteoporosis"):
        for line in open(os.path.join(ROOT, "data", f"qa_{key}.jsonl"), encoding="utf-8"):
            r = json.loads(line)
            u = r["question"]
            if not u or "。" not in u:
                u = u.rstrip("。") + "怎么办"
            plan = plan_of(u, override_cat=r["category"], override_intent=rule_intent(u))
            recs.append(build_example(u, plan))
    recs += gen_alert_realcat()
    recs += gen_intent()
    recs += gen_compound()

    # dedup by utterance
    seen = set()
    uniq = []
    for r in recs:
        if r["utterance"] in seen:
            continue
        seen.add(r["utterance"])
        uniq.append(r)

    random.shuffle(uniq)
    val_n = max(30, int(len(uniq) * 0.12))
    val = uniq[:val_n]
    train = uniq[val_n:]
    with open(os.path.join(OUT, "router_sft_train.jsonl"), "w", encoding="utf-8") as f:
        for r in train:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    with open(os.path.join(OUT, "router_sft_val.jsonl"), "w", encoding="utf-8") as f:
        for r in val:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")

    from collections import Counter
    modes = Counter(json.loads(r["messages"][1]["content"])["mode"] for r in uniq)
    cats = Counter()
    intents = Counter()
    for r in uniq:
        p = json.loads(r["messages"][1]["content"])
        for s in p["sub_intents"]:
            cats[s["category"]] += 1
            intents[s["intent"]] += 1
    print(f"total unique: {len(uniq)}  train={len(train)} val={len(val)}")
    print("modes:", dict(modes))
    print("categories:", dict(cats.most_common()))
    print("intents:", dict(intents.most_common()))


if __name__ == "__main__":
    main()