#!/usr/bin/env python3 """Parse the three diet/nutrition PDFs into structured Q/A records. Format of all three books (prose Q&A, not multi-column tables): 第N章 1. Uses a streaming line-by-line state machine: - a "第N章" line -> finalize current Q, set chapter/category - a numbered-title line -> finalize current Q, start a new (question, answer) - any other line -> append to the open question's answer This is robust to question titles that appear mid-block (no blank-line delimiters). Output: data/qa_.jsonl records: {category, book, chapter, question, answer, page, slug} category == the RAG index dir name (router must emit exactly this). """ import glob import hashlib import json import os import re from collections import Counter ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BOOKS = [ ("胃肠疾病营养与膳食指导", "gastro"), ("糖尿病营养与膳食指导", "diabetes"), ("骨质疏松症营养与膳食指导", "osteoporosis"), ] GASTRO_CAT = { "食管癌": "esophagus-cancer", "胃炎": "gastritis", "消化性溃疡": "peptic-ulcer", "胃癌": "stomach-cancer", "腹泻": "diarrhea", "消化道出血": "gi-bleeding", "大肠癌": "colorectal-cancer", "便秘": "constipation", "肠易激综合征": "ibs", "炎性肠病": "ibd", } _CHAPTER_RE = re.compile(r"^\s*第\s*[一二三四五六七八九十百\d]+\s*章\s*(.*)$") _QUESTION_RE = re.compile( r"^\s*(?P\d{1,2}|[一二三四五六七八九十百]+)\s*[.、.]\s*" r"(?P.{2,120}?)\s*[??]?\s*$") def strip_footer(t): t = re.sub(r"\n?下载.*?permission\.\n?", "\n", t, flags=re.DOTALL) t = re.sub(r"\n?Downloaded by \[.*?permission\.\n?", "\n", t, flags=re.DOTALL) return t def slug(q, page): return hashlib.md5((q + str(page)).encode("utf-8")).hexdigest()[:12] def is_question_line(line, book): """Heuristic: a numbered title that opens a new Q/A unit.""" if not re.match(r"^\s*\d|[一二三四五六七八九十百]", line): return False m = _QUESTION_RE.match(line) if not m: return False title = m.group("title").strip() # reject obvious page/header noise if title.isdigit() or len(title) < 4: return False # In gastro book questions are prose titles; require the line to look like a # heading (not a sentence running into body text). We accept short titles. return True def parse_one(pdf_path, book): import pymupdf doc = pymupdf.open(pdf_path) toc_cat = {} current_cat = None for lvl, title, page in doc.get_toc(): chm = re.match(r"^\s*第\s*[一二三四五六七八九十百\d]+\s*章\s*(.*)$", title) if lvl == 1 and chm: raw = chm.group(1).strip() if book == "gastro": current_cat = GASTRO_CAT.get(raw, current_cat) elif book == "diabetes": current_cat = "diabetes" elif book == "osteoporosis": current_cat = "osteoporosis" toc_cat[page] = current_cat def cat_at(page1): c = None for p in range(page1, 0, -1): if p in toc_cat and toc_cat[p]: c = toc_cat[p] break return c records = [] cur_question = None cur_answer = [] cur_page = None def flush(): nonlocal cur_question, cur_answer if cur_question and cur_answer: ans = "".join(cur_answer).strip() if len(ans) >= 10: records.append({ "category": cur_question[1], "book": book, "chapter": cur_question[2], "question": cur_question[0], "answer": ans, "page": cur_question[3], "slug": slug(cur_question[0], cur_question[3]), }) cur_question, cur_answer = None, [] for pno in range(doc.page_count): page1 = pno + 1 cat = cat_at(page1) text = strip_footer(doc.load_page(pno).get_text("text")) # chapter lines: set category from TOC line directly for line in text.splitlines(): line = line.strip() if not line: continue chm = _CHAPTER_RE.match(line) if chm: flush() raw = chm.group(1).strip() if book == "gastro": cat = GASTRO_CAT.get(raw, cat) elif book in ("diabetes", "osteoporosis"): cat = book continue if is_question_line(line, book): flush() m = _QUESTION_RE.match(line) title = m.group("title").strip() if not title.endswith(("?", "?", "。")): title += "。" cur_question = (title, cat, line, page1) cur_answer = [] else: if cur_question is None: # prelude text before first question (chapter intro) - skip continue cur_answer.append(line) flush() return records def main(): out_all = os.path.join(ROOT, "data") os.makedirs(out_all, exist_ok=True) for base, key in BOOKS: pdfs = glob.glob("/workspace/Med-Books-20260914/" + base + "*.pdf") if not pdfs: print("MISSING", base) continue recs = parse_one(pdfs[0], key) # drop preface-outline rows whose "answer" is just a following section heading recs = [r for r in recs if len(r["answer"]) >= 30] for r in recs: if not r["category"]: r["category"] = "diabetes" if key == "diabetes" else "osteoporosis" with open(os.path.join(out_all, f"qa_{key}.jsonl"), "w", encoding="utf-8") as f: for r in recs: f.write(json.dumps(r, ensure_ascii=False) + "\n") print(f"[{key}] records={len(recs)}") for c, n in Counter(r["category"] for r in recs).most_common(): print(f" {c}: {n}") if __name__ == "__main__": main()