diet-nutrition-qa / scripts /parse_pdfs.py
chenhaodev's picture
diet-nutrition QA harness: code, data, RAG indexes, router LoRA
709d759 verified
Raw History Blame Contribute Delete
6.27 kB
#!/usr/bin/env python3
"""Parse the three diet/nutrition PDFs into structured Q/A records.
Format of all three books (prose Q&A, not multi-column tables):
第N章
<chapter title>
1.<question title>
<prose answer...>
Uses a streaming line-by-line state machine:
- a "第N章" line -> finalize current Q, set chapter/category
- a numbered-title line -> finalize current Q, start a new (question, answer)
- any other line -> append to the open question's answer
This is robust to question titles that appear mid-block (no blank-line delimiters).
Output: data/qa_<book>.jsonl records:
{category, book, chapter, question, answer, page, slug}
category == the RAG index dir name (router must emit exactly this).
"""
import glob
import hashlib
import json
import os
import re
from collections import Counter
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BOOKS = [
("胃肠疾病营养与膳食指导", "gastro"),
("糖尿病营养与膳食指导", "diabetes"),
("骨质疏松症营养与膳食指导", "osteoporosis"),
]
GASTRO_CAT = {
"食管癌": "esophagus-cancer",
"胃炎": "gastritis",
"消化性溃疡": "peptic-ulcer",
"胃癌": "stomach-cancer",
"腹泻": "diarrhea",
"消化道出血": "gi-bleeding",
"大肠癌": "colorectal-cancer",
"便秘": "constipation",
"肠易激综合征": "ibs",
"炎性肠病": "ibd",
}
_CHAPTER_RE = re.compile(r"^\s*第\s*[一二三四五六七八九十百\d]+\s*章\s*(.*)$")
_QUESTION_RE = re.compile(
r"^\s*(?P<num>\d{1,2}|[一二三四五六七八九十百]+)\s*[.、.]\s*"
r"(?P<title>.{2,120}?)\s*[??]?\s*$")
def strip_footer(t):
t = re.sub(r"\n?下载.*?permission\.\n?", "\n", t, flags=re.DOTALL)
t = re.sub(r"\n?Downloaded by \[.*?permission\.\n?", "\n", t, flags=re.DOTALL)
return t
def slug(q, page):
return hashlib.md5((q + str(page)).encode("utf-8")).hexdigest()[:12]
def is_question_line(line, book):
"""Heuristic: a numbered title that opens a new Q/A unit."""
if not re.match(r"^\s*\d|[一二三四五六七八九十百]", line):
return False
m = _QUESTION_RE.match(line)
if not m:
return False
title = m.group("title").strip()
# reject obvious page/header noise
if title.isdigit() or len(title) < 4:
return False
# In gastro book questions are prose titles; require the line to look like a
# heading (not a sentence running into body text). We accept short titles.
return True
def parse_one(pdf_path, book):
import pymupdf
doc = pymupdf.open(pdf_path)
toc_cat = {}
current_cat = None
for lvl, title, page in doc.get_toc():
chm = re.match(r"^\s*第\s*[一二三四五六七八九十百\d]+\s*章\s*(.*)$", title)
if lvl == 1 and chm:
raw = chm.group(1).strip()
if book == "gastro":
current_cat = GASTRO_CAT.get(raw, current_cat)
elif book == "diabetes":
current_cat = "diabetes"
elif book == "osteoporosis":
current_cat = "osteoporosis"
toc_cat[page] = current_cat
def cat_at(page1):
c = None
for p in range(page1, 0, -1):
if p in toc_cat and toc_cat[p]:
c = toc_cat[p]
break
return c
records = []
cur_question = None
cur_answer = []
cur_page = None
def flush():
nonlocal cur_question, cur_answer
if cur_question and cur_answer:
ans = "".join(cur_answer).strip()
if len(ans) >= 10:
records.append({
"category": cur_question[1],
"book": book,
"chapter": cur_question[2],
"question": cur_question[0],
"answer": ans,
"page": cur_question[3],
"slug": slug(cur_question[0], cur_question[3]),
})
cur_question, cur_answer = None, []
for pno in range(doc.page_count):
page1 = pno + 1
cat = cat_at(page1)
text = strip_footer(doc.load_page(pno).get_text("text"))
# chapter lines: set category from TOC line directly
for line in text.splitlines():
line = line.strip()
if not line:
continue
chm = _CHAPTER_RE.match(line)
if chm:
flush()
raw = chm.group(1).strip()
if book == "gastro":
cat = GASTRO_CAT.get(raw, cat)
elif book in ("diabetes", "osteoporosis"):
cat = book
continue
if is_question_line(line, book):
flush()
m = _QUESTION_RE.match(line)
title = m.group("title").strip()
if not title.endswith(("?", "?", "。")):
title += "。"
cur_question = (title, cat, line, page1)
cur_answer = []
else:
if cur_question is None:
# prelude text before first question (chapter intro) - skip
continue
cur_answer.append(line)
flush()
return records
def main():
out_all = os.path.join(ROOT, "data")
os.makedirs(out_all, exist_ok=True)
for base, key in BOOKS:
pdfs = glob.glob("/workspace/Med-Books-20260914/" + base + "*.pdf")
if not pdfs:
print("MISSING", base)
continue
recs = parse_one(pdfs[0], key)
# drop preface-outline rows whose "answer" is just a following section heading
recs = [r for r in recs if len(r["answer"]) >= 30]
for r in recs:
if not r["category"]:
r["category"] = "diabetes" if key == "diabetes" else "osteoporosis"
with open(os.path.join(out_all, f"qa_{key}.jsonl"), "w", encoding="utf-8") as f:
for r in recs:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"[{key}] records={len(recs)}")
for c, n in Counter(r["category"] for r in recs).most_common():
print(f" {c}: {n}")
if __name__ == "__main__":
main()