diet-nutrition-qa / scripts /build_multi_rag.py
chenhaodev's picture
diet-nutrition QA harness: code, data, RAG indexes, router LoRA
0e2bdca verified
Raw History Blame Contribute Delete
3.1 kB
#!/usr/bin/env python3
"""Build per-category FAISS indexes (multi-RAG) from the parsed diet/nutrition QA.
One flat IndexFlatIP per category (12 indexes) using BAAI/bge-m3 (multilingual,
so Chinese patient queries embed fine). Each doc = question + answer.
Categories MUST equal the router's category output (sub_intents[].category):
gastro: esophagus-cancer, gastritis, peptic-ulcer, stomach-cancer, diarrhea,
gi-bleeding, colorectal-cancer, constipation, ibs, ibd
diabetes, osteoporosis
"""
import json
import os
import numpy as np
from collections import defaultdict
os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com")
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "data/rag")
MODEL_NAME = "BAAI/bge-m3"
from sentence_transformers import SentenceTransformer
import faiss
os.makedirs(OUT, exist_ok=True)
model = SentenceTransformer(MODEL_NAME)
model.max_seq_length = 512
print("embedder loaded:", MODEL_NAME, "| device:", model.device, flush=True)
rows = []
for key in ("gastro", "diabetes", "osteoporosis"):
p = os.path.join(ROOT, "data", f"qa_{key}.jsonl")
rows += [json.loads(l) for l in open(p, encoding="utf-8")]
print("total QA rows:", len(rows), flush=True)
bycat = defaultdict(list)
for r in rows:
if r.get("category"):
bycat[r["category"]].append(r)
print("categories:", sorted(bycat), flush=True)
# junk markers: front-matter / watermark / copyright glue that may carry book titles
_JUNK = ["z-library", "下载", "Downloaded by", "版权", "ISBN", "责任编辑",
"总前言", "《中国慢性疾病营养", "北京协和", "专家委员会", "出版"]
def _is_junk(d):
return any(k in d for k in _JUNK)
stats = {}
for cat, items in bycat.items():
seen = set()
uniq = []
for r in items:
if r["question"] in seen:
continue
seen.add(r["question"])
uniq.append(r)
docs = []
meta = []
for r in uniq:
d = f"Q:{r['question']}\nA:{r['answer']}"
if _is_junk(d):
continue
docs.append(d)
meta.append({"slug": r["slug"], "page": r["page"], "category": cat,
"question": r["question"], "book": r["book"],
"answer": r["answer"][:900]})
emb = model.encode(docs, normalize_embeddings=True, show_progress_bar=False, batch_size=32)
emb = np.asarray(emb, dtype="float32")
idx = faiss.IndexFlatIP(emb.shape[1])
idx.add(emb)
sub = os.path.join(OUT, cat)
os.makedirs(sub, exist_ok=True)
faiss.write_index(idx, os.path.join(sub, "index.faiss"))
with open(os.path.join(sub, "docs.json"), "w", encoding="utf-8") as f:
json.dump(docs, f, ensure_ascii=False)
with open(os.path.join(sub, "meta.json"), "w", encoding="utf-8") as f:
json.dump(meta, f, ensure_ascii=False)
stats[cat] = len(docs)
print(f" [{cat}] docs={len(docs)}", flush=True)
with open(os.path.join(OUT, "index_map.json"), "w", encoding="utf-8") as f:
json.dump(stats, f, ensure_ascii=False, indent=1)
print("DONE", stats)