#!/usr/bin/env python3 """Build per-category FAISS indexes (multi-RAG) from the parsed diet/nutrition QA. One flat IndexFlatIP per category (12 indexes) using BAAI/bge-m3 (multilingual, so Chinese patient queries embed fine). Each doc = question + answer. Categories MUST equal the router's category output (sub_intents[].category): gastro: esophagus-cancer, gastritis, peptic-ulcer, stomach-cancer, diarrhea, gi-bleeding, colorectal-cancer, constipation, ibs, ibd diabetes, osteoporosis """ import json import os import numpy as np from collections import defaultdict os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com") ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUT = os.path.join(ROOT, "data/rag") MODEL_NAME = "BAAI/bge-m3" from sentence_transformers import SentenceTransformer import faiss os.makedirs(OUT, exist_ok=True) model = SentenceTransformer(MODEL_NAME) model.max_seq_length = 512 print("embedder loaded:", MODEL_NAME, "| device:", model.device, flush=True) rows = [] for key in ("gastro", "diabetes", "osteoporosis"): p = os.path.join(ROOT, "data", f"qa_{key}.jsonl") rows += [json.loads(l) for l in open(p, encoding="utf-8")] print("total QA rows:", len(rows), flush=True) bycat = defaultdict(list) for r in rows: if r.get("category"): bycat[r["category"]].append(r) print("categories:", sorted(bycat), flush=True) # junk markers: front-matter / watermark / copyright glue that may carry book titles _JUNK = ["z-library", "下载", "Downloaded by", "版权", "ISBN", "责任编辑", "总前言", "《中国慢性疾病营养", "北京协和", "专家委员会", "出版"] def _is_junk(d): return any(k in d for k in _JUNK) stats = {} for cat, items in bycat.items(): seen = set() uniq = [] for r in items: if r["question"] in seen: continue seen.add(r["question"]) uniq.append(r) docs = [] meta = [] for r in uniq: d = f"Q:{r['question']}\nA:{r['answer']}" if _is_junk(d): continue docs.append(d) meta.append({"slug": r["slug"], "page": r["page"], "category": cat, "question": r["question"], "book": r["book"], "answer": r["answer"][:900]}) emb = model.encode(docs, normalize_embeddings=True, show_progress_bar=False, batch_size=32) emb = np.asarray(emb, dtype="float32") idx = faiss.IndexFlatIP(emb.shape[1]) idx.add(emb) sub = os.path.join(OUT, cat) os.makedirs(sub, exist_ok=True) faiss.write_index(idx, os.path.join(sub, "index.faiss")) with open(os.path.join(sub, "docs.json"), "w", encoding="utf-8") as f: json.dump(docs, f, ensure_ascii=False) with open(os.path.join(sub, "meta.json"), "w", encoding="utf-8") as f: json.dump(meta, f, ensure_ascii=False) stats[cat] = len(docs) print(f" [{cat}] docs={len(docs)}", flush=True) with open(os.path.join(OUT, "index_map.json"), "w", encoding="utf-8") as f: json.dump(stats, f, ensure_ascii=False, indent=1) print("DONE", stats)