groundwork / ingest.py
cynstl's picture
Fix HF Spaces build: upgrade sentence-transformers to >=5.2.0 for transformers 5.x / huggingface-hub 1.x compatibility with gradio 6.26.0
fe898d6
Raw History Blame Contribute Delete
3.17 kB
"""
Groundwork — Jury 1
Step 1: Ingest documents, chunk them (with traceable source metadata), embed, and index in FAISS.
Run this once (or whenever docs/ changes) to build the local vector index.
Usage: python ingest.py
"""
import os
from langchain_community.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import FAISS
from langchain_huggingface import HuggingFaceEmbeddings
DOCS_DIR = "docs"
INDEX_DIR = "faiss_index"
# Friendly display names for citations — maps filename -> (short label, full citation)
DOC_LABELS = {
"kaplan_scaling_laws.pdf": ("Kaplan et al. 2020", "Kaplan et al., 'Scaling Laws for Neural Language Models' (2020)"),
"hoffmann_chinchilla.pdf": ("Hoffmann et al. 2022 (Chinchilla)", "Hoffmann et al., 'Training Compute-Optimal Large Language Models' (2022)"),
"pearce_song_reconciling.pdf": ("Pearce & Song 2024", "Pearce & Song, 'Reconciling Kaplan and Chinchilla Scaling Laws' (2024)"),
"lewis_rag.pdf": ("Lewis et al. 2020 (RAG)", "Lewis et al., 'Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks' (2020)"),
"kalai_hallucinate.pdf": ("Kalai et al. 2025", "Kalai, Nachum, Vempala & Zhang, 'Why Language Models Hallucinate' (2025)"),
}
def load_and_chunk():
"""Load every PDF in docs/, tag each chunk with source + page, split into retrievable chunks."""
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
all_chunks = []
pdf_files = [f for f in os.listdir(DOCS_DIR) if f.endswith(".pdf")]
if not pdf_files:
raise FileNotFoundError(
f"No PDFs found in {DOCS_DIR}/. Download the 5 source papers there first (see README)."
)
for filename in pdf_files:
filepath = os.path.join(DOCS_DIR, filename)
label, full_citation = DOC_LABELS.get(filename, (filename, filename))
loader = PyPDFLoader(filepath)
pages = loader.load() # one Document per page, with metadata["page"]
chunks = splitter.split_documents(pages)
for i, chunk in enumerate(chunks):
chunk.metadata["source_label"] = label
chunk.metadata["source_citation"] = full_citation
chunk.metadata["source_file"] = filename
# page is 0-indexed from PyPDFLoader; make it human-readable
chunk.metadata["page_display"] = chunk.metadata.get("page", 0) + 1
chunk.metadata["chunk_id"] = f"{filename}::chunk{i}"
all_chunks.extend(chunks)
print(f" {filename}: {len(pages)} pages -> {len(chunks)} chunks")
return all_chunks
def build_index():
print("Loading and chunking documents...")
chunks = load_and_chunk()
print(f"Total chunks: {len(chunks)}")
print("Loading embedding model (all-MiniLM-L6-v2)...")
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
print("Building FAISS index...")
vectorstore = FAISS.from_documents(chunks, embeddings)
vectorstore.save_local(INDEX_DIR)
print(f"Index saved to {INDEX_DIR}/")
if __name__ == "__main__":
build_index()