""" Groundwork — Jury 1 Step 1: Ingest documents, chunk them (with traceable source metadata), embed, and index in FAISS. Run this once (or whenever docs/ changes) to build the local vector index. Usage: python ingest.py """ import os from langchain_community.document_loaders import PyPDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_community.vectorstores import FAISS from langchain_huggingface import HuggingFaceEmbeddings DOCS_DIR = "docs" INDEX_DIR = "faiss_index" # Friendly display names for citations — maps filename -> (short label, full citation) DOC_LABELS = { "kaplan_scaling_laws.pdf": ("Kaplan et al. 2020", "Kaplan et al., 'Scaling Laws for Neural Language Models' (2020)"), "hoffmann_chinchilla.pdf": ("Hoffmann et al. 2022 (Chinchilla)", "Hoffmann et al., 'Training Compute-Optimal Large Language Models' (2022)"), "pearce_song_reconciling.pdf": ("Pearce & Song 2024", "Pearce & Song, 'Reconciling Kaplan and Chinchilla Scaling Laws' (2024)"), "lewis_rag.pdf": ("Lewis et al. 2020 (RAG)", "Lewis et al., 'Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks' (2020)"), "kalai_hallucinate.pdf": ("Kalai et al. 2025", "Kalai, Nachum, Vempala & Zhang, 'Why Language Models Hallucinate' (2025)"), } def load_and_chunk(): """Load every PDF in docs/, tag each chunk with source + page, split into retrievable chunks.""" splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50) all_chunks = [] pdf_files = [f for f in os.listdir(DOCS_DIR) if f.endswith(".pdf")] if not pdf_files: raise FileNotFoundError( f"No PDFs found in {DOCS_DIR}/. Download the 5 source papers there first (see README)." ) for filename in pdf_files: filepath = os.path.join(DOCS_DIR, filename) label, full_citation = DOC_LABELS.get(filename, (filename, filename)) loader = PyPDFLoader(filepath) pages = loader.load() # one Document per page, with metadata["page"] chunks = splitter.split_documents(pages) for i, chunk in enumerate(chunks): chunk.metadata["source_label"] = label chunk.metadata["source_citation"] = full_citation chunk.metadata["source_file"] = filename # page is 0-indexed from PyPDFLoader; make it human-readable chunk.metadata["page_display"] = chunk.metadata.get("page", 0) + 1 chunk.metadata["chunk_id"] = f"{filename}::chunk{i}" all_chunks.extend(chunks) print(f" {filename}: {len(pages)} pages -> {len(chunks)} chunks") return all_chunks def build_index(): print("Loading and chunking documents...") chunks = load_and_chunk() print(f"Total chunks: {len(chunks)}") print("Loading embedding model (all-MiniLM-L6-v2)...") embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") print("Building FAISS index...") vectorstore = FAISS.from_documents(chunks, embeddings) vectorstore.save_local(INDEX_DIR) print(f"Index saved to {INDEX_DIR}/") if __name__ == "__main__": build_index()