Spaces:
Sleeping
Sleeping
Download ingest.py from cynstl/groundwork: direct link, hf CLI and curl.
- Browser
- Download file 3.17 kB
-
https://huggingface.co/spaces/cynstl/groundwork/resolve/main/ingest.py
- Command line
-
hf download hf://spaces/cynstl/groundwork/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/cynstl/groundwork/resolve/main/ingest.py
3.17 kB
| """ | |
| Groundwork — Jury 1 | |
| Step 1: Ingest documents, chunk them (with traceable source metadata), embed, and index in FAISS. | |
| Run this once (or whenever docs/ changes) to build the local vector index. | |
| Usage: python ingest.py | |
| """ | |
| import os | |
| from langchain_community.document_loaders import PyPDFLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain_community.vectorstores import FAISS | |
| from langchain_huggingface import HuggingFaceEmbeddings | |
| DOCS_DIR = "docs" | |
| INDEX_DIR = "faiss_index" | |
| # Friendly display names for citations — maps filename -> (short label, full citation) | |
| DOC_LABELS = { | |
| "kaplan_scaling_laws.pdf": ("Kaplan et al. 2020", "Kaplan et al., 'Scaling Laws for Neural Language Models' (2020)"), | |
| "hoffmann_chinchilla.pdf": ("Hoffmann et al. 2022 (Chinchilla)", "Hoffmann et al., 'Training Compute-Optimal Large Language Models' (2022)"), | |
| "pearce_song_reconciling.pdf": ("Pearce & Song 2024", "Pearce & Song, 'Reconciling Kaplan and Chinchilla Scaling Laws' (2024)"), | |
| "lewis_rag.pdf": ("Lewis et al. 2020 (RAG)", "Lewis et al., 'Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks' (2020)"), | |
| "kalai_hallucinate.pdf": ("Kalai et al. 2025", "Kalai, Nachum, Vempala & Zhang, 'Why Language Models Hallucinate' (2025)"), | |
| } | |
| def load_and_chunk(): | |
| """Load every PDF in docs/, tag each chunk with source + page, split into retrievable chunks.""" | |
| splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50) | |
| all_chunks = [] | |
| pdf_files = [f for f in os.listdir(DOCS_DIR) if f.endswith(".pdf")] | |
| if not pdf_files: | |
| raise FileNotFoundError( | |
| f"No PDFs found in {DOCS_DIR}/. Download the 5 source papers there first (see README)." | |
| ) | |
| for filename in pdf_files: | |
| filepath = os.path.join(DOCS_DIR, filename) | |
| label, full_citation = DOC_LABELS.get(filename, (filename, filename)) | |
| loader = PyPDFLoader(filepath) | |
| pages = loader.load() # one Document per page, with metadata["page"] | |
| chunks = splitter.split_documents(pages) | |
| for i, chunk in enumerate(chunks): | |
| chunk.metadata["source_label"] = label | |
| chunk.metadata["source_citation"] = full_citation | |
| chunk.metadata["source_file"] = filename | |
| # page is 0-indexed from PyPDFLoader; make it human-readable | |
| chunk.metadata["page_display"] = chunk.metadata.get("page", 0) + 1 | |
| chunk.metadata["chunk_id"] = f"{filename}::chunk{i}" | |
| all_chunks.extend(chunks) | |
| print(f" {filename}: {len(pages)} pages -> {len(chunks)} chunks") | |
| return all_chunks | |
| def build_index(): | |
| print("Loading and chunking documents...") | |
| chunks = load_and_chunk() | |
| print(f"Total chunks: {len(chunks)}") | |
| print("Loading embedding model (all-MiniLM-L6-v2)...") | |
| embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") | |
| print("Building FAISS index...") | |
| vectorstore = FAISS.from_documents(chunks, embeddings) | |
| vectorstore.save_local(INDEX_DIR) | |
| print(f"Index saved to {INDEX_DIR}/") | |
| if __name__ == "__main__": | |
| build_index() | |