Download src/ingest.py from prabhal/geneseek: direct link, hf CLI and curl.
- Browser
- Download file 3.38 kB
-
https://huggingface.co/spaces/prabhal/geneseek/resolve/main/src/ingest.py
- Command line
-
hf download hf://spaces/prabhal/geneseek/src/ingest.py
-
curl -L -o ingest.py https://huggingface.co/spaces/prabhal/geneseek/resolve/main/src/ingest.py
3.38 kB
| from langchain_community.document_loaders import PyMuPDFLoader, TextLoader | |
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from langchain_qdrant import QdrantVectorStore, FastEmbedSparse | |
| from langchain_openai import OpenAIEmbeddings | |
| from src import config | |
| import hashlib | |
| import os | |
| def ingest_file(file_path: str): | |
| """ | |
| Document ingestion pipeline for GeneSeek. | |
| Handles loading, chunking, embedding and upserting documents into the Qdrant vector store. | |
| Duplicate ingestion is prevented via MD5 hash tracking. | |
| """ | |
| # 1. Duplicate check | |
| with open(file_path, "rb") as f: | |
| file_hash = hashlib.md5(f.read()).hexdigest() | |
| hash_record_path = config.DATA_DIR / "ingested_hashes.txt" | |
| if hash_record_path.exists(): | |
| with open(hash_record_path, "r") as f: | |
| known_hashes = f.read().splitlines() | |
| if file_hash in known_hashes: | |
| print("File already ingested. Skipping.") | |
| return False | |
| # 2. Load file | |
| print(f"Loading file: {file_path}...") | |
| if file_path.endswith(".pdf"): | |
| loader = PyMuPDFLoader(file_path) | |
| else: | |
| loader = TextLoader(file_path) | |
| docs = loader.load() | |
| def extract_metadata(text): | |
| metadata = {} | |
| for line in text.split("\n"): | |
| if "Clinical Trial ID:" in line: | |
| metadata["trial_id"] = line.split(":")[-1].strip() | |
| elif "Study Title:" in line: | |
| metadata["title"] = line.split(":")[-1].strip() | |
| elif "Date:" in line: | |
| metadata["date"] = line.split(":")[-1].strip() | |
| return metadata | |
| # Attach metadata to each doc | |
| for doc in docs: | |
| extracted = extract_metadata(doc.page_content) | |
| doc.metadata.update(extracted) | |
| # 3. Split | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1000, | |
| chunk_overlap=200, | |
| separators=["\n\n", "\n", ".", " "] | |
| ) | |
| splits = text_splitter.split_documents(docs) | |
| print(f"Split into {len(splits)} chunks.") | |
| # 4. Initialize embeddings | |
| dense_embeddings = OpenAIEmbeddings( | |
| model=config.EMBEDDING_MODEL, | |
| openai_api_key=config.OPENAI_API_KEY | |
| ) | |
| sparse_embeddings = FastEmbedSparse(model_name=config.SPARSE_MODEL) | |
| # 5. Use shared client | |
| existing = [c.name for c in config.qdrant_client.get_collections().collections] | |
| if config.COLLECTION_NAME not in existing: | |
| print("Collection not found. Creating new collection...") | |
| QdrantVectorStore.from_documents( | |
| documents=splits, | |
| embedding=dense_embeddings, | |
| sparse_embedding=sparse_embeddings, | |
| url=os.getenv("QDRANT_URL"), | |
| api_key=os.getenv("QDRANT_API_KEY"), | |
| collection_name=config.COLLECTION_NAME, | |
| retrieval_mode="hybrid", | |
| ) | |
| else: | |
| print("Collection exists. Appending documents...") | |
| vector_store = QdrantVectorStore( | |
| client=config.qdrant_client, | |
| collection_name=config.COLLECTION_NAME, | |
| embedding=dense_embeddings, | |
| sparse_embedding=sparse_embeddings, | |
| retrieval_mode="hybrid" | |
| ) | |
| vector_store.add_documents(splits) | |
| # 6. Record hash | |
| with open(hash_record_path, "a") as f: | |
| f.write(file_hash + "\n") | |
| print("Ingestion complete.") | |
| return True |