""" S&C KNOWLEDGE INGESTOR — Separate Database for Switches & Crossings Only used when user asks S&C-related questions """ import os import sys from pathlib import Path from typing import List, Dict import chromadb from sentence_transformers import SentenceTransformer sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def read_pdf(file_path: Path) -> str: """Extract text from PDF""" try: from pypdf import PdfReader reader = PdfReader(file_path) content = "" for page_num, page in enumerate(reader.pages, 1): page_text = page.extract_text() if page_text: content += f"\n--- PAGE {page_num} ---\n" content += page_text + "\n" return content except Exception as e: print(f" āš ļø Failed to read {file_path.name}: {e}") return "" def get_sandc_documents() -> List[Dict]: """Get all S&C documents (PEPW drawings)""" documents = [] base_dir = Path(__file__).parent.parent # NR56 PEPW Drawings nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF" if nr56_path.exists(): pdf_files = list(nr56_path.glob("*.pdf")) print(f"šŸ“ NR56 PEPW Drawings: {len(pdf_files)} files") for file_path in pdf_files: content = read_pdf(file_path) if content.strip(): documents.append({ "content": content, "metadata": { "source": file_path.name, "standard": "NR56", "category": "pepw_drawing", "type": "turnout_crossover", "size_kb": round(file_path.stat().st_size / 1024, 1) } }) print(f" āœ… Ingested: {file_path.name}") # NR60 PEPW Drawings nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF" if nr60_path.exists(): pdf_files = list(nr60_path.glob("*.pdf")) print(f"šŸ“ NR60 PEPW Drawings: {len(pdf_files)} files") for file_path in pdf_files: content = read_pdf(file_path) if content.strip(): documents.append({ "content": content, "metadata": { "source": file_path.name, "standard": "NR60", "category": "pepw_drawing", "type": "turnout_crossover", "size_kb": round(file_path.stat().st_size / 1024, 1) } }) print(f" āœ… Ingested: {file_path.name}") return documents def ingest_sandc_knowledge(): """Ingest S&C knowledge into separate database""" print("\n" + "="*60) print(" šŸ”§ INGESTING S&C KNOWLEDGE (PEPW Drawings)") print("="*60 + "\n") base_dir = Path(__file__).parent.parent chroma_path = base_dir / "knowledge" / "sandc_chroma_db" print("šŸ“ Initializing S&C ChromaDB...") chroma_path.mkdir(parents=True, exist_ok=True) client = chromadb.PersistentClient(path=str(chroma_path)) try: client.delete_collection("sandc_knowledge") print(" šŸ—‘ļø Removed existing S&C collection") except Exception as e: pass collection = client.create_collection( name="sandc_knowledge", metadata={"hnsw:space": "cosine", "description": "S&C Design Knowledge"} ) print(" āœ… Created collection: sandc_knowledge\n") print("🧠 Loading embedding model...") model = SentenceTransformer('all-MiniLM-L6-v2') print(" āœ… Embedding model loaded\n") print("šŸ“„ Loading S&C documents...") documents = get_sandc_documents() if not documents: print(" āš ļø No S&C documents found!") print(" Place PEPW drawings in:") print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/") print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/") return print(f"\nšŸ“Š Total S&C documents: {len(documents)}") texts = [] metadatas = [] ids = [] for i, doc in enumerate(documents): texts.append(doc["content"]) metadatas.append(doc["metadata"]) ids.append(f"sandc_doc_{i+1}") print(f"\nšŸ”„ Generating embeddings for {len(texts)} documents...") embeddings = model.encode(texts).tolist() print(" āœ… Embeddings generated\n") print("šŸ’¾ Adding to ChromaDB...") collection.add( embeddings=embeddings, documents=texts, metadatas=metadatas, ids=ids ) print(f"\nāœ… Successfully ingested {len(texts)} S&C documents!") print(f"šŸ“ S&C ChromaDB location: {chroma_path}") # Summary nr56_count = len([d for d in documents if d["metadata"].get("standard") == "NR56"]) nr60_count = len([d for d in documents if d["metadata"].get("standard") == "NR60"]) print(f"\nšŸ”§ S&C Drawings Summary:") print(f" - NR56: {nr56_count} drawings") print(f" - NR60: {nr60_count} drawings") print(f" - TOTAL: {len(documents)} drawings") print("\n" + "="*60) print(" āœ… S&C KNOWLEDGE INGESTION COMPLETE") print("="*60 + "\n") if __name__ == "__main__": ingest_sandc_knowledge()