""" KNOWLEDGE INGESTOR — Complete with PEPW Drawings (74 files) NR56: 36 drawings | NR60: 38 drawings """ from pathlib import Path from typing import List, Dict, Any from core.bootstrap import ROOT from core.audit_logger import get_logger from core.config import ProductionConfig from core.rag import chunk_text import chromadb from sentence_transformers import SentenceTransformer logger = get_logger("scripts.ingest_knowledge") def read_pdf(file_path: Path) -> str: """Extract text from PDF""" try: from pypdf import PdfReader reader = PdfReader(file_path) content = "" for page_num, page in enumerate(reader.pages, 1): page_text = page.extract_text() if page_text: content += f"\n--- PAGE {page_num} ---\n" content += page_text + "\n" return content except Exception as e: logger.warning("Failed to read %s: %s", file_path.name, e) return "" def read_documents(doc_dir: str, category: str = "general", metadata_extra: Dict = None) -> List[Dict]: """Read all documents from directory""" documents = [] path = Path(doc_dir) if not path.exists(): return documents metadata_extra = metadata_extra or {} # Read PDF files try: from pypdf import PdfReader for file_path in path.glob("*.pdf"): content = read_pdf(file_path) if content.strip(): doc = { "content": content, "metadata": { "source": file_path.name, "type": "pdf_document", "category": category, "size_kb": round(file_path.stat().st_size / 1024, 1), "path": str(file_path) } } doc["metadata"].update(metadata_extra) documents.append(doc) except ImportError: print(" āš ļø pypdf not installed. PDF support disabled.") return documents def get_all_documents() -> List[Dict]: """Get all documents from all knowledge folders""" documents = [] base_dir = Path(__file__).parent.parent print("\nšŸ“„ Scanning all knowledge folders...\n") # ============================================ # PEPW DRAWINGS — NR56 (36 files) # ============================================ nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF" if nr56_path.exists(): print(f" šŸ“ NR56 PEPW Drawings: {len(list(nr56_path.glob('*.pdf')))} files") docs = read_documents( str(nr56_path), category="pepw_drawing", metadata_extra={"standard": "NR56", "type": "turnout_crossover"} ) documents.extend(docs) print(f" āœ… Ingested {len(docs)} NR56 drawings") else: print(" āš ļø NR56 path not found") # ============================================ # PEPW DRAWINGS — NR60 (38 files) # ============================================ nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF" if nr60_path.exists(): print(f" šŸ“ NR60 PEPW Drawings: {len(list(nr60_path.glob('*.pdf')))} files") docs = read_documents( str(nr60_path), category="pepw_drawing", metadata_extra={"standard": "NR60", "type": "turnout_crossover"} ) documents.extend(docs) print(f" āœ… Ingested {len(docs)} NR60 drawings") else: print(" āš ļø NR60 path not found") # ============================================ # SOP (Complete SOP) # ============================================ sop_path = base_dir / "knowledge/sops/complete_sop.txt" if sop_path.exists(): print(f" šŸ“ SOP: {sop_path.name}") try: content = sop_path.read_text(encoding='utf-8') documents.append({ "content": content, "metadata": { "source": sop_path.name, "type": "sop", "category": "sop", "size_kb": round(sop_path.stat().st_size / 1024, 1) } }) print(f" āœ… Ingested SOP ({len(content)} characters)") except Exception as e: print(f" āš ļø Failed to read SOP: {e}") # ============================================ # Standards (NR/2500, NR/2102, NR/2049) # ============================================ standards_folders = [ ("knowledge/standards/nr_2500", "NR/L2/TRK/2500"), ("knowledge/standards/nr_2102", "NR 2102"), ("knowledge/standards/nr_2049", "NR 2049"), ] for folder, standard_name in standards_folders: folder_path = base_dir / folder if folder_path.exists(): print(f" šŸ“ Standards: {standard_name}") docs = read_documents( str(folder_path), category="standard", metadata_extra={"standard": standard_name} ) documents.extend(docs) print(f" āœ… Ingested {len(docs)} documents") # ============================================ # Summary # ============================================ print(f"\nšŸ“Š TOTAL: {len(documents)} documents ready for ingestion") # Count by category categories = {} for doc in documents: cat = doc["metadata"].get("category", "unknown") categories[cat] = categories.get(cat, 0) + 1 if categories: print("\nšŸ“‹ Document Breakdown:") for cat, count in sorted(categories.items()): print(f" - {cat}: {count}") return documents def ingest_knowledge(): """Ingest all knowledge into ChromaDB""" print("\n" + "="*60) print(" šŸ“š INGESTING KNOWLEDGE BASE") print("="*60 + "\n") base_dir = Path(__file__).parent.parent chroma_path = base_dir / "knowledge" / "chroma_db" print("šŸ“ Initializing ChromaDB...") chroma_path.mkdir(parents=True, exist_ok=True) client = chromadb.PersistentClient(path=str(chroma_path)) try: client.delete_collection("track_design_knowledge") print(" šŸ—‘ļø Removed existing collection") except Exception as e: logger.debug("Collection delete skipped: %s", e) collection = client.create_collection( name="track_design_knowledge", metadata={"hnsw:space": "cosine"} ) print(" āœ… Created collection: track_design_knowledge\n") print("🧠 Loading embedding model...") model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) print(" āœ… Embedding model loaded\n") print("šŸ“„ Loading documents...") documents = get_all_documents() if not documents: print(" āš ļø No documents found! Creating samples...") from scripts.sample_docs import create_sample_documents documents = create_sample_documents() print(f"\nšŸ“Š Total documents: {len(documents)}") texts = [] metadatas = [] ids = [] n = 0 for doc in documents: chunks = chunk_text(doc["content"]) if not chunks: chunks = [doc["content"][:ProductionConfig.CHUNK_SIZE]] for j, chunk in enumerate(chunks): n += 1 texts.append(chunk) meta = dict(doc["metadata"]) meta["chunk"] = j metadatas.append(meta) ids.append(f"doc_{n}") print(f"\nšŸ”„ Generating embeddings for {len(texts)} documents...") embeddings = model.encode(texts).tolist() print(" āœ… Embeddings generated\n") print("šŸ’¾ Adding to ChromaDB...") collection.add( embeddings=embeddings, documents=texts, metadatas=metadatas, ids=ids ) print(f"\nāœ… Successfully ingested {len(texts)} documents!") print(f"šŸ“ ChromaDB location: {chroma_path}") # PEPW specific summary pepw_docs = [d for d in documents if d["metadata"].get("category") == "pepw_drawing"] if pepw_docs: nr56_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR56"]) nr60_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR60"]) print(f"\nšŸ”§ PEPW Drawings Ingested:") print(f" - NR56: {nr56_count} drawings") print(f" - NR60: {nr60_count} drawings") print(f" - TOTAL: {len(pepw_docs)} drawings") else: print("\nāš ļø No PEPW drawings found!") print(" Place drawings in:") print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/") print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/") print("\n" + "="*60) print(" āœ… KNOWLEDGE INGESTION COMPLETE") print("="*60 + "\n") if __name__ == "__main__": ingest_knowledge()