Download scripts/ingest_knowledge.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 9.05 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/scripts/ingest_knowledge.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/scripts/ingest_knowledge.py
-
curl -L -o ingest_knowledge.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/scripts/ingest_knowledge.py
9.05 kB
| """ | |
| KNOWLEDGE INGESTOR β Complete with PEPW Drawings (74 files) | |
| NR56: 36 drawings | NR60: 38 drawings | |
| """ | |
| from pathlib import Path | |
| from typing import List, Dict, Any | |
| from core.bootstrap import ROOT | |
| from core.audit_logger import get_logger | |
| from core.config import ProductionConfig | |
| from core.rag import chunk_text | |
| import chromadb | |
| from sentence_transformers import SentenceTransformer | |
| logger = get_logger("scripts.ingest_knowledge") | |
| def read_pdf(file_path: Path) -> str: | |
| """Extract text from PDF""" | |
| try: | |
| from pypdf import PdfReader | |
| reader = PdfReader(file_path) | |
| content = "" | |
| for page_num, page in enumerate(reader.pages, 1): | |
| page_text = page.extract_text() | |
| if page_text: | |
| content += f"\n--- PAGE {page_num} ---\n" | |
| content += page_text + "\n" | |
| return content | |
| except Exception as e: | |
| logger.warning("Failed to read %s: %s", file_path.name, e) | |
| return "" | |
| def read_documents(doc_dir: str, category: str = "general", metadata_extra: Dict = None) -> List[Dict]: | |
| """Read all documents from directory""" | |
| documents = [] | |
| path = Path(doc_dir) | |
| if not path.exists(): | |
| return documents | |
| metadata_extra = metadata_extra or {} | |
| # Read PDF files | |
| try: | |
| from pypdf import PdfReader | |
| for file_path in path.glob("*.pdf"): | |
| content = read_pdf(file_path) | |
| if content.strip(): | |
| doc = { | |
| "content": content, | |
| "metadata": { | |
| "source": file_path.name, | |
| "type": "pdf_document", | |
| "category": category, | |
| "size_kb": round(file_path.stat().st_size / 1024, 1), | |
| "path": str(file_path) | |
| } | |
| } | |
| doc["metadata"].update(metadata_extra) | |
| documents.append(doc) | |
| except ImportError: | |
| print(" β οΈ pypdf not installed. PDF support disabled.") | |
| return documents | |
| def get_all_documents() -> List[Dict]: | |
| """Get all documents from all knowledge folders""" | |
| documents = [] | |
| base_dir = Path(__file__).parent.parent | |
| print("\nπ Scanning all knowledge folders...\n") | |
| # ============================================ | |
| # PEPW DRAWINGS β NR56 (36 files) | |
| # ============================================ | |
| nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF" | |
| if nr56_path.exists(): | |
| print(f" π NR56 PEPW Drawings: {len(list(nr56_path.glob('*.pdf')))} files") | |
| docs = read_documents( | |
| str(nr56_path), | |
| category="pepw_drawing", | |
| metadata_extra={"standard": "NR56", "type": "turnout_crossover"} | |
| ) | |
| documents.extend(docs) | |
| print(f" β Ingested {len(docs)} NR56 drawings") | |
| else: | |
| print(" β οΈ NR56 path not found") | |
| # ============================================ | |
| # PEPW DRAWINGS β NR60 (38 files) | |
| # ============================================ | |
| nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF" | |
| if nr60_path.exists(): | |
| print(f" π NR60 PEPW Drawings: {len(list(nr60_path.glob('*.pdf')))} files") | |
| docs = read_documents( | |
| str(nr60_path), | |
| category="pepw_drawing", | |
| metadata_extra={"standard": "NR60", "type": "turnout_crossover"} | |
| ) | |
| documents.extend(docs) | |
| print(f" β Ingested {len(docs)} NR60 drawings") | |
| else: | |
| print(" β οΈ NR60 path not found") | |
| # ============================================ | |
| # SOP (Complete SOP) | |
| # ============================================ | |
| sop_path = base_dir / "knowledge/sops/complete_sop.txt" | |
| if sop_path.exists(): | |
| print(f" π SOP: {sop_path.name}") | |
| try: | |
| content = sop_path.read_text(encoding='utf-8') | |
| documents.append({ | |
| "content": content, | |
| "metadata": { | |
| "source": sop_path.name, | |
| "type": "sop", | |
| "category": "sop", | |
| "size_kb": round(sop_path.stat().st_size / 1024, 1) | |
| } | |
| }) | |
| print(f" β Ingested SOP ({len(content)} characters)") | |
| except Exception as e: | |
| print(f" β οΈ Failed to read SOP: {e}") | |
| # ============================================ | |
| # Standards (NR/2500, NR/2102, NR/2049) | |
| # ============================================ | |
| standards_folders = [ | |
| ("knowledge/standards/nr_2500", "NR/L2/TRK/2500"), | |
| ("knowledge/standards/nr_2102", "NR 2102"), | |
| ("knowledge/standards/nr_2049", "NR 2049"), | |
| ] | |
| for folder, standard_name in standards_folders: | |
| folder_path = base_dir / folder | |
| if folder_path.exists(): | |
| print(f" π Standards: {standard_name}") | |
| docs = read_documents( | |
| str(folder_path), | |
| category="standard", | |
| metadata_extra={"standard": standard_name} | |
| ) | |
| documents.extend(docs) | |
| print(f" β Ingested {len(docs)} documents") | |
| # ============================================ | |
| # Summary | |
| # ============================================ | |
| print(f"\nπ TOTAL: {len(documents)} documents ready for ingestion") | |
| # Count by category | |
| categories = {} | |
| for doc in documents: | |
| cat = doc["metadata"].get("category", "unknown") | |
| categories[cat] = categories.get(cat, 0) + 1 | |
| if categories: | |
| print("\nπ Document Breakdown:") | |
| for cat, count in sorted(categories.items()): | |
| print(f" - {cat}: {count}") | |
| return documents | |
| def ingest_knowledge(): | |
| """Ingest all knowledge into ChromaDB""" | |
| print("\n" + "="*60) | |
| print(" π INGESTING KNOWLEDGE BASE") | |
| print("="*60 + "\n") | |
| base_dir = Path(__file__).parent.parent | |
| chroma_path = base_dir / "knowledge" / "chroma_db" | |
| print("π Initializing ChromaDB...") | |
| chroma_path.mkdir(parents=True, exist_ok=True) | |
| client = chromadb.PersistentClient(path=str(chroma_path)) | |
| try: | |
| client.delete_collection("track_design_knowledge") | |
| print(" ποΈ Removed existing collection") | |
| except Exception as e: | |
| logger.debug("Collection delete skipped: %s", e) | |
| collection = client.create_collection( | |
| name="track_design_knowledge", | |
| metadata={"hnsw:space": "cosine"} | |
| ) | |
| print(" β Created collection: track_design_knowledge\n") | |
| print("π§ Loading embedding model...") | |
| model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL) | |
| print(" β Embedding model loaded\n") | |
| print("π Loading documents...") | |
| documents = get_all_documents() | |
| if not documents: | |
| print(" β οΈ No documents found! Creating samples...") | |
| from scripts.sample_docs import create_sample_documents | |
| documents = create_sample_documents() | |
| print(f"\nπ Total documents: {len(documents)}") | |
| texts = [] | |
| metadatas = [] | |
| ids = [] | |
| n = 0 | |
| for doc in documents: | |
| chunks = chunk_text(doc["content"]) | |
| if not chunks: | |
| chunks = [doc["content"][:ProductionConfig.CHUNK_SIZE]] | |
| for j, chunk in enumerate(chunks): | |
| n += 1 | |
| texts.append(chunk) | |
| meta = dict(doc["metadata"]) | |
| meta["chunk"] = j | |
| metadatas.append(meta) | |
| ids.append(f"doc_{n}") | |
| print(f"\nπ Generating embeddings for {len(texts)} documents...") | |
| embeddings = model.encode(texts).tolist() | |
| print(" β Embeddings generated\n") | |
| print("πΎ Adding to ChromaDB...") | |
| collection.add( | |
| embeddings=embeddings, | |
| documents=texts, | |
| metadatas=metadatas, | |
| ids=ids | |
| ) | |
| print(f"\nβ Successfully ingested {len(texts)} documents!") | |
| print(f"π ChromaDB location: {chroma_path}") | |
| # PEPW specific summary | |
| pepw_docs = [d for d in documents if d["metadata"].get("category") == "pepw_drawing"] | |
| if pepw_docs: | |
| nr56_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR56"]) | |
| nr60_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR60"]) | |
| print(f"\nπ§ PEPW Drawings Ingested:") | |
| print(f" - NR56: {nr56_count} drawings") | |
| print(f" - NR60: {nr60_count} drawings") | |
| print(f" - TOTAL: {len(pepw_docs)} drawings") | |
| else: | |
| print("\nβ οΈ No PEPW drawings found!") | |
| print(" Place drawings in:") | |
| print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/") | |
| print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/") | |
| print("\n" + "="*60) | |
| print(" β KNOWLEDGE INGESTION COMPLETE") | |
| print("="*60 + "\n") | |
| if __name__ == "__main__": | |
| ingest_knowledge() | |