flowmarg / scripts /ingest_knowledge.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
9.05 kB
"""
KNOWLEDGE INGESTOR β€” Complete with PEPW Drawings (74 files)
NR56: 36 drawings | NR60: 38 drawings
"""
from pathlib import Path
from typing import List, Dict, Any
from core.bootstrap import ROOT
from core.audit_logger import get_logger
from core.config import ProductionConfig
from core.rag import chunk_text
import chromadb
from sentence_transformers import SentenceTransformer
logger = get_logger("scripts.ingest_knowledge")
def read_pdf(file_path: Path) -> str:
"""Extract text from PDF"""
try:
from pypdf import PdfReader
reader = PdfReader(file_path)
content = ""
for page_num, page in enumerate(reader.pages, 1):
page_text = page.extract_text()
if page_text:
content += f"\n--- PAGE {page_num} ---\n"
content += page_text + "\n"
return content
except Exception as e:
logger.warning("Failed to read %s: %s", file_path.name, e)
return ""
def read_documents(doc_dir: str, category: str = "general", metadata_extra: Dict = None) -> List[Dict]:
"""Read all documents from directory"""
documents = []
path = Path(doc_dir)
if not path.exists():
return documents
metadata_extra = metadata_extra or {}
# Read PDF files
try:
from pypdf import PdfReader
for file_path in path.glob("*.pdf"):
content = read_pdf(file_path)
if content.strip():
doc = {
"content": content,
"metadata": {
"source": file_path.name,
"type": "pdf_document",
"category": category,
"size_kb": round(file_path.stat().st_size / 1024, 1),
"path": str(file_path)
}
}
doc["metadata"].update(metadata_extra)
documents.append(doc)
except ImportError:
print(" ⚠️ pypdf not installed. PDF support disabled.")
return documents
def get_all_documents() -> List[Dict]:
"""Get all documents from all knowledge folders"""
documents = []
base_dir = Path(__file__).parent.parent
print("\nπŸ“„ Scanning all knowledge folders...\n")
# ============================================
# PEPW DRAWINGS β€” NR56 (36 files)
# ============================================
nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF"
if nr56_path.exists():
print(f" πŸ“ NR56 PEPW Drawings: {len(list(nr56_path.glob('*.pdf')))} files")
docs = read_documents(
str(nr56_path),
category="pepw_drawing",
metadata_extra={"standard": "NR56", "type": "turnout_crossover"}
)
documents.extend(docs)
print(f" βœ… Ingested {len(docs)} NR56 drawings")
else:
print(" ⚠️ NR56 path not found")
# ============================================
# PEPW DRAWINGS β€” NR60 (38 files)
# ============================================
nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF"
if nr60_path.exists():
print(f" πŸ“ NR60 PEPW Drawings: {len(list(nr60_path.glob('*.pdf')))} files")
docs = read_documents(
str(nr60_path),
category="pepw_drawing",
metadata_extra={"standard": "NR60", "type": "turnout_crossover"}
)
documents.extend(docs)
print(f" βœ… Ingested {len(docs)} NR60 drawings")
else:
print(" ⚠️ NR60 path not found")
# ============================================
# SOP (Complete SOP)
# ============================================
sop_path = base_dir / "knowledge/sops/complete_sop.txt"
if sop_path.exists():
print(f" πŸ“ SOP: {sop_path.name}")
try:
content = sop_path.read_text(encoding='utf-8')
documents.append({
"content": content,
"metadata": {
"source": sop_path.name,
"type": "sop",
"category": "sop",
"size_kb": round(sop_path.stat().st_size / 1024, 1)
}
})
print(f" βœ… Ingested SOP ({len(content)} characters)")
except Exception as e:
print(f" ⚠️ Failed to read SOP: {e}")
# ============================================
# Standards (NR/2500, NR/2102, NR/2049)
# ============================================
standards_folders = [
("knowledge/standards/nr_2500", "NR/L2/TRK/2500"),
("knowledge/standards/nr_2102", "NR 2102"),
("knowledge/standards/nr_2049", "NR 2049"),
]
for folder, standard_name in standards_folders:
folder_path = base_dir / folder
if folder_path.exists():
print(f" πŸ“ Standards: {standard_name}")
docs = read_documents(
str(folder_path),
category="standard",
metadata_extra={"standard": standard_name}
)
documents.extend(docs)
print(f" βœ… Ingested {len(docs)} documents")
# ============================================
# Summary
# ============================================
print(f"\nπŸ“Š TOTAL: {len(documents)} documents ready for ingestion")
# Count by category
categories = {}
for doc in documents:
cat = doc["metadata"].get("category", "unknown")
categories[cat] = categories.get(cat, 0) + 1
if categories:
print("\nπŸ“‹ Document Breakdown:")
for cat, count in sorted(categories.items()):
print(f" - {cat}: {count}")
return documents
def ingest_knowledge():
"""Ingest all knowledge into ChromaDB"""
print("\n" + "="*60)
print(" πŸ“š INGESTING KNOWLEDGE BASE")
print("="*60 + "\n")
base_dir = Path(__file__).parent.parent
chroma_path = base_dir / "knowledge" / "chroma_db"
print("πŸ“ Initializing ChromaDB...")
chroma_path.mkdir(parents=True, exist_ok=True)
client = chromadb.PersistentClient(path=str(chroma_path))
try:
client.delete_collection("track_design_knowledge")
print(" πŸ—‘οΈ Removed existing collection")
except Exception as e:
logger.debug("Collection delete skipped: %s", e)
collection = client.create_collection(
name="track_design_knowledge",
metadata={"hnsw:space": "cosine"}
)
print(" βœ… Created collection: track_design_knowledge\n")
print("🧠 Loading embedding model...")
model = SentenceTransformer(ProductionConfig.EMBEDDING_MODEL)
print(" βœ… Embedding model loaded\n")
print("πŸ“„ Loading documents...")
documents = get_all_documents()
if not documents:
print(" ⚠️ No documents found! Creating samples...")
from scripts.sample_docs import create_sample_documents
documents = create_sample_documents()
print(f"\nπŸ“Š Total documents: {len(documents)}")
texts = []
metadatas = []
ids = []
n = 0
for doc in documents:
chunks = chunk_text(doc["content"])
if not chunks:
chunks = [doc["content"][:ProductionConfig.CHUNK_SIZE]]
for j, chunk in enumerate(chunks):
n += 1
texts.append(chunk)
meta = dict(doc["metadata"])
meta["chunk"] = j
metadatas.append(meta)
ids.append(f"doc_{n}")
print(f"\nπŸ”„ Generating embeddings for {len(texts)} documents...")
embeddings = model.encode(texts).tolist()
print(" βœ… Embeddings generated\n")
print("πŸ’Ύ Adding to ChromaDB...")
collection.add(
embeddings=embeddings,
documents=texts,
metadatas=metadatas,
ids=ids
)
print(f"\nβœ… Successfully ingested {len(texts)} documents!")
print(f"πŸ“ ChromaDB location: {chroma_path}")
# PEPW specific summary
pepw_docs = [d for d in documents if d["metadata"].get("category") == "pepw_drawing"]
if pepw_docs:
nr56_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR56"])
nr60_count = len([d for d in pepw_docs if d["metadata"].get("standard") == "NR60"])
print(f"\nπŸ”§ PEPW Drawings Ingested:")
print(f" - NR56: {nr56_count} drawings")
print(f" - NR60: {nr60_count} drawings")
print(f" - TOTAL: {len(pepw_docs)} drawings")
else:
print("\n⚠️ No PEPW drawings found!")
print(" Place drawings in:")
print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/")
print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/")
print("\n" + "="*60)
print(" βœ… KNOWLEDGE INGESTION COMPLETE")
print("="*60 + "\n")
if __name__ == "__main__":
ingest_knowledge()