flowmarg / scripts /ingest_sandc.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
5.45 kB
"""
S&C KNOWLEDGE INGESTOR β€” Separate Database for Switches & Crossings
Only used when user asks S&C-related questions
"""
import os
import sys
from pathlib import Path
from typing import List, Dict
import chromadb
from sentence_transformers import SentenceTransformer
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
def read_pdf(file_path: Path) -> str:
"""Extract text from PDF"""
try:
from pypdf import PdfReader
reader = PdfReader(file_path)
content = ""
for page_num, page in enumerate(reader.pages, 1):
page_text = page.extract_text()
if page_text:
content += f"\n--- PAGE {page_num} ---\n"
content += page_text + "\n"
return content
except Exception as e:
print(f" ⚠️ Failed to read {file_path.name}: {e}")
return ""
def get_sandc_documents() -> List[Dict]:
"""Get all S&C documents (PEPW drawings)"""
documents = []
base_dir = Path(__file__).parent.parent
# NR56 PEPW Drawings
nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF"
if nr56_path.exists():
pdf_files = list(nr56_path.glob("*.pdf"))
print(f"πŸ“ NR56 PEPW Drawings: {len(pdf_files)} files")
for file_path in pdf_files:
content = read_pdf(file_path)
if content.strip():
documents.append({
"content": content,
"metadata": {
"source": file_path.name,
"standard": "NR56",
"category": "pepw_drawing",
"type": "turnout_crossover",
"size_kb": round(file_path.stat().st_size / 1024, 1)
}
})
print(f" βœ… Ingested: {file_path.name}")
# NR60 PEPW Drawings
nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF"
if nr60_path.exists():
pdf_files = list(nr60_path.glob("*.pdf"))
print(f"πŸ“ NR60 PEPW Drawings: {len(pdf_files)} files")
for file_path in pdf_files:
content = read_pdf(file_path)
if content.strip():
documents.append({
"content": content,
"metadata": {
"source": file_path.name,
"standard": "NR60",
"category": "pepw_drawing",
"type": "turnout_crossover",
"size_kb": round(file_path.stat().st_size / 1024, 1)
}
})
print(f" βœ… Ingested: {file_path.name}")
return documents
def ingest_sandc_knowledge():
"""Ingest S&C knowledge into separate database"""
print("\n" + "="*60)
print(" πŸ”§ INGESTING S&C KNOWLEDGE (PEPW Drawings)")
print("="*60 + "\n")
base_dir = Path(__file__).parent.parent
chroma_path = base_dir / "knowledge" / "sandc_chroma_db"
print("πŸ“ Initializing S&C ChromaDB...")
chroma_path.mkdir(parents=True, exist_ok=True)
client = chromadb.PersistentClient(path=str(chroma_path))
try:
client.delete_collection("sandc_knowledge")
print(" πŸ—‘οΈ Removed existing S&C collection")
except Exception as e:
pass
collection = client.create_collection(
name="sandc_knowledge",
metadata={"hnsw:space": "cosine", "description": "S&C Design Knowledge"}
)
print(" βœ… Created collection: sandc_knowledge\n")
print("🧠 Loading embedding model...")
model = SentenceTransformer('all-MiniLM-L6-v2')
print(" βœ… Embedding model loaded\n")
print("πŸ“„ Loading S&C documents...")
documents = get_sandc_documents()
if not documents:
print(" ⚠️ No S&C documents found!")
print(" Place PEPW drawings in:")
print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/")
print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/")
return
print(f"\nπŸ“Š Total S&C documents: {len(documents)}")
texts = []
metadatas = []
ids = []
for i, doc in enumerate(documents):
texts.append(doc["content"])
metadatas.append(doc["metadata"])
ids.append(f"sandc_doc_{i+1}")
print(f"\nπŸ”„ Generating embeddings for {len(texts)} documents...")
embeddings = model.encode(texts).tolist()
print(" βœ… Embeddings generated\n")
print("πŸ’Ύ Adding to ChromaDB...")
collection.add(
embeddings=embeddings,
documents=texts,
metadatas=metadatas,
ids=ids
)
print(f"\nβœ… Successfully ingested {len(texts)} S&C documents!")
print(f"πŸ“ S&C ChromaDB location: {chroma_path}")
# Summary
nr56_count = len([d for d in documents if d["metadata"].get("standard") == "NR56"])
nr60_count = len([d for d in documents if d["metadata"].get("standard") == "NR60"])
print(f"\nπŸ”§ S&C Drawings Summary:")
print(f" - NR56: {nr56_count} drawings")
print(f" - NR60: {nr60_count} drawings")
print(f" - TOTAL: {len(documents)} drawings")
print("\n" + "="*60)
print(" βœ… S&C KNOWLEDGE INGESTION COMPLETE")
print("="*60 + "\n")
if __name__ == "__main__":
ingest_sandc_knowledge()