File size: 5,452 Bytes
2379054 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | """
S&C KNOWLEDGE INGESTOR β Separate Database for Switches & Crossings
Only used when user asks S&C-related questions
"""
import os
import sys
from pathlib import Path
from typing import List, Dict
import chromadb
from sentence_transformers import SentenceTransformer
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
def read_pdf(file_path: Path) -> str:
"""Extract text from PDF"""
try:
from pypdf import PdfReader
reader = PdfReader(file_path)
content = ""
for page_num, page in enumerate(reader.pages, 1):
page_text = page.extract_text()
if page_text:
content += f"\n--- PAGE {page_num} ---\n"
content += page_text + "\n"
return content
except Exception as e:
print(f" β οΈ Failed to read {file_path.name}: {e}")
return ""
def get_sandc_documents() -> List[Dict]:
"""Get all S&C documents (PEPW drawings)"""
documents = []
base_dir = Path(__file__).parent.parent
# NR56 PEPW Drawings
nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF"
if nr56_path.exists():
pdf_files = list(nr56_path.glob("*.pdf"))
print(f"π NR56 PEPW Drawings: {len(pdf_files)} files")
for file_path in pdf_files:
content = read_pdf(file_path)
if content.strip():
documents.append({
"content": content,
"metadata": {
"source": file_path.name,
"standard": "NR56",
"category": "pepw_drawing",
"type": "turnout_crossover",
"size_kb": round(file_path.stat().st_size / 1024, 1)
}
})
print(f" β
Ingested: {file_path.name}")
# NR60 PEPW Drawings
nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF"
if nr60_path.exists():
pdf_files = list(nr60_path.glob("*.pdf"))
print(f"π NR60 PEPW Drawings: {len(pdf_files)} files")
for file_path in pdf_files:
content = read_pdf(file_path)
if content.strip():
documents.append({
"content": content,
"metadata": {
"source": file_path.name,
"standard": "NR60",
"category": "pepw_drawing",
"type": "turnout_crossover",
"size_kb": round(file_path.stat().st_size / 1024, 1)
}
})
print(f" β
Ingested: {file_path.name}")
return documents
def ingest_sandc_knowledge():
"""Ingest S&C knowledge into separate database"""
print("\n" + "="*60)
print(" π§ INGESTING S&C KNOWLEDGE (PEPW Drawings)")
print("="*60 + "\n")
base_dir = Path(__file__).parent.parent
chroma_path = base_dir / "knowledge" / "sandc_chroma_db"
print("π Initializing S&C ChromaDB...")
chroma_path.mkdir(parents=True, exist_ok=True)
client = chromadb.PersistentClient(path=str(chroma_path))
try:
client.delete_collection("sandc_knowledge")
print(" ποΈ Removed existing S&C collection")
except Exception as e:
pass
collection = client.create_collection(
name="sandc_knowledge",
metadata={"hnsw:space": "cosine", "description": "S&C Design Knowledge"}
)
print(" β
Created collection: sandc_knowledge\n")
print("π§ Loading embedding model...")
model = SentenceTransformer('all-MiniLM-L6-v2')
print(" β
Embedding model loaded\n")
print("π Loading S&C documents...")
documents = get_sandc_documents()
if not documents:
print(" β οΈ No S&C documents found!")
print(" Place PEPW drawings in:")
print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/")
print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/")
return
print(f"\nπ Total S&C documents: {len(documents)}")
texts = []
metadatas = []
ids = []
for i, doc in enumerate(documents):
texts.append(doc["content"])
metadatas.append(doc["metadata"])
ids.append(f"sandc_doc_{i+1}")
print(f"\nπ Generating embeddings for {len(texts)} documents...")
embeddings = model.encode(texts).tolist()
print(" β
Embeddings generated\n")
print("πΎ Adding to ChromaDB...")
collection.add(
embeddings=embeddings,
documents=texts,
metadatas=metadatas,
ids=ids
)
print(f"\nβ
Successfully ingested {len(texts)} S&C documents!")
print(f"π S&C ChromaDB location: {chroma_path}")
# Summary
nr56_count = len([d for d in documents if d["metadata"].get("standard") == "NR56"])
nr60_count = len([d for d in documents if d["metadata"].get("standard") == "NR60"])
print(f"\nπ§ S&C Drawings Summary:")
print(f" - NR56: {nr56_count} drawings")
print(f" - NR60: {nr60_count} drawings")
print(f" - TOTAL: {len(documents)} drawings")
print("\n" + "="*60)
print(" β
S&C KNOWLEDGE INGESTION COMPLETE")
print("="*60 + "\n")
if __name__ == "__main__":
ingest_sandc_knowledge()
|