File size: 5,452 Bytes
2379054
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
"""
S&C KNOWLEDGE INGESTOR β€” Separate Database for Switches & Crossings
Only used when user asks S&C-related questions
"""

import os
import sys
from pathlib import Path
from typing import List, Dict
import chromadb
from sentence_transformers import SentenceTransformer

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def read_pdf(file_path: Path) -> str:
    """Extract text from PDF"""
    try:
        from pypdf import PdfReader
        reader = PdfReader(file_path)
        content = ""
        for page_num, page in enumerate(reader.pages, 1):
            page_text = page.extract_text()
            if page_text:
                content += f"\n--- PAGE {page_num} ---\n"
                content += page_text + "\n"
        return content
    except Exception as e:
        print(f"  ⚠️ Failed to read {file_path.name}: {e}")
        return ""

def get_sandc_documents() -> List[Dict]:
    """Get all S&C documents (PEPW drawings)"""
    documents = []
    base_dir = Path(__file__).parent.parent
    
    # NR56 PEPW Drawings
    nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF"
    if nr56_path.exists():
        pdf_files = list(nr56_path.glob("*.pdf"))
        print(f"πŸ“ NR56 PEPW Drawings: {len(pdf_files)} files")
        for file_path in pdf_files:
            content = read_pdf(file_path)
            if content.strip():
                documents.append({
                    "content": content,
                    "metadata": {
                        "source": file_path.name,
                        "standard": "NR56",
                        "category": "pepw_drawing",
                        "type": "turnout_crossover",
                        "size_kb": round(file_path.stat().st_size / 1024, 1)
                    }
                })
                print(f"    βœ… Ingested: {file_path.name}")
    
    # NR60 PEPW Drawings
    nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF"
    if nr60_path.exists():
        pdf_files = list(nr60_path.glob("*.pdf"))
        print(f"πŸ“ NR60 PEPW Drawings: {len(pdf_files)} files")
        for file_path in pdf_files:
            content = read_pdf(file_path)
            if content.strip():
                documents.append({
                    "content": content,
                    "metadata": {
                        "source": file_path.name,
                        "standard": "NR60",
                        "category": "pepw_drawing",
                        "type": "turnout_crossover",
                        "size_kb": round(file_path.stat().st_size / 1024, 1)
                    }
                })
                print(f"    βœ… Ingested: {file_path.name}")
    
    return documents

def ingest_sandc_knowledge():
    """Ingest S&C knowledge into separate database"""
    print("\n" + "="*60)
    print("  πŸ”§ INGESTING S&C KNOWLEDGE (PEPW Drawings)")
    print("="*60 + "\n")
    
    base_dir = Path(__file__).parent.parent
    chroma_path = base_dir / "knowledge" / "sandc_chroma_db"
    
    print("πŸ“ Initializing S&C ChromaDB...")
    chroma_path.mkdir(parents=True, exist_ok=True)
    client = chromadb.PersistentClient(path=str(chroma_path))
    
    try:
        client.delete_collection("sandc_knowledge")
        print("  πŸ—‘οΈ Removed existing S&C collection")
    except Exception as e:
        pass
    
    collection = client.create_collection(
        name="sandc_knowledge",
        metadata={"hnsw:space": "cosine", "description": "S&C Design Knowledge"}
    )
    print("  βœ… Created collection: sandc_knowledge\n")
    
    print("🧠 Loading embedding model...")
    model = SentenceTransformer('all-MiniLM-L6-v2')
    print("  βœ… Embedding model loaded\n")
    
    print("πŸ“„ Loading S&C documents...")
    documents = get_sandc_documents()
    
    if not documents:
        print("  ⚠️ No S&C documents found!")
        print("  Place PEPW drawings in:")
        print("  - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/")
        print("  - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/")
        return
    
    print(f"\nπŸ“Š Total S&C documents: {len(documents)}")
    
    texts = []
    metadatas = []
    ids = []
    
    for i, doc in enumerate(documents):
        texts.append(doc["content"])
        metadatas.append(doc["metadata"])
        ids.append(f"sandc_doc_{i+1}")
    
    print(f"\nπŸ”„ Generating embeddings for {len(texts)} documents...")
    embeddings = model.encode(texts).tolist()
    print("  βœ… Embeddings generated\n")
    
    print("πŸ’Ύ Adding to ChromaDB...")
    collection.add(
        embeddings=embeddings,
        documents=texts,
        metadatas=metadatas,
        ids=ids
    )
    
    print(f"\nβœ… Successfully ingested {len(texts)} S&C documents!")
    print(f"πŸ“ S&C ChromaDB location: {chroma_path}")
    
    # Summary
    nr56_count = len([d for d in documents if d["metadata"].get("standard") == "NR56"])
    nr60_count = len([d for d in documents if d["metadata"].get("standard") == "NR60"])
    
    print(f"\nπŸ”§ S&C Drawings Summary:")
    print(f"  - NR56: {nr56_count} drawings")
    print(f"  - NR60: {nr60_count} drawings")
    print(f"  - TOTAL: {len(documents)} drawings")
    
    print("\n" + "="*60)
    print("  βœ… S&C KNOWLEDGE INGESTION COMPLETE")
    print("="*60 + "\n")

if __name__ == "__main__":
    ingest_sandc_knowledge()