Download scripts/ingest_sandc.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 5.45 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/scripts/ingest_sandc.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/scripts/ingest_sandc.py
-
curl -L -o ingest_sandc.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/scripts/ingest_sandc.py
5.45 kB
| """ | |
| S&C KNOWLEDGE INGESTOR β Separate Database for Switches & Crossings | |
| Only used when user asks S&C-related questions | |
| """ | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from typing import List, Dict | |
| import chromadb | |
| from sentence_transformers import SentenceTransformer | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| def read_pdf(file_path: Path) -> str: | |
| """Extract text from PDF""" | |
| try: | |
| from pypdf import PdfReader | |
| reader = PdfReader(file_path) | |
| content = "" | |
| for page_num, page in enumerate(reader.pages, 1): | |
| page_text = page.extract_text() | |
| if page_text: | |
| content += f"\n--- PAGE {page_num} ---\n" | |
| content += page_text + "\n" | |
| return content | |
| except Exception as e: | |
| print(f" β οΈ Failed to read {file_path.name}: {e}") | |
| return "" | |
| def get_sandc_documents() -> List[Dict]: | |
| """Get all S&C documents (PEPW drawings)""" | |
| documents = [] | |
| base_dir = Path(__file__).parent.parent | |
| # NR56 PEPW Drawings | |
| nr56_path = base_dir / "knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF" | |
| if nr56_path.exists(): | |
| pdf_files = list(nr56_path.glob("*.pdf")) | |
| print(f"π NR56 PEPW Drawings: {len(pdf_files)} files") | |
| for file_path in pdf_files: | |
| content = read_pdf(file_path) | |
| if content.strip(): | |
| documents.append({ | |
| "content": content, | |
| "metadata": { | |
| "source": file_path.name, | |
| "standard": "NR56", | |
| "category": "pepw_drawing", | |
| "type": "turnout_crossover", | |
| "size_kb": round(file_path.stat().st_size / 1024, 1) | |
| } | |
| }) | |
| print(f" β Ingested: {file_path.name}") | |
| # NR60 PEPW Drawings | |
| nr60_path = base_dir / "knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF" | |
| if nr60_path.exists(): | |
| pdf_files = list(nr60_path.glob("*.pdf")) | |
| print(f"π NR60 PEPW Drawings: {len(pdf_files)} files") | |
| for file_path in pdf_files: | |
| content = read_pdf(file_path) | |
| if content.strip(): | |
| documents.append({ | |
| "content": content, | |
| "metadata": { | |
| "source": file_path.name, | |
| "standard": "NR60", | |
| "category": "pepw_drawing", | |
| "type": "turnout_crossover", | |
| "size_kb": round(file_path.stat().st_size / 1024, 1) | |
| } | |
| }) | |
| print(f" β Ingested: {file_path.name}") | |
| return documents | |
| def ingest_sandc_knowledge(): | |
| """Ingest S&C knowledge into separate database""" | |
| print("\n" + "="*60) | |
| print(" π§ INGESTING S&C KNOWLEDGE (PEPW Drawings)") | |
| print("="*60 + "\n") | |
| base_dir = Path(__file__).parent.parent | |
| chroma_path = base_dir / "knowledge" / "sandc_chroma_db" | |
| print("π Initializing S&C ChromaDB...") | |
| chroma_path.mkdir(parents=True, exist_ok=True) | |
| client = chromadb.PersistentClient(path=str(chroma_path)) | |
| try: | |
| client.delete_collection("sandc_knowledge") | |
| print(" ποΈ Removed existing S&C collection") | |
| except Exception as e: | |
| pass | |
| collection = client.create_collection( | |
| name="sandc_knowledge", | |
| metadata={"hnsw:space": "cosine", "description": "S&C Design Knowledge"} | |
| ) | |
| print(" β Created collection: sandc_knowledge\n") | |
| print("π§ Loading embedding model...") | |
| model = SentenceTransformer('all-MiniLM-L6-v2') | |
| print(" β Embedding model loaded\n") | |
| print("π Loading S&C documents...") | |
| documents = get_sandc_documents() | |
| if not documents: | |
| print(" β οΈ No S&C documents found!") | |
| print(" Place PEPW drawings in:") | |
| print(" - knowledge/sandc/pepw/nr56/Turnouts and Crossovers/PDF/") | |
| print(" - knowledge/sandc/pepw/nr60/Turnouts and Crossovers/PDF/") | |
| return | |
| print(f"\nπ Total S&C documents: {len(documents)}") | |
| texts = [] | |
| metadatas = [] | |
| ids = [] | |
| for i, doc in enumerate(documents): | |
| texts.append(doc["content"]) | |
| metadatas.append(doc["metadata"]) | |
| ids.append(f"sandc_doc_{i+1}") | |
| print(f"\nπ Generating embeddings for {len(texts)} documents...") | |
| embeddings = model.encode(texts).tolist() | |
| print(" β Embeddings generated\n") | |
| print("πΎ Adding to ChromaDB...") | |
| collection.add( | |
| embeddings=embeddings, | |
| documents=texts, | |
| metadatas=metadatas, | |
| ids=ids | |
| ) | |
| print(f"\nβ Successfully ingested {len(texts)} S&C documents!") | |
| print(f"π S&C ChromaDB location: {chroma_path}") | |
| # Summary | |
| nr56_count = len([d for d in documents if d["metadata"].get("standard") == "NR56"]) | |
| nr60_count = len([d for d in documents if d["metadata"].get("standard") == "NR60"]) | |
| print(f"\nπ§ S&C Drawings Summary:") | |
| print(f" - NR56: {nr56_count} drawings") | |
| print(f" - NR60: {nr60_count} drawings") | |
| print(f" - TOTAL: {len(documents)} drawings") | |
| print("\n" + "="*60) | |
| print(" β S&C KNOWLEDGE INGESTION COMPLETE") | |
| print("="*60 + "\n") | |
| if __name__ == "__main__": | |
| ingest_sandc_knowledge() | |