Download scripts/insert_data.py from hrrbb/ruhi: direct link, hf CLI and curl.
- Browser
- Download file 2.59 kB
-
https://huggingface.co/spaces/hrrbb/ruhi/resolve/main/scripts/insert_data.py
- Command line
-
hf download hf://spaces/hrrbb/ruhi/scripts/insert_data.py
-
curl -L -o insert_data.py https://huggingface.co/spaces/hrrbb/ruhi/resolve/main/scripts/insert_data.py
2.59 kB
| # Import required modules | |
| import sys | |
| import asyncio # For running asynchronous tasks | |
| import json # For parsing JSON input | |
| from pathlib import Path | |
| from config import EMBEDDING_MODEL | |
| # Add parent directory to Python path | |
| BASE_DIR = Path(__file__).parent.parent | |
| sys.path.append(str(BASE_DIR)) | |
| from sentence_transformers import SentenceTransformer # For generating text embeddings | |
| # Import database schemas for different collections | |
| from database.schemas import QuranVerse, Hadith, Dua | |
| from database.connection import init_db # Import function to initialize the database | |
| from config import MONGO_URI, DB_NAME, EMBEDDING_MODEL | |
| # ---------------------- Async Function to Insert Data ---------------------- | |
| async def insert_entry(data: dict, collection_type: str): | |
| """ | |
| Inserts a given entry into the appropriate MongoDB collection. | |
| Args: | |
| data (dict): A dictionary containing entry data. | |
| collection_type (str): The type of entry ("verse", "hadith", or "dua"). | |
| Steps: | |
| 1. Initialize the database connection. | |
| 2. Load the sentence embedding model. | |
| 3. Create an instance of the appropriate document schema. | |
| 4. Compute the embedding for the English text. | |
| 5. Insert the document into the database. | |
| """ | |
| # Initialize MongoDB database connection | |
| await init_db(MONGO_URI, DB_NAME) | |
| # Load a pre-trained sentence embedding model | |
| model = SentenceTransformer(EMBEDDING_MODEL) | |
| # Dictionary to map collection type to the corresponding schema | |
| entry_map = { | |
| "verse": QuranVerse, # Map "verse" to QuranVerse schema | |
| "hadith": Hadith, # Map "hadith" to Hadith schema | |
| "dua": Dua # Map "dua" to Dua schema | |
| } | |
| # Create an instance of the selected document schema | |
| doc = entry_map[collection_type]( | |
| **data, # Unpack data dictionary into the schema fields | |
| embedding=model.encode(data["english_text"]).tolist() # Generate embedding for English text | |
| ) | |
| # Insert the document into MongoDB | |
| await doc.insert() | |
| # Print success message with the document ID | |
| print(f"Inserted {doc.id}") | |
| # ---------------------- Command-Line Execution ---------------------- | |
| if __name__ == "__main__": | |
| try: | |
| collection_type = sys.argv[1] | |
| # Read JSON with UTF-8 encoding | |
| json_str = ' '.join(sys.argv[2:]).encode('utf-8').decode('unicode_escape') | |
| data = json.loads(json_str) | |
| asyncio.run(insert_entry(data, collection_type)) | |
| except Exception as e: | |
| print(f"❌ Critical error: {str(e)}") | |
| sys.exit(1) | |