ruhi / scripts /insert_data.py
hrrbb's picture
Add data import scripts
d11e4f7
Raw History Blame Contribute Delete
2.59 kB
# Import required modules
import sys
import asyncio # For running asynchronous tasks
import json # For parsing JSON input
from pathlib import Path
from config import EMBEDDING_MODEL
# Add parent directory to Python path
BASE_DIR = Path(__file__).parent.parent
sys.path.append(str(BASE_DIR))
from sentence_transformers import SentenceTransformer # For generating text embeddings
# Import database schemas for different collections
from database.schemas import QuranVerse, Hadith, Dua
from database.connection import init_db # Import function to initialize the database
from config import MONGO_URI, DB_NAME, EMBEDDING_MODEL
# ---------------------- Async Function to Insert Data ----------------------
async def insert_entry(data: dict, collection_type: str):
"""
Inserts a given entry into the appropriate MongoDB collection.
Args:
data (dict): A dictionary containing entry data.
collection_type (str): The type of entry ("verse", "hadith", or "dua").
Steps:
1. Initialize the database connection.
2. Load the sentence embedding model.
3. Create an instance of the appropriate document schema.
4. Compute the embedding for the English text.
5. Insert the document into the database.
"""
# Initialize MongoDB database connection
await init_db(MONGO_URI, DB_NAME)
# Load a pre-trained sentence embedding model
model = SentenceTransformer(EMBEDDING_MODEL)
# Dictionary to map collection type to the corresponding schema
entry_map = {
"verse": QuranVerse, # Map "verse" to QuranVerse schema
"hadith": Hadith, # Map "hadith" to Hadith schema
"dua": Dua # Map "dua" to Dua schema
}
# Create an instance of the selected document schema
doc = entry_map[collection_type](
**data, # Unpack data dictionary into the schema fields
embedding=model.encode(data["english_text"]).tolist() # Generate embedding for English text
)
# Insert the document into MongoDB
await doc.insert()
# Print success message with the document ID
print(f"Inserted {doc.id}")
# ---------------------- Command-Line Execution ----------------------
if __name__ == "__main__":
try:
collection_type = sys.argv[1]
# Read JSON with UTF-8 encoding
json_str = ' '.join(sys.argv[2:]).encode('utf-8').decode('unicode_escape')
data = json.loads(json_str)
asyncio.run(insert_entry(data, collection_type))
except Exception as e:
print(f"❌ Critical error: {str(e)}")
sys.exit(1)