ruhi / scripts /import_hadith.py
hrrbb's picture
Add data import scripts
d11e4f7
Raw History Blame Contribute Delete
7.06 kB
import csv
import asyncio
import sys
from pathlib import Path
from beanie import init_beanie
from motor.motor_asyncio import AsyncIOMotorClient
from pymongo.errors import DuplicateKeyError
PROJECT_ROOT = Path(__file__).parent.parent.parent
sys.path.append(str(PROJECT_ROOT))
from backend.database.connection import init_db
from backend.database.schemas import Theme, Chapter, Section, HadithSahih
THEME_MAPPING = {
"Acts of Worship": [2, 8, 30, 24, 25, 26],
"Social Conduct & Ethics": [78, 51, 34, 52],
"Family & Community Life": [67, 68, 69, 85],
"Spiritual Growth & Reflection": [80, 66, 82]
}
async def init_db():
client = AsyncIOMotorClient("mongodb://localhost:27017")
await init_beanie(
database=client.quranic_guide,
document_models=[Theme, Chapter, Section, HadithSahih]
)
return client
async def cleanup_invalid_chapters():
"""Remove any problematic chapters"""
# Delete chapters with null numbers
null_chapters = await Chapter.find({"Chapter_Number": None}).to_list()
for chapter in null_chapters:
print(f"🚮 Deleting invalid chapter: {chapter.id}")
await chapter.delete()
async def setup_themes():
for order, (theme_name, chapters) in enumerate(THEME_MAPPING.items(), start=1):
theme = await Theme.find_one({"name": theme_name})
if not theme:
theme = Theme(
name=theme_name,
order=order,
chapters=chapters
)
await theme.insert()
print(f"✅ Created theme: {theme_name}")
else:
if set(theme.chapters) != set(chapters):
theme.chapters = chapters
await theme.save()
print(f"🔄 Updated theme: {theme_name}")
async def process_chapter(csv_path: Path):
try:
with open(csv_path, "r", encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
rows = list(reader)
if not rows:
print(f"⚠️ Empty file: {csv_path.name}")
return
first_row = rows[0]
# Validate chapter number
try:
chapter_num = int(first_row["Chapter_Number"])
if chapter_num <= 0:
print(f"❌ Invalid chapter number: {chapter_num}")
return
except (KeyError, ValueError, TypeError):
print(f"❌ Invalid chapter number in {csv_path.name}")
return
chapter_name = first_row["Chapter_English"].strip()
print(f"\n📘 Processing Chapter {chapter_num}: {chapter_name}")
# Validate theme mapping
theme = await Theme.find_one({"chapters": chapter_num})
if not theme:
print(f"❌ Chapter {chapter_num} not mapped to any theme")
return
# Check for existing chapter
existing_chapter = await Chapter.find_one(
{"number": chapter_num, "name": chapter_name}
)
if not existing_chapter:
# Create new chapter if no exact match exists
chapter = Chapter(
number=chapter_num,
name=chapter_name,
theme=theme
)
await chapter.insert()
print(f"✅ Created chapter {chapter_num}: {chapter_name}")
else:
chapter = existing_chapter
print(f"🔄 Using existing chapter {chapter_num}")
# Process sections
sections = {}
for row in rows:
try:
# Get raw values
raw_hadith_number = row.get("Hadith_number", "").strip()
raw_section_number = row.get("Section_Number", "").strip()
# Skip rows without hadith text
if not row.get("English_Hadith") or not raw_hadith_number:
continue
# Generate unique hadith ID
hadith_id = f"{chapter_num}-{raw_section_number}-{raw_hadith_number}"
# Create section if needed
try:
section_num = int(raw_section_number) if raw_section_number else 1
except ValueError:
section_num = 1
if section_num not in sections:
section_name = row.get("Section_English", "").strip() or "General Principles"
section = await Section.find_one(
{"number": section_num, "chapter": chapter.id}
)
if not section:
section = Section(
number=section_num,
name=section_name,
chapter=chapter
)
await section.insert()
print(f" ✅ Created section {section_num}: {section_name}")
sections[section_num] = section
# Create hadith with unique composite ID
hadith = HadithSahih(
hadith_id=hadith_id,
number=raw_hadith_number,
text=row["English_Hadith"].replace('(?)', 'ﷺ'),
section=sections[section_num]
)
await hadith.insert()
print(f" ✅ Added hadith {hadith_id}")
except DuplicateKeyError:
# Skip duplicate hadiths
continue
except Exception as e:
print(f" ⚠️ Hadith error: {str(e)}")
continue
except Exception as e:
print(f"🔥 Error processing {csv_path.name}: {str(e)}")
import traceback
traceback.print_exc()
finally:
print(f"✅ Chapter {chapter_num} processing complete")
async def main():
client = await init_db()
await cleanup_invalid_chapters()
await setup_themes()
data_dir = Path("data/sahih_bukhari")
for csv_file in data_dir.glob("Chapter*.csv"):
print(f"\n{'='*50}")
print(f"📥 Processing file: {csv_file.name}")
await process_chapter(csv_file)
# Final verification
chapters = await Chapter.find_all().to_list()
print("\n📚 Chapters in database:")
for chapter in chapters:
theme = await chapter.theme.fetch()
print(f" - Chapter {chapter.number}: {chapter.name} (Theme: {theme.name})")
if __name__ == "__main__":
asyncio.run(main())