import csv import asyncio import sys from pathlib import Path from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient from pymongo.errors import DuplicateKeyError PROJECT_ROOT = Path(__file__).parent.parent.parent sys.path.append(str(PROJECT_ROOT)) from backend.database.connection import init_db from backend.database.schemas import Theme, Chapter, Section, HadithSahih THEME_MAPPING = { "Acts of Worship": [2, 8, 30, 24, 25, 26], "Social Conduct & Ethics": [78, 51, 34, 52], "Family & Community Life": [67, 68, 69, 85], "Spiritual Growth & Reflection": [80, 66, 82] } async def init_db(): client = AsyncIOMotorClient("mongodb://localhost:27017") await init_beanie( database=client.quranic_guide, document_models=[Theme, Chapter, Section, HadithSahih] ) return client async def cleanup_invalid_chapters(): """Remove any problematic chapters""" # Delete chapters with null numbers null_chapters = await Chapter.find({"Chapter_Number": None}).to_list() for chapter in null_chapters: print(f"🚮 Deleting invalid chapter: {chapter.id}") await chapter.delete() async def setup_themes(): for order, (theme_name, chapters) in enumerate(THEME_MAPPING.items(), start=1): theme = await Theme.find_one({"name": theme_name}) if not theme: theme = Theme( name=theme_name, order=order, chapters=chapters ) await theme.insert() print(f"āœ… Created theme: {theme_name}") else: if set(theme.chapters) != set(chapters): theme.chapters = chapters await theme.save() print(f"šŸ”„ Updated theme: {theme_name}") async def process_chapter(csv_path: Path): try: with open(csv_path, "r", encoding="utf-8-sig") as f: reader = csv.DictReader(f) rows = list(reader) if not rows: print(f"āš ļø Empty file: {csv_path.name}") return first_row = rows[0] # Validate chapter number try: chapter_num = int(first_row["Chapter_Number"]) if chapter_num <= 0: print(f"āŒ Invalid chapter number: {chapter_num}") return except (KeyError, ValueError, TypeError): print(f"āŒ Invalid chapter number in {csv_path.name}") return chapter_name = first_row["Chapter_English"].strip() print(f"\nšŸ“˜ Processing Chapter {chapter_num}: {chapter_name}") # Validate theme mapping theme = await Theme.find_one({"chapters": chapter_num}) if not theme: print(f"āŒ Chapter {chapter_num} not mapped to any theme") return # Check for existing chapter existing_chapter = await Chapter.find_one( {"number": chapter_num, "name": chapter_name} ) if not existing_chapter: # Create new chapter if no exact match exists chapter = Chapter( number=chapter_num, name=chapter_name, theme=theme ) await chapter.insert() print(f"āœ… Created chapter {chapter_num}: {chapter_name}") else: chapter = existing_chapter print(f"šŸ”„ Using existing chapter {chapter_num}") # Process sections sections = {} for row in rows: try: # Get raw values raw_hadith_number = row.get("Hadith_number", "").strip() raw_section_number = row.get("Section_Number", "").strip() # Skip rows without hadith text if not row.get("English_Hadith") or not raw_hadith_number: continue # Generate unique hadith ID hadith_id = f"{chapter_num}-{raw_section_number}-{raw_hadith_number}" # Create section if needed try: section_num = int(raw_section_number) if raw_section_number else 1 except ValueError: section_num = 1 if section_num not in sections: section_name = row.get("Section_English", "").strip() or "General Principles" section = await Section.find_one( {"number": section_num, "chapter": chapter.id} ) if not section: section = Section( number=section_num, name=section_name, chapter=chapter ) await section.insert() print(f" āœ… Created section {section_num}: {section_name}") sections[section_num] = section # Create hadith with unique composite ID hadith = HadithSahih( hadith_id=hadith_id, number=raw_hadith_number, text=row["English_Hadith"].replace('(?)', 'ļ·ŗ'), section=sections[section_num] ) await hadith.insert() print(f" āœ… Added hadith {hadith_id}") except DuplicateKeyError: # Skip duplicate hadiths continue except Exception as e: print(f" āš ļø Hadith error: {str(e)}") continue except Exception as e: print(f"šŸ”„ Error processing {csv_path.name}: {str(e)}") import traceback traceback.print_exc() finally: print(f"āœ… Chapter {chapter_num} processing complete") async def main(): client = await init_db() await cleanup_invalid_chapters() await setup_themes() data_dir = Path("data/sahih_bukhari") for csv_file in data_dir.glob("Chapter*.csv"): print(f"\n{'='*50}") print(f"šŸ“„ Processing file: {csv_file.name}") await process_chapter(csv_file) # Final verification chapters = await Chapter.find_all().to_list() print("\nšŸ“š Chapters in database:") for chapter in chapters: theme = await chapter.theme.fetch() print(f" - Chapter {chapter.number}: {chapter.name} (Theme: {theme.name})") if __name__ == "__main__": asyncio.run(main())