Download scripts/import_hadith.py from hrrbb/ruhi: direct link, hf CLI and curl.
- Browser
- Download file 7.06 kB
-
https://huggingface.co/spaces/hrrbb/ruhi/resolve/main/scripts/import_hadith.py
- Command line
-
hf download hf://spaces/hrrbb/ruhi/scripts/import_hadith.py
-
curl -L -o import_hadith.py https://huggingface.co/spaces/hrrbb/ruhi/resolve/main/scripts/import_hadith.py
7.06 kB
| import csv | |
| import asyncio | |
| import sys | |
| from pathlib import Path | |
| from beanie import init_beanie | |
| from motor.motor_asyncio import AsyncIOMotorClient | |
| from pymongo.errors import DuplicateKeyError | |
| PROJECT_ROOT = Path(__file__).parent.parent.parent | |
| sys.path.append(str(PROJECT_ROOT)) | |
| from backend.database.connection import init_db | |
| from backend.database.schemas import Theme, Chapter, Section, HadithSahih | |
| THEME_MAPPING = { | |
| "Acts of Worship": [2, 8, 30, 24, 25, 26], | |
| "Social Conduct & Ethics": [78, 51, 34, 52], | |
| "Family & Community Life": [67, 68, 69, 85], | |
| "Spiritual Growth & Reflection": [80, 66, 82] | |
| } | |
| async def init_db(): | |
| client = AsyncIOMotorClient("mongodb://localhost:27017") | |
| await init_beanie( | |
| database=client.quranic_guide, | |
| document_models=[Theme, Chapter, Section, HadithSahih] | |
| ) | |
| return client | |
| async def cleanup_invalid_chapters(): | |
| """Remove any problematic chapters""" | |
| # Delete chapters with null numbers | |
| null_chapters = await Chapter.find({"Chapter_Number": None}).to_list() | |
| for chapter in null_chapters: | |
| print(f"🚮 Deleting invalid chapter: {chapter.id}") | |
| await chapter.delete() | |
| async def setup_themes(): | |
| for order, (theme_name, chapters) in enumerate(THEME_MAPPING.items(), start=1): | |
| theme = await Theme.find_one({"name": theme_name}) | |
| if not theme: | |
| theme = Theme( | |
| name=theme_name, | |
| order=order, | |
| chapters=chapters | |
| ) | |
| await theme.insert() | |
| print(f"✅ Created theme: {theme_name}") | |
| else: | |
| if set(theme.chapters) != set(chapters): | |
| theme.chapters = chapters | |
| await theme.save() | |
| print(f"🔄 Updated theme: {theme_name}") | |
| async def process_chapter(csv_path: Path): | |
| try: | |
| with open(csv_path, "r", encoding="utf-8-sig") as f: | |
| reader = csv.DictReader(f) | |
| rows = list(reader) | |
| if not rows: | |
| print(f"⚠️ Empty file: {csv_path.name}") | |
| return | |
| first_row = rows[0] | |
| # Validate chapter number | |
| try: | |
| chapter_num = int(first_row["Chapter_Number"]) | |
| if chapter_num <= 0: | |
| print(f"❌ Invalid chapter number: {chapter_num}") | |
| return | |
| except (KeyError, ValueError, TypeError): | |
| print(f"❌ Invalid chapter number in {csv_path.name}") | |
| return | |
| chapter_name = first_row["Chapter_English"].strip() | |
| print(f"\n📘 Processing Chapter {chapter_num}: {chapter_name}") | |
| # Validate theme mapping | |
| theme = await Theme.find_one({"chapters": chapter_num}) | |
| if not theme: | |
| print(f"❌ Chapter {chapter_num} not mapped to any theme") | |
| return | |
| # Check for existing chapter | |
| existing_chapter = await Chapter.find_one( | |
| {"number": chapter_num, "name": chapter_name} | |
| ) | |
| if not existing_chapter: | |
| # Create new chapter if no exact match exists | |
| chapter = Chapter( | |
| number=chapter_num, | |
| name=chapter_name, | |
| theme=theme | |
| ) | |
| await chapter.insert() | |
| print(f"✅ Created chapter {chapter_num}: {chapter_name}") | |
| else: | |
| chapter = existing_chapter | |
| print(f"🔄 Using existing chapter {chapter_num}") | |
| # Process sections | |
| sections = {} | |
| for row in rows: | |
| try: | |
| # Get raw values | |
| raw_hadith_number = row.get("Hadith_number", "").strip() | |
| raw_section_number = row.get("Section_Number", "").strip() | |
| # Skip rows without hadith text | |
| if not row.get("English_Hadith") or not raw_hadith_number: | |
| continue | |
| # Generate unique hadith ID | |
| hadith_id = f"{chapter_num}-{raw_section_number}-{raw_hadith_number}" | |
| # Create section if needed | |
| try: | |
| section_num = int(raw_section_number) if raw_section_number else 1 | |
| except ValueError: | |
| section_num = 1 | |
| if section_num not in sections: | |
| section_name = row.get("Section_English", "").strip() or "General Principles" | |
| section = await Section.find_one( | |
| {"number": section_num, "chapter": chapter.id} | |
| ) | |
| if not section: | |
| section = Section( | |
| number=section_num, | |
| name=section_name, | |
| chapter=chapter | |
| ) | |
| await section.insert() | |
| print(f" ✅ Created section {section_num}: {section_name}") | |
| sections[section_num] = section | |
| # Create hadith with unique composite ID | |
| hadith = HadithSahih( | |
| hadith_id=hadith_id, | |
| number=raw_hadith_number, | |
| text=row["English_Hadith"].replace('(?)', 'ﷺ'), | |
| section=sections[section_num] | |
| ) | |
| await hadith.insert() | |
| print(f" ✅ Added hadith {hadith_id}") | |
| except DuplicateKeyError: | |
| # Skip duplicate hadiths | |
| continue | |
| except Exception as e: | |
| print(f" ⚠️ Hadith error: {str(e)}") | |
| continue | |
| except Exception as e: | |
| print(f"🔥 Error processing {csv_path.name}: {str(e)}") | |
| import traceback | |
| traceback.print_exc() | |
| finally: | |
| print(f"✅ Chapter {chapter_num} processing complete") | |
| async def main(): | |
| client = await init_db() | |
| await cleanup_invalid_chapters() | |
| await setup_themes() | |
| data_dir = Path("data/sahih_bukhari") | |
| for csv_file in data_dir.glob("Chapter*.csv"): | |
| print(f"\n{'='*50}") | |
| print(f"📥 Processing file: {csv_file.name}") | |
| await process_chapter(csv_file) | |
| # Final verification | |
| chapters = await Chapter.find_all().to_list() | |
| print("\n📚 Chapters in database:") | |
| for chapter in chapters: | |
| theme = await chapter.theme.fetch() | |
| print(f" - Chapter {chapter.number}: {chapter.name} (Theme: {theme.name})") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) |