#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ ============================================================================= MASTER QURAN TAFSEER COMPILER & GENERATOR ============================================================================= Compiles verified multi-scholar Tafseer JSON files ayah-by-ayah. Supports generating a single Surah at a time (e.g. --surah 2) or all 114 Surahs with rigorous consistency, validation, and multi-directory synchronization. Strictly Curated 17 Sunni Orthodox Scholars & Commentaries: 1. en-tafsir-taqi-usmani (Mufti Muhammad Taqi Usmani) 2. en-tafisr-ibn-kathir (Hafiz Ibn Kathir) 3. ur-tafseer-ibn-e-kaseer (Hafiz Ibn Kathir) 4. ur-tafsir-bayan-ul-quran (Dr. Israr Ahmed) 5. ur-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes) 6. en-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes) 7. en-tafsir-maarif-ul-quran (Mufti Muhammad Shafi) 8. ur-tafsir-fee-zilal-il-quran (Sayyid Qutb) 9. ur-tafsir-tazkirul-quran (Maulana Wahiduddin Khan) 10. en-tafsir-tazkirul-quran (Maulana Wahiduddin Khan) 11. ur-tafsir-as-saadi-urdu (Shaykh Abdur Rahman al-Sa'di) 12. en-al-jalalayn (Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti) 13. en-tafsir-al-mukhtasar (Tafsir Center for Quranic Studies, Riyadh) 14. ar-tafsir-muyassar (King Fahd Complex, Medina) 15. ar-tafsir-as-saadi (Shaykh Abdur Rahman al-Sa'di) 16. ar-tafsir-ibn-kathir (Hafiz Ibn Kathir) 17. bn-tafsir-ahsanul-bayan (Salahuddin Yusuf / King Fahd Complex) ============================================================================= """ import argparse import html import json import os import re import sqlite3 import sys import time import urllib.request from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Dict, List, Optional, Tuple sys.stdout.reconfigure(encoding="utf-8") # Root directory paths PROJECT_ROOT = Path(r"d:\Project-x") DB_PATH = PROJECT_ROOT / "BACKEND" / "data" / "tafseers.db" CACHE_DIR = PROJECT_ROOT / "BACKEND" / "data" / "cache" CACHE_DIR.mkdir(parents=True, exist_ok=True) FN_CACHE_DIR = CACHE_DIR / "footnotes" FN_CACHE_DIR.mkdir(parents=True, exist_ok=True) SYNC_DIRS = [ PROJECT_ROOT / "quran_tafseer", PROJECT_ROOT / "BACKEND" / "data" / "quran_tafseer", PROJECT_ROOT / "FRONTEND" / "assets" / "tafsir", ] for d in SYNC_DIRS: d.mkdir(parents=True, exist_ok=True) # ----------------------------------------------------------------------------- # SURAH METADATA DICTIONARY (1-114) # ----------------------------------------------------------------------------- SURAH_METADATA = { 1: {"name_ar": "سورة الفاتحة", "name_en": "Al-Faatihah", "file_name": "Al_Fatihah_Tafseer.json", "ayahs": 7}, 2: {"name_ar": "سورة البقرة", "name_en": "Al-Baqarah", "file_name": "Al_Baqarah_Tafseer.json", "ayahs": 286}, 3: {"name_ar": "سورة آل عمران", "name_en": "Aal-i-Imraan", "file_name": "Aal_Imran_Tafseer.json", "ayahs": 200}, 4: {"name_ar": "سورة النساء", "name_en": "An-Nisaa", "file_name": "An_Nisa_Tafseer.json", "ayahs": 176}, 5: {"name_ar": "سورة المائدة", "name_en": "Al-Maaida", "file_name": "Al_Maidah_Tafseer.json", "ayahs": 120}, 6: {"name_ar": "سورة الأنعام", "name_en": "Al-An'aam", "file_name": "Al_Anam_Tafseer.json", "ayahs": 165}, 7: {"name_ar": "سورة الأعراف", "name_en": "Al-A'raaf", "file_name": "Al_Araf_Tafseer.json", "ayahs": 206}, 8: {"name_ar": "سورة الأنفال", "name_en": "Al-Anfaal", "file_name": "Al_Anfal_Tafseer.json", "ayahs": 75}, 9: {"name_ar": "سورة التوبة", "name_en": "At-Tawba", "file_name": "At_Tawbah_Tafseer.json", "ayahs": 129}, 10: {"name_ar": "سورة يونس", "name_en": "Yunus", "file_name": "Yunus_Tafseer.json", "ayahs": 109}, 11: {"name_ar": "سورة هود", "name_en": "Hud", "file_name": "Hud_Tafseer.json", "ayahs": 123}, 12: {"name_ar": "سورة يوسف", "name_en": "Yusuf", "file_name": "Yusuf_Tafseer.json", "ayahs": 111}, 13: {"name_ar": "سورة الرعد", "name_en": "Ar-Ra'd", "file_name": "Ar_Rad_Tafseer.json", "ayahs": 43}, 14: {"name_ar": "سورة إبراهيم", "name_en": "Ibrahim", "file_name": "Ibrahim_Tafseer.json", "ayahs": 52}, 15: {"name_ar": "سورة الحجر", "name_en": "Al-Hijr", "file_name": "Al_Hijr_Tafseer.json", "ayahs": 99}, 16: {"name_ar": "سورة النحل", "name_en": "An-Nahl", "file_name": "An_Nahl_Tafseer.json", "ayahs": 128}, 17: {"name_ar": "سورة الإسراء", "name_en": "Al-Israa", "file_name": "Al_Isra_Tafseer.json", "ayahs": 111}, 18: {"name_ar": "سورة الكهف", "name_en": "Al-Kahf", "file_name": "Al_Kahf_Tafseer.json", "ayahs": 110}, 19: {"name_ar": "سورة مريم", "name_en": "Maryam", "file_name": "Maryam_Tafseer.json", "ayahs": 98}, 20: {"name_ar": "سورة طه", "name_en": "Taa-Haa", "file_name": "Ta_Ha_Tafseer.json", "ayahs": 135}, 21: {"name_ar": "سورة الأنبياء", "name_en": "Al-Anbiyaa", "file_name": "Al_Anbiya_Tafseer.json", "ayahs": 112}, 22: {"name_ar": "سورة الحج", "name_en": "Al-Hajj", "file_name": "Al_Hajj_Tafseer.json", "ayahs": 78}, 23: {"name_ar": "سورة المؤمنون", "name_en": "Al-Muminoon", "file_name": "Al_Muminun_Tafseer.json", "ayahs": 118}, 24: {"name_ar": "سورة النور", "name_en": "An-Noor", "file_name": "An_Nur_Tafseer.json", "ayahs": 64}, 25: {"name_ar": "سورة الفرقان", "name_en": "Al-Furqaan", "file_name": "Al_Furqan_Tafseer.json", "ayahs": 77}, 26: {"name_ar": "سورة الشعراء", "name_en": "Ash-Shu'araa", "file_name": "Ash_Shuara_Tafseer.json", "ayahs": 227}, 27: {"name_ar": "سورة النمل", "name_en": "An-Naml", "file_name": "An_Naml_Tafseer.json", "ayahs": 93}, 28: {"name_ar": "سورة القصص", "name_en": "Al-Qasas", "file_name": "Al_Qasas_Tafseer.json", "ayahs": 88}, 29: {"name_ar": "سورة العنكبوت", "name_en": "Al-Ankaboot", "file_name": "Al_Ankabut_Tafseer.json", "ayahs": 69}, 30: {"name_ar": "سورة الروم", "name_en": "Ar-Room", "file_name": "Ar_Rum_Tafseer.json", "ayahs": 60}, 31: {"name_ar": "سورة لقمان", "name_en": "Luqman", "file_name": "Luqman_Tafseer.json", "ayahs": 34}, 32: {"name_ar": "سورة السجدة", "name_en": "As-Sajda", "file_name": "As_Sajdah_Tafseer.json", "ayahs": 30}, 33: {"name_ar": "سورة الأحزاب", "name_en": "Al-Ahzaab", "file_name": "Al_Ahzab_Tafseer.json", "ayahs": 73}, 34: {"name_ar": "سورة سبإ", "name_en": "Saba", "file_name": "Saba_Tafseer.json", "ayahs": 54}, 35: {"name_ar": "سورة فاطر", "name_en": "Faatir", "file_name": "Fatir_Tafseer.json", "ayahs": 45}, 36: {"name_ar": "سورة يس", "name_en": "Yaseen", "file_name": "Ya_Sin_Tafseer.json", "ayahs": 83}, 37: {"name_ar": "سورة الصافات", "name_en": "As-Saaffaat", "file_name": "As_Saffat_Tafseer.json", "ayahs": 182}, 38: {"name_ar": "سورة ص", "name_en": "Saad", "file_name": "Sad_Tafseer.json", "ayahs": 88}, 39: {"name_ar": "سورة الزمر", "name_en": "Az-Zumar", "file_name": "Az_Zumar_Tafseer.json", "ayahs": 75}, 40: {"name_ar": "سورة غافر", "name_en": "Ghafir", "file_name": "Ghafir_Tafseer.json", "ayahs": 85}, 41: {"name_ar": "سورة فصلت", "name_en": "Fussilat", "file_name": "Fussilat_Tafseer.json", "ayahs": 54}, 42: {"name_ar": "سورة الشورى", "name_en": "Ash-Shura", "file_name": "Ash_Shura_Tafseer.json", "ayahs": 53}, 43: {"name_ar": "سورة الزخرف", "name_en": "Az-Zukhruf", "file_name": "Az_Zukhruf_Tafseer.json", "ayahs": 89}, 44: {"name_ar": "سورة الدخان", "name_en": "Ad-Dukhaan", "file_name": "Ad_Dukhan_Tafseer.json", "ayahs": 59}, 45: {"name_ar": "سورة الجاثية", "name_en": "Al-Jaathiya", "file_name": "Al_Jathiyah_Tafseer.json", "ayahs": 37}, 46: {"name_ar": "سورة الأحقاف", "name_en": "Al-Ahqaaf", "file_name": "Al_Ahqaf_Tafseer.json", "ayahs": 35}, 47: {"name_ar": "سورة محمد", "name_en": "Muhammad", "file_name": "Muhammad_Tafseer.json", "ayahs": 38}, 48: {"name_ar": "سورة الفتح", "name_en": "Al-Fath", "file_name": "Al_Fath_Tafseer.json", "ayahs": 29}, 49: {"name_ar": "سورة الحجرات", "name_en": "Al-Hujuraat", "file_name": "Al_Hujurat_Tafseer.json", "ayahs": 18}, 50: {"name_ar": "سورة ق", "name_en": "Qaaf", "file_name": "Qaf_Tafseer.json", "ayahs": 45}, 51: {"name_ar": "سورة الذاريات", "name_en": "Adh-Dhaariyaat", "file_name": "Adh_Dhariyat_Tafseer.json", "ayahs": 60}, 52: {"name_ar": "سورة الطور", "name_en": "At-Toor", "file_name": "At_Tur_Tafseer.json", "ayahs": 49}, 53: {"name_ar": "سورة النجم", "name_en": "An-Najm", "file_name": "An_Najm_Tafseer.json", "ayahs": 62}, 54: {"name_ar": "سورة القمر", "name_en": "Al-Qamar", "file_name": "Al_Qamar_Tafseer.json", "ayahs": 55}, 55: {"name_ar": "سورة الرحمن", "name_en": "Ar-Rahmaan", "file_name": "Ar_Rahman_Tafseer.json", "ayahs": 78}, 56: {"name_ar": "سورة الواقعة", "name_en": "Al-Waaqia", "file_name": "Al_Waqiah_Tafseer.json", "ayahs": 96}, 57: {"name_ar": "سورة الحديد", "name_en": "Al-Hadid", "file_name": "Al_Hadid_Tafseer.json", "ayahs": 29}, 58: {"name_ar": "سورة المجادلة", "name_en": "Al-Mujaadila", "file_name": "Al_Mujadilah_Tafseer.json", "ayahs": 22}, 59: {"name_ar": "سورة الحشر", "name_en": "Al-Hashr", "file_name": "Al_Hashr_Tafseer.json", "ayahs": 24}, 60: {"name_ar": "سورة الممتحنة", "name_en": "Al-Mumtahana", "file_name": "Al_Mumtahanah_Tafseer.json", "ayahs": 13}, 61: {"name_ar": "سورة الصف", "name_en": "As-Saff", "file_name": "As_Saff_Tafseer.json", "ayahs": 14}, 62: {"name_ar": "سورة الجمعة", "name_en": "Al-Jumu'a", "file_name": "Al_Jumuah_Tafseer.json", "ayahs": 11}, 63: {"name_ar": "سورة المنافقون", "name_en": "Al-Munaafiqoon", "file_name": "Al_Munafiqun_Tafseer.json", "ayahs": 11}, 64: {"name_ar": "سورة التغابن", "name_en": "At-Taghaabun", "file_name": "At_Taghabun_Tafseer.json", "ayahs": 18}, 65: {"name_ar": "سورة الطلاق", "name_en": "At-Talaaq", "file_name": "At_Talaq_Tafseer.json", "ayahs": 12}, 66: {"name_ar": "سورة التحريم", "name_en": "At-Tahrim", "file_name": "At_Tahrim_Tafseer.json", "ayahs": 12}, 67: {"name_ar": "سورة الملك", "name_en": "Al-Mulk", "file_name": "Al_Mulk_Tafseer.json", "ayahs": 30}, 68: {"name_ar": "سورة القلم", "name_en": "Al-Qalam", "file_name": "Al_Qalam_Tafseer.json", "ayahs": 52}, 69: {"name_ar": "سورة الحاقة", "name_en": "Al-Haaqqa", "file_name": "Al_Haqqah_Tafseer.json", "ayahs": 52}, 70: {"name_ar": "سورة المعارج", "name_en": "Al-Ma'aarij", "file_name": "Al_Maarij_Tafseer.json", "ayahs": 44}, 71: {"name_ar": "سورة نوح", "name_en": "Nooh", "file_name": "Nuh_Tafseer.json", "ayahs": 28}, 72: {"name_ar": "سورة الجن", "name_en": "Al-Jinn", "file_name": "Al_Jinn_Tafseer.json", "ayahs": 28}, 73: {"name_ar": "سورة المزمل", "name_en": "Al-Muzzammil", "file_name": "Al_Muzzammil_Tafseer.json", "ayahs": 20}, 74: {"name_ar": "سورة المدثر", "name_en": "Al-Muddaththir", "file_name": "Al_Muddaththir_Tafseer.json", "ayahs": 56}, 75: {"name_ar": "سورة القيامة", "name_en": "Al-Qiyaama", "file_name": "Al_Qiyamah_Tafseer.json", "ayahs": 40}, 76: {"name_ar": "سورة الإنسان", "name_en": "Al-Insaan", "file_name": "Al_Insan_Tafseer.json", "ayahs": 31}, 77: {"name_ar": "سورة المرسلات", "name_en": "Al-Mursalaat", "file_name": "Al_Mursalat_Tafseer.json", "ayahs": 50}, 78: {"name_ar": "سورة النبإ", "name_en": "An-Naba", "file_name": "An_Naba_Tafseer.json", "ayahs": 40}, 79: {"name_ar": "سورة النازعات", "name_en": "An-Naazi'aat", "file_name": "An_Naziat_Tafseer.json", "ayahs": 46}, 80: {"name_ar": "سورة عبس", "name_en": "Abasa", "file_name": "Abasa_Tafseer.json", "ayahs": 42}, 81: {"name_ar": "سورة التكوير", "name_en": "At-Takwir", "file_name": "At_Takwir_Tafseer.json", "ayahs": 29}, 82: {"name_ar": "سورة الانفطار", "name_en": "Al-Infitaar", "file_name": "Al_Infitar_Tafseer.json", "ayahs": 19}, 83: {"name_ar": "سورة المطففين", "name_en": "Al-Mutaffifin", "file_name": "Al_Mutaffifin_Tafseer.json", "ayahs": 36}, 84: {"name_ar": "سورة الانشقاق", "name_en": "Al-Inshiqaaq", "file_name": "Al_Inshiqaq_Tafseer.json", "ayahs": 25}, 85: {"name_ar": "سورة البروج", "name_en": "Al-Burooj", "file_name": "Al_Buruj_Tafseer.json", "ayahs": 22}, 86: {"name_ar": "سورة الطارق", "name_en": "At-Taariq", "file_name": "At_Tariq_Tafseer.json", "ayahs": 17}, 87: {"name_ar": "سورة الأعلى", "name_en": "Al-A'laa", "file_name": "Al_Ala_Tafseer.json", "ayahs": 19}, 88: {"name_ar": "سورة الغاشية", "name_en": "Al-Ghaashiya", "file_name": "Al_Ghashiyah_Tafseer.json", "ayahs": 26}, 89: {"name_ar": "سورة الفجر", "name_en": "Al-Fajr", "file_name": "Al_Fajr_Tafseer.json", "ayahs": 30}, 90: {"name_ar": "سورة البلد", "name_en": "Al-Balad", "file_name": "Al_Balad_Tafseer.json", "ayahs": 20}, 91: {"name_ar": "سورة الشمس", "name_en": "Ash-Shams", "file_name": "Ash_Shams_Tafseer.json", "ayahs": 15}, 92: {"name_ar": "سورة الليل", "name_en": "Al-Layl", "file_name": "Al_Lail_Tafseer.json", "ayahs": 21}, 93: {"name_ar": "سورة الضحى", "name_en": "Ad-Dhuhaa", "file_name": "Ad_Duha_Tafseer.json", "ayahs": 11}, 94: {"name_ar": "سورة الشرح", "name_en": "Ash-Sharh", "file_name": "Ash_Sharh_Tafseer.json", "ayahs": 8}, 95: {"name_ar": "سورة التين", "name_en": "At-Tin", "file_name": "At_Tin_Tafseer.json", "ayahs": 8}, 96: {"name_ar": "سورة العلق", "name_en": "Al-Alaq", "file_name": "Al_Alaq_Tafseer.json", "ayahs": 19}, 97: {"name_ar": "سورة القدر", "name_en": "Al-Qadr", "file_name": "Al_Qadr_Tafseer.json", "ayahs": 5}, 98: {"name_ar": "سورة البينة", "name_en": "Al-Bayyina", "file_name": "Al_Bayyinah_Tafseer.json", "ayahs": 8}, 99: {"name_ar": "سورة الزلزلة", "name_en": "Az-Zalzala", "file_name": "Az_Zalzalah_Tafseer.json", "ayahs": 8}, 100: {"name_ar": "سورة العاديات", "name_en": "Al-Aadiyaat", "file_name": "Al_Adiyat_Tafseer.json", "ayahs": 11}, 101: {"name_ar": "سورة القارعة", "name_en": "Al-Qaari'a", "file_name": "Al_Qariah_Tafseer.json", "ayahs": 11}, 102: {"name_ar": "سورة التكاثر", "name_en": "At-Takaathur", "file_name": "At_Takathur_Tafseer.json", "ayahs": 8}, 103: {"name_ar": "سورة العصر", "name_en": "Al-Asr", "file_name": "Al_Asr_Tafseer.json", "ayahs": 3}, 104: {"name_ar": "سورة الهمزة", "name_en": "Al-Humaza", "file_name": "Al_Humazah_Tafseer.json", "ayahs": 9}, 105: {"name_ar": "سورة الفيل", "name_en": "Al-Fil", "file_name": "Al_Fil_Tafseer.json", "ayahs": 5}, 106: {"name_ar": "سورة قريش", "name_en": "Quraysh", "file_name": "Quraish_Tafseer.json", "ayahs": 4}, 107: {"name_ar": "سورة الماعون", "name_en": "Al-Maa'oon", "file_name": "Al_Maun_Tafseer.json", "ayahs": 7}, 108: {"name_ar": "سورة الكوثر", "name_en": "Al-Kawthar", "file_name": "Al_Kauthar_Tafseer.json", "ayahs": 3}, 109: {"name_ar": "سورة الكافرون", "name_en": "Al-Kaafiroon", "file_name": "Al_Kafirun_Tafseer.json", "ayahs": 6}, 110: {"name_ar": "سورة النصر", "name_en": "An-Nasr", "file_name": "An_Nasr_Tafseer.json", "ayahs": 3}, 111: {"name_ar": "سورة المسد", "name_en": "Al-Masad", "file_name": "Al_Masad_Tafseer.json", "ayahs": 5}, 112: {"name_ar": "سورة الإخلاص", "name_en": "Al-Ikhlaas", "file_name": "Al_Ikhlas_Tafseer.json", "ayahs": 4}, 113: {"name_ar": "سورة الفلق", "name_en": "Al-Falaq", "file_name": "Al_Falaq_Tafseer.json", "ayahs": 5}, 114: {"name_ar": "سورة الناس", "name_en": "An-Naas", "file_name": "An_Nas_Tafseer.json", "ayahs": 6}, } # ----------------------------------------------------------------------------- # EDITIONS CONFIGURATION (17 VERIFIED SCHOLARS) # ----------------------------------------------------------------------------- EDITIONS_META = [ { "tafsir_id": "en-tafsir-taqi-usmani", "scholar": "Mufti Muhammad Taqi Usmani", "tafsir_name": "The Noble Quran (English)", "language": "en", "source": "The Noble Quran - Mufti Taqi Usmani", "fetcher": "fawaz_en_taqi", }, { "tafsir_id": "en-tafisr-ibn-kathir", "scholar": "Hafiz Ibn Kathir", "tafsir_name": "Tafsir Ibn Kathir (English)", "language": "en", "source": "Darussalam Tafsir Ibn Kathir (Abridged)", "fetcher": "sqlite", "db_key": "en-tafisr-ibn-kathir", }, { "tafsir_id": "ur-tafseer-ibn-e-kaseer", "scholar": "Hafiz Ibn Kathir", "tafsir_name": "Tafsir Ibn Kathir (تفسیر ابن کثیر اردو)", "language": "ur", "source": "Tafsir Ibn Kathir Urdu Translation", "fetcher": "sqlite", "db_key": "tafseer-ibn-e-kaseer-urdu", }, { "tafsir_id": "ur-tafsir-bayan-ul-quran", "scholar": "Dr. Israr Ahmed", "tafsir_name": "Bayan-ul-Quran (بیان القرآن)", "language": "ur", "source": "Markazi Anjuman Khuddam-ul-Quran Lahore", "fetcher": "sqlite", "db_key": "tafsir-bayan-ul-quran", }, { "tafsir_id": "ur-tafsir-tafheem-ul-quran", "scholar": "Syed Abul A'la Maududi", "tafsir_name": "Tafheem-ul-Quran (تفہیم القرآن مع تفسیری حواشی)", "language": "ur", "source": "Tafheem-ul-Quran (Maududi Complete Commentary)", "fetcher": "qurancom_tafheem_ur", }, { "tafsir_id": "en-tafsir-tafheem-ul-quran", "scholar": "Syed Abul A'la Maududi", "tafsir_name": "Towards Understanding the Quran (Tafhim Commentary)", "language": "en", "source": "Towards Understanding the Quran (Maududi Complete Commentary)", "fetcher": "qurancom_tafheem_en", }, { "tafsir_id": "en-tafsir-maarif-ul-quran", "scholar": "Mufti Muhammad Shafi", "tafsir_name": "Ma’ariful Quran (English)", "language": "en", "source": "Maktaba Darul Uloom Karachi", "fetcher": "sqlite", "db_key": "en-tafsir-maarif-ul-quran", }, { "tafsir_id": "ur-tafsir-fee-zilal-il-quran", "scholar": "Sayyid Qutb", "tafsir_name": "Fi Zilal al-Quran (فی ظلال القرآن)", "language": "ur", "source": "Fi Zilal al-Quran Urdu", "fetcher": "spa5k", "spa5k_slug": "tafsir-fe-zalul-quran-syed-qatab", }, { "tafsir_id": "ur-tafsir-tazkirul-quran", "scholar": "Maulana Wahiduddin Khan", "tafsir_name": "Tazkirul Quran (تذکیر القرآن)", "language": "ur", "source": "Goodword Books / Tazkirul Quran", "fetcher": "spa5k", "spa5k_slug": "ur-tazkirul-quran", }, { "tafsir_id": "en-tafsir-tazkirul-quran", "scholar": "Maulana Wahiduddin Khan", "tafsir_name": "Tazkirul Quran (English)", "language": "en", "source": "Goodword Books / Tazkirul Quran", "fetcher": "spa5k", "spa5k_slug": "en-tazkirul-quran", }, { "tafsir_id": "ur-tafsir-as-saadi-urdu", "scholar": "Shaykh Abdur Rahman al-Sa'di", "tafsir_name": "Tafsir As-Sa’di (تفسیر السعدی اردو)", "language": "ur", "source": "Tafsir As-Sa'di Urdu Translation", "fetcher": "sqlite", "db_key": "ur-tafsir-as-saadi-urdu", }, { "tafsir_id": "en-al-jalalayn", "scholar": "Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti", "tafsir_name": "Tafsir Al-Jalalayn (English)", "language": "en", "source": "Firas Hamza / Royal Aal al-Bayt Institute", "fetcher": "sqlite", "db_key": "tafsir-al-jalalayn", }, { "tafsir_id": "en-tafsir-al-mukhtasar", "scholar": "Tafsir Center for Quranic Studies", "tafsir_name": "Al-Mukhtasar in Interpreting the Noble Quran", "language": "en", "source": "Tafsir Center for Quranic Studies, Riyadh", "fetcher": "sqlite", "db_key": "en-tafsir-al-mukhtasar", }, { "tafsir_id": "ar-tafsir-muyassar", "scholar": "King Fahd Complex, Medina", "tafsir_name": "Al-Tafsir Al-Muyassar (التفسير الميسر)", "language": "ar", "source": "King Fahd Glorious Quran Printing Complex", "fetcher": "sqlite", "db_key": "ar-tafsir-muyassar", }, { "tafsir_id": "ar-tafsir-as-saadi", "scholar": "Shaykh Abdur Rahman al-Sa'di", "tafsir_name": "Tafsir As-Sa’di (تيسير الكريم الرحمن)", "language": "ar", "source": "Taysir al-Karim al-Rahman fi Tafsir Kalam al-Mannan", "fetcher": "sqlite", "db_key": "ar-tafsir-as-saadi", }, { "tafsir_id": "ar-tafsir-ibn-kathir", "scholar": "Hafiz Ibn Kathir", "tafsir_name": "Tafsir Ibn Kathir (تفسير القرآن العظيم)", "language": "ar", "source": "Tafsir al-Quran al-Azim", "fetcher": "sqlite", "db_key": "ar-tafsir-ibn-kathir", }, { "tafsir_id": "bn-tafsir-ahsanul-bayan", "scholar": "Salahuddin Yusuf (King Fahd Complex)", "tafsir_name": "Ahsanul Bayaan (তাফসীর আহসানুল বায়ান)", "language": "bn", "source": "King Fahd Complex / Tawheed Publication", "fetcher": "sqlite", "db_key": "bn-tafsir-ahsanul-bayaan", }, ] # ----------------------------------------------------------------------------- # HELPER FUNCTIONS # ----------------------------------------------------------------------------- def clean_html(raw: str) -> str: if not raw: return "" # Strip HTML tags text = re.sub(r"<[^>]+>", " ", raw) text = html.unescape(text) # Clean multiple whitespace & newlines text = re.sub(r"[ \t]+", " ", text) text = re.sub(r"\n\s*\n+", "\n\n", text) return text.strip() def http_get_json(url: str, timeout: int = 15) -> Any: req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}) with urllib.request.urlopen(req, timeout=timeout) as resp: return json.loads(resp.read().decode("utf-8")) # Global in-memory cache for Fawaz Ahmed Taqi Usmani English _FAWAZ_TAQI_EN_CACHE: Dict[int, Dict[int, str]] = {} def load_fawaz_taqi_en() -> Dict[int, Dict[int, str]]: global _FAWAZ_TAQI_EN_CACHE if _FAWAZ_TAQI_EN_CACHE: return _FAWAZ_TAQI_EN_CACHE cache_file = CACHE_DIR / "eng-muftitaqiusmani.min.json" if cache_file.exists(): with open(cache_file, "r", encoding="utf-8") as f: data = json.load(f) else: print(" Downloading Taqi Usmani English (Fawaz Ahmed repository)...") url = "https://cdn.jsdelivr.net/gh/fawazahmed0/quran-api@1/editions/eng-muftitaqiusmani.min.json" data = http_get_json(url) with open(cache_file, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False) for item in data.get("quran", []): s = item["chapter"] a = item["verse"] t = item["text"] if s not in _FAWAZ_TAQI_EN_CACHE: _FAWAZ_TAQI_EN_CACHE[s] = {} _FAWAZ_TAQI_EN_CACHE[s][a] = clean_html(t) return _FAWAZ_TAQI_EN_CACHE # Footnote fetcher with disk cache & retry def fetch_single_footnote(fid: str) -> str: cache_path = FN_CACHE_DIR / f"{fid}.txt" if cache_path.exists(): try: with open(cache_path, "r", encoding="utf-8") as f: return f.read() except Exception: pass url = f"https://api.quran.com/api/v4/foot_notes/{fid}" for attempt in range(3): try: req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=10) as r: d = json.loads(r.read().decode("utf-8")) text = d.get("foot_note", {}).get("text", "") cleaned = clean_html(text) with open(cache_path, "w", encoding="utf-8") as f: f.write(cleaned) return cleaned except Exception as e: time.sleep(0.5 * (attempt + 1)) return "" def fetch_qurancom_tafheem(surah: int, res_id: int, is_urdu: bool, total_ayahs: int) -> Dict[int, str]: """Fetches full Tafheem-ul-Quran translation and merges footnotes per Ayah.""" cache_file = CACHE_DIR / f"tafheem_{'ur' if is_urdu else 'en'}_{surah}.json" if cache_file.exists(): with open(cache_file, "r", encoding="utf-8") as f: cached_map = json.load(f) return {int(k): v for k, v in cached_map.items()} url = f"https://api.quran.com/api/v4/quran/translations/{res_id}?chapter_number={surah}" print(f" Fetching Tafheem-ul-Quran ({'Urdu' if is_urdu else 'English'}) for Surah {surah} from Quran.com...") data = http_get_json(url) translations = data.get("translations", []) # Map of ayah_no -> list of foot_note IDs ayah_footnotes: Dict[int, List[str]] = {} ayah_trans: Dict[int, str] = {} all_fn_ids: List[str] = [] for idx, item in enumerate(translations): a_no = idx + 1 raw_text = item.get("text", "") fn_ids = re.findall(r'foot_note[ =\"\\\']+([0-9]+)', raw_text) ayah_footnotes[a_no] = fn_ids all_fn_ids.extend(fn_ids) ayah_trans[a_no] = clean_html(raw_text) # Fetch unique footnotes concurrently unique_fn_ids = list(set(all_fn_ids)) fn_text_map: Dict[str, str] = {} if unique_fn_ids: print(f" Fetching {len(unique_fn_ids)} footnotes concurrently...") with ThreadPoolExecutor(max_workers=16) as pool: results = pool.map(fetch_single_footnote, unique_fn_ids) for fid, ftxt in zip(unique_fn_ids, results): fn_text_map[fid] = ftxt # Merge translation + footnotes merged: Dict[int, str] = {} for a_no in range(1, total_ayahs + 1): trans = ayah_trans.get(a_no, "") fn_ids = ayah_footnotes.get(a_no, []) notes = [fn_text_map[fid] for fid in fn_ids if fid in fn_text_map and fn_text_map[fid].strip()] if notes: if is_urdu: notes_block = "\n\n【تفسیر و حواشی】:\n" + "\n\n".join(notes) else: notes_block = "\n\n[Tafsir & Commentary Notes]:\n" + "\n\n".join(notes) merged[a_no] = (trans + notes_block).strip() else: merged[a_no] = trans.strip() # Cache locally with open(cache_file, "w", encoding="utf-8") as f: json.dump(merged, f, ensure_ascii=False) return merged def fetch_spa5k_surah(slug: str, surah: int) -> Dict[int, str]: """Fetches Surah tafseer from spa5k CDN with local disk caching.""" cache_file = CACHE_DIR / f"spa5k_{slug}_{surah}.json" if cache_file.exists(): with open(cache_file, "r", encoding="utf-8") as f: cached_map = json.load(f) return {int(k): v for k, v in cached_map.items()} url = f"https://cdn.jsdelivr.net/gh/spa5k/tafsir_api@main/tafsir/{slug}/{surah}.json" data = http_get_json(url) result: Dict[int, str] = {} if isinstance(data, dict) and "ayahs" in data: items = data["ayahs"] elif isinstance(data, list): items = data else: items = [] for idx, item in enumerate(items): ayah_no = item.get("ayah", idx + 1) txt = clean_html(item.get("text", "")) result[ayah_no] = txt with open(cache_file, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False) return result def fetch_sqlite_surah(conn: sqlite3.Connection, db_key: str, surah: int) -> Dict[int, str]: """Queries SQLite database for pre-ingested classical tafseers.""" cur = conn.cursor() cur.execute("SELECT ayah, text FROM tafseers WHERE tafsir_id=? AND surah=? ORDER BY ayah ASC;", (db_key, surah)) rows = cur.fetchall() return {row[0]: clean_html(row[1]) for row in rows} # ----------------------------------------------------------------------------- # MASTER COMPILER FOR A SINGLE SURAH # ----------------------------------------------------------------------------- def compile_surah(surah: int, conn: sqlite3.Connection) -> Dict[str, Any]: if surah not in SURAH_METADATA: raise ValueError(f"Invalid surah number {surah}. Must be 1-114.") meta = SURAH_METADATA[surah] total_ayahs = meta["ayahs"] print(f"\n=======================================================") print(f"COMPILING SURAH {surah}: {meta['name_en']} ({meta['name_ar']})") print(f"Total Ayahs: {total_ayahs} | Expected Editions: {len(EDITIONS_META)}") print(f"=======================================================") # 1. Pre-fetch all 17 edition maps for this surah edition_ayah_maps: Dict[str, Dict[int, str]] = {} # SQLite editions print("→ Querying pre-indexed classical editions from local database...") for ed in EDITIONS_META: if ed["fetcher"] == "sqlite": edition_ayah_maps[ed["tafsir_id"]] = fetch_sqlite_surah(conn, ed["db_key"], surah) # Fawaz Taqi Usmani English print("→ Loading Mufti Taqi Usmani English...") taqi_en_all = load_fawaz_taqi_en() edition_ayah_maps["en-tafsir-taqi-usmani"] = taqi_en_all.get(surah, {}) # spa5k editions for ed in EDITIONS_META: if ed["fetcher"] == "spa5k": print(f"→ Loading {ed['scholar']} ({ed['tafsir_name']})...") edition_ayah_maps[ed["tafsir_id"]] = fetch_spa5k_surah(ed["spa5k_slug"], surah) # Quran.com Tafheem editions print("→ Loading Tafheem-ul-Quran Urdu with complete commentary notes...") edition_ayah_maps["ur-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 97, True, total_ayahs) print("→ Loading Towards Understanding the Quran English with complete commentary notes...") edition_ayah_maps["en-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 95, False, total_ayahs) # 2. Assemble Ayah by Ayah JSON ayahs_list: List[Dict[str, Any]] = [] missing_report: List[str] = [] for a_no in range(1, total_ayahs + 1): ayah_tafsirs: List[Dict[str, Any]] = [] for ed in EDITIONS_META: tid = ed["tafsir_id"] emap = edition_ayah_maps.get(tid, {}) text = emap.get(a_no, "").strip() if not text: missing_report.append(f"Ayah {surah}:{a_no} missing '{tid}' ({ed['scholar']})") ayah_tafsirs.append({ "tafsir_id": tid, "scholar": ed["scholar"], "tafsir_name": ed["tafsir_name"], "language": ed["language"], "text": text, "source": ed["source"], }) ayahs_list.append({ "ayah_no": a_no, "reference": f"{surah}:{a_no}", "tafsirs": ayah_tafsirs, }) # Consistency check if missing_report: print(f"\n[WARNING] Found {len(missing_report)} missing text items!") for m in missing_report[:10]: print(f" - {m}") if len(missing_report) > 10: print(f" ... and {len(missing_report) - 10} more.") else: print(f"\n✓ 101% Verification Passed! All {total_ayahs} Ayahs have complete text for all {len(EDITIONS_META)} scholars.") result_json = { "schema_version": "1.0", "surahs": [ { "surah_no": surah, "surah_name_arabic": meta["name_ar"], "surah_name_english": meta["name_en"], "total_ayahs": total_ayahs, "ayahs": ayahs_list, } ], } # 3. Synchronize to all target directories file_name = meta["file_name"] for out_dir in SYNC_DIRS: target_file = out_dir / file_name with open(target_file, "w", encoding="utf-8") as f: json.dump(result_json, f, ensure_ascii=False, indent=2) print(f"✓ Saved: {target_file} ({os.path.getsize(target_file) / (1024 * 1024):.2f} MB)") return result_json def compile_surah_safe(surah: int, conn: sqlite3.Connection, max_retries: int = 3) -> bool: for attempt in range(max_retries): try: compile_surah(surah, conn) return True except Exception as e: print(f"[RETRY {attempt+1}/{max_retries}] Error compiling Surah {surah}: {e}") time.sleep(2 * (attempt + 1)) print(f"[FAILED] Surah {surah} failed after {max_retries} attempts.") return False # ----------------------------------------------------------------------------- # MAIN CLI ENTRYPOINT # ----------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser( description="Master Quran Tafseer Compiler & Generator", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: python master_tafseer_generator.py --surah 3 # Compiles Surah 3 only python master_tafseer_generator.py --all # Compiles all 114 Surahs (skips existing) python master_tafseer_generator.py --start 3 --end 114 # Compiles Surahs 3 to 114 python master_tafseer_generator.py --all --force # Re-compiles all Surahs even if existing """, ) parser.add_argument("--surah", type=int, default=None, help="Specific Surah number to compile (1-114).") parser.add_argument("--all", action="store_true", help="Compile all 114 Surahs in batch mode.") parser.add_argument("--start", type=int, default=1, help="Start Surah number for range (1-114).") parser.add_argument("--end", type=int, default=114, help="End Surah number for range (1-114).") parser.add_argument("--force", action="store_true", help="Overwrite existing files even if already present.") args = parser.parse_args() if not DB_PATH.exists(): print(f"ERROR: SQLite database not found at {DB_PATH}!") sys.exit(1) conn = sqlite3.connect(str(DB_PATH)) if args.surah is not None: compile_surah_safe(args.surah, conn) print(f"\n🎉 SURAH {args.surah} COMPILATION & SYNCHRONIZATION COMPLETE!") elif args.all or (args.start != 1 or args.end != 114): start_surah = args.start end_surah = args.end print(f"\n=======================================================") print(f"STARTING BATCH COMPILATION: Surahs {start_surah} to {end_surah}") print(f"=======================================================") success_list = [] skipped_list = [] failed_list = [] for s in range(start_surah, end_surah + 1): meta = SURAH_METADATA.get(s) if not meta: continue target_file = PROJECT_ROOT / "quran_tafseer" / meta["file_name"] if target_file.exists() and not args.force: try: # Quick integrity check with open(target_file, "r", encoding="utf-8") as f: existing = json.load(f) if len(existing.get("surahs", [{}])[0].get("ayahs", [])) == meta["ayahs"]: print(f"⏩ Skipping Surah {s} ({meta['name_en']}) - already compiled & verified.") skipped_list.append(s) continue except Exception: pass ok = compile_surah_safe(s, conn) if ok: success_list.append(s) else: failed_list.append(s) print("\n=======================================================") print("BATCH COMPILATION SUMMARY") print(f"✓ Successfully compiled: {len(success_list)} surahs") print(f"⏩ Already existed (skipped): {len(skipped_list)} surahs") if failed_list: print(f"✗ Failed surahs: {failed_list}") else: print("🎉 ALL REQUESTED SURAHS VERIFIED & GENERATED WITH 100% SUCCESS!") print("=======================================================") else: # Default single surah instruction if neither flag is passed compile_surah_safe(2, conn) print("\n🎉 SURAH 2 COMPILATION COMPLETE! (Pass --all to batch compile all Surahs)") conn.close() if __name__ == "__main__": main()