Spaces:
Running on Zero
Running on Zero
feat: integrate Cloudflare R2 Quran translations, add translation endpoints and edge caching
3c6e2f1 Download scripts/master_tafseer_generator.py from sahihislam/superadmin: direct link, hf CLI and curl.
- Browser
- Download file 37.3 kB
-
https://huggingface.co/spaces/sahihislam/superadmin/resolve/main/scripts/master_tafseer_generator.py
- Command line
-
hf download hf://spaces/sahihislam/superadmin/scripts/master_tafseer_generator.py
-
curl -L -o master_tafseer_generator.py https://huggingface.co/spaces/sahihislam/superadmin/resolve/main/scripts/master_tafseer_generator.py
37.3 kB
| #!/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| """ | |
| ============================================================================= | |
| MASTER QURAN TAFSEER COMPILER & GENERATOR | |
| ============================================================================= | |
| Compiles verified multi-scholar Tafseer JSON files ayah-by-ayah. | |
| Supports generating a single Surah at a time (e.g. --surah 2) or all 114 Surahs | |
| with rigorous consistency, validation, and multi-directory synchronization. | |
| Strictly Curated 17 Sunni Orthodox Scholars & Commentaries: | |
| 1. en-tafsir-taqi-usmani (Mufti Muhammad Taqi Usmani) | |
| 2. en-tafisr-ibn-kathir (Hafiz Ibn Kathir) | |
| 3. ur-tafseer-ibn-e-kaseer (Hafiz Ibn Kathir) | |
| 4. ur-tafsir-bayan-ul-quran (Dr. Israr Ahmed) | |
| 5. ur-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes) | |
| 6. en-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes) | |
| 7. en-tafsir-maarif-ul-quran (Mufti Muhammad Shafi) | |
| 8. ur-tafsir-fee-zilal-il-quran (Sayyid Qutb) | |
| 9. ur-tafsir-tazkirul-quran (Maulana Wahiduddin Khan) | |
| 10. en-tafsir-tazkirul-quran (Maulana Wahiduddin Khan) | |
| 11. ur-tafsir-as-saadi-urdu (Shaykh Abdur Rahman al-Sa'di) | |
| 12. en-al-jalalayn (Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti) | |
| 13. en-tafsir-al-mukhtasar (Tafsir Center for Quranic Studies, Riyadh) | |
| 14. ar-tafsir-muyassar (King Fahd Complex, Medina) | |
| 15. ar-tafsir-as-saadi (Shaykh Abdur Rahman al-Sa'di) | |
| 16. ar-tafsir-ibn-kathir (Hafiz Ibn Kathir) | |
| 17. bn-tafsir-ahsanul-bayan (Salahuddin Yusuf / King Fahd Complex) | |
| ============================================================================= | |
| """ | |
| import argparse | |
| import html | |
| import json | |
| import os | |
| import re | |
| import sqlite3 | |
| import sys | |
| import time | |
| import urllib.request | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Tuple | |
| sys.stdout.reconfigure(encoding="utf-8") | |
| # Root directory paths | |
| PROJECT_ROOT = Path(r"d:\Project-x") | |
| DB_PATH = PROJECT_ROOT / "BACKEND" / "data" / "tafseers.db" | |
| CACHE_DIR = PROJECT_ROOT / "BACKEND" / "data" / "cache" | |
| CACHE_DIR.mkdir(parents=True, exist_ok=True) | |
| FN_CACHE_DIR = CACHE_DIR / "footnotes" | |
| FN_CACHE_DIR.mkdir(parents=True, exist_ok=True) | |
| SYNC_DIRS = [ | |
| PROJECT_ROOT / "quran_tafseer", | |
| PROJECT_ROOT / "BACKEND" / "data" / "quran_tafseer", | |
| PROJECT_ROOT / "FRONTEND" / "assets" / "tafsir", | |
| ] | |
| for d in SYNC_DIRS: | |
| d.mkdir(parents=True, exist_ok=True) | |
| # ----------------------------------------------------------------------------- | |
| # SURAH METADATA DICTIONARY (1-114) | |
| # ----------------------------------------------------------------------------- | |
| SURAH_METADATA = { | |
| 1: {"name_ar": "سورة الفاتحة", "name_en": "Al-Faatihah", "file_name": "Al_Fatihah_Tafseer.json", "ayahs": 7}, | |
| 2: {"name_ar": "سورة البقرة", "name_en": "Al-Baqarah", "file_name": "Al_Baqarah_Tafseer.json", "ayahs": 286}, | |
| 3: {"name_ar": "سورة آل عمران", "name_en": "Aal-i-Imraan", "file_name": "Aal_Imran_Tafseer.json", "ayahs": 200}, | |
| 4: {"name_ar": "سورة النساء", "name_en": "An-Nisaa", "file_name": "An_Nisa_Tafseer.json", "ayahs": 176}, | |
| 5: {"name_ar": "سورة المائدة", "name_en": "Al-Maaida", "file_name": "Al_Maidah_Tafseer.json", "ayahs": 120}, | |
| 6: {"name_ar": "سورة الأنعام", "name_en": "Al-An'aam", "file_name": "Al_Anam_Tafseer.json", "ayahs": 165}, | |
| 7: {"name_ar": "سورة الأعراف", "name_en": "Al-A'raaf", "file_name": "Al_Araf_Tafseer.json", "ayahs": 206}, | |
| 8: {"name_ar": "سورة الأنفال", "name_en": "Al-Anfaal", "file_name": "Al_Anfal_Tafseer.json", "ayahs": 75}, | |
| 9: {"name_ar": "سورة التوبة", "name_en": "At-Tawba", "file_name": "At_Tawbah_Tafseer.json", "ayahs": 129}, | |
| 10: {"name_ar": "سورة يونس", "name_en": "Yunus", "file_name": "Yunus_Tafseer.json", "ayahs": 109}, | |
| 11: {"name_ar": "سورة هود", "name_en": "Hud", "file_name": "Hud_Tafseer.json", "ayahs": 123}, | |
| 12: {"name_ar": "سورة يوسف", "name_en": "Yusuf", "file_name": "Yusuf_Tafseer.json", "ayahs": 111}, | |
| 13: {"name_ar": "سورة الرعد", "name_en": "Ar-Ra'd", "file_name": "Ar_Rad_Tafseer.json", "ayahs": 43}, | |
| 14: {"name_ar": "سورة إبراهيم", "name_en": "Ibrahim", "file_name": "Ibrahim_Tafseer.json", "ayahs": 52}, | |
| 15: {"name_ar": "سورة الحجر", "name_en": "Al-Hijr", "file_name": "Al_Hijr_Tafseer.json", "ayahs": 99}, | |
| 16: {"name_ar": "سورة النحل", "name_en": "An-Nahl", "file_name": "An_Nahl_Tafseer.json", "ayahs": 128}, | |
| 17: {"name_ar": "سورة الإسراء", "name_en": "Al-Israa", "file_name": "Al_Isra_Tafseer.json", "ayahs": 111}, | |
| 18: {"name_ar": "سورة الكهف", "name_en": "Al-Kahf", "file_name": "Al_Kahf_Tafseer.json", "ayahs": 110}, | |
| 19: {"name_ar": "سورة مريم", "name_en": "Maryam", "file_name": "Maryam_Tafseer.json", "ayahs": 98}, | |
| 20: {"name_ar": "سورة طه", "name_en": "Taa-Haa", "file_name": "Ta_Ha_Tafseer.json", "ayahs": 135}, | |
| 21: {"name_ar": "سورة الأنبياء", "name_en": "Al-Anbiyaa", "file_name": "Al_Anbiya_Tafseer.json", "ayahs": 112}, | |
| 22: {"name_ar": "سورة الحج", "name_en": "Al-Hajj", "file_name": "Al_Hajj_Tafseer.json", "ayahs": 78}, | |
| 23: {"name_ar": "سورة المؤمنون", "name_en": "Al-Muminoon", "file_name": "Al_Muminun_Tafseer.json", "ayahs": 118}, | |
| 24: {"name_ar": "سورة النور", "name_en": "An-Noor", "file_name": "An_Nur_Tafseer.json", "ayahs": 64}, | |
| 25: {"name_ar": "سورة الفرقان", "name_en": "Al-Furqaan", "file_name": "Al_Furqan_Tafseer.json", "ayahs": 77}, | |
| 26: {"name_ar": "سورة الشعراء", "name_en": "Ash-Shu'araa", "file_name": "Ash_Shuara_Tafseer.json", "ayahs": 227}, | |
| 27: {"name_ar": "سورة النمل", "name_en": "An-Naml", "file_name": "An_Naml_Tafseer.json", "ayahs": 93}, | |
| 28: {"name_ar": "سورة القصص", "name_en": "Al-Qasas", "file_name": "Al_Qasas_Tafseer.json", "ayahs": 88}, | |
| 29: {"name_ar": "سورة العنكبوت", "name_en": "Al-Ankaboot", "file_name": "Al_Ankabut_Tafseer.json", "ayahs": 69}, | |
| 30: {"name_ar": "سورة الروم", "name_en": "Ar-Room", "file_name": "Ar_Rum_Tafseer.json", "ayahs": 60}, | |
| 31: {"name_ar": "سورة لقمان", "name_en": "Luqman", "file_name": "Luqman_Tafseer.json", "ayahs": 34}, | |
| 32: {"name_ar": "سورة السجدة", "name_en": "As-Sajda", "file_name": "As_Sajdah_Tafseer.json", "ayahs": 30}, | |
| 33: {"name_ar": "سورة الأحزاب", "name_en": "Al-Ahzaab", "file_name": "Al_Ahzab_Tafseer.json", "ayahs": 73}, | |
| 34: {"name_ar": "سورة سبإ", "name_en": "Saba", "file_name": "Saba_Tafseer.json", "ayahs": 54}, | |
| 35: {"name_ar": "سورة فاطر", "name_en": "Faatir", "file_name": "Fatir_Tafseer.json", "ayahs": 45}, | |
| 36: {"name_ar": "سورة يس", "name_en": "Yaseen", "file_name": "Ya_Sin_Tafseer.json", "ayahs": 83}, | |
| 37: {"name_ar": "سورة الصافات", "name_en": "As-Saaffaat", "file_name": "As_Saffat_Tafseer.json", "ayahs": 182}, | |
| 38: {"name_ar": "سورة ص", "name_en": "Saad", "file_name": "Sad_Tafseer.json", "ayahs": 88}, | |
| 39: {"name_ar": "سورة الزمر", "name_en": "Az-Zumar", "file_name": "Az_Zumar_Tafseer.json", "ayahs": 75}, | |
| 40: {"name_ar": "سورة غافر", "name_en": "Ghafir", "file_name": "Ghafir_Tafseer.json", "ayahs": 85}, | |
| 41: {"name_ar": "سورة فصلت", "name_en": "Fussilat", "file_name": "Fussilat_Tafseer.json", "ayahs": 54}, | |
| 42: {"name_ar": "سورة الشورى", "name_en": "Ash-Shura", "file_name": "Ash_Shura_Tafseer.json", "ayahs": 53}, | |
| 43: {"name_ar": "سورة الزخرف", "name_en": "Az-Zukhruf", "file_name": "Az_Zukhruf_Tafseer.json", "ayahs": 89}, | |
| 44: {"name_ar": "سورة الدخان", "name_en": "Ad-Dukhaan", "file_name": "Ad_Dukhan_Tafseer.json", "ayahs": 59}, | |
| 45: {"name_ar": "سورة الجاثية", "name_en": "Al-Jaathiya", "file_name": "Al_Jathiyah_Tafseer.json", "ayahs": 37}, | |
| 46: {"name_ar": "سورة الأحقاف", "name_en": "Al-Ahqaaf", "file_name": "Al_Ahqaf_Tafseer.json", "ayahs": 35}, | |
| 47: {"name_ar": "سورة محمد", "name_en": "Muhammad", "file_name": "Muhammad_Tafseer.json", "ayahs": 38}, | |
| 48: {"name_ar": "سورة الفتح", "name_en": "Al-Fath", "file_name": "Al_Fath_Tafseer.json", "ayahs": 29}, | |
| 49: {"name_ar": "سورة الحجرات", "name_en": "Al-Hujuraat", "file_name": "Al_Hujurat_Tafseer.json", "ayahs": 18}, | |
| 50: {"name_ar": "سورة ق", "name_en": "Qaaf", "file_name": "Qaf_Tafseer.json", "ayahs": 45}, | |
| 51: {"name_ar": "سورة الذاريات", "name_en": "Adh-Dhaariyaat", "file_name": "Adh_Dhariyat_Tafseer.json", "ayahs": 60}, | |
| 52: {"name_ar": "سورة الطور", "name_en": "At-Toor", "file_name": "At_Tur_Tafseer.json", "ayahs": 49}, | |
| 53: {"name_ar": "سورة النجم", "name_en": "An-Najm", "file_name": "An_Najm_Tafseer.json", "ayahs": 62}, | |
| 54: {"name_ar": "سورة القمر", "name_en": "Al-Qamar", "file_name": "Al_Qamar_Tafseer.json", "ayahs": 55}, | |
| 55: {"name_ar": "سورة الرحمن", "name_en": "Ar-Rahmaan", "file_name": "Ar_Rahman_Tafseer.json", "ayahs": 78}, | |
| 56: {"name_ar": "سورة الواقعة", "name_en": "Al-Waaqia", "file_name": "Al_Waqiah_Tafseer.json", "ayahs": 96}, | |
| 57: {"name_ar": "سورة الحديد", "name_en": "Al-Hadid", "file_name": "Al_Hadid_Tafseer.json", "ayahs": 29}, | |
| 58: {"name_ar": "سورة المجادلة", "name_en": "Al-Mujaadila", "file_name": "Al_Mujadilah_Tafseer.json", "ayahs": 22}, | |
| 59: {"name_ar": "سورة الحشر", "name_en": "Al-Hashr", "file_name": "Al_Hashr_Tafseer.json", "ayahs": 24}, | |
| 60: {"name_ar": "سورة الممتحنة", "name_en": "Al-Mumtahana", "file_name": "Al_Mumtahanah_Tafseer.json", "ayahs": 13}, | |
| 61: {"name_ar": "سورة الصف", "name_en": "As-Saff", "file_name": "As_Saff_Tafseer.json", "ayahs": 14}, | |
| 62: {"name_ar": "سورة الجمعة", "name_en": "Al-Jumu'a", "file_name": "Al_Jumuah_Tafseer.json", "ayahs": 11}, | |
| 63: {"name_ar": "سورة المنافقون", "name_en": "Al-Munaafiqoon", "file_name": "Al_Munafiqun_Tafseer.json", "ayahs": 11}, | |
| 64: {"name_ar": "سورة التغابن", "name_en": "At-Taghaabun", "file_name": "At_Taghabun_Tafseer.json", "ayahs": 18}, | |
| 65: {"name_ar": "سورة الطلاق", "name_en": "At-Talaaq", "file_name": "At_Talaq_Tafseer.json", "ayahs": 12}, | |
| 66: {"name_ar": "سورة التحريم", "name_en": "At-Tahrim", "file_name": "At_Tahrim_Tafseer.json", "ayahs": 12}, | |
| 67: {"name_ar": "سورة الملك", "name_en": "Al-Mulk", "file_name": "Al_Mulk_Tafseer.json", "ayahs": 30}, | |
| 68: {"name_ar": "سورة القلم", "name_en": "Al-Qalam", "file_name": "Al_Qalam_Tafseer.json", "ayahs": 52}, | |
| 69: {"name_ar": "سورة الحاقة", "name_en": "Al-Haaqqa", "file_name": "Al_Haqqah_Tafseer.json", "ayahs": 52}, | |
| 70: {"name_ar": "سورة المعارج", "name_en": "Al-Ma'aarij", "file_name": "Al_Maarij_Tafseer.json", "ayahs": 44}, | |
| 71: {"name_ar": "سورة نوح", "name_en": "Nooh", "file_name": "Nuh_Tafseer.json", "ayahs": 28}, | |
| 72: {"name_ar": "سورة الجن", "name_en": "Al-Jinn", "file_name": "Al_Jinn_Tafseer.json", "ayahs": 28}, | |
| 73: {"name_ar": "سورة المزمل", "name_en": "Al-Muzzammil", "file_name": "Al_Muzzammil_Tafseer.json", "ayahs": 20}, | |
| 74: {"name_ar": "سورة المدثر", "name_en": "Al-Muddaththir", "file_name": "Al_Muddaththir_Tafseer.json", "ayahs": 56}, | |
| 75: {"name_ar": "سورة القيامة", "name_en": "Al-Qiyaama", "file_name": "Al_Qiyamah_Tafseer.json", "ayahs": 40}, | |
| 76: {"name_ar": "سورة الإنسان", "name_en": "Al-Insaan", "file_name": "Al_Insan_Tafseer.json", "ayahs": 31}, | |
| 77: {"name_ar": "سورة المرسلات", "name_en": "Al-Mursalaat", "file_name": "Al_Mursalat_Tafseer.json", "ayahs": 50}, | |
| 78: {"name_ar": "سورة النبإ", "name_en": "An-Naba", "file_name": "An_Naba_Tafseer.json", "ayahs": 40}, | |
| 79: {"name_ar": "سورة النازعات", "name_en": "An-Naazi'aat", "file_name": "An_Naziat_Tafseer.json", "ayahs": 46}, | |
| 80: {"name_ar": "سورة عبس", "name_en": "Abasa", "file_name": "Abasa_Tafseer.json", "ayahs": 42}, | |
| 81: {"name_ar": "سورة التكوير", "name_en": "At-Takwir", "file_name": "At_Takwir_Tafseer.json", "ayahs": 29}, | |
| 82: {"name_ar": "سورة الانفطار", "name_en": "Al-Infitaar", "file_name": "Al_Infitar_Tafseer.json", "ayahs": 19}, | |
| 83: {"name_ar": "سورة المطففين", "name_en": "Al-Mutaffifin", "file_name": "Al_Mutaffifin_Tafseer.json", "ayahs": 36}, | |
| 84: {"name_ar": "سورة الانشقاق", "name_en": "Al-Inshiqaaq", "file_name": "Al_Inshiqaq_Tafseer.json", "ayahs": 25}, | |
| 85: {"name_ar": "سورة البروج", "name_en": "Al-Burooj", "file_name": "Al_Buruj_Tafseer.json", "ayahs": 22}, | |
| 86: {"name_ar": "سورة الطارق", "name_en": "At-Taariq", "file_name": "At_Tariq_Tafseer.json", "ayahs": 17}, | |
| 87: {"name_ar": "سورة الأعلى", "name_en": "Al-A'laa", "file_name": "Al_Ala_Tafseer.json", "ayahs": 19}, | |
| 88: {"name_ar": "سورة الغاشية", "name_en": "Al-Ghaashiya", "file_name": "Al_Ghashiyah_Tafseer.json", "ayahs": 26}, | |
| 89: {"name_ar": "سورة الفجر", "name_en": "Al-Fajr", "file_name": "Al_Fajr_Tafseer.json", "ayahs": 30}, | |
| 90: {"name_ar": "سورة البلد", "name_en": "Al-Balad", "file_name": "Al_Balad_Tafseer.json", "ayahs": 20}, | |
| 91: {"name_ar": "سورة الشمس", "name_en": "Ash-Shams", "file_name": "Ash_Shams_Tafseer.json", "ayahs": 15}, | |
| 92: {"name_ar": "سورة الليل", "name_en": "Al-Layl", "file_name": "Al_Lail_Tafseer.json", "ayahs": 21}, | |
| 93: {"name_ar": "سورة الضحى", "name_en": "Ad-Dhuhaa", "file_name": "Ad_Duha_Tafseer.json", "ayahs": 11}, | |
| 94: {"name_ar": "سورة الشرح", "name_en": "Ash-Sharh", "file_name": "Ash_Sharh_Tafseer.json", "ayahs": 8}, | |
| 95: {"name_ar": "سورة التين", "name_en": "At-Tin", "file_name": "At_Tin_Tafseer.json", "ayahs": 8}, | |
| 96: {"name_ar": "سورة العلق", "name_en": "Al-Alaq", "file_name": "Al_Alaq_Tafseer.json", "ayahs": 19}, | |
| 97: {"name_ar": "سورة القدر", "name_en": "Al-Qadr", "file_name": "Al_Qadr_Tafseer.json", "ayahs": 5}, | |
| 98: {"name_ar": "سورة البينة", "name_en": "Al-Bayyina", "file_name": "Al_Bayyinah_Tafseer.json", "ayahs": 8}, | |
| 99: {"name_ar": "سورة الزلزلة", "name_en": "Az-Zalzala", "file_name": "Az_Zalzalah_Tafseer.json", "ayahs": 8}, | |
| 100: {"name_ar": "سورة العاديات", "name_en": "Al-Aadiyaat", "file_name": "Al_Adiyat_Tafseer.json", "ayahs": 11}, | |
| 101: {"name_ar": "سورة القارعة", "name_en": "Al-Qaari'a", "file_name": "Al_Qariah_Tafseer.json", "ayahs": 11}, | |
| 102: {"name_ar": "سورة التكاثر", "name_en": "At-Takaathur", "file_name": "At_Takathur_Tafseer.json", "ayahs": 8}, | |
| 103: {"name_ar": "سورة العصر", "name_en": "Al-Asr", "file_name": "Al_Asr_Tafseer.json", "ayahs": 3}, | |
| 104: {"name_ar": "سورة الهمزة", "name_en": "Al-Humaza", "file_name": "Al_Humazah_Tafseer.json", "ayahs": 9}, | |
| 105: {"name_ar": "سورة الفيل", "name_en": "Al-Fil", "file_name": "Al_Fil_Tafseer.json", "ayahs": 5}, | |
| 106: {"name_ar": "سورة قريش", "name_en": "Quraysh", "file_name": "Quraish_Tafseer.json", "ayahs": 4}, | |
| 107: {"name_ar": "سورة الماعون", "name_en": "Al-Maa'oon", "file_name": "Al_Maun_Tafseer.json", "ayahs": 7}, | |
| 108: {"name_ar": "سورة الكوثر", "name_en": "Al-Kawthar", "file_name": "Al_Kauthar_Tafseer.json", "ayahs": 3}, | |
| 109: {"name_ar": "سورة الكافرون", "name_en": "Al-Kaafiroon", "file_name": "Al_Kafirun_Tafseer.json", "ayahs": 6}, | |
| 110: {"name_ar": "سورة النصر", "name_en": "An-Nasr", "file_name": "An_Nasr_Tafseer.json", "ayahs": 3}, | |
| 111: {"name_ar": "سورة المسد", "name_en": "Al-Masad", "file_name": "Al_Masad_Tafseer.json", "ayahs": 5}, | |
| 112: {"name_ar": "سورة الإخلاص", "name_en": "Al-Ikhlaas", "file_name": "Al_Ikhlas_Tafseer.json", "ayahs": 4}, | |
| 113: {"name_ar": "سورة الفلق", "name_en": "Al-Falaq", "file_name": "Al_Falaq_Tafseer.json", "ayahs": 5}, | |
| 114: {"name_ar": "سورة الناس", "name_en": "An-Naas", "file_name": "An_Nas_Tafseer.json", "ayahs": 6}, | |
| } | |
| # ----------------------------------------------------------------------------- | |
| # EDITIONS CONFIGURATION (17 VERIFIED SCHOLARS) | |
| # ----------------------------------------------------------------------------- | |
| EDITIONS_META = [ | |
| { | |
| "tafsir_id": "en-tafsir-taqi-usmani", | |
| "scholar": "Mufti Muhammad Taqi Usmani", | |
| "tafsir_name": "The Noble Quran (English)", | |
| "language": "en", | |
| "source": "The Noble Quran - Mufti Taqi Usmani", | |
| "fetcher": "fawaz_en_taqi", | |
| }, | |
| { | |
| "tafsir_id": "en-tafisr-ibn-kathir", | |
| "scholar": "Hafiz Ibn Kathir", | |
| "tafsir_name": "Tafsir Ibn Kathir (English)", | |
| "language": "en", | |
| "source": "Darussalam Tafsir Ibn Kathir (Abridged)", | |
| "fetcher": "sqlite", | |
| "db_key": "en-tafisr-ibn-kathir", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafseer-ibn-e-kaseer", | |
| "scholar": "Hafiz Ibn Kathir", | |
| "tafsir_name": "Tafsir Ibn Kathir (تفسیر ابن کثیر اردو)", | |
| "language": "ur", | |
| "source": "Tafsir Ibn Kathir Urdu Translation", | |
| "fetcher": "sqlite", | |
| "db_key": "tafseer-ibn-e-kaseer-urdu", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafsir-bayan-ul-quran", | |
| "scholar": "Dr. Israr Ahmed", | |
| "tafsir_name": "Bayan-ul-Quran (بیان القرآن)", | |
| "language": "ur", | |
| "source": "Markazi Anjuman Khuddam-ul-Quran Lahore", | |
| "fetcher": "sqlite", | |
| "db_key": "tafsir-bayan-ul-quran", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafsir-tafheem-ul-quran", | |
| "scholar": "Syed Abul A'la Maududi", | |
| "tafsir_name": "Tafheem-ul-Quran (تفہیم القرآن مع تفسیری حواشی)", | |
| "language": "ur", | |
| "source": "Tafheem-ul-Quran (Maududi Complete Commentary)", | |
| "fetcher": "qurancom_tafheem_ur", | |
| }, | |
| { | |
| "tafsir_id": "en-tafsir-tafheem-ul-quran", | |
| "scholar": "Syed Abul A'la Maududi", | |
| "tafsir_name": "Towards Understanding the Quran (Tafhim Commentary)", | |
| "language": "en", | |
| "source": "Towards Understanding the Quran (Maududi Complete Commentary)", | |
| "fetcher": "qurancom_tafheem_en", | |
| }, | |
| { | |
| "tafsir_id": "en-tafsir-maarif-ul-quran", | |
| "scholar": "Mufti Muhammad Shafi", | |
| "tafsir_name": "Ma’ariful Quran (English)", | |
| "language": "en", | |
| "source": "Maktaba Darul Uloom Karachi", | |
| "fetcher": "sqlite", | |
| "db_key": "en-tafsir-maarif-ul-quran", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafsir-fee-zilal-il-quran", | |
| "scholar": "Sayyid Qutb", | |
| "tafsir_name": "Fi Zilal al-Quran (فی ظلال القرآن)", | |
| "language": "ur", | |
| "source": "Fi Zilal al-Quran Urdu", | |
| "fetcher": "spa5k", | |
| "spa5k_slug": "tafsir-fe-zalul-quran-syed-qatab", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafsir-tazkirul-quran", | |
| "scholar": "Maulana Wahiduddin Khan", | |
| "tafsir_name": "Tazkirul Quran (تذکیر القرآن)", | |
| "language": "ur", | |
| "source": "Goodword Books / Tazkirul Quran", | |
| "fetcher": "spa5k", | |
| "spa5k_slug": "ur-tazkirul-quran", | |
| }, | |
| { | |
| "tafsir_id": "en-tafsir-tazkirul-quran", | |
| "scholar": "Maulana Wahiduddin Khan", | |
| "tafsir_name": "Tazkirul Quran (English)", | |
| "language": "en", | |
| "source": "Goodword Books / Tazkirul Quran", | |
| "fetcher": "spa5k", | |
| "spa5k_slug": "en-tazkirul-quran", | |
| }, | |
| { | |
| "tafsir_id": "ur-tafsir-as-saadi-urdu", | |
| "scholar": "Shaykh Abdur Rahman al-Sa'di", | |
| "tafsir_name": "Tafsir As-Sa’di (تفسیر السعدی اردو)", | |
| "language": "ur", | |
| "source": "Tafsir As-Sa'di Urdu Translation", | |
| "fetcher": "sqlite", | |
| "db_key": "ur-tafsir-as-saadi-urdu", | |
| }, | |
| { | |
| "tafsir_id": "en-al-jalalayn", | |
| "scholar": "Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti", | |
| "tafsir_name": "Tafsir Al-Jalalayn (English)", | |
| "language": "en", | |
| "source": "Firas Hamza / Royal Aal al-Bayt Institute", | |
| "fetcher": "sqlite", | |
| "db_key": "tafsir-al-jalalayn", | |
| }, | |
| { | |
| "tafsir_id": "en-tafsir-al-mukhtasar", | |
| "scholar": "Tafsir Center for Quranic Studies", | |
| "tafsir_name": "Al-Mukhtasar in Interpreting the Noble Quran", | |
| "language": "en", | |
| "source": "Tafsir Center for Quranic Studies, Riyadh", | |
| "fetcher": "sqlite", | |
| "db_key": "en-tafsir-al-mukhtasar", | |
| }, | |
| { | |
| "tafsir_id": "ar-tafsir-muyassar", | |
| "scholar": "King Fahd Complex, Medina", | |
| "tafsir_name": "Al-Tafsir Al-Muyassar (التفسير الميسر)", | |
| "language": "ar", | |
| "source": "King Fahd Glorious Quran Printing Complex", | |
| "fetcher": "sqlite", | |
| "db_key": "ar-tafsir-muyassar", | |
| }, | |
| { | |
| "tafsir_id": "ar-tafsir-as-saadi", | |
| "scholar": "Shaykh Abdur Rahman al-Sa'di", | |
| "tafsir_name": "Tafsir As-Sa’di (تيسير الكريم الرحمن)", | |
| "language": "ar", | |
| "source": "Taysir al-Karim al-Rahman fi Tafsir Kalam al-Mannan", | |
| "fetcher": "sqlite", | |
| "db_key": "ar-tafsir-as-saadi", | |
| }, | |
| { | |
| "tafsir_id": "ar-tafsir-ibn-kathir", | |
| "scholar": "Hafiz Ibn Kathir", | |
| "tafsir_name": "Tafsir Ibn Kathir (تفسير القرآن العظيم)", | |
| "language": "ar", | |
| "source": "Tafsir al-Quran al-Azim", | |
| "fetcher": "sqlite", | |
| "db_key": "ar-tafsir-ibn-kathir", | |
| }, | |
| { | |
| "tafsir_id": "bn-tafsir-ahsanul-bayan", | |
| "scholar": "Salahuddin Yusuf (King Fahd Complex)", | |
| "tafsir_name": "Ahsanul Bayaan (তাফসীর আহসানুল বায়ান)", | |
| "language": "bn", | |
| "source": "King Fahd Complex / Tawheed Publication", | |
| "fetcher": "sqlite", | |
| "db_key": "bn-tafsir-ahsanul-bayaan", | |
| }, | |
| ] | |
| # ----------------------------------------------------------------------------- | |
| # HELPER FUNCTIONS | |
| # ----------------------------------------------------------------------------- | |
| def clean_html(raw: str) -> str: | |
| if not raw: | |
| return "" | |
| # Strip HTML tags | |
| text = re.sub(r"<[^>]+>", " ", raw) | |
| text = html.unescape(text) | |
| # Clean multiple whitespace & newlines | |
| text = re.sub(r"[ \t]+", " ", text) | |
| text = re.sub(r"\n\s*\n+", "\n\n", text) | |
| return text.strip() | |
| def http_get_json(url: str, timeout: int = 15) -> Any: | |
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}) | |
| with urllib.request.urlopen(req, timeout=timeout) as resp: | |
| return json.loads(resp.read().decode("utf-8")) | |
| # Global in-memory cache for Fawaz Ahmed Taqi Usmani English | |
| _FAWAZ_TAQI_EN_CACHE: Dict[int, Dict[int, str]] = {} | |
| def load_fawaz_taqi_en() -> Dict[int, Dict[int, str]]: | |
| global _FAWAZ_TAQI_EN_CACHE | |
| if _FAWAZ_TAQI_EN_CACHE: | |
| return _FAWAZ_TAQI_EN_CACHE | |
| cache_file = CACHE_DIR / "eng-muftitaqiusmani.min.json" | |
| if cache_file.exists(): | |
| with open(cache_file, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| else: | |
| print(" Downloading Taqi Usmani English (Fawaz Ahmed repository)...") | |
| url = "https://cdn.jsdelivr.net/gh/fawazahmed0/quran-api@1/editions/eng-muftitaqiusmani.min.json" | |
| data = http_get_json(url) | |
| with open(cache_file, "w", encoding="utf-8") as f: | |
| json.dump(data, f, ensure_ascii=False) | |
| for item in data.get("quran", []): | |
| s = item["chapter"] | |
| a = item["verse"] | |
| t = item["text"] | |
| if s not in _FAWAZ_TAQI_EN_CACHE: | |
| _FAWAZ_TAQI_EN_CACHE[s] = {} | |
| _FAWAZ_TAQI_EN_CACHE[s][a] = clean_html(t) | |
| return _FAWAZ_TAQI_EN_CACHE | |
| # Footnote fetcher with disk cache & retry | |
| def fetch_single_footnote(fid: str) -> str: | |
| cache_path = FN_CACHE_DIR / f"{fid}.txt" | |
| if cache_path.exists(): | |
| try: | |
| with open(cache_path, "r", encoding="utf-8") as f: | |
| return f.read() | |
| except Exception: | |
| pass | |
| url = f"https://api.quran.com/api/v4/foot_notes/{fid}" | |
| for attempt in range(3): | |
| try: | |
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) | |
| with urllib.request.urlopen(req, timeout=10) as r: | |
| d = json.loads(r.read().decode("utf-8")) | |
| text = d.get("foot_note", {}).get("text", "") | |
| cleaned = clean_html(text) | |
| with open(cache_path, "w", encoding="utf-8") as f: | |
| f.write(cleaned) | |
| return cleaned | |
| except Exception as e: | |
| time.sleep(0.5 * (attempt + 1)) | |
| return "" | |
| def fetch_qurancom_tafheem(surah: int, res_id: int, is_urdu: bool, total_ayahs: int) -> Dict[int, str]: | |
| """Fetches full Tafheem-ul-Quran translation and merges footnotes per Ayah.""" | |
| cache_file = CACHE_DIR / f"tafheem_{'ur' if is_urdu else 'en'}_{surah}.json" | |
| if cache_file.exists(): | |
| with open(cache_file, "r", encoding="utf-8") as f: | |
| cached_map = json.load(f) | |
| return {int(k): v for k, v in cached_map.items()} | |
| url = f"https://api.quran.com/api/v4/quran/translations/{res_id}?chapter_number={surah}" | |
| print(f" Fetching Tafheem-ul-Quran ({'Urdu' if is_urdu else 'English'}) for Surah {surah} from Quran.com...") | |
| data = http_get_json(url) | |
| translations = data.get("translations", []) | |
| # Map of ayah_no -> list of foot_note IDs | |
| ayah_footnotes: Dict[int, List[str]] = {} | |
| ayah_trans: Dict[int, str] = {} | |
| all_fn_ids: List[str] = [] | |
| for idx, item in enumerate(translations): | |
| a_no = idx + 1 | |
| raw_text = item.get("text", "") | |
| fn_ids = re.findall(r'foot_note[ =\"\\\']+([0-9]+)', raw_text) | |
| ayah_footnotes[a_no] = fn_ids | |
| all_fn_ids.extend(fn_ids) | |
| ayah_trans[a_no] = clean_html(raw_text) | |
| # Fetch unique footnotes concurrently | |
| unique_fn_ids = list(set(all_fn_ids)) | |
| fn_text_map: Dict[str, str] = {} | |
| if unique_fn_ids: | |
| print(f" Fetching {len(unique_fn_ids)} footnotes concurrently...") | |
| with ThreadPoolExecutor(max_workers=16) as pool: | |
| results = pool.map(fetch_single_footnote, unique_fn_ids) | |
| for fid, ftxt in zip(unique_fn_ids, results): | |
| fn_text_map[fid] = ftxt | |
| # Merge translation + footnotes | |
| merged: Dict[int, str] = {} | |
| for a_no in range(1, total_ayahs + 1): | |
| trans = ayah_trans.get(a_no, "") | |
| fn_ids = ayah_footnotes.get(a_no, []) | |
| notes = [fn_text_map[fid] for fid in fn_ids if fid in fn_text_map and fn_text_map[fid].strip()] | |
| if notes: | |
| if is_urdu: | |
| notes_block = "\n\n【تفسیر و حواشی】:\n" + "\n\n".join(notes) | |
| else: | |
| notes_block = "\n\n[Tafsir & Commentary Notes]:\n" + "\n\n".join(notes) | |
| merged[a_no] = (trans + notes_block).strip() | |
| else: | |
| merged[a_no] = trans.strip() | |
| # Cache locally | |
| with open(cache_file, "w", encoding="utf-8") as f: | |
| json.dump(merged, f, ensure_ascii=False) | |
| return merged | |
| def fetch_spa5k_surah(slug: str, surah: int) -> Dict[int, str]: | |
| """Fetches Surah tafseer from spa5k CDN with local disk caching.""" | |
| cache_file = CACHE_DIR / f"spa5k_{slug}_{surah}.json" | |
| if cache_file.exists(): | |
| with open(cache_file, "r", encoding="utf-8") as f: | |
| cached_map = json.load(f) | |
| return {int(k): v for k, v in cached_map.items()} | |
| url = f"https://cdn.jsdelivr.net/gh/spa5k/tafsir_api@main/tafsir/{slug}/{surah}.json" | |
| data = http_get_json(url) | |
| result: Dict[int, str] = {} | |
| if isinstance(data, dict) and "ayahs" in data: | |
| items = data["ayahs"] | |
| elif isinstance(data, list): | |
| items = data | |
| else: | |
| items = [] | |
| for idx, item in enumerate(items): | |
| ayah_no = item.get("ayah", idx + 1) | |
| txt = clean_html(item.get("text", "")) | |
| result[ayah_no] = txt | |
| with open(cache_file, "w", encoding="utf-8") as f: | |
| json.dump(result, f, ensure_ascii=False) | |
| return result | |
| def fetch_sqlite_surah(conn: sqlite3.Connection, db_key: str, surah: int) -> Dict[int, str]: | |
| """Queries SQLite database for pre-ingested classical tafseers.""" | |
| cur = conn.cursor() | |
| cur.execute("SELECT ayah, text FROM tafseers WHERE tafsir_id=? AND surah=? ORDER BY ayah ASC;", (db_key, surah)) | |
| rows = cur.fetchall() | |
| return {row[0]: clean_html(row[1]) for row in rows} | |
| # ----------------------------------------------------------------------------- | |
| # MASTER COMPILER FOR A SINGLE SURAH | |
| # ----------------------------------------------------------------------------- | |
| def compile_surah(surah: int, conn: sqlite3.Connection) -> Dict[str, Any]: | |
| if surah not in SURAH_METADATA: | |
| raise ValueError(f"Invalid surah number {surah}. Must be 1-114.") | |
| meta = SURAH_METADATA[surah] | |
| total_ayahs = meta["ayahs"] | |
| print(f"\n=======================================================") | |
| print(f"COMPILING SURAH {surah}: {meta['name_en']} ({meta['name_ar']})") | |
| print(f"Total Ayahs: {total_ayahs} | Expected Editions: {len(EDITIONS_META)}") | |
| print(f"=======================================================") | |
| # 1. Pre-fetch all 17 edition maps for this surah | |
| edition_ayah_maps: Dict[str, Dict[int, str]] = {} | |
| # SQLite editions | |
| print("→ Querying pre-indexed classical editions from local database...") | |
| for ed in EDITIONS_META: | |
| if ed["fetcher"] == "sqlite": | |
| edition_ayah_maps[ed["tafsir_id"]] = fetch_sqlite_surah(conn, ed["db_key"], surah) | |
| # Fawaz Taqi Usmani English | |
| print("→ Loading Mufti Taqi Usmani English...") | |
| taqi_en_all = load_fawaz_taqi_en() | |
| edition_ayah_maps["en-tafsir-taqi-usmani"] = taqi_en_all.get(surah, {}) | |
| # spa5k editions | |
| for ed in EDITIONS_META: | |
| if ed["fetcher"] == "spa5k": | |
| print(f"→ Loading {ed['scholar']} ({ed['tafsir_name']})...") | |
| edition_ayah_maps[ed["tafsir_id"]] = fetch_spa5k_surah(ed["spa5k_slug"], surah) | |
| # Quran.com Tafheem editions | |
| print("→ Loading Tafheem-ul-Quran Urdu with complete commentary notes...") | |
| edition_ayah_maps["ur-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 97, True, total_ayahs) | |
| print("→ Loading Towards Understanding the Quran English with complete commentary notes...") | |
| edition_ayah_maps["en-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 95, False, total_ayahs) | |
| # 2. Assemble Ayah by Ayah JSON | |
| ayahs_list: List[Dict[str, Any]] = [] | |
| missing_report: List[str] = [] | |
| for a_no in range(1, total_ayahs + 1): | |
| ayah_tafsirs: List[Dict[str, Any]] = [] | |
| for ed in EDITIONS_META: | |
| tid = ed["tafsir_id"] | |
| emap = edition_ayah_maps.get(tid, {}) | |
| text = emap.get(a_no, "").strip() | |
| if not text: | |
| missing_report.append(f"Ayah {surah}:{a_no} missing '{tid}' ({ed['scholar']})") | |
| ayah_tafsirs.append({ | |
| "tafsir_id": tid, | |
| "scholar": ed["scholar"], | |
| "tafsir_name": ed["tafsir_name"], | |
| "language": ed["language"], | |
| "text": text, | |
| "source": ed["source"], | |
| }) | |
| ayahs_list.append({ | |
| "ayah_no": a_no, | |
| "reference": f"{surah}:{a_no}", | |
| "tafsirs": ayah_tafsirs, | |
| }) | |
| # Consistency check | |
| if missing_report: | |
| print(f"\n[WARNING] Found {len(missing_report)} missing text items!") | |
| for m in missing_report[:10]: | |
| print(f" - {m}") | |
| if len(missing_report) > 10: | |
| print(f" ... and {len(missing_report) - 10} more.") | |
| else: | |
| print(f"\n✓ 101% Verification Passed! All {total_ayahs} Ayahs have complete text for all {len(EDITIONS_META)} scholars.") | |
| result_json = { | |
| "schema_version": "1.0", | |
| "surahs": [ | |
| { | |
| "surah_no": surah, | |
| "surah_name_arabic": meta["name_ar"], | |
| "surah_name_english": meta["name_en"], | |
| "total_ayahs": total_ayahs, | |
| "ayahs": ayahs_list, | |
| } | |
| ], | |
| } | |
| # 3. Synchronize to all target directories | |
| file_name = meta["file_name"] | |
| for out_dir in SYNC_DIRS: | |
| target_file = out_dir / file_name | |
| with open(target_file, "w", encoding="utf-8") as f: | |
| json.dump(result_json, f, ensure_ascii=False, indent=2) | |
| print(f"✓ Saved: {target_file} ({os.path.getsize(target_file) / (1024 * 1024):.2f} MB)") | |
| return result_json | |
| def compile_surah_safe(surah: int, conn: sqlite3.Connection, max_retries: int = 3) -> bool: | |
| for attempt in range(max_retries): | |
| try: | |
| compile_surah(surah, conn) | |
| return True | |
| except Exception as e: | |
| print(f"[RETRY {attempt+1}/{max_retries}] Error compiling Surah {surah}: {e}") | |
| time.sleep(2 * (attempt + 1)) | |
| print(f"[FAILED] Surah {surah} failed after {max_retries} attempts.") | |
| return False | |
| # ----------------------------------------------------------------------------- | |
| # MAIN CLI ENTRYPOINT | |
| # ----------------------------------------------------------------------------- | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Master Quran Tafseer Compiler & Generator", | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| epilog=""" | |
| Examples: | |
| python master_tafseer_generator.py --surah 3 # Compiles Surah 3 only | |
| python master_tafseer_generator.py --all # Compiles all 114 Surahs (skips existing) | |
| python master_tafseer_generator.py --start 3 --end 114 # Compiles Surahs 3 to 114 | |
| python master_tafseer_generator.py --all --force # Re-compiles all Surahs even if existing | |
| """, | |
| ) | |
| parser.add_argument("--surah", type=int, default=None, help="Specific Surah number to compile (1-114).") | |
| parser.add_argument("--all", action="store_true", help="Compile all 114 Surahs in batch mode.") | |
| parser.add_argument("--start", type=int, default=1, help="Start Surah number for range (1-114).") | |
| parser.add_argument("--end", type=int, default=114, help="End Surah number for range (1-114).") | |
| parser.add_argument("--force", action="store_true", help="Overwrite existing files even if already present.") | |
| args = parser.parse_args() | |
| if not DB_PATH.exists(): | |
| print(f"ERROR: SQLite database not found at {DB_PATH}!") | |
| sys.exit(1) | |
| conn = sqlite3.connect(str(DB_PATH)) | |
| if args.surah is not None: | |
| compile_surah_safe(args.surah, conn) | |
| print(f"\n🎉 SURAH {args.surah} COMPILATION & SYNCHRONIZATION COMPLETE!") | |
| elif args.all or (args.start != 1 or args.end != 114): | |
| start_surah = args.start | |
| end_surah = args.end | |
| print(f"\n=======================================================") | |
| print(f"STARTING BATCH COMPILATION: Surahs {start_surah} to {end_surah}") | |
| print(f"=======================================================") | |
| success_list = [] | |
| skipped_list = [] | |
| failed_list = [] | |
| for s in range(start_surah, end_surah + 1): | |
| meta = SURAH_METADATA.get(s) | |
| if not meta: | |
| continue | |
| target_file = PROJECT_ROOT / "quran_tafseer" / meta["file_name"] | |
| if target_file.exists() and not args.force: | |
| try: | |
| # Quick integrity check | |
| with open(target_file, "r", encoding="utf-8") as f: | |
| existing = json.load(f) | |
| if len(existing.get("surahs", [{}])[0].get("ayahs", [])) == meta["ayahs"]: | |
| print(f"⏩ Skipping Surah {s} ({meta['name_en']}) - already compiled & verified.") | |
| skipped_list.append(s) | |
| continue | |
| except Exception: | |
| pass | |
| ok = compile_surah_safe(s, conn) | |
| if ok: | |
| success_list.append(s) | |
| else: | |
| failed_list.append(s) | |
| print("\n=======================================================") | |
| print("BATCH COMPILATION SUMMARY") | |
| print(f"✓ Successfully compiled: {len(success_list)} surahs") | |
| print(f"⏩ Already existed (skipped): {len(skipped_list)} surahs") | |
| if failed_list: | |
| print(f"✗ Failed surahs: {failed_list}") | |
| else: | |
| print("🎉 ALL REQUESTED SURAHS VERIFIED & GENERATED WITH 100% SUCCESS!") | |
| print("=======================================================") | |
| else: | |
| # Default single surah instruction if neither flag is passed | |
| compile_surah_safe(2, conn) | |
| print("\n🎉 SURAH 2 COMPILATION COMPLETE! (Pass --all to batch compile all Surahs)") | |
| conn.close() | |
| if __name__ == "__main__": | |
| main() | |