superadmin / scripts /master_tafseer_generator.py
suhailshafi007's picture
feat: integrate Cloudflare R2 Quran translations, add translation endpoints and edge caching
3c6e2f1
Raw History Blame Contribute Delete
37.3 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
=============================================================================
MASTER QURAN TAFSEER COMPILER & GENERATOR
=============================================================================
Compiles verified multi-scholar Tafseer JSON files ayah-by-ayah.
Supports generating a single Surah at a time (e.g. --surah 2) or all 114 Surahs
with rigorous consistency, validation, and multi-directory synchronization.
Strictly Curated 17 Sunni Orthodox Scholars & Commentaries:
1. en-tafsir-taqi-usmani (Mufti Muhammad Taqi Usmani)
2. en-tafisr-ibn-kathir (Hafiz Ibn Kathir)
3. ur-tafseer-ibn-e-kaseer (Hafiz Ibn Kathir)
4. ur-tafsir-bayan-ul-quran (Dr. Israr Ahmed)
5. ur-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes)
6. en-tafsir-tafheem-ul-quran (Syed Abul A'la Maududi - With Commentary Notes)
7. en-tafsir-maarif-ul-quran (Mufti Muhammad Shafi)
8. ur-tafsir-fee-zilal-il-quran (Sayyid Qutb)
9. ur-tafsir-tazkirul-quran (Maulana Wahiduddin Khan)
10. en-tafsir-tazkirul-quran (Maulana Wahiduddin Khan)
11. ur-tafsir-as-saadi-urdu (Shaykh Abdur Rahman al-Sa'di)
12. en-al-jalalayn (Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti)
13. en-tafsir-al-mukhtasar (Tafsir Center for Quranic Studies, Riyadh)
14. ar-tafsir-muyassar (King Fahd Complex, Medina)
15. ar-tafsir-as-saadi (Shaykh Abdur Rahman al-Sa'di)
16. ar-tafsir-ibn-kathir (Hafiz Ibn Kathir)
17. bn-tafsir-ahsanul-bayan (Salahuddin Yusuf / King Fahd Complex)
=============================================================================
"""
import argparse
import html
import json
import os
import re
import sqlite3
import sys
import time
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
sys.stdout.reconfigure(encoding="utf-8")
# Root directory paths
PROJECT_ROOT = Path(r"d:\Project-x")
DB_PATH = PROJECT_ROOT / "BACKEND" / "data" / "tafseers.db"
CACHE_DIR = PROJECT_ROOT / "BACKEND" / "data" / "cache"
CACHE_DIR.mkdir(parents=True, exist_ok=True)
FN_CACHE_DIR = CACHE_DIR / "footnotes"
FN_CACHE_DIR.mkdir(parents=True, exist_ok=True)
SYNC_DIRS = [
PROJECT_ROOT / "quran_tafseer",
PROJECT_ROOT / "BACKEND" / "data" / "quran_tafseer",
PROJECT_ROOT / "FRONTEND" / "assets" / "tafsir",
]
for d in SYNC_DIRS:
d.mkdir(parents=True, exist_ok=True)
# -----------------------------------------------------------------------------
# SURAH METADATA DICTIONARY (1-114)
# -----------------------------------------------------------------------------
SURAH_METADATA = {
1: {"name_ar": "سورة الفاتحة", "name_en": "Al-Faatihah", "file_name": "Al_Fatihah_Tafseer.json", "ayahs": 7},
2: {"name_ar": "سورة البقرة", "name_en": "Al-Baqarah", "file_name": "Al_Baqarah_Tafseer.json", "ayahs": 286},
3: {"name_ar": "سورة آل عمران", "name_en": "Aal-i-Imraan", "file_name": "Aal_Imran_Tafseer.json", "ayahs": 200},
4: {"name_ar": "سورة النساء", "name_en": "An-Nisaa", "file_name": "An_Nisa_Tafseer.json", "ayahs": 176},
5: {"name_ar": "سورة المائدة", "name_en": "Al-Maaida", "file_name": "Al_Maidah_Tafseer.json", "ayahs": 120},
6: {"name_ar": "سورة الأنعام", "name_en": "Al-An'aam", "file_name": "Al_Anam_Tafseer.json", "ayahs": 165},
7: {"name_ar": "سورة الأعراف", "name_en": "Al-A'raaf", "file_name": "Al_Araf_Tafseer.json", "ayahs": 206},
8: {"name_ar": "سورة الأنفال", "name_en": "Al-Anfaal", "file_name": "Al_Anfal_Tafseer.json", "ayahs": 75},
9: {"name_ar": "سورة التوبة", "name_en": "At-Tawba", "file_name": "At_Tawbah_Tafseer.json", "ayahs": 129},
10: {"name_ar": "سورة يونس", "name_en": "Yunus", "file_name": "Yunus_Tafseer.json", "ayahs": 109},
11: {"name_ar": "سورة هود", "name_en": "Hud", "file_name": "Hud_Tafseer.json", "ayahs": 123},
12: {"name_ar": "سورة يوسف", "name_en": "Yusuf", "file_name": "Yusuf_Tafseer.json", "ayahs": 111},
13: {"name_ar": "سورة الرعد", "name_en": "Ar-Ra'd", "file_name": "Ar_Rad_Tafseer.json", "ayahs": 43},
14: {"name_ar": "سورة إبراهيم", "name_en": "Ibrahim", "file_name": "Ibrahim_Tafseer.json", "ayahs": 52},
15: {"name_ar": "سورة الحجر", "name_en": "Al-Hijr", "file_name": "Al_Hijr_Tafseer.json", "ayahs": 99},
16: {"name_ar": "سورة النحل", "name_en": "An-Nahl", "file_name": "An_Nahl_Tafseer.json", "ayahs": 128},
17: {"name_ar": "سورة الإسراء", "name_en": "Al-Israa", "file_name": "Al_Isra_Tafseer.json", "ayahs": 111},
18: {"name_ar": "سورة الكهف", "name_en": "Al-Kahf", "file_name": "Al_Kahf_Tafseer.json", "ayahs": 110},
19: {"name_ar": "سورة مريم", "name_en": "Maryam", "file_name": "Maryam_Tafseer.json", "ayahs": 98},
20: {"name_ar": "سورة طه", "name_en": "Taa-Haa", "file_name": "Ta_Ha_Tafseer.json", "ayahs": 135},
21: {"name_ar": "سورة الأنبياء", "name_en": "Al-Anbiyaa", "file_name": "Al_Anbiya_Tafseer.json", "ayahs": 112},
22: {"name_ar": "سورة الحج", "name_en": "Al-Hajj", "file_name": "Al_Hajj_Tafseer.json", "ayahs": 78},
23: {"name_ar": "سورة المؤمنون", "name_en": "Al-Muminoon", "file_name": "Al_Muminun_Tafseer.json", "ayahs": 118},
24: {"name_ar": "سورة النور", "name_en": "An-Noor", "file_name": "An_Nur_Tafseer.json", "ayahs": 64},
25: {"name_ar": "سورة الفرقان", "name_en": "Al-Furqaan", "file_name": "Al_Furqan_Tafseer.json", "ayahs": 77},
26: {"name_ar": "سورة الشعراء", "name_en": "Ash-Shu'araa", "file_name": "Ash_Shuara_Tafseer.json", "ayahs": 227},
27: {"name_ar": "سورة النمل", "name_en": "An-Naml", "file_name": "An_Naml_Tafseer.json", "ayahs": 93},
28: {"name_ar": "سورة القصص", "name_en": "Al-Qasas", "file_name": "Al_Qasas_Tafseer.json", "ayahs": 88},
29: {"name_ar": "سورة العنكبوت", "name_en": "Al-Ankaboot", "file_name": "Al_Ankabut_Tafseer.json", "ayahs": 69},
30: {"name_ar": "سورة الروم", "name_en": "Ar-Room", "file_name": "Ar_Rum_Tafseer.json", "ayahs": 60},
31: {"name_ar": "سورة لقمان", "name_en": "Luqman", "file_name": "Luqman_Tafseer.json", "ayahs": 34},
32: {"name_ar": "سورة السجدة", "name_en": "As-Sajda", "file_name": "As_Sajdah_Tafseer.json", "ayahs": 30},
33: {"name_ar": "سورة الأحزاب", "name_en": "Al-Ahzaab", "file_name": "Al_Ahzab_Tafseer.json", "ayahs": 73},
34: {"name_ar": "سورة سبإ", "name_en": "Saba", "file_name": "Saba_Tafseer.json", "ayahs": 54},
35: {"name_ar": "سورة فاطر", "name_en": "Faatir", "file_name": "Fatir_Tafseer.json", "ayahs": 45},
36: {"name_ar": "سورة يس", "name_en": "Yaseen", "file_name": "Ya_Sin_Tafseer.json", "ayahs": 83},
37: {"name_ar": "سورة الصافات", "name_en": "As-Saaffaat", "file_name": "As_Saffat_Tafseer.json", "ayahs": 182},
38: {"name_ar": "سورة ص", "name_en": "Saad", "file_name": "Sad_Tafseer.json", "ayahs": 88},
39: {"name_ar": "سورة الزمر", "name_en": "Az-Zumar", "file_name": "Az_Zumar_Tafseer.json", "ayahs": 75},
40: {"name_ar": "سورة غافر", "name_en": "Ghafir", "file_name": "Ghafir_Tafseer.json", "ayahs": 85},
41: {"name_ar": "سورة فصلت", "name_en": "Fussilat", "file_name": "Fussilat_Tafseer.json", "ayahs": 54},
42: {"name_ar": "سورة الشورى", "name_en": "Ash-Shura", "file_name": "Ash_Shura_Tafseer.json", "ayahs": 53},
43: {"name_ar": "سورة الزخرف", "name_en": "Az-Zukhruf", "file_name": "Az_Zukhruf_Tafseer.json", "ayahs": 89},
44: {"name_ar": "سورة الدخان", "name_en": "Ad-Dukhaan", "file_name": "Ad_Dukhan_Tafseer.json", "ayahs": 59},
45: {"name_ar": "سورة الجاثية", "name_en": "Al-Jaathiya", "file_name": "Al_Jathiyah_Tafseer.json", "ayahs": 37},
46: {"name_ar": "سورة الأحقاف", "name_en": "Al-Ahqaaf", "file_name": "Al_Ahqaf_Tafseer.json", "ayahs": 35},
47: {"name_ar": "سورة محمد", "name_en": "Muhammad", "file_name": "Muhammad_Tafseer.json", "ayahs": 38},
48: {"name_ar": "سورة الفتح", "name_en": "Al-Fath", "file_name": "Al_Fath_Tafseer.json", "ayahs": 29},
49: {"name_ar": "سورة الحجرات", "name_en": "Al-Hujuraat", "file_name": "Al_Hujurat_Tafseer.json", "ayahs": 18},
50: {"name_ar": "سورة ق", "name_en": "Qaaf", "file_name": "Qaf_Tafseer.json", "ayahs": 45},
51: {"name_ar": "سورة الذاريات", "name_en": "Adh-Dhaariyaat", "file_name": "Adh_Dhariyat_Tafseer.json", "ayahs": 60},
52: {"name_ar": "سورة الطور", "name_en": "At-Toor", "file_name": "At_Tur_Tafseer.json", "ayahs": 49},
53: {"name_ar": "سورة النجم", "name_en": "An-Najm", "file_name": "An_Najm_Tafseer.json", "ayahs": 62},
54: {"name_ar": "سورة القمر", "name_en": "Al-Qamar", "file_name": "Al_Qamar_Tafseer.json", "ayahs": 55},
55: {"name_ar": "سورة الرحمن", "name_en": "Ar-Rahmaan", "file_name": "Ar_Rahman_Tafseer.json", "ayahs": 78},
56: {"name_ar": "سورة الواقعة", "name_en": "Al-Waaqia", "file_name": "Al_Waqiah_Tafseer.json", "ayahs": 96},
57: {"name_ar": "سورة الحديد", "name_en": "Al-Hadid", "file_name": "Al_Hadid_Tafseer.json", "ayahs": 29},
58: {"name_ar": "سورة المجادلة", "name_en": "Al-Mujaadila", "file_name": "Al_Mujadilah_Tafseer.json", "ayahs": 22},
59: {"name_ar": "سورة الحشر", "name_en": "Al-Hashr", "file_name": "Al_Hashr_Tafseer.json", "ayahs": 24},
60: {"name_ar": "سورة الممتحنة", "name_en": "Al-Mumtahana", "file_name": "Al_Mumtahanah_Tafseer.json", "ayahs": 13},
61: {"name_ar": "سورة الصف", "name_en": "As-Saff", "file_name": "As_Saff_Tafseer.json", "ayahs": 14},
62: {"name_ar": "سورة الجمعة", "name_en": "Al-Jumu'a", "file_name": "Al_Jumuah_Tafseer.json", "ayahs": 11},
63: {"name_ar": "سورة المنافقون", "name_en": "Al-Munaafiqoon", "file_name": "Al_Munafiqun_Tafseer.json", "ayahs": 11},
64: {"name_ar": "سورة التغابن", "name_en": "At-Taghaabun", "file_name": "At_Taghabun_Tafseer.json", "ayahs": 18},
65: {"name_ar": "سورة الطلاق", "name_en": "At-Talaaq", "file_name": "At_Talaq_Tafseer.json", "ayahs": 12},
66: {"name_ar": "سورة التحريم", "name_en": "At-Tahrim", "file_name": "At_Tahrim_Tafseer.json", "ayahs": 12},
67: {"name_ar": "سورة الملك", "name_en": "Al-Mulk", "file_name": "Al_Mulk_Tafseer.json", "ayahs": 30},
68: {"name_ar": "سورة القلم", "name_en": "Al-Qalam", "file_name": "Al_Qalam_Tafseer.json", "ayahs": 52},
69: {"name_ar": "سورة الحاقة", "name_en": "Al-Haaqqa", "file_name": "Al_Haqqah_Tafseer.json", "ayahs": 52},
70: {"name_ar": "سورة المعارج", "name_en": "Al-Ma'aarij", "file_name": "Al_Maarij_Tafseer.json", "ayahs": 44},
71: {"name_ar": "سورة نوح", "name_en": "Nooh", "file_name": "Nuh_Tafseer.json", "ayahs": 28},
72: {"name_ar": "سورة الجن", "name_en": "Al-Jinn", "file_name": "Al_Jinn_Tafseer.json", "ayahs": 28},
73: {"name_ar": "سورة المزمل", "name_en": "Al-Muzzammil", "file_name": "Al_Muzzammil_Tafseer.json", "ayahs": 20},
74: {"name_ar": "سورة المدثر", "name_en": "Al-Muddaththir", "file_name": "Al_Muddaththir_Tafseer.json", "ayahs": 56},
75: {"name_ar": "سورة القيامة", "name_en": "Al-Qiyaama", "file_name": "Al_Qiyamah_Tafseer.json", "ayahs": 40},
76: {"name_ar": "سورة الإنسان", "name_en": "Al-Insaan", "file_name": "Al_Insan_Tafseer.json", "ayahs": 31},
77: {"name_ar": "سورة المرسلات", "name_en": "Al-Mursalaat", "file_name": "Al_Mursalat_Tafseer.json", "ayahs": 50},
78: {"name_ar": "سورة النبإ", "name_en": "An-Naba", "file_name": "An_Naba_Tafseer.json", "ayahs": 40},
79: {"name_ar": "سورة النازعات", "name_en": "An-Naazi'aat", "file_name": "An_Naziat_Tafseer.json", "ayahs": 46},
80: {"name_ar": "سورة عبس", "name_en": "Abasa", "file_name": "Abasa_Tafseer.json", "ayahs": 42},
81: {"name_ar": "سورة التكوير", "name_en": "At-Takwir", "file_name": "At_Takwir_Tafseer.json", "ayahs": 29},
82: {"name_ar": "سورة الانفطار", "name_en": "Al-Infitaar", "file_name": "Al_Infitar_Tafseer.json", "ayahs": 19},
83: {"name_ar": "سورة المطففين", "name_en": "Al-Mutaffifin", "file_name": "Al_Mutaffifin_Tafseer.json", "ayahs": 36},
84: {"name_ar": "سورة الانشقاق", "name_en": "Al-Inshiqaaq", "file_name": "Al_Inshiqaq_Tafseer.json", "ayahs": 25},
85: {"name_ar": "سورة البروج", "name_en": "Al-Burooj", "file_name": "Al_Buruj_Tafseer.json", "ayahs": 22},
86: {"name_ar": "سورة الطارق", "name_en": "At-Taariq", "file_name": "At_Tariq_Tafseer.json", "ayahs": 17},
87: {"name_ar": "سورة الأعلى", "name_en": "Al-A'laa", "file_name": "Al_Ala_Tafseer.json", "ayahs": 19},
88: {"name_ar": "سورة الغاشية", "name_en": "Al-Ghaashiya", "file_name": "Al_Ghashiyah_Tafseer.json", "ayahs": 26},
89: {"name_ar": "سورة الفجر", "name_en": "Al-Fajr", "file_name": "Al_Fajr_Tafseer.json", "ayahs": 30},
90: {"name_ar": "سورة البلد", "name_en": "Al-Balad", "file_name": "Al_Balad_Tafseer.json", "ayahs": 20},
91: {"name_ar": "سورة الشمس", "name_en": "Ash-Shams", "file_name": "Ash_Shams_Tafseer.json", "ayahs": 15},
92: {"name_ar": "سورة الليل", "name_en": "Al-Layl", "file_name": "Al_Lail_Tafseer.json", "ayahs": 21},
93: {"name_ar": "سورة الضحى", "name_en": "Ad-Dhuhaa", "file_name": "Ad_Duha_Tafseer.json", "ayahs": 11},
94: {"name_ar": "سورة الشرح", "name_en": "Ash-Sharh", "file_name": "Ash_Sharh_Tafseer.json", "ayahs": 8},
95: {"name_ar": "سورة التين", "name_en": "At-Tin", "file_name": "At_Tin_Tafseer.json", "ayahs": 8},
96: {"name_ar": "سورة العلق", "name_en": "Al-Alaq", "file_name": "Al_Alaq_Tafseer.json", "ayahs": 19},
97: {"name_ar": "سورة القدر", "name_en": "Al-Qadr", "file_name": "Al_Qadr_Tafseer.json", "ayahs": 5},
98: {"name_ar": "سورة البينة", "name_en": "Al-Bayyina", "file_name": "Al_Bayyinah_Tafseer.json", "ayahs": 8},
99: {"name_ar": "سورة الزلزلة", "name_en": "Az-Zalzala", "file_name": "Az_Zalzalah_Tafseer.json", "ayahs": 8},
100: {"name_ar": "سورة العاديات", "name_en": "Al-Aadiyaat", "file_name": "Al_Adiyat_Tafseer.json", "ayahs": 11},
101: {"name_ar": "سورة القارعة", "name_en": "Al-Qaari'a", "file_name": "Al_Qariah_Tafseer.json", "ayahs": 11},
102: {"name_ar": "سورة التكاثر", "name_en": "At-Takaathur", "file_name": "At_Takathur_Tafseer.json", "ayahs": 8},
103: {"name_ar": "سورة العصر", "name_en": "Al-Asr", "file_name": "Al_Asr_Tafseer.json", "ayahs": 3},
104: {"name_ar": "سورة الهمزة", "name_en": "Al-Humaza", "file_name": "Al_Humazah_Tafseer.json", "ayahs": 9},
105: {"name_ar": "سورة الفيل", "name_en": "Al-Fil", "file_name": "Al_Fil_Tafseer.json", "ayahs": 5},
106: {"name_ar": "سورة قريش", "name_en": "Quraysh", "file_name": "Quraish_Tafseer.json", "ayahs": 4},
107: {"name_ar": "سورة الماعون", "name_en": "Al-Maa'oon", "file_name": "Al_Maun_Tafseer.json", "ayahs": 7},
108: {"name_ar": "سورة الكوثر", "name_en": "Al-Kawthar", "file_name": "Al_Kauthar_Tafseer.json", "ayahs": 3},
109: {"name_ar": "سورة الكافرون", "name_en": "Al-Kaafiroon", "file_name": "Al_Kafirun_Tafseer.json", "ayahs": 6},
110: {"name_ar": "سورة النصر", "name_en": "An-Nasr", "file_name": "An_Nasr_Tafseer.json", "ayahs": 3},
111: {"name_ar": "سورة المسد", "name_en": "Al-Masad", "file_name": "Al_Masad_Tafseer.json", "ayahs": 5},
112: {"name_ar": "سورة الإخلاص", "name_en": "Al-Ikhlaas", "file_name": "Al_Ikhlas_Tafseer.json", "ayahs": 4},
113: {"name_ar": "سورة الفلق", "name_en": "Al-Falaq", "file_name": "Al_Falaq_Tafseer.json", "ayahs": 5},
114: {"name_ar": "سورة الناس", "name_en": "An-Naas", "file_name": "An_Nas_Tafseer.json", "ayahs": 6},
}
# -----------------------------------------------------------------------------
# EDITIONS CONFIGURATION (17 VERIFIED SCHOLARS)
# -----------------------------------------------------------------------------
EDITIONS_META = [
{
"tafsir_id": "en-tafsir-taqi-usmani",
"scholar": "Mufti Muhammad Taqi Usmani",
"tafsir_name": "The Noble Quran (English)",
"language": "en",
"source": "The Noble Quran - Mufti Taqi Usmani",
"fetcher": "fawaz_en_taqi",
},
{
"tafsir_id": "en-tafisr-ibn-kathir",
"scholar": "Hafiz Ibn Kathir",
"tafsir_name": "Tafsir Ibn Kathir (English)",
"language": "en",
"source": "Darussalam Tafsir Ibn Kathir (Abridged)",
"fetcher": "sqlite",
"db_key": "en-tafisr-ibn-kathir",
},
{
"tafsir_id": "ur-tafseer-ibn-e-kaseer",
"scholar": "Hafiz Ibn Kathir",
"tafsir_name": "Tafsir Ibn Kathir (تفسیر ابن کثیر اردو)",
"language": "ur",
"source": "Tafsir Ibn Kathir Urdu Translation",
"fetcher": "sqlite",
"db_key": "tafseer-ibn-e-kaseer-urdu",
},
{
"tafsir_id": "ur-tafsir-bayan-ul-quran",
"scholar": "Dr. Israr Ahmed",
"tafsir_name": "Bayan-ul-Quran (بیان القرآن)",
"language": "ur",
"source": "Markazi Anjuman Khuddam-ul-Quran Lahore",
"fetcher": "sqlite",
"db_key": "tafsir-bayan-ul-quran",
},
{
"tafsir_id": "ur-tafsir-tafheem-ul-quran",
"scholar": "Syed Abul A'la Maududi",
"tafsir_name": "Tafheem-ul-Quran (تفہیم القرآن مع تفسیری حواشی)",
"language": "ur",
"source": "Tafheem-ul-Quran (Maududi Complete Commentary)",
"fetcher": "qurancom_tafheem_ur",
},
{
"tafsir_id": "en-tafsir-tafheem-ul-quran",
"scholar": "Syed Abul A'la Maududi",
"tafsir_name": "Towards Understanding the Quran (Tafhim Commentary)",
"language": "en",
"source": "Towards Understanding the Quran (Maududi Complete Commentary)",
"fetcher": "qurancom_tafheem_en",
},
{
"tafsir_id": "en-tafsir-maarif-ul-quran",
"scholar": "Mufti Muhammad Shafi",
"tafsir_name": "Ma’ariful Quran (English)",
"language": "en",
"source": "Maktaba Darul Uloom Karachi",
"fetcher": "sqlite",
"db_key": "en-tafsir-maarif-ul-quran",
},
{
"tafsir_id": "ur-tafsir-fee-zilal-il-quran",
"scholar": "Sayyid Qutb",
"tafsir_name": "Fi Zilal al-Quran (فی ظلال القرآن)",
"language": "ur",
"source": "Fi Zilal al-Quran Urdu",
"fetcher": "spa5k",
"spa5k_slug": "tafsir-fe-zalul-quran-syed-qatab",
},
{
"tafsir_id": "ur-tafsir-tazkirul-quran",
"scholar": "Maulana Wahiduddin Khan",
"tafsir_name": "Tazkirul Quran (تذکیر القرآن)",
"language": "ur",
"source": "Goodword Books / Tazkirul Quran",
"fetcher": "spa5k",
"spa5k_slug": "ur-tazkirul-quran",
},
{
"tafsir_id": "en-tafsir-tazkirul-quran",
"scholar": "Maulana Wahiduddin Khan",
"tafsir_name": "Tazkirul Quran (English)",
"language": "en",
"source": "Goodword Books / Tazkirul Quran",
"fetcher": "spa5k",
"spa5k_slug": "en-tazkirul-quran",
},
{
"tafsir_id": "ur-tafsir-as-saadi-urdu",
"scholar": "Shaykh Abdur Rahman al-Sa'di",
"tafsir_name": "Tafsir As-Sa’di (تفسیر السعدی اردو)",
"language": "ur",
"source": "Tafsir As-Sa'di Urdu Translation",
"fetcher": "sqlite",
"db_key": "ur-tafsir-as-saadi-urdu",
},
{
"tafsir_id": "en-al-jalalayn",
"scholar": "Jalal al-Din al-Mahalli & Jalal al-Din al-Suyuti",
"tafsir_name": "Tafsir Al-Jalalayn (English)",
"language": "en",
"source": "Firas Hamza / Royal Aal al-Bayt Institute",
"fetcher": "sqlite",
"db_key": "tafsir-al-jalalayn",
},
{
"tafsir_id": "en-tafsir-al-mukhtasar",
"scholar": "Tafsir Center for Quranic Studies",
"tafsir_name": "Al-Mukhtasar in Interpreting the Noble Quran",
"language": "en",
"source": "Tafsir Center for Quranic Studies, Riyadh",
"fetcher": "sqlite",
"db_key": "en-tafsir-al-mukhtasar",
},
{
"tafsir_id": "ar-tafsir-muyassar",
"scholar": "King Fahd Complex, Medina",
"tafsir_name": "Al-Tafsir Al-Muyassar (التفسير الميسر)",
"language": "ar",
"source": "King Fahd Glorious Quran Printing Complex",
"fetcher": "sqlite",
"db_key": "ar-tafsir-muyassar",
},
{
"tafsir_id": "ar-tafsir-as-saadi",
"scholar": "Shaykh Abdur Rahman al-Sa'di",
"tafsir_name": "Tafsir As-Sa’di (تيسير الكريم الرحمن)",
"language": "ar",
"source": "Taysir al-Karim al-Rahman fi Tafsir Kalam al-Mannan",
"fetcher": "sqlite",
"db_key": "ar-tafsir-as-saadi",
},
{
"tafsir_id": "ar-tafsir-ibn-kathir",
"scholar": "Hafiz Ibn Kathir",
"tafsir_name": "Tafsir Ibn Kathir (تفسير القرآن العظيم)",
"language": "ar",
"source": "Tafsir al-Quran al-Azim",
"fetcher": "sqlite",
"db_key": "ar-tafsir-ibn-kathir",
},
{
"tafsir_id": "bn-tafsir-ahsanul-bayan",
"scholar": "Salahuddin Yusuf (King Fahd Complex)",
"tafsir_name": "Ahsanul Bayaan (তাফসীর আহসানুল বায়ান)",
"language": "bn",
"source": "King Fahd Complex / Tawheed Publication",
"fetcher": "sqlite",
"db_key": "bn-tafsir-ahsanul-bayaan",
},
]
# -----------------------------------------------------------------------------
# HELPER FUNCTIONS
# -----------------------------------------------------------------------------
def clean_html(raw: str) -> str:
if not raw:
return ""
# Strip HTML tags
text = re.sub(r"<[^>]+>", " ", raw)
text = html.unescape(text)
# Clean multiple whitespace & newlines
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n\s*\n+", "\n\n", text)
return text.strip()
def http_get_json(url: str, timeout: int = 15) -> Any:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
with urllib.request.urlopen(req, timeout=timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
# Global in-memory cache for Fawaz Ahmed Taqi Usmani English
_FAWAZ_TAQI_EN_CACHE: Dict[int, Dict[int, str]] = {}
def load_fawaz_taqi_en() -> Dict[int, Dict[int, str]]:
global _FAWAZ_TAQI_EN_CACHE
if _FAWAZ_TAQI_EN_CACHE:
return _FAWAZ_TAQI_EN_CACHE
cache_file = CACHE_DIR / "eng-muftitaqiusmani.min.json"
if cache_file.exists():
with open(cache_file, "r", encoding="utf-8") as f:
data = json.load(f)
else:
print(" Downloading Taqi Usmani English (Fawaz Ahmed repository)...")
url = "https://cdn.jsdelivr.net/gh/fawazahmed0/quran-api@1/editions/eng-muftitaqiusmani.min.json"
data = http_get_json(url)
with open(cache_file, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False)
for item in data.get("quran", []):
s = item["chapter"]
a = item["verse"]
t = item["text"]
if s not in _FAWAZ_TAQI_EN_CACHE:
_FAWAZ_TAQI_EN_CACHE[s] = {}
_FAWAZ_TAQI_EN_CACHE[s][a] = clean_html(t)
return _FAWAZ_TAQI_EN_CACHE
# Footnote fetcher with disk cache & retry
def fetch_single_footnote(fid: str) -> str:
cache_path = FN_CACHE_DIR / f"{fid}.txt"
if cache_path.exists():
try:
with open(cache_path, "r", encoding="utf-8") as f:
return f.read()
except Exception:
pass
url = f"https://api.quran.com/api/v4/foot_notes/{fid}"
for attempt in range(3):
try:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=10) as r:
d = json.loads(r.read().decode("utf-8"))
text = d.get("foot_note", {}).get("text", "")
cleaned = clean_html(text)
with open(cache_path, "w", encoding="utf-8") as f:
f.write(cleaned)
return cleaned
except Exception as e:
time.sleep(0.5 * (attempt + 1))
return ""
def fetch_qurancom_tafheem(surah: int, res_id: int, is_urdu: bool, total_ayahs: int) -> Dict[int, str]:
"""Fetches full Tafheem-ul-Quran translation and merges footnotes per Ayah."""
cache_file = CACHE_DIR / f"tafheem_{'ur' if is_urdu else 'en'}_{surah}.json"
if cache_file.exists():
with open(cache_file, "r", encoding="utf-8") as f:
cached_map = json.load(f)
return {int(k): v for k, v in cached_map.items()}
url = f"https://api.quran.com/api/v4/quran/translations/{res_id}?chapter_number={surah}"
print(f" Fetching Tafheem-ul-Quran ({'Urdu' if is_urdu else 'English'}) for Surah {surah} from Quran.com...")
data = http_get_json(url)
translations = data.get("translations", [])
# Map of ayah_no -> list of foot_note IDs
ayah_footnotes: Dict[int, List[str]] = {}
ayah_trans: Dict[int, str] = {}
all_fn_ids: List[str] = []
for idx, item in enumerate(translations):
a_no = idx + 1
raw_text = item.get("text", "")
fn_ids = re.findall(r'foot_note[ =\"\\\']+([0-9]+)', raw_text)
ayah_footnotes[a_no] = fn_ids
all_fn_ids.extend(fn_ids)
ayah_trans[a_no] = clean_html(raw_text)
# Fetch unique footnotes concurrently
unique_fn_ids = list(set(all_fn_ids))
fn_text_map: Dict[str, str] = {}
if unique_fn_ids:
print(f" Fetching {len(unique_fn_ids)} footnotes concurrently...")
with ThreadPoolExecutor(max_workers=16) as pool:
results = pool.map(fetch_single_footnote, unique_fn_ids)
for fid, ftxt in zip(unique_fn_ids, results):
fn_text_map[fid] = ftxt
# Merge translation + footnotes
merged: Dict[int, str] = {}
for a_no in range(1, total_ayahs + 1):
trans = ayah_trans.get(a_no, "")
fn_ids = ayah_footnotes.get(a_no, [])
notes = [fn_text_map[fid] for fid in fn_ids if fid in fn_text_map and fn_text_map[fid].strip()]
if notes:
if is_urdu:
notes_block = "\n\n【تفسیر و حواشی】:\n" + "\n\n".join(notes)
else:
notes_block = "\n\n[Tafsir & Commentary Notes]:\n" + "\n\n".join(notes)
merged[a_no] = (trans + notes_block).strip()
else:
merged[a_no] = trans.strip()
# Cache locally
with open(cache_file, "w", encoding="utf-8") as f:
json.dump(merged, f, ensure_ascii=False)
return merged
def fetch_spa5k_surah(slug: str, surah: int) -> Dict[int, str]:
"""Fetches Surah tafseer from spa5k CDN with local disk caching."""
cache_file = CACHE_DIR / f"spa5k_{slug}_{surah}.json"
if cache_file.exists():
with open(cache_file, "r", encoding="utf-8") as f:
cached_map = json.load(f)
return {int(k): v for k, v in cached_map.items()}
url = f"https://cdn.jsdelivr.net/gh/spa5k/tafsir_api@main/tafsir/{slug}/{surah}.json"
data = http_get_json(url)
result: Dict[int, str] = {}
if isinstance(data, dict) and "ayahs" in data:
items = data["ayahs"]
elif isinstance(data, list):
items = data
else:
items = []
for idx, item in enumerate(items):
ayah_no = item.get("ayah", idx + 1)
txt = clean_html(item.get("text", ""))
result[ayah_no] = txt
with open(cache_file, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False)
return result
def fetch_sqlite_surah(conn: sqlite3.Connection, db_key: str, surah: int) -> Dict[int, str]:
"""Queries SQLite database for pre-ingested classical tafseers."""
cur = conn.cursor()
cur.execute("SELECT ayah, text FROM tafseers WHERE tafsir_id=? AND surah=? ORDER BY ayah ASC;", (db_key, surah))
rows = cur.fetchall()
return {row[0]: clean_html(row[1]) for row in rows}
# -----------------------------------------------------------------------------
# MASTER COMPILER FOR A SINGLE SURAH
# -----------------------------------------------------------------------------
def compile_surah(surah: int, conn: sqlite3.Connection) -> Dict[str, Any]:
if surah not in SURAH_METADATA:
raise ValueError(f"Invalid surah number {surah}. Must be 1-114.")
meta = SURAH_METADATA[surah]
total_ayahs = meta["ayahs"]
print(f"\n=======================================================")
print(f"COMPILING SURAH {surah}: {meta['name_en']} ({meta['name_ar']})")
print(f"Total Ayahs: {total_ayahs} | Expected Editions: {len(EDITIONS_META)}")
print(f"=======================================================")
# 1. Pre-fetch all 17 edition maps for this surah
edition_ayah_maps: Dict[str, Dict[int, str]] = {}
# SQLite editions
print("→ Querying pre-indexed classical editions from local database...")
for ed in EDITIONS_META:
if ed["fetcher"] == "sqlite":
edition_ayah_maps[ed["tafsir_id"]] = fetch_sqlite_surah(conn, ed["db_key"], surah)
# Fawaz Taqi Usmani English
print("→ Loading Mufti Taqi Usmani English...")
taqi_en_all = load_fawaz_taqi_en()
edition_ayah_maps["en-tafsir-taqi-usmani"] = taqi_en_all.get(surah, {})
# spa5k editions
for ed in EDITIONS_META:
if ed["fetcher"] == "spa5k":
print(f"→ Loading {ed['scholar']} ({ed['tafsir_name']})...")
edition_ayah_maps[ed["tafsir_id"]] = fetch_spa5k_surah(ed["spa5k_slug"], surah)
# Quran.com Tafheem editions
print("→ Loading Tafheem-ul-Quran Urdu with complete commentary notes...")
edition_ayah_maps["ur-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 97, True, total_ayahs)
print("→ Loading Towards Understanding the Quran English with complete commentary notes...")
edition_ayah_maps["en-tafsir-tafheem-ul-quran"] = fetch_qurancom_tafheem(surah, 95, False, total_ayahs)
# 2. Assemble Ayah by Ayah JSON
ayahs_list: List[Dict[str, Any]] = []
missing_report: List[str] = []
for a_no in range(1, total_ayahs + 1):
ayah_tafsirs: List[Dict[str, Any]] = []
for ed in EDITIONS_META:
tid = ed["tafsir_id"]
emap = edition_ayah_maps.get(tid, {})
text = emap.get(a_no, "").strip()
if not text:
missing_report.append(f"Ayah {surah}:{a_no} missing '{tid}' ({ed['scholar']})")
ayah_tafsirs.append({
"tafsir_id": tid,
"scholar": ed["scholar"],
"tafsir_name": ed["tafsir_name"],
"language": ed["language"],
"text": text,
"source": ed["source"],
})
ayahs_list.append({
"ayah_no": a_no,
"reference": f"{surah}:{a_no}",
"tafsirs": ayah_tafsirs,
})
# Consistency check
if missing_report:
print(f"\n[WARNING] Found {len(missing_report)} missing text items!")
for m in missing_report[:10]:
print(f" - {m}")
if len(missing_report) > 10:
print(f" ... and {len(missing_report) - 10} more.")
else:
print(f"\n✓ 101% Verification Passed! All {total_ayahs} Ayahs have complete text for all {len(EDITIONS_META)} scholars.")
result_json = {
"schema_version": "1.0",
"surahs": [
{
"surah_no": surah,
"surah_name_arabic": meta["name_ar"],
"surah_name_english": meta["name_en"],
"total_ayahs": total_ayahs,
"ayahs": ayahs_list,
}
],
}
# 3. Synchronize to all target directories
file_name = meta["file_name"]
for out_dir in SYNC_DIRS:
target_file = out_dir / file_name
with open(target_file, "w", encoding="utf-8") as f:
json.dump(result_json, f, ensure_ascii=False, indent=2)
print(f"✓ Saved: {target_file} ({os.path.getsize(target_file) / (1024 * 1024):.2f} MB)")
return result_json
def compile_surah_safe(surah: int, conn: sqlite3.Connection, max_retries: int = 3) -> bool:
for attempt in range(max_retries):
try:
compile_surah(surah, conn)
return True
except Exception as e:
print(f"[RETRY {attempt+1}/{max_retries}] Error compiling Surah {surah}: {e}")
time.sleep(2 * (attempt + 1))
print(f"[FAILED] Surah {surah} failed after {max_retries} attempts.")
return False
# -----------------------------------------------------------------------------
# MAIN CLI ENTRYPOINT
# -----------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Master Quran Tafseer Compiler & Generator",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
python master_tafseer_generator.py --surah 3 # Compiles Surah 3 only
python master_tafseer_generator.py --all # Compiles all 114 Surahs (skips existing)
python master_tafseer_generator.py --start 3 --end 114 # Compiles Surahs 3 to 114
python master_tafseer_generator.py --all --force # Re-compiles all Surahs even if existing
""",
)
parser.add_argument("--surah", type=int, default=None, help="Specific Surah number to compile (1-114).")
parser.add_argument("--all", action="store_true", help="Compile all 114 Surahs in batch mode.")
parser.add_argument("--start", type=int, default=1, help="Start Surah number for range (1-114).")
parser.add_argument("--end", type=int, default=114, help="End Surah number for range (1-114).")
parser.add_argument("--force", action="store_true", help="Overwrite existing files even if already present.")
args = parser.parse_args()
if not DB_PATH.exists():
print(f"ERROR: SQLite database not found at {DB_PATH}!")
sys.exit(1)
conn = sqlite3.connect(str(DB_PATH))
if args.surah is not None:
compile_surah_safe(args.surah, conn)
print(f"\n🎉 SURAH {args.surah} COMPILATION & SYNCHRONIZATION COMPLETE!")
elif args.all or (args.start != 1 or args.end != 114):
start_surah = args.start
end_surah = args.end
print(f"\n=======================================================")
print(f"STARTING BATCH COMPILATION: Surahs {start_surah} to {end_surah}")
print(f"=======================================================")
success_list = []
skipped_list = []
failed_list = []
for s in range(start_surah, end_surah + 1):
meta = SURAH_METADATA.get(s)
if not meta:
continue
target_file = PROJECT_ROOT / "quran_tafseer" / meta["file_name"]
if target_file.exists() and not args.force:
try:
# Quick integrity check
with open(target_file, "r", encoding="utf-8") as f:
existing = json.load(f)
if len(existing.get("surahs", [{}])[0].get("ayahs", [])) == meta["ayahs"]:
print(f"⏩ Skipping Surah {s} ({meta['name_en']}) - already compiled & verified.")
skipped_list.append(s)
continue
except Exception:
pass
ok = compile_surah_safe(s, conn)
if ok:
success_list.append(s)
else:
failed_list.append(s)
print("\n=======================================================")
print("BATCH COMPILATION SUMMARY")
print(f"✓ Successfully compiled: {len(success_list)} surahs")
print(f"⏩ Already existed (skipped): {len(skipped_list)} surahs")
if failed_list:
print(f"✗ Failed surahs: {failed_list}")
else:
print("🎉 ALL REQUESTED SURAHS VERIFIED & GENERATED WITH 100% SUCCESS!")
print("=======================================================")
else:
# Default single surah instruction if neither flag is passed
compile_surah_safe(2, conn)
print("\n🎉 SURAH 2 COMPILATION COMPLETE! (Pass --all to batch compile all Surahs)")
conn.close()
if __name__ == "__main__":
main()