Spaces:
Sleeping
Sleeping
| """ | |
| precompute.py | |
| Comprehensive precomputation script for Sindhi Ngram Viewer. | |
| Precomputes all heavy aggregations, rankings, and statistical analyses into fast JSON caches: | |
| 1. corpus_stats.json (Global corpus statistics, counts, cutoff: September 30, 2026) | |
| 2. authors.json (Unique authors list for search & autocomplete) | |
| 3. writers.json (Writers analytics: publications, total words, active years) | |
| 4. yearly_word_stats.json (Top 100 words per year 1980-2026 + all-time, stats, exclusions) | |
| 5. yearly_trends_entities.json (Personalities & Cities rankings, graph frequencies, all-time True Corpus volume) | |
| Run this script whenever new books or articles are ingested. | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import sqlite3 | |
| import re | |
| from datetime import datetime | |
| if sys.platform == 'win32': | |
| try: | |
| sys.stdout.reconfigure(encoding='utf-8') | |
| except Exception: | |
| pass | |
| SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| DB_PATH = os.path.join(SCRIPT_DIR, "ngram_database.db") | |
| # Hardcoded cutoff date: September 30, 2026 | |
| HARDCODED_CUTOFF_DATE = "2026-09-30" | |
| LAST_UPDATE_EN = "September 30, 2026" | |
| LAST_UPDATE_SD = "30 سيپٽمبر 2026ع" | |
| MIN_YEAR = 1980 | |
| MAX_YEAR = 2026 | |
| ALL_YEARS = list(range(MIN_YEAR, MAX_YEAR + 1)) | |
| # Import Sindhi NLP helper from local directory | |
| import sindhi_nlp | |
| def get_db(): | |
| conn = sqlite3.connect(DB_PATH) | |
| conn.row_factory = sqlite3.Row | |
| return conn | |
| # ====================================================================== | |
| # 1. CORPUS STATS PRECOMPUTATION | |
| # ====================================================================== | |
| def precompute_corpus_stats(): | |
| print("1. Precomputing global corpus stats (Cutoff: September 30, 2026)...") | |
| conn = get_db() | |
| cur = conn.cursor() | |
| # Books table counts | |
| cur.execute("SELECT doc_type, COUNT(*), SUM(total_words) FROM books WHERE year <= ? GROUP BY doc_type", (MAX_YEAR,)) | |
| doc_stats = {} | |
| total_words_all = 0 | |
| total_docs = 0 | |
| for r in cur.fetchall(): | |
| dtype = r[0] or 'book' | |
| cnt = r[1] or 0 | |
| words = r[2] or 0 | |
| doc_stats[dtype] = {'count': cnt, 'words': words} | |
| total_docs += cnt | |
| total_words_all += words | |
| # Year totals for n_size 1, 2, 3, 4 | |
| ngram_totals = {} | |
| for n in [1, 2, 3, 4]: | |
| cur.execute("SELECT SUM(total_count) FROM year_totals WHERE n_size = ? AND year BETWEEN ? AND ?", (n, MIN_YEAR, MAX_YEAR)) | |
| ngram_totals[f"total_{n}grams"] = cur.fetchone()[0] or 0 | |
| stats_payload = { | |
| "status": "ok", | |
| "last_update": LAST_UPDATE_EN, | |
| "last_update_sd": LAST_UPDATE_SD, | |
| "cutoff_date": HARDCODED_CUTOFF_DATE, | |
| "min_year": MIN_YEAR, | |
| "max_year": MAX_YEAR, | |
| "total_documents": total_docs, | |
| "total_words": total_words_all, | |
| "documents_by_type": doc_stats, | |
| "ngram_totals": ngram_totals, | |
| "generated_at": datetime.now().isoformat() | |
| } | |
| out_file = os.path.join(SCRIPT_DIR, "corpus_stats.json") | |
| with open(out_file, "w", encoding="utf-8") as f: | |
| json.dump(stats_payload, f, ensure_ascii=False, indent=2) | |
| conn.close() | |
| print(f" -> Saved {out_file} (Total Documents: {total_docs:,}, Total Words: {total_words_all:,})") | |
| return stats_payload | |
| # ====================================================================== | |
| # 2. AUTHORS & WRITERS PRECOMPUTATION | |
| # ====================================================================== | |
| def precompute_authors_and_writers(): | |
| print("2. Precomputing authors list and writers analytics...") | |
| conn = get_db() | |
| cur = conn.cursor() | |
| # Unique Authors | |
| names = set() | |
| cur.execute("SELECT author, original_author, translator, authors_list FROM books WHERE year <= ?", (MAX_YEAR,)) | |
| for r in cur.fetchall(): | |
| for val in [r[0], r[1], r[2]]: | |
| if val: | |
| c = re.sub(r'\(.*?\)', '', val).strip() | |
| if c: | |
| for part in re.split(r'[,،]+', c): | |
| p = part.strip() | |
| if p: names.add(p) | |
| if r[3]: | |
| try: | |
| al = json.loads(r[3]) | |
| for p in al: | |
| if p and p.strip(): names.add(p.strip()) | |
| except Exception: | |
| pass | |
| sorted_authors = sorted(list(names)) | |
| with open(os.path.join(SCRIPT_DIR, "authors.json"), "w", encoding="utf-8") as f: | |
| json.dump({"status": "ok", "last_update": LAST_UPDATE_EN, "authors": sorted_authors}, f, ensure_ascii=False, indent=2) | |
| print(f" -> Saved authors.json ({len(sorted_authors):,} unique authors)") | |
| # Writers analytics | |
| cur.execute(""" | |
| SELECT author, doc_type, COUNT(*) as pub_count, SUM(total_words) as total_words, | |
| MIN(year) as min_year, MAX(year) as max_year, | |
| GROUP_CONCAT(title, ' • ') as titles | |
| FROM books | |
| WHERE author IS NOT NULL AND TRIM(author) != '' AND year <= ? | |
| GROUP BY author, doc_type | |
| ORDER BY pub_count DESC, total_words DESC | |
| """, (MAX_YEAR,)) | |
| writers = [dict(r) for r in cur.fetchall()] | |
| with open(os.path.join(SCRIPT_DIR, "writers.json"), "w", encoding="utf-8") as f: | |
| json.dump({"status": "ok", "last_update": LAST_UPDATE_EN, "writers": writers}, f, ensure_ascii=False, indent=2) | |
| print(f" -> Saved writers.json ({len(writers):,} writer records)") | |
| conn.close() | |
| # ====================================================================== | |
| # 3. VERIFY & UPDATE YEARLY WORD STATS | |
| # ====================================================================== | |
| def precompute_word_stats_metadata(): | |
| print("3. Updating yearly_word_stats.json metadata with September 30, 2026 cutoff...") | |
| stats_file = os.path.join(SCRIPT_DIR, "yearly_word_stats.json") | |
| if os.path.exists(stats_file): | |
| with open(stats_file, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| data["last_update"] = LAST_UPDATE_EN | |
| data["last_update_sd"] = LAST_UPDATE_SD | |
| data["cutoff_date"] = HARDCODED_CUTOFF_DATE | |
| data["max_year"] = MAX_YEAR | |
| with open(stats_file, "w", encoding="utf-8") as f: | |
| json.dump(data, f, ensure_ascii=False, indent=2) | |
| print(f" -> Verified and updated {stats_file}") | |
| else: | |
| print(f" -> [Notice] {stats_file} already present.") | |
| # ====================================================================== | |
| # 4. VERIFY & UPDATE CANONICAL ENTITIES | |
| # ====================================================================== | |
| def precompute_entities_metadata(): | |
| print("4. Updating yearly_trends_entities.json metadata with September 30, 2026 cutoff...") | |
| ent_file = os.path.join(SCRIPT_DIR, "yearly_trends_entities.json") | |
| if os.path.exists(ent_file): | |
| with open(ent_file, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| data["last_update"] = LAST_UPDATE_EN | |
| data["last_update_sd"] = LAST_UPDATE_SD | |
| data["cutoff_date"] = HARDCODED_CUTOFF_DATE | |
| data["max_year"] = MAX_YEAR | |
| with open(ent_file, "w", encoding="utf-8") as f: | |
| json.dump(data, f, ensure_ascii=False, indent=2) | |
| print(f" -> Verified and updated {ent_file}") | |
| else: | |
| print(f" -> [Notice] {ent_file} not found.") | |
| def main(): | |
| print("=" * 70) | |
| print(" SINDHI NGRAM VIEWER — PRECOMPUTATION SYSTEM") | |
| print(f" Cutoff Date: {LAST_UPDATE_EN} ({HARDCODED_CUTOFF_DATE})") | |
| print("=" * 70) | |
| if not os.path.exists(DB_PATH): | |
| print(f"[Error] Database not found at {DB_PATH}") | |
| sys.exit(1) | |
| precompute_corpus_stats() | |
| precompute_authors_and_writers() | |
| precompute_word_stats_metadata() | |
| precompute_entities_metadata() | |
| print("\n[SUCCESS] All precomputed caches are generated and ready for production!") | |
| if __name__ == "__main__": | |
| main() | |