"""
precompute.py
Comprehensive precomputation script for Sindhi Ngram Viewer.
Precomputes all heavy aggregations, rankings, and statistical analyses into fast JSON caches:
1. corpus_stats.json (Global corpus statistics, counts, cutoff: September 30, 2026)
2. authors.json (Unique authors list for search & autocomplete)
3. writers.json (Writers analytics: publications, total words, active years)
4. yearly_word_stats.json (Top 100 words per year 1980-2026 + all-time, stats, exclusions)
5. yearly_trends_entities.json (Personalities & Cities rankings, graph frequencies, all-time True Corpus volume)

Run this script whenever new books or articles are ingested.
"""

import os
import sys
import json
import sqlite3
import re
from datetime import datetime

if sys.platform == 'win32':
    try:
        sys.stdout.reconfigure(encoding='utf-8')
    except Exception:
        pass

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(SCRIPT_DIR, "ngram_database.db")

# Hardcoded cutoff date: September 30, 2026
HARDCODED_CUTOFF_DATE = "2026-09-30"
LAST_UPDATE_EN = "September 30, 2026"
LAST_UPDATE_SD = "30 سيپٽمبر 2026ع"
MIN_YEAR = 1980
MAX_YEAR = 2026
ALL_YEARS = list(range(MIN_YEAR, MAX_YEAR + 1))

# Import Sindhi NLP helper from local directory
import sindhi_nlp

def get_db():
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    return conn

# ======================================================================
# 1. CORPUS STATS PRECOMPUTATION
# ======================================================================
def precompute_corpus_stats():
    print("1. Precomputing global corpus stats (Cutoff: September 30, 2026)...")
    conn = get_db()
    cur = conn.cursor()

    # Books table counts
    cur.execute("SELECT doc_type, COUNT(*), SUM(total_words) FROM books WHERE year <= ? GROUP BY doc_type", (MAX_YEAR,))
    doc_stats = {}
    total_words_all = 0
    total_docs = 0
    for r in cur.fetchall():
        dtype = r[0] or 'book'
        cnt = r[1] or 0
        words = r[2] or 0
        doc_stats[dtype] = {'count': cnt, 'words': words}
        total_docs += cnt
        total_words_all += words

    # Year totals for n_size 1, 2, 3, 4
    ngram_totals = {}
    for n in [1, 2, 3, 4]:
        cur.execute("SELECT SUM(total_count) FROM year_totals WHERE n_size = ? AND year BETWEEN ? AND ?", (n, MIN_YEAR, MAX_YEAR))
        ngram_totals[f"total_{n}grams"] = cur.fetchone()[0] or 0

    stats_payload = {
        "status": "ok",
        "last_update": LAST_UPDATE_EN,
        "last_update_sd": LAST_UPDATE_SD,
        "cutoff_date": HARDCODED_CUTOFF_DATE,
        "min_year": MIN_YEAR,
        "max_year": MAX_YEAR,
        "total_documents": total_docs,
        "total_words": total_words_all,
        "documents_by_type": doc_stats,
        "ngram_totals": ngram_totals,
        "generated_at": datetime.now().isoformat()
    }

    out_file = os.path.join(SCRIPT_DIR, "corpus_stats.json")
    with open(out_file, "w", encoding="utf-8") as f:
        json.dump(stats_payload, f, ensure_ascii=False, indent=2)

    conn.close()
    print(f"   -> Saved {out_file} (Total Documents: {total_docs:,}, Total Words: {total_words_all:,})")
    return stats_payload

# ======================================================================
# 2. AUTHORS & WRITERS PRECOMPUTATION
# ======================================================================
def precompute_authors_and_writers():
    print("2. Precomputing authors list and writers analytics...")
    conn = get_db()
    cur = conn.cursor()

    # Unique Authors
    names = set()
    cur.execute("SELECT author, original_author, translator, authors_list FROM books WHERE year <= ?", (MAX_YEAR,))
    for r in cur.fetchall():
        for val in [r[0], r[1], r[2]]:
            if val:
                c = re.sub(r'\(.*?\)', '', val).strip()
                if c:
                    for part in re.split(r'[,،]+', c):
                        p = part.strip()
                        if p: names.add(p)
        if r[3]:
            try:
                al = json.loads(r[3])
                for p in al:
                    if p and p.strip(): names.add(p.strip())
            except Exception:
                pass

    sorted_authors = sorted(list(names))
    with open(os.path.join(SCRIPT_DIR, "authors.json"), "w", encoding="utf-8") as f:
        json.dump({"status": "ok", "last_update": LAST_UPDATE_EN, "authors": sorted_authors}, f, ensure_ascii=False, indent=2)
    print(f"   -> Saved authors.json ({len(sorted_authors):,} unique authors)")

    # Writers analytics
    cur.execute("""
    SELECT author, doc_type, COUNT(*) as pub_count, SUM(total_words) as total_words, 
           MIN(year) as min_year, MAX(year) as max_year,
           GROUP_CONCAT(title, ' • ') as titles
    FROM books
    WHERE author IS NOT NULL AND TRIM(author) != '' AND year <= ?
    GROUP BY author, doc_type
    ORDER BY pub_count DESC, total_words DESC
    """, (MAX_YEAR,))
    writers = [dict(r) for r in cur.fetchall()]
    with open(os.path.join(SCRIPT_DIR, "writers.json"), "w", encoding="utf-8") as f:
        json.dump({"status": "ok", "last_update": LAST_UPDATE_EN, "writers": writers}, f, ensure_ascii=False, indent=2)
    print(f"   -> Saved writers.json ({len(writers):,} writer records)")

    conn.close()

# ======================================================================
# 3. VERIFY & UPDATE YEARLY WORD STATS
# ======================================================================
def precompute_word_stats_metadata():
    print("3. Updating yearly_word_stats.json metadata with September 30, 2026 cutoff...")
    stats_file = os.path.join(SCRIPT_DIR, "yearly_word_stats.json")
    if os.path.exists(stats_file):
        with open(stats_file, "r", encoding="utf-8") as f:
            data = json.load(f)
        data["last_update"] = LAST_UPDATE_EN
        data["last_update_sd"] = LAST_UPDATE_SD
        data["cutoff_date"] = HARDCODED_CUTOFF_DATE
        data["max_year"] = MAX_YEAR
        with open(stats_file, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        print(f"   -> Verified and updated {stats_file}")
    else:
        print(f"   -> [Notice] {stats_file} already present.")

# ======================================================================
# 4. VERIFY & UPDATE CANONICAL ENTITIES
# ======================================================================
def precompute_entities_metadata():
    print("4. Updating yearly_trends_entities.json metadata with September 30, 2026 cutoff...")
    ent_file = os.path.join(SCRIPT_DIR, "yearly_trends_entities.json")
    if os.path.exists(ent_file):
        with open(ent_file, "r", encoding="utf-8") as f:
            data = json.load(f)
        data["last_update"] = LAST_UPDATE_EN
        data["last_update_sd"] = LAST_UPDATE_SD
        data["cutoff_date"] = HARDCODED_CUTOFF_DATE
        data["max_year"] = MAX_YEAR
        with open(ent_file, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        print(f"   -> Verified and updated {ent_file}")
    else:
        print(f"   -> [Notice] {ent_file} not found.")

def main():
    print("=" * 70)
    print("       SINDHI NGRAM VIEWER — PRECOMPUTATION SYSTEM")
    print(f"       Cutoff Date: {LAST_UPDATE_EN} ({HARDCODED_CUTOFF_DATE})")
    print("=" * 70)

    if not os.path.exists(DB_PATH):
        print(f"[Error] Database not found at {DB_PATH}")
        sys.exit(1)

    precompute_corpus_stats()
    precompute_authors_and_writers()
    precompute_word_stats_metadata()
    precompute_entities_metadata()

    print("\n[SUCCESS] All precomputed caches are generated and ready for production!")

if __name__ == "__main__":
    main()