data/parse_corpus.py

"""
Full Quranic Corpus Parser — builds the complete root graph from real data.

Data sources:
  1. roots_lane.json — 1,651 roots with Lane's Lexicon definitions,
     frequencies, and morphological forms
  2. quran-morphology.txt — Quranic Arabic Corpus v0.4, per-word
     morphological annotation for all 77,430 words

Output:
  - Complete NetworkX graph with:
    - ALL root nodes (with frequency, semantic field, confidence)
    - ALL lemma/form nodes (with pattern, POS, frequency)
    - Derivation edges (root → lemma)
    - Semantic co-occurrence edges (root ↔ root, weighted by verse co-occurrence)
"""

import json
import re
from collections import defaultdict, Counter
from pathlib import Path
import networkx as nx

CORPUS_DIR = Path(__file__).parent / "corpus"


def parse_roots_lane():
    """Parse roots_lane.json — all 1,651 roots with morphological forms."""
    path = CORPUS_DIR / "roots_lane.json"
    with open(path, "r", encoding="utf-8") as f:
        data = json.load(f)

    roots = {}
    for entry in data["roots"]:
        root_ar = entry["root"]
        roots[root_ar] = {
            "buckwalter": entry.get("root_buckwalter", ""),
            "frequency": entry.get("quran_frequency", 0),
            "summary_en": entry.get("summary_en", ""),
            "confidence": entry.get("confidence", ""),
            "source": entry.get("source", ""),
            "forms": [],
        }
        for form in entry.get("morphological_forms", []):
            roots[root_ar]["forms"].append({
                "pattern": form.get("form_pattern", ""),
                "arabic": form.get("form_arabic", ""),
                "name": form.get("form_name", ""),
                "category": form.get("form_category", ""),
                "example": form.get("example_word", ""),
                "occurrences": form.get("occurrences", 0),
            })

    return roots, data.get("metadata", {})


def parse_morphology_corpus():
    """
    Parse quran-morphology.txt — every word in the Quran with root + lemma.

    Returns:
      word_data: list of (chapter, verse, word_pos, arabic, pos, root, lemma, features_dict)
      verse_roots: dict mapping (chapter, verse) → set of roots in that verse
      root_lemmas: dict mapping root → Counter of lemma occurrences
      root_forms: dict mapping root → Counter of verb form (VF:N) occurrences
    """
    path = CORPUS_DIR / "quran-morphology.txt"

    word_data = []
    verse_roots = defaultdict(set)
    root_lemmas = defaultdict(Counter)
    root_forms = defaultdict(Counter)
    root_pos_counts = defaultdict(Counter)

    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#"):
                continue

            parts = line.split("\t")
            if len(parts) < 4:
                continue

            position = parts[0]
            arabic = parts[1]
            pos = parts[2]
            features_str = parts[3]

            pos_parts = position.split(":")
            if len(pos_parts) < 3:
                continue
            chapter = int(pos_parts[0])
            verse = int(pos_parts[1])
            word_pos = int(pos_parts[2])

            features = {}
            for feat in features_str.split("|"):
                if ":" in feat:
                    key, val = feat.split(":", 1)
                    features[key] = val
                else:
                    features[feat] = True

            root = features.get("ROOT", None)
            lemma = features.get("LEM", None)
            verb_form = features.get("VF", None)

            if root:
                verse_roots[(chapter, verse)].add(root)
                if lemma:
                    root_lemmas[root][lemma] += 1
                if verb_form:
                    root_forms[root][verb_form] += 1
                root_pos_counts[root][pos] += 1

            word_data.append((chapter, verse, word_pos, arabic, pos, root, lemma, features))

    return word_data, dict(verse_roots), dict(root_lemmas), dict(root_forms), dict(root_pos_counts)


def build_cooccurrence_edges(verse_roots, min_cooccurrence=3):
    """
    Build semantic co-occurrence edges from verse-level root co-occurrence.

    Two roots that appear in the same verse are semantically related.
    Weight = number of verses they co-occur in.
    """
    cooccurrence = Counter()

    for (chapter, verse), roots in verse_roots.items():
        roots_list = sorted(roots)
        for i in range(len(roots_list)):
            for j in range(i + 1, len(roots_list)):
                pair = (roots_list[i], roots_list[j])
                cooccurrence[pair] += 1

    edges = []
    for (root_a, root_b), count in cooccurrence.items():
        if count >= min_cooccurrence:
            edges.append((root_a, root_b, count))

    edges.sort(key=lambda x: -x[2])
    return edges


def build_full_graph(min_cooccurrence=3):
    """Build the complete Quranic root graph from all data sources."""

    print("Parsing roots_lane.json...")
    roots_data, metadata = parse_roots_lane()
    print(f"  {len(roots_data)} roots loaded")

    print("Parsing quran-morphology.txt...")
    word_data, verse_roots, root_lemmas, root_forms, root_pos = parse_morphology_corpus()
    print(f"  {len(word_data)} word tokens parsed")
    print(f"  {len(verse_roots)} verses processed")
    print(f"  {len(root_lemmas)} unique roots found in morphology corpus")

    print("Building co-occurrence edges...")
    cooc_edges = build_cooccurrence_edges(verse_roots, min_cooccurrence)
    print(f"  {len(cooc_edges)} co-occurrence edges (min {min_cooccurrence} shared verses)")

    G = nx.DiGraph()

    all_roots_in_corpus = set(root_lemmas.keys())
    all_roots_in_lane = set(roots_data.keys())
    corpus_root_freq = Counter()
    for (ch, v), roots in verse_roots.items():
        for r in roots:
            corpus_root_freq[r] += 1

    for root_ar, data in roots_data.items():
        G.add_node(root_ar, type="root", depth=0,
                   frequency=data["frequency"],
                   summary=data["summary_en"],
                   buckwalter=data["buckwalter"],
                   confidence=data["confidence"],
                   num_forms=len(data["forms"]),
                   num_lemmas=len(root_lemmas.get(root_ar, {})))

    for root_ar in all_roots_in_corpus - all_roots_in_lane:
        freq = corpus_root_freq.get(root_ar, 0)
        if freq >= 2:
            G.add_node(root_ar, type="root", depth=0,
                       frequency=freq, summary="", buckwalter="",
                       confidence="corpus_only", num_forms=0,
                       num_lemmas=len(root_lemmas.get(root_ar, {})))

    for root_ar, data in roots_data.items():
        for form in data["forms"]:
            form_name = form["example"] or form["arabic"]
            if not form_name:
                continue
            node_id = f"{root_ar}:{form_name}"
            depth = 1 if form["pattern"].startswith("form_") else 2
            G.add_node(node_id, type="derived", depth=depth,
                       arabic=form_name, pattern=form["pattern"],
                       pattern_name=form["name"],
                       category=form["category"],
                       frequency=form["occurrences"],
                       source_root=root_ar)
            G.add_edge(root_ar, node_id, relation="derivation",
                       pattern=form["pattern"])

    for root_ar, lemmas in root_lemmas.items():
        if not G.has_node(root_ar):
            continue
        for lemma, count in lemmas.most_common():
            node_id = f"{root_ar}:{lemma}"
            if G.has_node(node_id):
                continue
            G.add_node(node_id, type="derived", depth=2,
                       arabic=lemma, pattern="corpus",
                       frequency=count, source_root=root_ar)
            G.add_edge(root_ar, node_id, relation="derivation",
                       pattern="corpus", weight=count)

    for root_a, root_b, count in cooc_edges:
        if G.has_node(root_a) and G.has_node(root_b):
            G.add_edge(root_a, root_b, relation="semantic",
                       co_occurrence=count)
            G.add_edge(root_b, root_a, relation="semantic",
                       co_occurrence=count)

    return G, metadata


def print_stats(G):
    """Print comprehensive graph statistics."""
    roots = [n for n, d in G.nodes(data=True) if d.get("type") == "root"]
    derived = [n for n, d in G.nodes(data=True) if d.get("type") == "derived"]

    deriv_edges = [(u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "derivation"]
    sem_edges = [(u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "semantic"]

    root_freqs = sorted([G.nodes[n].get("frequency", 0) for n in roots], reverse=True)

    print(f"\n{'='*60}")
    print(f"QURANIC ROOT GRAPH — COMPLETE")
    print(f"{'='*60}")
    print(f"  Root nodes:        {len(roots)}")
    print(f"  Derived nodes:     {len(derived)}")
    print(f"  Total nodes:       {G.number_of_nodes()}")
    print(f"  Derivation edges:  {len(deriv_edges)}")
    print(f"  Semantic edges:    {len(sem_edges)} (co-occurrence)")
    print(f"  Total edges:       {G.number_of_edges()}")
    print(f"\n  Top 20 roots by Quranic frequency:")
    top_roots = sorted(roots, key=lambda n: G.nodes[n].get("frequency", 0), reverse=True)[:20]
    for i, r in enumerate(top_roots, 1):
        d = G.nodes[r]
        summ = d.get("summary", "")[:60]
        print(f"    {i:2d}. {r:8s}  freq={d['frequency']:5d}  forms={d.get('num_forms',0):2d}  {summ}")

    print(f"\n  Frequency distribution:")
    print(f"    Roots with freq > 100: {sum(1 for f in root_freqs if f > 100)}")
    print(f"    Roots with freq 10-100: {sum(1 for f in root_freqs if 10 <= f <= 100)}")
    print(f"    Roots with freq 1-9:    {sum(1 for f in root_freqs if 1 <= f < 10)}")

    top_cooc = sorted(
        [(u, v, d["co_occurrence"]) for u, v, d in G.edges(data=True)
         if d.get("relation") == "semantic" and u < v],
        key=lambda x: -x[2]
    )[:15]
    print(f"\n  Top 15 co-occurring root pairs:")
    for r1, r2, count in top_cooc:
        s1 = G.nodes[r1].get("summary", "")[:25]
        s2 = G.nodes[r2].get("summary", "")[:25]
        print(f"    {r1:8s} <-> {r2:8s}  co_occ={count:4d}  ({s1} / {s2})")


def save_graph(G, path):
    """Save graph as JSON for reloading."""
    nodes = {}
    for n, d in G.nodes(data=True):
        nodes[n] = dict(d)

    edges = []
    for u, v, d in G.edges(data=True):
        edges.append({"source": u, "target": v, **d})

    with open(path, "w", encoding="utf-8") as f:
        json.dump({"nodes": nodes, "edges": edges}, f, ensure_ascii=False, indent=1)
    print(f"\n  Graph saved to {path}")


if __name__ == "__main__":
    import sys
    min_co = int(sys.argv[1]) if len(sys.argv) > 1 else 3
    G, meta = build_full_graph(min_cooccurrence=min_co)
    print_stats(G)
    save_graph(G, "E:/ABU/NEOCORTEX/data/full_graph.json")