app.py

import spaces
import gradio as gr
import pandas as pd
import numpy as np
import os
import re
import hashlib
import plotly.express as px
import plotly.graph_objects as go
import networkx as nx
from datetime import datetime
from huggingface_hub import HfApi, hf_hub_download
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity, cosine_distances
from sklearn.manifold import TSNE
from sklearn.cluster import AgglomerativeClustering

# Try loading advanced density-based clustering; fallback gracefully if unavailable
try:
    import hdbscan
    HAS_HDBSCAN = True
except ImportError:
    HAS_HDBSCAN = False

# =====================================================================
# 1. GLOBAL CORE CONFIGURATION & COGNITIVE SCORING SYSTEM
# =====================================================================
MODEL_NAME = 'all-MiniLM-L6-v2'
model = SentenceTransformer(MODEL_NAME)

DATASET_REPO_ID = "Masterogon/dream-database"
DB_FILE = "dream_database.csv"
HF_TOKEN = os.environ.get("HF_TOKEN")
api = HfApi()

# Precise analytical weights assigned to Altered States of Consciousness (ASC)
ASC_WEIGHTS = {
    "NDE": 1.0,
    "OBE": 0.85,
    "Lucid Dream (LD)": 0.65,
    "Ordinary Dream": 0.4,
    "Other": 0.3
}

# In-memory global embedding cache to protect processing resources across large workloads
GLOBAL_EMBEDDING_CACHE = {}

def get_text_hash(text):
    return hashlib.sha256(text.strip().encode('utf-8')).hexdigest()

def get_cached_embeddings(texts):
    """
    Batched execution system that pulls processed embeddings from cache 
    and only computes missing elements via SentenceTransformer.
    """
    if not texts:
        return np.empty((0, model.get_sentence_embedding_dimension()))
        
    hashes = [get_text_hash(t) for t in texts]
    missing_texts = []
    missing_indices = []
    
    embeddings = [None] * len(texts)
    
    for idx, h in enumerate(hashes):
        if h in GLOBAL_EMBEDDING_CACHE:
            embeddings[idx] = GLOBAL_EMBEDDING_CACHE[h]
        else:
            missing_texts.append(texts[idx])
            missing_indices.append(idx)
            
    if missing_texts:
        computed = model.encode(missing_texts, batch_size=64, show_progress_bar=False)
        for idx, comp_idx in enumerate(missing_indices):
            h_val = hashes[comp_idx]
            GLOBAL_EMBEDDING_CACHE[h_val] = computed[idx]
            embeddings[comp_idx] = computed[idx]
            
    return np.array(embeddings)

def split_into_sentences(text):
    if not text or not isinstance(text, str):
        return []
    sentences = re.split(r'(?<=[.!?])\s+', text.strip())
    return [s.strip() for s in sentences if len(s.strip()) > 10]

def extract_semantic_fragments(sentence, window_size=5):
    """Generates localized semantic sub-phrases for granular token testing."""
    words = sentence.split()
    if len(words) <= window_size:
        return [sentence]
    fragments = []
    for i in range(len(words) - window_size + 1):
        fragments.append(" ".join(words[i:i+window_size]))
    return fragments

# =====================================================================
# 2. HIGH-AVAILABILITY FAULT-TOLERANT DATABASE ARCHITECTURE
# =====================================================================
DATASET_REPO_ID = "Masterogon/dream-database" 
DB_FILE = "dream_database.csv"
HF_TOKEN = os.environ.get("HF_TOKEN")

api = HfApi()

def init_db():
    """Synchronizes with Hugging Face storage or initializes clean data matrix locally."""
    if HF_TOKEN:
        try:
            print("🔄 Attempting to pull database from Hugging Face Hub...")
            file_path = hf_hub_download(
                repo_id=DATASET_REPO_ID, 
                filename=DB_FILE, 
                repo_type="dataset", 
                token=HF_TOKEN,
                force_download=True
            )
            import shutil
            shutil.copy(file_path, DB_FILE)
            print(f"✅ Target database synchronized. Rows loaded: {len(pd.read_csv(DB_FILE))}")
            return
        except Exception as e:
            print(f"⚠️ Cloud retrieval failed. Evaluating fallback arrays. Reason: {e}")
            
    if not os.path.exists(DB_FILE):
        df = pd.DataFrame(columns=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])
        df.to_csv(DB_FILE, index=False)
        print("📁 Initialized clean localized database container.")

init_db()

def safe_read_db():
    """Reads transactional data entries while safely stripping missing records."""
    try:
        if os.path.exists(DB_FILE):
            df = pd.read_csv(DB_FILE)
            return df.dropna(subset=['Narrative']).reset_index(drop=True)
        return pd.DataFrame(columns=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])
    except Exception as e:
        print(f"❌ Database execution exception: {e}")
        return pd.DataFrame(columns=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])

# =====================================================================
# 3. DATA INGESTION ENGINE
# =====================================================================
def process_entry(alias, asc_type, emotion, intensity, narrative):
    if not narrative or not narrative.strip():
        return "❌ Error: System requires a written narrative payload.", safe_read_db().tail(10)
    
    try:
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        new_row = pd.DataFrame([{
            "Timestamp": timestamp,
            "Alias": alias or "Anonymous",
            "ASC_Type": asc_type,
            "Emotion": emotion,
            "Intensity": int(intensity),
            "Narrative": narrative.strip()
        }])
        
        new_row.to_csv(DB_FILE, mode='a', header=False, index=False)
        backup_status = "Saved locally."
        
        if HF_TOKEN:
            try:
                api.upload_file(
                    path_or_fileobj=DB_FILE,
                    path_in_repo=DB_FILE,
                    repo_id=DATASET_REPO_ID,
                    repo_type="dataset",
                    token=HF_TOKEN,
                    commit_message=f"Ingested report entry by [{alias or 'Anonymous'}]"
                )
                backup_status = "Successfully mirrored to cloud storage repository."
            except Exception as e:
                backup_status = f"Mirrored locally (Cloud upload exception: {e})"
                
        return f"✅ Transmission Complete. Status: {backup_status}", view_database().tail(10)
    except Exception as e:
        return f"❌ Ingestion Error: {str(e)}", safe_read_db().tail(5)

def view_database():
    df = safe_read_db()
    if len(df) == 0:
        return pd.DataFrame(columns=['Report_ID', 'Timestamp', 'ASC_Type', 'Emotion', 'Intensity', 'Narrative'])
    df_display = df.copy()
    df_display.insert(0, 'Report_ID', [f"Report #{i+1}" for i in range(len(df_display))])
    return df_display[['Report_ID', 'Timestamp', 'ASC_Type', 'Emotion', 'Intensity', 'Narrative']]

# =====================================================================
# 4. COMPOSITE STATISTICAL SCORING ENGINE
# =====================================================================
def calculate_composite_score(semantic_sim, asc_type, intensity, cluster_support, uniqueness, cross_report_conf):
    """
    Executes a high-dimensional composite geometric calculation to verify 
    the systemic significance of a given pattern.
    """
    asc_w = ASC_WEIGHTS.get(asc_type, 0.3)
    intens_w = intensity / 3.0
    
    score = (semantic_sim * 0.35) + \
            (asc_w * 0.15) + \
            (intens_w * 0.10) + \
            (cluster_support * 0.15) + \
            (uniqueness * 0.10) + \
            (cross_report_conf * 0.15)
    return float(score)

# =====================================================================
# 5. HIGH-DENSITY CLUSTERING ENGINE
# =====================================================================
def execute_advanced_clustering(embeddings, min_cluster_size=2):
    """
    Executes clustering via HDBSCAN if available, falling back to 
    AgglomerativeClustering or KMeans dynamically based on data density.
    """
    n_samples = len(embeddings)
    if n_samples < 2:
        return np.zeros(n_samples, dtype=int), 1
        
    if HAS_HDBSCAN:
        try:
            clusterer = hdbscan.HDBSCAN(min_cluster_size=max(2, min_cluster_size), metric='euclidean', prediction_data=True)
            labels = clusterer.fit_predict(embeddings)
            # Route outliers (-1) to a dedicated cluster index to avoid structural breakdown
            if -1 in labels:
                labels[labels == -1] = labels.max() + 1
            n_clusters = len(set(labels))
            return labels, n_clusters
        except:
            pass
            
    # Fallback to Agglomerative Clustering
    n_clusters = max(2, min(8, n_samples // 3))
    if n_clusters >= n_samples:
        n_clusters = n_samples - 1 if n_samples > 1 else 1
        
    agg = AgglomerativeClustering(n_clusters=n_clusters, metric='euclidean', linkage='ward')
    labels = agg.fit_predict(embeddings)
    return labels, n_clusters

# =====================================================================
# 6. TAB 2: MACRO-LEVEL COMPREHENSIVE ANALYSIS
# =====================================================================
@spaces.GPU
def macro_analysis(similarity_threshold=0.40):
    df = safe_read_db()
    if len(df) < 2:
        return go.Figure(), go.Figure(), "### System Status\nInsufficient reports available to calculate structural distance matrix."
        
    texts = df['Narrative'].tolist()
    report_ids = [f"Report #{i+1}" for i in range(len(df))]
    
    # Process Embeddings
    embeddings = get_cached_embeddings(texts)
    sim_matrix = cosine_similarity(embeddings)
    
    # 1. Interactive Heatmap Map
    fig_heat = px.imshow(
        sim_matrix,
        labels=dict(x="Target Matrix ID", y="Source Matrix ID", color="Cosine Metric"),
        x=report_ids,
        y=report_ids,
        color_continuous_scale="Viridis",
        aspect="auto",
        title="Document-Level Cosine Similarity Heatmap"
    )
    fig_heat.update_layout(height=600, template="plotly_dark")
    
    # 2. Network Topology Model
    G = nx.Graph()
    for idx, r_id in enumerate(report_ids):
        # Calculate localized baseline parameters
        row = df.iloc[idx]
        asc_w = ASC_WEIGHTS.get(row['ASC_Type'], 0.3)
        node_size = float(30 * asc_w * (row['Intensity'] / 2.0))
        G.add_node(r_id, size=max(10, node_size), asc=row['ASC_Type'], state=row['Emotion'])
        
    for i in range(len(report_ids)):
        for j in range(i + 1, len(report_ids)):
            if sim_matrix[i, j] >= similarity_threshold:
                G.add_edge(report_ids[i], report_ids[j], weight=float(sim_matrix[i, j]))
                
    pos = nx.kamada_kawai_layout(G) if len(G.edges()) > 0 else nx.circular_layout(G)
    
    edge_x = []
    edge_y = []
    edge_text = []
    
    for edge in G.edges(data=True):
        x0, y0 = pos[edge[0]]
        x1, y1 = pos[edge[1]]
        edge_x.extend([x0, x1, None])
        edge_y.extend([y0, y1, None])
        edge_text.append(f"Strength: {edge[2]['weight']:.3f}")
        
    edge_trace = go.Scatter(
        x=edge_x, y=edge_y,
        line=dict(width=1.5, color='rgba(150,150,150,0.4)'),
        hoverinfo='none',
        mode='lines'
    )
    
    node_x = []
    node_y = []
    node_sizes = []
    node_colors = []
    node_text = []
    
    color_map = {"Positive": "#00ff88", "Neutral": "#00bfff", "Negative": "#ff0055"}
    
    for node in G.nodes():
        x, y = pos[node]
        node_x.append(x)
        node_y.append(y)
        node_sizes.append(G.nodes[node]['size'])
        node_colors.append(color_map.get(G.nodes[node]['state'], "#ffffff"))
        node_text.append(f"<b>{node}</b><br>State: {G.nodes[node]['asc']}<br>Emotion: {G.nodes[node]['state']}")
        
    node_trace = go.Scatter(
        x=node_x, y=node_y,
        mode='markers+text',
        text=[n.replace("Report ", "") for n in G.nodes()],
        textposition="top center",
        hoverinfo='text',
        hovertext=node_text,
        marker=dict(
            showscale=False,
            color=node_colors,
            size=node_sizes,
            line=dict(width=2, color='#ffffff')
        )
    )
    
    fig_net = go.Figure(data=[edge_trace, node_trace])
    fig_net.update_layout(
        title=f"Semantic Topological Network Graph (Threshold >= {similarity_threshold})",
        showlegend=False,
        hovermode='closest',
        margin=dict(b=20, l=5, r=5, t=40),
        xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
        yaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
        height=600,
        template="plotly_dark"
    )
    
    summary_md = f"### System Matrix Analysis Complete\n* Total active reports: **{len(df)}**\n* Extracted topological edges: **{len(G.edges())}**"
    return fig_heat, fig_net, summary_md

# =====================================================================
# 7. TAB 3: MICRO-LEVEL MICRO ANALYSIS
# =====================================================================
@spaces.GPU
def micro_analysis():
    df = safe_read_db()
    if len(df) < 1:
        return go.Figure(), "### Narrative Array Empty\nIngest more source content to generate structural fragments map."
        
    all_sentences = []
    sentence_metadata = []
    
    for idx, row in df.iterrows():
        sents = split_into_sentences(row['Narrative'])
        for s in sents:
            all_sentences.append(s)
            sentence_metadata.append({
                "Report_ID": f"Report #{idx+1}",
                "ASC_Type": row["ASC_Type"],
                "Emotion": row["Emotion"],
                "Intensity": row["Intensity"]
            })
            
    if len(all_sentences) < 3:
        return go.Figure(), "### Data Slicing Constraint\nExtracted structural components are insufficient. Submit detailed prose."
        
    sent_embeddings = get_cached_embeddings(all_sentences)
    labels, n_clusters = execute_advanced_clustering(sent_embeddings)
    
    # High-dimensional space transformation via t-SNE
    perplexity = min(30, max(2, len(all_sentences) - 1))
    tsne = TSNE(n_components=2, random_state=42, perplexity=perplexity)
    
    vecs_2d = tsne.fit_transform(sent_embeddings)
    
    sim_matrix = cosine_similarity(sent_embeddings)
    
    plot_df = pd.DataFrame({
        "X": vecs_2d[:, 0],
        "Y": vecs_2d[:, 1],
        "Cluster": [f"Cluster {l}" for l in labels],
        "Report": [m["Report_ID"] for m in sentence_metadata],
        "ASC": [m["ASC_Type"] for m in sentence_metadata],
        "Sentence": all_sentences
    })
    
    fig_tsne = px.scatter(
        plot_df, x="X", y="Y",
        color="Cluster",
        symbol="ASC",
        hover_data=["Report", "Sentence"],
        title="Granular Micro-Level Semantic Structural Vector Map (t-SNE Projection)"
    )
    fig_tsne.update_layout(height=600, template="plotly_dark", showlegend=False, margin=dict(l=10, r=10, t=40, b=10))
    #fig_tsne.update_layout(height=600, template="plotly_dark")
    
    # Evaluate Centrality Metrics across groups
    centrality_scores = sim_matrix.mean(axis=1)
    top_indices = centrality_scores.argsort()[-5:][::-1]
    
    central_text = "### 🎯 Verified Central Sentence Fragments\n"
    for rank, idx in enumerate(top_indices):
        meta = sentence_metadata[idx]
        central_text += f"{rank+1}. **{meta['Report_ID']}** ({meta['ASC_Type']}): \n" \
                        f"  > \"{all_sentences[idx]}\"\n" \
                        f"  *Centrality Weight Coefficient: `{centrality_scores[idx]:.3f}`*\n\n"
                        
    return fig_tsne, central_text

# =====================================================================
# 8. TAB 4: ADVANCED AI PATTERN DISCOVERY & HYBRID SEARCH ENGINE
# =====================================================================
@spaces.GPU
def hybrid_pattern_discovery(mode, custom_motifs_text):
    df = safe_read_db()
    if len(df) < 2:
        return "### Analysis Engine Halted\nThe system requires a minimum database configuration of **2 distinct reports** to run unsupervised blind grouping algorithms."
        
    all_sentences = []
    sentence_metadata = []
    
    for idx, row in df.iterrows():
        sents = split_into_sentences(row['Narrative'])
        for s in sents:
            all_sentences.append(s)
            sentence_metadata.append({
                "Report_Idx": idx,
                "Report_ID": f"Report #{idx+1}",
                "ASC_Type": row["ASC_Type"],
                "Intensity": row["Intensity"],
                "Emotion": row["Emotion"]
            })
            
    if len(all_sentences) < 4:
        return "### Text Normalization Constraint\nInsufficient atomic text components found. Enter longer, multi-sentence descriptions."
        
    sent_embeddings = get_cached_embeddings(all_sentences)
    
    # -----------------------------------------------------------------
    # RUN UNSUPERVISED BLIND EXTRACTION MODE
    # -----------------------------------------------------------------
    if mode == "Blind Extraction (Unsupervised)":
        labels, n_clusters = execute_advanced_clustering(sent_embeddings)
        
        report_output = f"# 👁️ Unsupervised Pattern Discovery Report\n" \
                        f"**Executed Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} | **Analytic Dimension:** {len(all_sentences)} Parsed Fragments\n" \
                        f"**Target Mathematical Linkage Engine:** { 'HDBSCAN (Density Engine)' if HAS_HDBSCAN else 'Agglomerative Ward Hierarchy' }\n" \
                        f"**Total Extracted Thematic Formations:** {n_clusters}\n\n---\n"
                        
        global_sim_matrix = cosine_similarity(sent_embeddings)
        
        for c_id in sorted(list(set(labels))):
            cluster_indices = np.where(labels == c_id)[0]
            if len(cluster_indices) < 2: 
                continue  # Filter out noise components lacking cross-report verification
                
            c_embeddings = sent_embeddings[cluster_indices]
            centroid = c_embeddings.mean(axis=0).reshape(1, -1)
            
            # Find the closest sentence to the cluster centroid to serve as the structural anchor
            distances = cosine_distances(centroid, c_embeddings)[0]
            local_center_idx = np.argmin(distances)
            global_anchor_idx = cluster_indices[local_center_idx]
            
            representative_fragment = all_sentences[global_anchor_idx]
            anchor_meta = sentence_metadata[global_anchor_idx]
            
            # Cross-report verification metric calculations
            associated_reports = set([sentence_metadata[i]["Report_ID"] for i in cluster_indices])
            cross_report_conf = len(associated_reports) / len(df)
            
            cluster_sims = cosine_similarity(centroid, c_embeddings)[0]
            avg_semantic_sim = float(cluster_sims.mean())
            
            # Compute systemic uniqueness vs alternative configurations
            outer_indices = np.where(labels != c_id)[0]
            if len(outer_indices) > 0:
                uniqueness = float(1.0 - cosine_similarity(centroid, sent_embeddings[outer_indices]).mean())
            else:
                uniqueness = 0.5
                
            cluster_support_ratio = len(cluster_indices) / len(all_sentences)
            
            composite_confidence = calculate_composite_score(
                semantic_sim=avg_semantic_sim,
                asc_type=anchor_meta["ASC_Type"],
                intensity=anchor_meta["Intensity"],
                cluster_support=cluster_support_ratio,
                uniqueness=uniqueness,
                cross_report_conf=cross_report_conf
            )
            
            report_output += f"## 🟢 Detected Archetype Cluster #{c_id + 1}\n" \
                             f"* **Statistical Confidence Metric:** `{composite_confidence:.3f}`\n" \
                             f"* **Cross-Report Support:** {len(associated_reports)} Independent Narrative Matrices\n" \
                             f"* **Internal Cohesion Density:** `{avg_semantic_sim:.3f}`\n" \
                             f"* **Systemic Divergence (Uniqueness):** `{uniqueness:.3f}`\n\n" \
                             f"### 📍 Structural Anchor Element\n" \
                             f"> \"{representative_fragment}\" — *Verified via source baseline: {anchor_meta['Report_ID']} ({anchor_meta['ASC_Type']})*\n\n" \
                             f"### 🧪 Secondary Verification Cross-Matches\n"
                             
            printed_matches = 0
            for idx in cluster_indices:
                if idx != global_anchor_idx and printed_matches < 3:
                    m_meta = sentence_metadata[idx]
                    report_output += f"* **{m_meta['Report_ID']}**: \"{all_sentences[idx]}\" *(Local Distance Vector Match: `{cluster_sims[np.where(cluster_indices == idx)[0][0]]:.3f}`)*\n"
                    printed_matches += 1
            report_output += "\n---\n"
            
        return report_output

    # -----------------------------------------------------------------
    # RUN TARGETED HYPOTHESIS TESTING MODE
    # -----------------------------------------------------------------
    else:
        seed_motifs = [m.strip() for m in re.split(r'[,|\n]', custom_motifs_text) if m.strip()]
        if not seed_motifs:
            return "### Execution Blocked\nPlease provide targeted structural search parameters (phrases or semantic tokens) to verify."
            
        motif_embeddings = model.encode(seed_motifs)
        search_report = f"# 🎯 High-Fidelity Zero-Shot Structural Search Matrix\n" \
                        f"**Evaluated Hypotheses:** {len(seed_motifs)} Target Search Fields\n\n---\n"
                        
        for m_idx, motif in enumerate(seed_motifs):
            m_emb = motif_embeddings[m_idx].reshape(1, -1)
            similarities = cosine_similarity(m_emb, sent_embeddings)[0]
            
            matched_indices = np.where(similarities >= 0.35)[0]
            
            search_report += f"## 🔎 Evaluation Array: '{motif}'\n"
            
            if len(matched_indices) == 0:
                search_report += "*No semantic matches found above validation parameters (`>=0.35`). Hypothesis unverified across current dataset.*\n\n---\n"
                continue
                
            # Filter and rank based on verification parameters
            ranked_matches = matched_indices[np.argsort(similarities[matched_indices])[::-1]]
            unique_reports = set([sentence_metadata[i]["Report_ID"] for i in ranked_matches])
            
            search_report += f"* **Empirical Signal Status:** Verified \n" \
                             f"* **Cross-Report Proximity Network:** Detected inside {len(unique_reports)} unique documents\n" \
                             f"* **Top Empirical Amplitude Correlation:** `{similarities[ranked_matches[0]]:.3f}`\n\n" \
                             f"### 📊 Extraction Array Ranked by Composite Proximity\n"
                             
            for count, idx in enumerate(ranked_matches[:5]):
                meta = sentence_metadata[idx]
                search_report += f"{count+1}. **{meta['Report_ID']}** (`{meta['ASC_Type']}`): \"{all_sentences[idx]}\"\n" \
                                 f"   * Raw Proximity: `{similarities[idx]:.3f}` | Internal Context Amplitude Level: {meta['Intensity']}/3*\n"
            search_report += "\n---\n"
            
        return search_report

# =====================================================================
# 9. TAB 5: ADVANCED COMPOSITE ANOMALY DETECTION ENGINE
# =====================================================================
def calculate_anomalies():
    """
    Identifies rare semantic motifs and isolated responses using dimensional distance variations.
    """
    df = safe_read_db()
    if len(df) < 3:
        return pd.DataFrame(columns=["Rank", "Report_ID", "Isolation Score", "ASC", "Narrative Anchor Snippet"])
        
    texts = df['Narrative'].tolist()
    embeddings = get_cached_embeddings(texts)
    
    # Measure macro-level isolation using average cosine distances
    sim_matrix = cosine_similarity(embeddings)
    isolation_scores = 1.0 - sim_matrix.mean(axis=1)
    
    # Factor in semantic fragment isolation
    fragment_isolation_penalties = np.zeros(len(df))
    all_sentences = []
    sentence_map = []
    
    for idx, row in df.iterrows():
        sents = split_into_sentences(row['Narrative'])
        for s in sents:
            all_sentences.append(s)
            sentence_map.append(idx)
            
    if len(all_sentences) >= 5:
        sent_embs = get_cached_embeddings(all_sentences)
        sent_sims = cosine_similarity(sent_embs)
        sent_isolation = 1.0 - sent_sims.mean(axis=1)
        
        for idx in range(len(df)):
            sub_indices = [i for i, r_idx in enumerate(sentence_map) if r_idx == idx]
            if sub_indices:
                fragment_isolation_penalties[idx] = sent_isolation[sub_indices].max()
                
    # Calculate composite anomaly ranking matrices
    composite_anomaly_vectors = (isolation_scores * 0.6) + (fragment_isolation_penalties * 0.4)
    ranked_indices = composite_anomaly_vectors.argsort()[::-1]
    
    anomaly_records = []
    for rank, idx in enumerate(ranked_indices):
        row = df.iloc[idx]
        anomaly_records.append({
            "Rank": rank + 1,
            "Report_ID": f"Report #{idx+1}",
            "Anomaly Metric": f"{composite_anomaly_vectors[idx]:.3f}",
            "ASC State": row["ASC_Type"],
            "Emotion Index": row["Emotion"],
            "Narrative Excerpt": row["Narrative"][:140] + "..."
        })
        
    return pd.DataFrame(anomaly_records)

# =====================================================================
# 10. TAB 6: MODERN USER INTERFACE ARCHITECTURE
# =====================================================================
with gr.Blocks(title="DreamCode v3 Engine", theme=gr.themes.Monochrome()) as app:
    gr.Markdown(
        "# 🌌 DreamCode — Advanced Cognitive Pattern Discovery Platform\n"
        "### High-Throughput Unsupervised Neural Extraction Pipeline for Scientific Analysis"
    )
    
    with gr.Tabs():
        # TAB 1: Data Ingestion Pipeline
        with gr.TabItem("1. Data Ingestion Pipeline"):
            with gr.Row():
                with gr.Column(scale=2):
                    gr.Markdown("### 📥 Neural Signal Entry Protocol")
                    alias = gr.Textbox(label="Researcher / Participant Pseudonym Anchor", placeholder="e.g. MasterOgon", value="Anonymous")
                    asc_type = gr.Dropdown(choices=["Ordinary Dream", "Lucid Dream (LD)", "OBE", "NDE", "Other"], label="State Target Parameter", value="Ordinary Dream")
                    emotion = gr.Radio(choices=["Positive", "Neutral", "Negative"], label="Inherent Affective Valence Vector", value="Neutral")
                    intensity = gr.Slider(minimum=1, maximum=3, step=1, label="Subjective Amplitude Level (Intensity)", value=2)
                    narrative = gr.Textbox(label="Verbatim Narrative Phenomenological Payload", lines=8, placeholder="Provide detailed operational accounts of the imagery, spatial transitions, or entities encountered...")
                    submit_btn = gr.Button("Ingest Sequence Stream", variant="primary")
                    
                with gr.Column(scale=3):
                    gr.Markdown("### 📡 Global Transaction Monitor")
                    status_output = gr.Textbox(label="Operational Output Status Code", interactive=False)
                    data_preview = gr.Dataframe(label="System Queue Monitor (Latest Ingested Rows)")
                    
            submit_btn.click(
                fn=process_entry, 
                inputs=[alias, asc_type, emotion, intensity, narrative], 
                outputs=[status_output, data_preview]
            )

        # TAB 2: Macro-Level Analysis Topology
        with gr.TabItem("2. Document Similarity Matrix"):
            gr.Markdown("### 🌍 Macroscopic Semantic Network Mapping")
            thresh_slider = gr.Slider(minimum=0.20, maximum=0.90, step=0.05, value=0.45, label="Cosine Similarity Adjacency Threshold ($W_{ij}$)")
            analyze_macro_btn = gr.Button("Compute Structural Matrices", variant="primary")
            
            with gr.Row():
                heat_plot = gr.Plot(label="Document Inter-Proximity Density Matrix")
                network_plot = gr.Plot(label="Topological Projection Model Graph")
                
            macro_status = gr.Markdown("### System Status\nEngine idle. Awaiting structural computation triggers.")
            analyze_macro_btn.click(
                fn=macro_analysis, 
                inputs=[thresh_slider], 
                outputs=[heat_plot, network_plot, macro_status]
            )
            
        # TAB 3: Sentence-Level Dimensional Projections
        with gr.TabItem("3. Scene & Sentence Mapping"):
            gr.Markdown("### 🔬 Micro-Level Token Mapping & Centrality Evaluation")
            analyze_micro_btn = gr.Button("Extract Structural Components", variant="primary")

            
            with gr.Column():
                tsne_plot = gr.Plot(label="Non-Parametric t-SNE Clustering Projection Map")
                central_text_md = gr.Markdown(label="Calculated Centroid Vector Array Coordinates")
                
            analyze_micro_btn.click(
                fn=micro_analysis, 
                inputs=[], 
                outputs=[tsne_plot, central_text_md]
            )

        # TAB 4: Semantic Pattern Discovery Engine
        with gr.TabItem("4. AI Pattern Discovery Engine"):
            gr.Markdown("### 🧠 Unsupervised Extraction & Hypothesis Validation Arrays")
            
            search_mode = gr.Radio(
                choices=["Blind Extraction (Unsupervised)", "Targeted Search (Zero-Shot)"],
                value="Blind Extraction (Unsupervised)",
                label="Selected Neural Engine Operational Domain"
            )
            
            motif_input = gr.Textbox(
                label="Target Search Expressions (Separated by commas or line returns)", 
                lines=3, 
                value="flying vehicle, red sky, city of robots, cosmic catastrophe, mechanical device",
                visible=False 
            )
            
            def toggle_input_field(mode_selection):
                return gr.update(visible=(mode_selection == "Targeted Search (Zero-Shot)"))
                    
            search_mode.change(fn=toggle_input_field, inputs=[search_mode], outputs=[motif_input])
            
            analyze_engine_btn = gr.Button("Execute Discovery Matrix Engine", variant="primary")
            output_md_report = gr.Markdown(value="*Awaiting algorithm run parameters...*")
            
            analyze_engine_btn.click(
                fn=hybrid_pattern_discovery, 
                inputs=[search_mode, motif_input], 
                outputs=[output_md_report]
            )

        # TAB 5: High-Dimensional Anomaly Detection Engine
        with gr.TabItem("5. Anomaly Detection Engine"):
            gr.Markdown("### 🚨 Identification of Rare Metaphorical Configurations and Isolated Signatures")
            run_anomalies_btn = gr.Button("Scan System for Structural Outliers", variant="primary")
            anomaly_display_grid = gr.Dataframe(label="Ranked Divergent Anomalies Matrix")
            
            run_anomalies_btn.click(
                fn=calculate_anomalies, 
                inputs=[], 
                outputs=[anomaly_display_grid]
            )

        # TAB 6: Complete Database Explorer
        with gr.TabItem("6. Complete Database Explorer"):
            gr.Markdown("### 📖 Structural Repository Ledger")
            refresh_db_btn = gr.Button("Query Ledger System State")
            db_display_table = gr.Dataframe(wrap=True, interactive=False) 
            
            refresh_db_btn.click(fn=view_database, inputs=[], outputs=[db_display_table])
            app.load(fn=view_database, inputs=[], outputs=[db_display_table])

if __name__ == "__main__":
    app.queue().launch()