Update app.py

app.py

@@ -1,4 +1,4 @@
-import spaces
+import spaces  
 import gradio as gr
 import pandas as pd
 import numpy as np
@@ -6,7 +6,7 @@ import os
 import re
 
 import matplotlib
-matplotlib.use('Agg')
+matplotlib.use('Agg') 
 import matplotlib.pyplot as plt
 import seaborn as sns
 import networkx as nx
@@ -18,18 +18,19 @@ from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.manifold import TSNE
 
 # ==========================================
-# INITIALIZATION
+# 1. ИНИЦИАЛИЗАЦИЯ ИИ
 # ==========================================
 model = SentenceTransformer('all-MiniLM-L6-v2')
 
 def split_into_sentences(text):
+    # Разбиваем по точкам, восклицательным и вопросительным знакам
     sentences = re.split(r'(?<=[.!?])\s+', text.strip())
     return [s for s in sentences if len(s) > 10]
 
 # ==========================================
-# DATABASE
+# 2. КОНФИГУРАЦИЯ БАЗЫ ДАННЫХ
 # ==========================================
-DATASET_REPO_ID = "Masterogon/dream-database"
+DATASET_REPO_ID = "Masterogon/dream-database" 
 DB_FILE = "dream_database.csv"
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
@@ -38,202 +39,340 @@ api = HfApi()
 def init_db():
     if HF_TOKEN:
         try:
+            print("Downloading database from Hugging Face Hub...")
             file_path = hf_hub_download(repo_id=DATASET_REPO_ID, filename=DB_FILE, repo_type="dataset", token=HF_TOKEN)
             import shutil
             shutil.copy(file_path, DB_FILE)
-            print("Database loaded.")
+            print("Database successfully loaded.")
             return
         except Exception as e:
-            print("Download failed:", e)
+            print("Could not download DB. It might be empty or missing. Error:", e)
     
     if not os.path.exists(DB_FILE):
         df = pd.DataFrame(columns=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])
         df.to_csv(DB_FILE, index=False)
+        print("Created a fresh local database.")
 
 init_db()
 
 # ==========================================
-# SAVE ENTRY
+# 3. ФУНКЦИЯ ЗАПИСИ (Data Ingestion)
 # ==========================================
 def process_entry(alias, asc_type, emotion, intensity, narrative):
     if not narrative.strip():
-        return "Error: Please enter a narrative.", pd.read_csv(DB_FILE).tail(5)
+        return "Error: Please describe your experience.", pd.read_csv(DB_FILE).tail(5)
     
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    new_data = pd.DataFrame([[timestamp, alias or "Anonymous", asc_type, emotion, intensity, narrative]],
+    new_data = pd.DataFrame([[timestamp, alias, asc_type, emotion, intensity, narrative]], 
                             columns=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])
     new_data.to_csv(DB_FILE, mode='a', header=False, index=False)
     
-    backup_status = "Synced to cloud." if HF_TOKEN else "Saved locally."
-    return f"✅ Entry added successfully. {backup_status}", pd.read_csv(DB_FILE).tail(8)
+    if HF_TOKEN:
+        try:
+            api.upload_file(
+                path_or_fileobj=DB_FILE,
+                path_in_repo=DB_FILE,
+                repo_id=DATASET_REPO_ID,
+                repo_type="dataset",
+                token=HF_TOKEN,
+                commit_message=f"Added new report by {alias}"
+            )
+            backup_status = "Successfully synced to cloud."
+        except Exception as e:
+            backup_status = f"Warning: Cloud sync failed ({e})"
+    else:
+        backup_status = "Warning: No HF_TOKEN. Data is local."
+        
+    return f"Success! Report added. {backup_status}", pd.read_csv(DB_FILE).tail(10)
+
+# ==========================================
+# 4. ФУНКЦИЯ ДЛЯ ПРОСМОТРА ПОЛНОЙ БАЗЫ (Tab 5)
+# ==========================================
+def view_database():
+    if not os.path.exists(DB_FILE):
+        return pd.DataFrame()
+    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative']).reset_index(drop=True)
+    if len(df) == 0:
+        return pd.DataFrame()
+    
+    # Создаем анонимные ID для удобного чтения
+    df.insert(0, 'Report_ID', [f"Report #{i+1}" for i in range(len(df))])
+    # Возвращаем нужные столбцы, скрывая реальные имена (Alias)
+    return df[['Report_ID', 'Timestamp', 'ASC_Type', 'Emotion', 'Intensity', 'Narrative']]
 
 # ==========================================
-# ANALYSES (без ссылок)
+# 5. ФУНКЦИИ МАКРО-АНАЛИЗА (Tab 2: Heatmap & Graph)
 # ==========================================
 @spaces.GPU
 def macro_analysis():
-    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative'])
-    if len(df) < 3:
-        return None, None, "Not enough data (minimum 3 reports)"
+    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative']).reset_index(drop=True)
+    if len(df) < 2:
+        return None, None
     
     texts = df['Narrative'].tolist()
+    # Заменяем имена на анонимные номера отчетов
+    report_ids = [f"Report #{i+1}" for i in range(len(df))]
+    
+    # Векторизация целых документов для общей картины
     embeddings = model.encode(texts)
     sim_matrix = cosine_similarity(embeddings)
-    labels = [f"R{i+1}" for i in range(len(texts))]
     
-    fig_heat, ax = plt.subplots(figsize=(9, 7))
-    sns.heatmap(sim_matrix, xticklabels=labels, yticklabels=labels, annot=True, cmap="YlOrRd", fmt=".2f", ax=ax)
-    ax.set_title("Report Similarity Matrix")
+    # Построение Heatmap
+    fig_heat, ax_heat = plt.subplots(figsize=(8, 6))
+    sns.heatmap(sim_matrix, xticklabels=report_ids, yticklabels=report_ids, 
+                annot=True, cmap="YlOrRd", fmt=".2f", ax=ax_heat)
+    ax_heat.set_title("Document Cosine Similarity Matrix")
     plt.tight_layout()
     
-    fig_graph, ax_g = plt.subplots(figsize=(9, 7))
+    # Построение Network Graph
+    fig_graph, ax_graph = plt.subplots(figsize=(8, 6))
     G = nx.Graph()
-    threshold = 0.45
-    for i in range(len(texts)):
-        G.add_node(f"R{i+1}")
-    for i in range(len(texts)):
-        for j in range(i + 1, len(texts)):
-            if sim_matrix[i, j] > threshold:
-                G.add_edge(f"R{i+1}", f"R{j+1}", weight=sim_matrix[i, j])
     
+    for i, r_id in enumerate(report_ids):
+        G.add_node(r_id)
+        
+    threshold = 0.40 # Порог сходства для создания связи
+    for i in range(len(report_ids)):
+        for j in range(i + 1, len(report_ids)):
+            if sim_matrix[i, j] > threshold:
+                G.add_edge(report_ids[i], report_ids[j], weight=sim_matrix[i, j])
+                
     pos = nx.spring_layout(G, seed=42)
-    nx.draw_networkx_nodes(G, pos, node_color="lightblue", node_size=1400, ax=ax_g)
-    nx.draw_networkx_labels(G, pos, font_size=10, ax=ax_g)
-    weights = [G[u][v]['weight'] * 7 for u, v in G.edges()]
-    nx.draw_networkx_edges(G, pos, width=weights, edge_color='darkred', alpha=0.7, ax=ax_g)
-    ax_g.set_title("Network of Shared Semantic Structures")
-    ax_g.axis('off')
+
+    # Рисуем только вершины
+    nx.draw_networkx_nodes(
+        G, pos, node_color="skyblue", node_size=2000, ax=ax_graph
+    )
+
+    # Подписи
+    nx.draw_networkx_labels(
+        G, pos, font_size=10, font_weight="bold", ax=ax_graph
+    )
     
-    return fig_heat, fig_graph, f"Analyzed {len(df)} reports."
+    # Рисуем толщину линий в зависимости от силы сходства
+    edges = G.edges()
+    weights = [G[u][v]['weight'] * 5 for u,v in edges]
+    nx.draw_networkx_edges(G, pos, edgelist=edges, width=weights, edge_color='red', alpha=0.5, ax=ax_graph)
+    ax_graph.set_title(f"Semantic Network Graph (Threshold > {threshold})")
 
+    return fig_heat, fig_graph
+
+# ==========================================
+# 6. ФУНКЦИИ МИКРО-АНАЛИЗА (Tab 3: Sentence t-SNE)
+# ==========================================
 @spaces.GPU
 def micro_analysis():
-    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative'])
+    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative']).reset_index(drop=True)
     if len(df) < 2:
-        return None, "Not enough data"
+        return None, "Not enough data."
     
     all_sentences = []
-    parent_report = []
+    parent_report_ids = []
     
     for idx, row in df.iterrows():
-        rid = f"R{idx+1}"
         sents = split_into_sentences(row['Narrative'])
         all_sentences.extend(sents)
-        parent_report.extend([rid] * len(sents))
-    
+        # Привязываем предложение к анонимному номеру отчета
+        parent_report_ids.extend([f"Report #{idx+1}"] * len(sents))
+        
     if len(all_sentences) < 5:
-        return None, "Not enough sentences"
-    
+        return None, "Not enough sentences extracted."
+        
     sent_embeddings = model.encode(all_sentences)
-    tsne = TSNE(n_components=2, random_state=42, perplexity=min(30, len(all_sentences)-1))
-    vecs = tsne.fit_transform(sent_embeddings)
     
-    fig, ax = plt.subplots(figsize=(10, 8))
-    sns.scatterplot(x=vecs[:,0], y=vecs[:,1], hue=parent_report, palette="tab10", s=90, ax=ax)
-    ax.set_title("Semantic Fragments Clustering (t-SNE)")
-    plt.legend(bbox_to_anchor=(1.05, 1))
+    perplexity = min(30, len(all_sentences) - 1)
+    tsne = TSNE(n_components=2, random_state=42, perplexity=perplexity)
+    vecs_2d = tsne.fit_transform(sent_embeddings)
+    
+    fig_tsne, ax_tsne = plt.subplots(figsize=(10, 8))
+    sns.scatterplot(x=vecs_2d[:,0], y=vecs_2d[:,1], hue=parent_report_ids, palette="tab10", s=100, ax=ax_tsne)
+    ax_tsne.set_title("Sentence-Level Semantic Projections (t-SNE)")
+    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
     plt.tight_layout()
     
     sim_matrix = cosine_similarity(sent_embeddings)
     mean_sims = sim_matrix.mean(axis=1)
-    top_idx = mean_sims.argsort()[-8:][::-1]
-    
-    text = "### �� Most Central Reproducible Fragments\n\n"
-    for i in top_idx:
-        rid = parent_report[i]
-        text += f"**{rid}** (Centrality: {mean_sims[i]:.3f})\n"
-        text += f"\"{all_sentences[i]}\"\n\n"
-        text += f"→ Go to Tab 5 and enter **{rid}** to read the full report.\n---\n"
+    top_indices = mean_sims.argsort()[-5:][::-1]
     
-    return fig, text
-
-@spaces.GPU
-def hybrid_pattern_discovery(mode, custom_motifs_text):
-    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative'])
-    if len(df) < 3:
-        return "Not enough data (minimum 3 reports needed)"
-    return "✅ Analysis completed.\n\nFull original reports can be viewed in **Tab 5 → Full Reports Explorer**."
+    central_text = "### Top 5 Most Central Semantic Fragments\n\n"
+    for idx in top_indices:
+        central_text += f"> **{parent_report_ids[idx]}**: \"{all_sentences[idx]}\" *(Centrality Score: {mean_sims[idx]:.2f})*\n\n"
+        
+    return fig_tsne, central_text
 
 # ==========================================
-# FULL REPORT VIEWER
+# 7. ГИБРИДНЫЙ ДВИЖОК ПОИСКА (Tab 4)
 # ==========================================
-def show_full_report(report_id):
-    try:
-        df = pd.read_csv(DB_FILE)
-        idx = int(report_id.strip().replace("R", "").replace("r", "")) - 1
-        if 0 <= idx < len(df):
-            row = df.iloc[idx]
-            return f"""**Report ID:** {report_id.upper()}
-**Date:** {row['Timestamp']}
-**Participant:** {row['Alias']}
-**Type:** {row['ASC_Type']} | **Emotion:** {row['Emotion']} | **Intensity:** {row['Intensity']}
-
----
-
-{row['Narrative']}
-"""
-        return "❌ Report not found. Check the ID in Tab 1 or Tab 5 table."
-    except:
-        return "❌ Invalid format. Please enter ID like: R1, R2, R5"
+@spaces.GPU
+def hybrid_pattern_discovery(mode, custom_motifs_text):
+    df = pd.read_csv(DB_FILE).dropna(subset=['Narrative']).reset_index(drop=True)
+    if len(df) < 2:
+        return "Not enough data. Need at least 2 reports."
+        
+    all_sentences = []
+    parent_report_ids = []
+    
+    for idx, row in df.iterrows():
+        sents = split_into_sentences(row['Narrative'])
+        all_sentences.extend(sents)
+        parent_report_ids.extend([f"Report #{idx+1}"] * len(sents))
+        
+    if len(all_sentences) < 5:
+        return "Not enough detailed sentences."
+        
+    # РЕЖИМ 1: СЛЕПОЙ ПОИСК
+    if mode == "Blind Extraction (Unsupervised)":
+        sent_embeddings = model.encode(all_sentences)
+        num_clusters = max(2, min(8, len(all_sentences) // 10))
+        
+        from sklearn.cluster import KMeans
+        kmeans = KMeans(n_clusters=num_clusters, random_state=42)
+        labels = kmeans.fit_predict(sent_embeddings)
+        
+        results = "### 👁️ Blind AI Extraction: Emergent Signals from Noise\n"
+        results += f"*Analyzed {len(all_sentences)} sentences. Extracted {num_clusters} hidden thematic clusters without human prompts.*\n\n"
+        
+        for i in range(num_clusters):
+            cluster_indices = np.where(labels == i)[0]
+            if len(cluster_indices) < 2: continue
+                
+            cluster_embeddings = sent_embeddings[cluster_indices]
+            centroid = kmeans.cluster_centers_[i]
+            
+            from sklearn.metrics.pairwise import cosine_distances
+            distances = cosine_distances([centroid], cluster_embeddings)[0]
+            global_central_idx = cluster_indices[np.argmin(distances)]
+            central_sentence = all_sentences[global_central_idx]
+            
+            authors_in_cluster = set([parent_report_ids[idx] for idx in cluster_indices])
+            
+            if len(authors_in_cluster) > 1:
+                results += f"#### 🟢 Discovered Archetype: *«{central_sentence[:100]}...»*\n"
+                results += f"**Cross-validation:** Found in {len(authors_in_cluster)} independent reports.\n"
+                count = 0
+                for idx in cluster_indices:
+                    if idx != global_central_idx and count < 3:
+                        results += f"- *{parent_report_ids[idx]}*: \"{all_sentences[idx]}\"\n"
+                        count += 1
+                results += "---\n"
+        return results
 
-# ==========================================
-# INTERFACE
-# ==========================================
-with gr.Blocks(theme=gr.themes.Monochrome()) as app:
-    gr.Markdown("# 🌌 DreamCode — Detector of Reproducible Informational Signals")
+    # РЕЖИМ 2: ЦЕЛЕВОЙ ПОИСК
+    else:
+        seed_motifs = [m.strip() for m in re.split(r'[,|\n]', custom_motifs_text) if m.strip()]
+        if not seed_motifs:
+            return "Please enter at least one motif to search for."
+            
+        motif_embeddings = model.encode(seed_motifs)
+        results = "### 🎯 Targeted AI Search: Hypothesis Testing\n\n"
+        
+        for idx_motif, motif in enumerate(seed_motifs):
+            motif_emb = motif_embeddings[idx_motif]
+            reports_with_motif = 0
+            best_matches = []
+            
+            for idx_row, row in df.iterrows():
+                sents = split_into_sentences(row['Narrative'])
+                if not sents: continue
+                
+                sent_embs = model.encode(sents)
+                sims = cosine_similarity([motif_emb], sent_embs)[0]
+                
+                max_sim = np.max(sims)
+                if max_sim > 0.40: 
+                    reports_with_motif += 1
+                    best_idx = np.argmax(sims)
+                    best_matches.append(f"*Report #{idx_row+1}*: \"{sents[best_idx]}\"")
+                    
+            results += f"#### ✔ Target: '{motif}'\n"
+            results += f"**Found in {reports_with_motif} out of {len(df)} reports.**\n"
+            if best_matches:
+                for match in best_matches[:4]:
+                    results += f"- {match}\n"
+            results += "---\n"
+            
+        return results
 
+# ----------------- ИНТЕРФЕЙС GRADIO -----------------
+with gr.Blocks() as app:
+    gr.Markdown("# 🌌 DreamCode: Research Platform")
+    
     with gr.Tabs():
+        # TAB 1: Data Ingestion
         with gr.TabItem("1. Data Ingestion"):
             with gr.Row():
                 with gr.Column():
-                    alias = gr.Textbox(label="Participant ID / Alias (optional)")
-                    asc_type = gr.Dropdown(choices=["Ordinary Dream", "Lucid Dream (LD)", "OBE", "NDE", "Other"], label="Altered State Type")
-                    emotion = gr.Radio(choices=["Positive", "Neutral", "Negative"], label="Dominant Emotion")
-                    intensity = gr.Slider(1, 3, value=2, step=1, label="Intensity (1-3)")
-                    narrative = gr.Textbox(label="Full Narrative / Experience Description", lines=8)
+                    alias = gr.Textbox(label="Participant ID (Kept private during analysis)")
+                    asc_type = gr.Dropdown(choices=["Ordinary Dream", "Lucid Dream (LD)", "OBE", "NDE", "Other"], label="State")
+                    emotion = gr.Radio(choices=["Positive", "Neutral", "Negative"], label="Emotional Tone")
+                    intensity = gr.Slider(minimum=1, maximum=3, step=1, label="Intensity")
+                    narrative = gr.Textbox(label="Narrative", lines=7)
                     submit_btn = gr.Button("Submit Experience", variant="primary")
-                
+                    
                 with gr.Column():
                     status_output = gr.Textbox(label="Status")
-                    recent_data = gr.Dataframe(label="Recent Entries")
-            
-            submit_btn.click(process_entry, inputs=[alias, asc_type, emotion, intensity, narrative],
-                             outputs=[status_output, recent_data])
+                    data_preview = gr.Dataframe(headers=["Timestamp", "Alias", "ASC_Type", "Emotion", "Intensity", "Narrative"])
+                    
+            submit_btn.click(fn=process_entry, inputs=[alias, asc_type, emotion, intensity, narrative], outputs=[status_output, data_preview])
 
-        with gr.TabItem("2. Document-Level Structures"):
-            gr.Markdown("Similarity between full reports")
-            btn_macro = gr.Button("Run Macro Analysis", variant="primary")
+        # TAB 2: Document Level Analysis
+        with gr.TabItem("2. Document Similarity"):
+            gr.Markdown("Analyze macro-connections between entire reports using Cosine Similarity (Anonymized).")
+            analyze_macro_btn = gr.Button("Generate Matrix & Graph", variant="primary")
             with gr.Row():
-                heat = gr.Plot(label="Similarity Heatmap")
-                net = gr.Plot(label="Semantic Network")
-            summary = gr.Textbox(label="Summary")
-            btn_macro.click(macro_analysis, outputs=[heat, net, summary])
-
-        with gr.TabItem("3. Scene & Fragment Clustering"):
-            gr.Markdown("Sentence-level semantic analysis")
-            btn_micro = gr.Button("Run Sentence Analysis", variant="primary")
+                heat_plot = gr.Plot(label="Cosine Similarity Heatmap")
+                network_plot = gr.Plot(label="Semantic Network Graph")
+                
+            analyze_macro_btn.click(fn=macro_analysis, inputs=[], outputs=[heat_plot, network_plot])
+            
+        # TAB 3: Sentence Level Analysis
+        with gr.TabItem("3. Scene & Sentence t-SNE"):
+            gr.Markdown("AI splits narratives into individual sentences to cluster specific scenes.")
+            analyze_micro_btn = gr.Button("Process Sentences", variant="primary")
             with gr.Row():
-                tsne_plot = gr.Plot()
-                central_md = gr.Markdown()
-            btn_micro.click(micro_analysis, outputs=[tsne_plot, central_md])
+                tsne_plot = gr.Plot(label="Sentence t-SNE")
+                central_text = gr.Markdown(label="Central Fragments")
+                
+            analyze_micro_btn.click(fn=micro_analysis, inputs=[], outputs=[tsne_plot, central_text])
 
+        # TAB 4: AI Pattern Discovery
         with gr.TabItem("4. AI Pattern Discovery"):
-            gr.Markdown("### Main Analysis Engine")
-            mode = gr.Radio(["Blind Extraction (Unsupervised)", "Targeted Search (Zero-Shot)"], value="Blind Extraction (Unsupervised)")
-            motifs = gr.Textbox(label="Custom motifs (optional for Targeted mode)", lines=3, visible=False)
-            btn_analyze = gr.Button("Run Analysis Engine", variant="primary")
-            result_md = gr.Markdown()
-            btn_analyze.click(hybrid_pattern_discovery, inputs=[mode, motifs], outputs=[result_md])
-
-        with gr.TabItem("5. Full Reports Explorer"):
-            gr.Markdown("### All Reports & Full Text Viewer")
-            gr.Dataframe(get_all_reports(), label="All Submitted Reports")
+            gr.Markdown("### Semantic Search Engine\nChoose between extracting unknown signals automatically or testing specific hypotheses against the database.")
             
-            with gr.Row():
-                id_input = gr.Textbox(label="Enter Report ID (example: R1, R3, R7)", placeholder="R1")
-                view_button = gr.Button("Show Full Report", variant="primary", size="large")
+            search_mode = gr.Radio(
+                choices=["Blind Extraction (Unsupervised)", "Targeted Search (Zero-Shot)"],
+                value="Blind Extraction (Unsupervised)",
+                label="Select Analysis Mode"
+            )
+            
+            motif_input = gr.Textbox(
+                label="Enter custom phrases, motifs, or fragments from your own dream (separated by commas or new lines)", 
+                lines=3, 
+                value="A red celestial body, Huge planet in the sky, Feeling of global catastrophe",
+                visible=False 
+            )
+            
+            def toggle_input(mode):
+                if mode == "Targeted Search (Zero-Shot)":
+                    return gr.update(visible=True)
+                else:
+                    return gr.update(visible=False)
+                    
+            search_mode.change(fn=toggle_input, inputs=[search_mode], outputs=[motif_input])
+            
+            analyze_btn = gr.Button("Run Analysis Engine", variant="primary")
+            output_md = gr.Markdown()
+            
+            analyze_btn.click(fn=hybrid_pattern_discovery, inputs=[search_mode, motif_input], outputs=[output_md])
+
+        # TAB 5: Database Explorer (NEW)
+        with gr.TabItem("5. Database Explorer"):
+            gr.Markdown("### 📖 Full Reports Viewer\nCross-reference the **Report ID** from the analysis tabs to read the full context of the experience here.")
+            refresh_btn = gr.Button("Refresh Database")
+            # wrap=True позволяет тексту переноситься на новые строки, чтобы читать абзацы целиком
+            db_display = gr.Dataframe(wrap=True) 
             
-            full_text = gr.Markdown(label="Full Report Content")
-            view_button.click(show_full_report, inputs=id_input, outputs=full_text)
+            refresh_btn.click(fn=view_database, inputs=[], outputs=[db_display])
+            app.load(fn=view_database, inputs=[], outputs=[db_display])
 
-app.launch()
+app.launch(theme=gr.themes.Monochrome())