Fix gradio version conflict

Backup code/app_21.py

@@ -0,0 +1,942 @@
+# -*- coding:utf-8 -*-
+
+import os
+import sys
+import json
+import time
+import datetime
+import copy
+import shutil
+import signal
+import logging
+import platform
+import re
+
+import gradio as gr
+import nltk
+import torch
+import torch.nn.functional as F
+import tiktoken
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+# ==========================================
+# Hugging Face Dataset Integration
+# ==========================================
+from huggingface_hub import hf_hub_download, HfApi
+
+from openai import OpenAI as OpenAIClient
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.llms.openai import OpenAI as LlamaIndexOpenAI
+from llama_index.core import Settings
+from llama_index.core.indices.prompt_helper import PromptHelper
+
+prompt_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../')
+sys.path.append(prompt_path)
+
+# Local module imports
+from utils.sys_args import data_args, model_args
+from utils.app_modules.utils import *
+from utils.app_modules.presets import *
+from utils.app_modules.overwrites import *
+from utils.prompt_utils import *
+from utils.memory_utils import (
+    enter_name_llamaindex,
+    summarize_memory_event_personality,
+    save_local_memory,
+    extract_session_summary,
+    extract_semantic_memory,
+)
+
+# Ensure NLTK data path
+nltk.data.path = [os.path.join(os.path.dirname(__file__), "nltk_data")] + nltk.data.path
+
+tokenizer = tiktoken.get_encoding("cl100k_base")
+
+GAPGPT_BASE_URL = os.getenv("GAPGPT_BASE_URL", "https://api.gapgpt.app/v1")
+openai_client_cache = {}
+
+HF_MODEL_ID = "Keyvan1986/Emma-Classification_Model"
+HF_TOKEN = os.getenv("HF_TOKEN")  # در صورت Private بودن مدل
+
+print(f"Loading classifier model from Hugging Face: {HF_MODEL_ID}")
+
+try:
+    _tokenizer = AutoTokenizer.from_pretrained(
+        HF_MODEL_ID, 
+        token=HF_TOKEN
+    )
+    _classifier_model = AutoModelForSequenceClassification.from_pretrained(
+        HF_MODEL_ID, 
+        token=HF_TOKEN
+    )
+    _classifier_model.eval()
+    print("Classifier model loaded successfully from Hugging Face Hub.")
+except Exception as e:
+    print(f"Error loading classifier from Hugging Face: {e}")
+    _tokenizer = None
+    _classifier_model = None
+
+
+
+def get_gapgpt_client(api_key: str) -> OpenAIClient:
+    if not api_key:
+        raise ValueError("API key is None while attempting to create a GapGPT client.")
+    if api_key not in openai_client_cache:
+        openai_client_cache[api_key] = OpenAIClient(api_key=api_key, base_url=GAPGPT_BASE_URL)
+    return openai_client_cache[api_key]
+
+
+os_name = platform.system()
+clear_command = 'cls' if os_name == 'Windows' else 'clear'
+stop_stream = False
+
+
+def signal_handler(signal_number, frame):
+    global stop_stream
+    stop_stream = True
+
+
+VECTOR_SEARCH_TOP_K = 2
+
+# Update this path to your actual file location
+api_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "api_key_list.txt")
+
+
+def read_apis(path):
+    api_keys_local = []
+    if os.path.exists(path):
+        with open(path, 'r', encoding='utf8') as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    api_keys_local.append(line)
+    return api_keys_local
+
+
+# ==========================================
+# پیکربندی حافظه و اتصال به Hugging Face Dataset
+# ==========================================
+HF_TOKEN = os.getenv("HF_TOKEN")  # توکن Write هاگینگ‌فیس
+HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "YOUR_HF_USERNAME/YOUR_DATASET_NAME") # نام دیتاست خود را وارد کنید
+HF_MEMORY_FILENAME = "update_memory_0512_eng.json"
+
+base_data_dir = os.getenv("EMMA_DATA_DIR", os.path.join(os.path.dirname(os.path.abspath(__file__)), "data"))
+memory_dir = os.path.join(base_data_dir, "memories", HF_MEMORY_FILENAME)
+os.makedirs(os.path.dirname(memory_dir), exist_ok=True)
+
+# ۱. دانلود فایل از دیتاست در ابتدای اجرای برنامه
+try:
+    print(f"Fetching {HF_MEMORY_FILENAME} from Hugging Face Dataset: {HF_DATASET_REPO}...")
+    hf_hub_download(
+        repo_id=HF_DATASET_REPO,
+        filename=HF_MEMORY_FILENAME,
+        repo_type="dataset",
+        local_dir=os.path.dirname(memory_dir),
+        token=HF_TOKEN
+    )
+    print("Memory file successfully downloaded from Hugging Face Dataset.")
+except Exception as e:
+    print(f"Warning: Could not fetch memory from Hugging Face ({e}). Checking local fallback...")
+    if not os.path.exists(memory_dir):
+        with open(memory_dir, "w", encoding="utf-8") as f:
+            json.dump({}, f)
+
+# لود کردن حافظه در رم
+global memory
+try:
+    with open(memory_dir, "r", encoding="utf-8") as f:
+        memory = json.load(f)
+except Exception as e:
+    print(f"Error parsing memory JSON: {e}. Initializing empty memory.")
+    memory = {}
+
+
+def sync_memory_to_hf():
+    """ذخیره لوکال و آپلود همزمان به دیتاست Hugging Face"""
+    try:
+        # ۱. ذخیره روی دیسک کانتینر
+        with open(memory_dir, "w", encoding="utf-8") as f:
+            json.dump(memory, f, ensure_ascii=False, indent=4)
+
+        # ۲. آپلود به دیتاست
+        if HF_TOKEN and HF_DATASET_REPO:
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=memory_dir,
+                path_in_repo=HF_MEMORY_FILENAME,
+                repo_id=HF_DATASET_REPO,
+                repo_type="dataset",
+                token=HF_TOKEN,
+                commit_message=f"Update user memory - {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+            )
+            print("Memory successfully synced to Hugging Face Dataset.")
+    except Exception as e:
+        print(f"Error syncing memory to HF Dataset: {e}")
+
+
+language = 'en'
+user_keyword = generate_user_keyword()[language]
+ai_keyword = generate_ai_keyword()[language]
+boot_name = boot_name_dict[language]
+boot_actual_name = boot_actual_name_dict[language]
+meta_prompt = generate_meta_prompt_dict_chatgpt()[language]
+meta_prompt_semantic = generate_meta_prompt_dict_semantic_chatgpt()[language]
+meta_prompt_semantic_episodic = generate_meta_prompt_dict_semantic_episodic_chatgpt()[language]
+new_user_meta_prompt = generate_new_user_meta_prompt_dict_chatgpt()[language]
+
+api_keys = read_apis(api_path)
+if not api_keys:
+    print(f"Warning: No API keys found in {api_path}.")
+
+new_conversation = False
+chatgpt_config = {
+    "model": "gpt-4o",
+    "temperature": 1,
+    "max_tokens": 1024,
+    "top_p": 0.95,
+    "frequency_penalty": 0.4,
+    "presence_penalty": 0.2,
+    "n": 1,
+}
+
+deactivated_keys = []
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s",
+)
+
+
+def chatgpt_chat(prompt, system, history, gpt_config, api_index=0):
+    retry_times, count = 5, 0
+    response = None
+
+    while response is None and count < retry_times:
+        try:
+            request = copy.deepcopy(gpt_config)
+            
+            if data_args.language == 'en':
+                message = [
+                    {"role": "system", "content": system.strip()},
+                    {"role": "user", "content": "Hi!"},
+                    {"role": "assistant", "content": f"Hi! I'm {boot_actual_name}! I will give you warm companion!"},
+                ]
+            else:
+                message = [
+                    {"role": "system", "content": system.strip()},
+                    {"role": "user", "content": "Hi"},
+                    {"role": "assistant", "content": f"Hi! I'm {boot_actual_name}! I will give you warm companion!"},
+                ]
+
+            if history:
+                if isinstance(history[0], list):
+                    for q, a in history:
+                        message.append({"role": "user", "content": str(q)})
+                        message.append({"role": "assistant", "content": str(a)})
+                elif isinstance(history[0], dict):
+                    for msg in history:
+                        if msg.get('role') in ['user', 'assistant']:
+                            message.append(msg)
+
+            message.append({"role": "user", "content": f"{prompt}"})
+
+            if not api_keys:
+                return "Error: No API Keys available."
+                
+            client = get_gapgpt_client(api_keys[api_index])
+            response = client.chat.completions.create(messages=message, **request)
+
+        except Exception as e:
+            print(f"Chat Error: {e}")
+            if 'This key is associated with a deactivated account' in str(e):
+                deactivated_keys.append(api_keys[api_index])
+
+            if api_keys:
+                api_index = api_index + 1 if api_index < len(api_keys) - 1 else 0
+                loop_check = 0
+                while api_keys[api_index] in deactivated_keys and loop_check < len(api_keys):
+                    api_index = api_index + 1 if api_index < len(api_keys) - 1 else 0
+                    loop_check += 1
+            count += 1
+
+    if response:
+        response = response.choices[0].message.content
+    else:
+        response = ''
+    return response
+
+
+def classify_query_local(text):
+    id2label_map = {
+        0: "episodic_memory",
+        1: "semantic_memory",
+        2: "semantic-episodic",  
+        3: "unknown"             
+    }
+
+    try:
+        inputs = _tokenizer(
+            text, 
+            return_tensors="pt", 
+            truncation=True, 
+            padding=True, 
+            max_length=512
+        )
+        
+        with torch.no_grad():
+            outputs = _classifier_model(**inputs)
+        
+        logits = outputs.logits
+        probabilities = F.softmax(logits, dim=-1)
+        predicted_class_id = torch.argmax(probabilities, dim=-1).item()
+        
+        category = id2label_map.get(predicted_class_id, "unknown")
+        print(f"Local Classifier: '{text}' -> {category} (Class ID: {predicted_class_id})")
+        return category
+
+    except Exception as e:
+        print(f"Error in local classification: {e}")
+        return "unknown"
+
+
+def classify_query_openai(text, gpt_config, api_index=0, retry_times=5):
+    response = None
+    count = 0
+    local_deactivated = []
+    print("text********:", text)
+
+    system_prompt = """
+You are an AI that classifies user queries into one of the following memory types:
+- 'episodic_memory': Queries about past personal events, daily life logs, or specific experiences the user has shared (e.g., "What did I eat yesterday?", "Tell me about my trip").
+- 'semantic_memory': Queries about facts, preferences, general knowledge the user has taught you, or summaries of their personality (e.g., "What is my favorite color?", "Do I like sci-fi movies?").
+- 'semantic-episodic': Complex queries requiring both specific past events and general facts/preferences (e.g., "Based on my food preferences, did I enjoy the dinner last night?").
+
+Output ONLY one of these three strings: 'episodic_memory', 'semantic_memory', or 'semantic-episodic'. If unsure, output 'episodic_memory'.
+    """.strip()
+    
+    if not api_keys:
+        return "unknown"
+
+    while response is None and count < retry_times:
+        try:
+            client = get_gapgpt_client(api_keys[api_index])
+            response = client.chat.completions.create(
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": text.strip()},
+                ],
+                **copy.deepcopy(gpt_config),
+            )
+        except Exception as e:
+            print(f"Classify Error: {e}")
+            if "This key is associated with a deactivated account" in str(e):
+                local_deactivated.append(api_keys[api_index])
+
+            api_index = api_index + 1 if api_index < len(api_keys) - 1 else 0
+            while api_keys[api_index] in local_deactivated:
+                api_index = api_index + 1 if api_index < len(api_keys) - 1 else 0
+            count += 1
+
+    if response:
+        category = response.choices[0].message.content.strip().lower()
+        if "semantic_memory" in category: category = "semantic_memory"
+        elif "semantic-episodic" in category: category = "semantic-episodic"
+        elif "episodic_memory" in category: category = "episodic_memory"
+        else: category = "unknown"
+    else:
+        category = "unknown"
+
+    return category
+
+
+def predict_new(
+    text,
+    history,
+    top_p,
+    temperature,
+    max_length_tokens,
+    max_context_length_tokens,
+    user_name,
+    user_memory,
+    user_memory_index,
+    service_context,
+    api_index,
+    semantic_memory_text,
+    query_category,
+    user_profile=None
+):
+    chatgpt_cfg = {
+        "model": "gpt-4o",
+        "temperature": temperature,
+        "max_tokens": max_length_tokens,
+        "top_p": top_p,
+        "frequency_penalty": 0.4,
+        "presence_penalty": 0.2,
+        "n": 1,
+    }
+
+    if text == "":
+        return history, history, "Empty context."
+        
+    if history is None:
+        history = []
+
+    system_prompt, related_memos = build_prompt_with_search_memory_llamaindex(
+        history=history,
+        query=text,
+        user_memory=user_memory,
+        user_name=user_name,
+        user_memory_index=user_memory_index,
+        service_context=service_context,
+        api_keys=api_keys,
+        api_index=api_index,
+        meta_prompt=meta_prompt,
+        new_user_meta_prompt=new_user_meta_prompt,
+        data_args=data_args,
+        boot_actual_name=boot_actual_name,
+        semantic_memory_text=semantic_memory_text,
+        query_category=query_category,
+        meta_prompt_semantic=meta_prompt_semantic,
+        meta_prompt_semantic_episodic=meta_prompt_semantic_episodic,
+    )
+
+    if user_profile:
+        profile_str = (
+            f"\n\n[User Profile Information]:\n"
+            f"- Age: {user_profile.get('age', 'Unknown')}\n"
+            f"- Gender: {user_profile.get('gender', 'Unknown')}\n"
+            f"- Occupation: {user_profile.get('occupation', 'Unknown')}\n"
+            f"- Residence: {user_profile.get('residence', 'Unknown')}\n"
+            f"Use this context subtly when talking to the user."
+        )
+        system_prompt += profile_str
+
+    current_history_for_llm = history
+    if len(history) > data_args.max_history * 2:
+        current_history_for_llm = history[-(data_args.max_history * 2):]
+
+    response = chatgpt_chat(
+        prompt=text,
+        system=system_prompt,
+        history=current_history_for_llm,
+        gpt_config=chatgpt_cfg,
+        api_index=api_index,
+    )
+
+    torch.cuda.empty_cache()
+
+    new_history = history + [
+        {"role": "user", "content": text},
+        {"role": "assistant", "content": response}
+    ]
+
+    if user_name:
+        save_local_memory(memory, new_history, user_name, data_args)
+        # همگام‌سازی بلافاصله با دیتاست Hugging Face
+        sync_memory_to_hf()
+
+    return new_history, new_history, "Generating..."
+
+
+def create_gradio_interface(service_context, api_keys):
+    custom_css = """
+.send-btn {
+    border: 3px solid #0066cc !important;
+    border-radius: 12px !important;
+    background-color: white !important;
+    color: black !important;
+    font-weight: bold !important;
+    font-size: 16px !important;
+    transition: all 0.3s ease !important;
+}
+.send-btn:hover {
+    background-color: #0066cc !important;
+    color: white !important;
+}
+
+.bottom-actions-row {
+    background-color: #e5e7eb !important;
+    border-radius: 8px !important;
+    padding: 5px !important;
+    margin-top: 10px !important;
+}
+
+.action-btn {
+    border: none !important;
+    background: transparent !important;
+    color: black !important;
+    font-weight: bold !important;
+    box-shadow: none !important;
+    transition: all 0.3s ease !important;
+}
+
+.action-btn:hover {
+    background-color: #d1d5db !important;
+    border-radius: 8px !important;
+}
+
+.progress-level, .progress-text, .progress-level svg, .progress-level img {
+    display: none !important;
+}
+
+body:has(.progress-level) {
+    pointer-events: none !important;
+}
+
+body:has(.progress-level)::after {
+    content: "";
+    position: fixed;
+    top: 0; left: 0;
+    width: 100vw; height: 100vh;
+    background-color: rgba(255, 255, 255, 0.7);
+    z-index: 9998;
+    pointer-events: all;
+    cursor: wait;
+}
+
+body:has(.progress-level)::before {
+    content: "";
+    position: fixed;
+    top: 50%; left: 50%;
+    transform: translate(-50%, -50%);
+    width: 60px; height: 60px;
+    border: 6px solid #e5e7eb;
+    border-top-color: #0066cc;
+    border-radius: 50%;
+    animation: spin 1s linear infinite;
+    z-index: 9999;
+}
+
+@keyframes spin {
+    0% { transform: translate(-50%, -50%) rotate(0deg); }
+    100% { transform: translate(-50%, -50%) rotate(360deg); }
+}
+
+.typing-dots::after {
+    content: '';
+    animation: typing 1.5s infinite;
+}
+@keyframes typing {
+    0% { content: ''; }
+    25% { content: ' .'; }
+    50% { content: ' . .'; }
+    75% { content: ' . . .'; }
+    100% { content: ''; }
+}
+"""
+
+    with gr.Blocks(title="EMMA") as demo:
+        gr.HTML(f"<style>{custom_css}</style>")
+        
+        state = gr.State({
+            "history": [], 
+            "user_name": None,
+            "memory": memory,
+            "data_args": data_args,
+            "service_context": service_context,
+            "api_keys": api_keys,
+            "api_index": 0,
+            "semantic_memory_text": "",
+            "new_conversation": True,
+            "initialized": False
+        })
+
+        with gr.Row():
+            # ==========================================
+            # 1. LOGIN PAGE (Left Sidebar)
+            # ==========================================
+            with gr.Sidebar(open=True) as login_page:
+                gr.Markdown("<h1 style='text-align: center;'>🧠 EMMA</h1><h3 style='text-align: center;'>Your Empathetic Mental Health Assistant</h3><br><p style='text-align: center;'>Please login or register to begin.</p>")
+                
+                username_input = gr.Textbox(label="Your Name (Required)", placeholder="e.g., Alex")
+                password_input = gr.Textbox(label="Password (Required)", placeholder="Enter your password", type="password") 
+
+                login_status = gr.Textbox(
+                    visible=False, 
+                    label="Status", 
+                    interactive=False, 
+                    lines=1,         
+                    max_lines=10      
+                )
+                
+                check_user_btn = gr.Button("🔍 Login / Check", variant="primary", elem_classes=["login-btn"])
+                
+                with gr.Column(visible=False) as registration_fields:
+                    gr.Markdown("### ✨ New User Registration")
+                    age_input = gr.Textbox(label="Age", placeholder="e.g., 28")
+                    gender_input = gr.Dropdown(label="Gender", choices=["Male", "Female", "Other"])
+                    occupation_input = gr.Textbox(label="Occupation", placeholder="e.g., Student...")
+                    residence_input = gr.Textbox(label="Place of Residence", placeholder="e.g., Berlin")
+                    register_btn = gr.Button("🎯 Complete Registration", variant="primary", elem_classes=["login-btn"])
+
+            # ==========================================
+            # 2. CHAT PAGE (Main Column)
+            # ==========================================
+            with gr.Column(scale=3, elem_classes=["chat-box"]) as chat_page:
+                active_header = gr.Markdown("<h2 style='text-align: center; color: #333;'>🧠 EMMA: Session</h2>")
+
+                with gr.Group():
+                    chatbot = gr.Chatbot(label="💬 EMMA Conversation", height=350)
+
+                    with gr.Row():
+                        user_input = gr.Textbox(placeholder="Type your message here...", show_label=False, scale=6)
+                        submit_btn = gr.Button("📤 Send", elem_classes=["send-btn"], scale=1)
+
+                    with gr.Row(equal_height=True, elem_classes=["bottom-actions-row"]):
+                        clear_btn = gr.Button("🧹 Clear", elem_classes=["action-btn"])
+                        new_session_btn = gr.Button("🔄 New Session", elem_classes=["action-btn"])
+                        switch_user_btn = gr.Button("👥 Logout", elem_classes=["action-btn"])
+                system_msg = gr.Textbox(label="🔔 System Messages", interactive=False, max_lines=2)
+
+        # -------------------------------------------------------
+        # Internal Functions
+        # -------------------------------------------------------
+        def handle_login_check(name, password, state):
+            if not name.strip() or not password.strip():
+                return (gr.update(), gr.update(), gr.update(), gr.update(visible=True, value="⚠️ Please enter both name and password."), gr.update(), gr.update(), state)
+            
+            if name in memory:
+                stored_password = memory[name].get("profile", {}).get("password")
+                
+                if stored_password and stored_password != password:
+                    return (gr.update(), gr.update(), gr.update(), gr.update(visible=True, value="⚠️ Incorrect password!"), gr.update(), gr.update(), state)
+                
+                hello_msg, user_memory, sessions_memory, episodic_memory, semantic_memory = enter_name_llamaindex(name, memory, data_args)
+                user_memory = summarize_memory_event_personality(data_args, memory, name)
+                
+                new_state = state.copy()
+                new_state["user_name"] = name
+                new_state["memory"] = memory
+                new_state["initialized"] = True
+                new_state["semantic_memory_text"] = user_memory.get("semantic_memory", {})
+                new_state["semantic_memory_index"] = semantic_memory
+                new_state["user_memory_index"] = sessions_memory
+                new_state["episodic_memory_index"] = episodic_memory
+                new_state["history"] = []
+                
+                welcome_msg = hello_msg if hello_msg else f"Welcome back, {name}!"
+                
+                return (
+                    gr.update(open=False),
+                    gr.update(visible=False),
+                    gr.update(visible=True),
+                    gr.update(visible=False, value=""),
+                    gr.update(value=f"<h2 style='text-align: center; color: #333;'>🧠 EMMA: Session for {name}</h2>"),
+                    gr.update(value=welcome_msg),
+                    new_state
+                )
+            else:
+                return (
+                    gr.update(),
+                    gr.update(visible=True),
+                    gr.update(visible=False),
+                    gr.update(visible=True, value="📝 New user detected. Please fill the details below to register with this password."), 
+                    gr.update(),
+                    gr.update(),
+                    state
+                )
+
+        def handle_register(name, password, age, gender, occupation, residence, state):
+            name = str(name).strip()
+
+            if not name or not password.strip():
+                raise gr.Error("Please enter a valid user name and password.")
+
+            if name in memory:
+                raise gr.Error(f"User '{name}' already exists. Please use the login form.")
+
+            memory[name] = {
+                "profile": {
+                    "password": password, 
+                    "age": age,
+                    "gender": gender,
+                    "occupation": occupation,
+                    "residence": residence,
+                },
+                "sessions": [],
+                "episodic_memory": [],
+                "semantic_memory": {},
+            }
+
+            try:
+                (
+                    _,
+                    user_memory,
+                    sessions_memory,
+                    episodic_memory,
+                    semantic_memory,
+                ) = enter_name_llamaindex(
+                    name=name,
+                    memory=memory,
+                    data_args=data_args,
+                    update_memory_index=True,
+                )
+            except Exception:
+                memory.pop(name, None)
+                raise
+
+            # همگام‌سازی کاربر جدید با HF Dataset
+            sync_memory_to_hf()
+
+            new_state = dict(state or {})
+            new_state.update(
+                {
+                    "user_name": name,
+                    "memory": memory,
+                    "initialized": True,
+                    "history": [],
+                    "semantic_memory_text": user_memory.get("semantic_memory", {}),
+                    "user_memory_index": sessions_memory,
+                    "episodic_memory_index": episodic_memory,
+                    "semantic_memory_index": semantic_memory,
+                }
+            )
+
+            welcome_msg = f"Welcome {name}! Registration complete."
+
+            return (
+                gr.update(open=False),
+                gr.update(visible=False),
+                gr.update(visible=True),
+                gr.update(visible=False, value=""),
+                gr.update(
+                    value=(
+                        "<h2 style='text-align: center; color: #333;'>"
+                        f"🧠 EMMA: Session for {name}</h2>"
+                    )
+                ),
+                gr.update(value=welcome_msg),
+                new_state,
+            )
+
+        def switch_user(state):
+            if state["initialized"] and state["user_name"] in state["memory"]:
+                if state["memory"][state["user_name"]]["sessions"]:
+                    previous_session = state["memory"][state["user_name"]]["sessions"][-1]
+                    summary = extract_session_summary(previous_session["conversation"], previous_session["date"], len(state["memory"][state["user_name"]]["sessions"]) - 1)
+                    state["memory"][state["user_name"]]["episodic_memory"].append(summary)
+                    
+                    try:
+                        user_mem = state["memory"][state["user_name"]]
+                        existing_semantic = user_mem.get("semantic_memory", {})
+                        updated_semantic = extract_semantic_memory(summary, existing_semantic)
+                        state["memory"][state["user_name"]]["semantic_memory"] = updated_semantic
+                    except Exception as e:
+                        print(f"Error updating semantic memory on logout: {e}")
+
+                # آپلود تغییرات ایجادشده در زمان لاگ‌اوت به HF Dataset
+                sync_memory_to_hf()
+
+            new_state = state.copy()
+            new_state.update({"history": [], "user_name": None, "semantic_memory_text": "", "initialized": False})
+
+            return (
+                gr.update(open=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(value="Logged out successfully.", visible=True),
+                gr.update(value=""), 
+                gr.update(value=""),
+                new_state,
+                gr.update(value=[]),
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value="")
+            )
+
+        def handle_chat(user_message, state):
+            if not user_message.strip():
+                yield gr.update(), state.get("history", []), state
+                return
+
+            current_history = state.get("history", [])
+            temp_history = current_history + [
+                {"role": "user", "content": user_message},
+                {"role": "assistant", "content": "<span class='typing-dots'>Generating</span>"}
+            ]
+            
+            yield gr.update(value=""), temp_history, state
+
+            user_name = state.get("user_name")
+            user_memory_index = state.get("user_memory_index")
+            user_memory = state.get("user_memory", {})
+            semantic_memory_text = state.get("semantic_memory_text", "")
+            service_context = state.get("service_context")
+            api_index = state.get("api_index", 0)
+
+            user_profile = {}
+            if user_name and state.get("memory") and user_name in state["memory"]:
+                user_profile = state["memory"][user_name].get("profile", {})
+
+            query_category = classify_query_local(user_message)
+
+            new_history, _, status_msg = predict_new(
+                text=user_message,
+                history=current_history,
+                top_p=0.95,
+                temperature=1.0,
+                max_length_tokens=1024,
+                max_context_length_tokens=4096,
+                user_name=user_name,
+                user_memory=user_memory,
+                user_memory_index=user_memory_index,
+                service_context=service_context,
+                api_index=api_index,
+                semantic_memory_text=semantic_memory_text,
+                query_category=query_category,
+                user_profile=user_profile 
+            )
+
+            state["history"] = new_history
+            yield gr.update(), new_history, state
+                    
+        def clear_history(state):
+            state["history"] = []
+            return [], state
+        
+        def handle_new_session(state):
+            user_name = state.get("user_name")
+            if not user_name or "memory" not in state or user_name not in state["memory"]:
+                print("Error: User not logged in or memory None.")
+                return [], state, gr.update()
+
+            user_data = state["memory"][user_name]
+            
+            if user_data.get("sessions") and len(user_data["sessions"]) > 0:
+                last_session = user_data["sessions"][-1]
+                
+                if len(last_session.get("conversation", [])) > 0:
+                    session_date = last_session.get("date", "")
+                    session_id = last_session.get("session_id", 0)
+                    
+                    ep_summary = extract_session_summary(last_session["conversation"], session_date, session_id)
+                    
+                    if "episodic_memory" not in user_data:
+                        user_data["episodic_memory"] = []
+                    user_data["episodic_memory"].append(ep_summary)
+
+                    try:
+                        existing_semantic = user_data.get("semantic_memory", {})
+                        updated_semantic = extract_semantic_memory(ep_summary, existing_semantic)
+                        user_data["semantic_memory"] = updated_semantic 
+                    except Exception as e:
+                        print(f"Error updating semantic memory: {e}")
+
+            new_session_id = len(user_data.get("sessions", []))
+            new_session = {
+                "session_id": new_session_id,
+                "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                "conversation": []
+            }
+            
+            if "sessions" not in user_data:
+                user_data["sessions"] = []
+                
+            user_data["sessions"].append(new_session)
+            
+            # ذخیره و سینک با شروع سشن جدید
+            sync_memory_to_hf()
+
+            state["history"] = []
+            session_number = len(user_data["sessions"])
+            new_header_text = f"<h2 style='text-align: center; color: #333;'>🧠 EMMA: Session {session_number} for {user_name}</h2>"
+            
+            return [], state, gr.update(value=new_header_text)
+
+        check_user_btn.click(
+            handle_login_check,
+            inputs=[username_input, password_input, state],
+            outputs=[login_page, registration_fields, chat_page, login_status, active_header, system_msg, state],
+        )
+
+        register_btn.click(
+            handle_register,
+            inputs=[username_input, password_input, age_input, gender_input, occupation_input, residence_input, state],
+            outputs=[login_page, registration_fields, chat_page, login_status, active_header, system_msg, state],
+        )
+
+        switch_user_btn.click(
+            switch_user,
+            inputs=[state],
+            outputs=[login_page, registration_fields, chat_page, login_status, username_input, password_input, state, chatbot, age_input, gender_input, occupation_input, residence_input, system_msg], 
+        )
+
+        submit_btn.click(
+            handle_chat,
+            inputs=[user_input, state],
+            outputs=[user_input, chatbot, state],
+        )
+        
+        user_input.submit(
+            handle_chat,
+            inputs=[user_input, state],
+            outputs=[user_input, chatbot, state],
+        )
+        
+        clear_btn.click(
+            clear_history,
+            inputs=[state],
+            outputs=[chatbot, state],
+        )
+        
+        new_session_btn.click(
+            handle_new_session,
+            inputs=[state],
+            outputs=[chatbot, state, active_header],
+        )
+
+    return demo
+
+
+def main():
+    global api_keys 
+    gapgpt_key = os.getenv("GAPGPT_API_KEY")
+    
+    if not gapgpt_key:
+        print("Warning: GAPGPT_API_KEY environment variable is not set. Proceeding with keys from file.")
+    else:
+        os.environ["OPENAI_API_KEY"] = gapgpt_key
+        os.environ["OPENAI_API_BASE"] = GAPGPT_BASE_URL  
+        os.environ["OPENAI_BASE_URL"] = GAPGPT_BASE_URL  
+        
+        if gapgpt_key not in api_keys:
+            api_keys.insert(0, gapgpt_key)
+
+    llm = LlamaIndexOpenAI(
+        model="gpt-4o",
+        temperature=1,
+        max_tokens=1024,
+        top_p=0.95,
+        frequency_penalty=0.4,
+        presence_penalty=0.2,
+        api_key=gapgpt_key,
+        api_base=GAPGPT_BASE_URL,
+    )
+
+    embed_model = OpenAIEmbedding(
+        api_key=gapgpt_key,
+        api_base=GAPGPT_BASE_URL,
+        model="text-embedding-ada-002"
+    )
+
+    Settings.llm = llm
+    Settings.embed_model = embed_model 
+
+    Settings.prompt_helper = PromptHelper(
+        context_window=4096,
+        num_output=256,
+        chunk_overlap_ratio=20 / 4096,
+        tokenizer=tokenizer,
+    )
+
+    demo = create_gradio_interface(Settings, api_keys)
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False 
+    )
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGINT, signal_handler)
+    main()

Backup code/build_memory_index_21.py

@@ -0,0 +1,259 @@
+#
+# Memory Indexing Utility for LlamaIndex
+#
+# This script provides functionalities to build and manage memory indices for a conversational AI
+# using the LlamaIndex framework. It supports creating structured memory documents from raw
+# conversation data and persisting them as vector indices for efficient retrieval.
+#
+
+# --- Standard Library Imports ---
+import json
+import os
+
+# --- Third-Party Imports ---
+import tiktoken
+from llama_index.core import (
+    Document,
+    PromptHelper,
+    Settings,           # Replaces the deprecated ServiceContext
+    StorageContext,
+    VectorStoreIndex,
+    load_index_from_storage,
+)
+from llama_index.llms.openai import OpenAI
+# If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
+# from llama_index.embeddings.openai import OpenAIEmbedding
+
+
+# --- Global Variables ---
+# A dictionary to hold the loaded or newly created indices in memory.
+index_set = {}
+
+_INDEX_BASE_DIR = os.getenv(
+    "EMMA_INDEX_DIR",
+    os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "memories", "memory_index", "llamaindex")
+)
+
+# --- Core Functions ---
+
+def setup_global_settings():
+    """
+    Configures global settings for LlamaIndex (v0.10+).
+
+    This function replaces the deprecated `build_service_context` pattern by directly
+    configuring the global `Settings` object for the LLM, tokenizer, and other components.
+    """
+    # 1. Set Tokenizer (to resolve potential tiktoken errors)
+    try:
+        # This encoding is commonly used by OpenAI models like GPT-4.
+        Settings.tokenizer = tiktoken.get_encoding("cl100k_base").encode
+    except Exception as e:
+        print(f"Warning: Could not set the tokenizer. This might lead to issues. Error: {e}")
+
+    # 2. Configure the Language Model (LLM)
+    # Use the LlamaIndex-specific OpenAI class to avoid conflicts with other libraries.
+    # It automatically reads the API key from the "OPENAI_API_KEY" environment variable.
+    Settings.llm = OpenAI(
+        model="gpt-4o",
+        temperature=1.0,
+        max_tokens=1024,
+        api_key=os.environ.get("GAPGPT_API_KEY"),
+        api_base="https://api.gapgpt.app/v1",
+        additional_kwargs={
+            "top_p": 0.95,
+            "frequency_penalty": 0.4,
+            "presence_penalty": 0.2
+        }
+    )
+
+    # 3. Configure the Prompt Helper (with updated parameter names)
+    # This helps manage context window size, number of outputs, etc.
+    Settings.prompt_helper = PromptHelper(
+        context_window=4096,
+        num_output=256,
+        chunk_overlap_ratio=0.1
+    )
+
+    # 4. Configure Chunk Size for document processing
+    Settings.chunk_size = 512
+    Settings.chunk_overlap = 20
+
+    # Optional: Configure the embedding model if needed.
+    # If not set, LlamaIndex defaults to a compatible model (e.g., OpenAI's text-embedding-ada-002).
+    # Settings.embed_model = OpenAIEmbedding(api_key=..., api_base=...)
+
+
+def generate_memory_docs(data, language):
+    """
+    Generates structured LlamaIndex Document objects from a nested dictionary of user memories.
+
+    This function processes different types of memory (sessions, episodic, semantic) and
+    formats them into Document objects suitable for indexing.
+
+    Args:
+        data (dict): A dictionary where keys are user names and values contain their memories.
+        language (str): The language code (e.g., "en"), currently used for formatting.
+
+    Returns:
+        dict: A dictionary where keys are user names and values are dicts containing
+              lists of Document objects for each memory type.
+    """
+    all_user_memories = {}
+
+    for user_name, user_memory in data.items():
+        all_user_memories[user_name] = {
+            "sessions": [],
+            "episodic_memory": [],
+            "semantic_memory": [],
+        }
+
+        # Process session history
+        if "sessions" in user_memory:
+            for session in user_memory["sessions"]:
+                date = session["date"]
+                content = session["conversation"]
+
+                memory_str = f"Session on {date}:\n"
+                for dialog in content:
+                    query = dialog.get("query", "")
+                    response = dialog.get("response", "")
+                    memory_str += f"\n{user_name}: {query.strip()}"
+                    memory_str += f"\nAI: {response.strip()}"
+                memory_str += "\n"
+
+                if "summary" in user_memory and date in user_memory["summary"]:
+                    summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
+                    memory_str += summary
+
+                # Use the 'text' argument for the Document object
+                all_user_memories[user_name]["sessions"].append(Document(text=memory_str))
+
+        # Process episodic memory
+        if "episodic_memory" in user_memory:
+            episodic_str = "Recent experiences:\n"
+            for event in user_memory["episodic_memory"]:
+                episodic_str += f"- {event}\n"
+            all_user_memories[user_name]["episodic_memory"].append(Document(text=episodic_str))
+
+        # Process semantic memory (long-term facts and personality)
+        if "semantic_memory" in user_memory:
+            semantic_str = "Long-term personality traits and facts:\n"
+            if isinstance(user_memory["semantic_memory"], dict):
+                for trait, value in user_memory["semantic_memory"].items():
+                    semantic_str += f"- {trait}: {value}\n"
+            elif isinstance(user_memory["semantic_memory"], list):
+                for item in user_memory["semantic_memory"]:
+                    semantic_str += f"- {item}\n"
+            else:
+                print(f"WARNING: semantic_memory for user '{user_name}' has an unexpected format: {type(user_memory['semantic_memory'])}")
+
+            all_user_memories[user_name]["semantic_memory"].append(Document(text=semantic_str))
+
+    return all_user_memories
+
+
+def build_memory_index(all_user_memories, data_args, name=None):
+    """
+    Build and persist memory indices for each user and memory type.
+    """
+    structured_docs = generate_memory_docs(
+        all_user_memories, data_args.language
+    )
+
+    setup_global_settings()
+
+    for user_name, memories_by_type in structured_docs.items():
+        if name and user_name != name:
+            continue
+
+        print(f"Building indices for user '{user_name}'...")
+
+        base_path = os.path.join(_INDEX_BASE_DIR, user_name)
+
+        path_map = {
+            "sessions": os.path.join(base_path, "sessions"),
+            "episodic_memory": os.path.join(base_path, "episodic_memory"),
+            "semantic_memory": os.path.join(base_path, "semantic_memory"),
+        }
+
+        for memory_type, docs in memories_by_type.items():
+            if not docs:
+                print(f"  -> Skipping '{memory_type}' index (no documents found).")
+                continue
+
+            if memory_type not in path_map:
+                print(f"  -> Skipping unknown memory type: '{memory_type}'")
+                continue
+
+            print(f"  -> Building '{memory_type}' index...")
+
+            cur_index = VectorStoreIndex.from_documents(docs)
+
+            index_dir = path_map[memory_type]
+            os.makedirs(index_dir, exist_ok=True)
+
+            cur_index.storage_context.persist(persist_dir=index_dir)
+            print(f"  + Saved '{memory_type}' index to: {index_dir}")
+
+            index_set[f"{user_name}_{memory_type}"] = cur_index
+
+
+# --- Deprecated Functions (Kept for reference) ---
+
+def generate_memory_docs_old(data, language):
+    """
+    DEPRECATED: An older version of the document generation function.
+
+    This function had a simpler structure and contained Chinese string literals.
+    It is replaced by `generate_memory_docs`.
+    """
+    all_user_memories = {}
+    for user_name, user_memory in data.items():
+        all_user_memories[user_name] = []
+        if "history" not in user_memory:
+            continue
+        for date, content in user_memory["history"].items():
+            memory_str = f"Conversation on {date}:"
+
+            for dialog in content:
+                query = dialog.get("query", "")
+                response = dialog.get("response", "")
+                memory_str += f"\n{user_name}: {query.strip()}"
+                memory_str += f"\nAI: {response.strip()}"
+            memory_str += "\n"
+            if "summary" in user_memory and date in user_memory["summary"]:
+                summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
+                memory_str += summary
+
+            all_user_memories[user_name].append(Document(text=memory_str))
+    return all_user_memories
+
+
+def build_memory_index_old(all_user_memories, data_args, name=None):
+    """
+    DEPRECATED: An older version of the index building function.
+
+    This function used a flatter memory structure and is replaced by the more
+    comprehensive `build_memory_index`.
+    """
+    # The function name is a typo in the original code, corrected here for clarity.
+    all_user_memories_docs = generate_memory_docs(
+        all_user_memories, data_args.language
+    )
+
+    # Apply global settings
+    setup_global_settings()
+
+    for user_name, memories in all_user_memories_docs.items():
+        if name and user_name != name:
+            continue
+        print(f"Building index for user {user_name} (using old method)...")
+
+        # The service_context argument is removed in newer versions.
+        cur_index = VectorStoreIndex.from_documents(memories)
+        index_set[user_name] = cur_index
+
+        # Corrected save path; the new persist method requires a directory.
+        save_dir = f"../memories/memory_index/llamaindex/{user_name}_store"
+        os.makedirs(save_dir, exist_ok=True)
+        cur_index.storage_context.persist(persist_dir=save_dir)

Backup code/summarize_memory_21.py

@@ -0,0 +1,433 @@
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import json
+import time
+import copy
+import argparse
+from typing import Dict, Any
+from openai import OpenAI
+import re
+import openai
+
+# Add memory bank path
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'memory_bank'))
+
+# Configuration for OpenAI API
+CHATGPT_CONFIG = {
+    "model": "gpt-4o",
+    "temperature": 0.7,
+    "max_tokens": 400,
+    "top_p": 1.0,
+    "frequency_penalty": 0.4,
+    "presence_penalty": 0.2,
+    "stop": ["<|im_end|>", "¬Human"]
+}
+
+# Default memory directory
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+MEMORY_DIR = os.path.join(BASE_DIR, "memories", "update_memory_0512_eng.json")
+
+class LLMClientSimple:
+    def __init__(self, config, gen_config=None):
+        self.config = config
+        self.client = OpenAI(
+            api_key=self.config.get("api_key") or os.environ.get("GAPGPT_API_KEY"),
+            base_url=self.config.get("base_url") or os.environ.get(
+                "GAPGPT_BASE_URL", "https://api.gapgpt.app/v1"
+            ),
+        )
+        self.disable_tqdm = False
+        self.gen_config = copy.deepcopy(
+            gen_config if gen_config is not None else config
+        )
+
+        self.disable_tqdm = False
+        self.gen_config = gen_config
+
+    def generate_text_simple(self, prompt, prompt_num, language='en'):
+        self.gen_config['n'] = prompt_num
+        retry_times, count = 5, 0
+        response = None
+        
+        while response is None and count < retry_times:
+            try:
+                request = copy.deepcopy(self.gen_config)
+                
+                # System prompt setup based on language
+                if language == 'cn':
+                    message = [
+                        {"role": "system", "content": "The following is a transcript of a conversation between a human and a smart, psychology-savvy AI assistant."},
+                        {"role": "user", "content": "Hello! Please help me summarize the conversation"},
+                        {"role": "system", "content": "OK, I'll try to help you."},
+                        {"role": "user", "content": f"{prompt}"}
+                    ]
+                else:
+                    message = [
+                        {"role": "system", "content": "Below is a transcript of a conversation between a human and an AI assistant that is intelligent and knowledgeable in psychology."},
+                        {"role": "user", "content": "Hello! Please help me summarize the content of the conversation."},
+                        {"role": "system", "content": "Sure, I will do my best to assist you."},
+                        {"role": "user", "content": f"{prompt}"}
+                    ]
+
+                # Call the API method
+                response = self.client.chat.completions.create(**request, messages=message)
+
+            except Exception as e:
+                print(e)
+                # Check for context limit errors
+                if 'context' in str(e).lower() or 'maximum' in str(e).lower():
+                    cut_length = 1800 - 200 * count
+                    print('Max context length reached, cut to {}'.format(cut_length))
+                    prompt = prompt[-cut_length:]
+                    response = None
+                count += 1
+        
+        if response:
+            # Get response content using dot notation
+            task_desc = response.choices[0].message.content
+        else:
+            task_desc = ''
+            
+        return task_desc
+
+
+# Lazy initialization instead of module-level
+_llm_client = None
+
+def get_llm_client():
+    global _llm_client
+    if _llm_client is None:
+        _llm_client = LLMClientSimple(CHATGPT_CONFIG, gen_config=CHATGPT_CONFIG)
+    return _llm_client
+
+
+
+# --- Prompt Generation Functions ---
+
+def summarize_content_prompt(content, user_name, boot_name, language='en'):
+    # Base prompt adjusted based on psychology
+    prompt = 'Please summarize the following dialogue from a psychological perspective, focusing on the user\'s emotional state, key concerns, and potential therapeutic insights. Dialogue content:\n'
+    
+    for dialog in content:
+        query = dialog['query']
+        response = dialog['response']
+        prompt += f"\n{user_name}: {query.strip()}"
+        prompt += f"\n{boot_name}: {response.strip()}"
+    
+    prompt += ('\nSummary:' if language == 'cn' else '\nSummarization:')
+    return prompt
+
+
+def summarize_user_issues_prompt(content, user_name, boot_name, language='en'):
+    """
+    Analyze the conversation from a psychological perspective, identify user problems and their causes,
+    and organize them into a structured JSON format.
+    """
+    prompt = (
+        "Please analyze the following dialogue from a psychological perspective. Identify the user's key "
+        "problems and the underlying causes behind them. Structure the results in a clear and concise way. "
+        "Dialogue content:\n"
+    )
+
+    for dialog in content:
+        query = dialog['query']
+        response = dialog['response']
+        prompt += f"\n{user_name}: {query.strip()}"
+        prompt += f"\n{boot_name}: {response.strip()}"
+
+    prompt += '\n\nPlease list the user’s problems and their causes in JSON format, like this:\n'
+    prompt += '{\n  "problems": [\n    {\n      "problem": "<problem_1>",\n      "cause": "<cause_1>"\n    },\n    {\n      "problem": "<problem_2>",\n      "cause": "<cause_2>"\n    }\n  ]\n}'
+
+    return prompt
+
+
+def summarize_overall_prompt(content, language='en'):
+    prompt = 'Please summarize the following events in a high-level manner, and try to be as concise as possible, summarizing and retaining the core key information:\n' if language == 'cn' else "Please provide a highly concise summary of the following event, capturing the essential key information as succinctly as possible. Summarize the event:\n"
+    
+    for date, summary_dict in content:
+        summary = summary_dict['content']
+        prompt += (f"\nTime {date} The event that occurred was {summary.strip()}" if language == 'cn' else f"At {date}, the events are {summary.strip()}")
+    
+    prompt += ('\nSummary:' if language == 'cn' else '\nSummarization:')
+    return prompt
+
+
+def summarize_overall_personality(content, language='en'):
+    prompt = 'Below are the personality traits and moods of users in multiple conversations, as well as the appropriate response strategies at the time:\n' if language == 'cn' else "The following are the user's exhibited personality traits and emotions throughout multiple dialogues, along with appropriate response strategies for the current situation:"
+    
+    for date, summary in content:
+        prompt += (f"\n in time {date} The analysis is {summary.strip()}" if language == 'cn' else f"At {date}, the analysis shows {summary.strip()}")
+    
+    prompt += ('\nPlease give a general summary of the users personality and the most appropriate response strategy for the AI lover, and try to be concise and highly summarized. The summary is:' if language == 'cn' else "Please provide a highly concise and general summary of the user's personality and the most appropriate response strategy for the AI lover, summarized as:")
+    return prompt
+
+
+def summarize_person_prompt(content, user_name, boot_name, language):
+    prompt = f"Based on the following dialogue, analyze {user_name}'s emotional state, cognitive patterns, and potential psychological needs. Suggest therapeutic strategies for the AI to respond empathetically and effectively. Dialogue content:\n"
+    
+    for dialog in content:
+        query = dialog['query']
+        response = dialog['response']
+        prompt += f"\n{user_name}: {query.strip()}"
+        prompt += f"\n{boot_name}: {response.strip()}"
+
+    prompt += (f'\n{user_name} personality traits, mood, {boot_name} The response strategy is:' if language == 'cn' else f"\n{user_name}'s personality traits, emotions, and {boot_name}'s response strategy are:")
+    return prompt
+
+
+# --- Memory Extraction Functions ---
+
+def extract_session_summary(conversation_text, session_date, session_id):
+    """
+    Extracts session-level insights and creates an episodic memory entry.
+
+    Args:
+        conversation_text (str): The full conversation from the session.
+        session_date (str): The date of the session.
+        session_id (int): The index of the session.
+
+    Returns:
+        dict: Extracted episodic memory entry.
+    """
+    # Prompt to the LLM to extract insights
+    insight_prompt = f"""
+Please analyze the following conversation and extract key information:
+
+Conversation:
+{conversation_text}
+
+1. Topics discussed (as a list)
+2. User's emotional state (e.g., happy, anxious, stressed)
+3. Key insights or takeaways (what the user is feeling or thinking)
+
+Format response as JSON:
+{{
+  "topics_discussed": [],
+  "emotional_state": "",
+  "insights": ""
+}}
+"""
+
+    response = get_llm_client().generate_text_simple(
+        prompt=insight_prompt,
+        prompt_num=1,
+        language="en",
+    )
+
+    # Remove extra whitespace
+    response = response.strip()
+
+    # حذف تگ شروع بلاک کد مارک‌داون
+    if response.startswith("```json"):response = response[7:]
+    elif response.startswith("```"):response = response[3:]
+
+    # حذف تگ پایان بلاک کد مارک‌داون
+    if response.endswith("```"):
+        response = response[:-3]
+
+        response = response.strip()
+
+    try:
+        episodic_memory = json.loads(response)
+    except json.JSONDecodeError:
+        print("⚠️ Failed to parse episodic memory response!")
+        episodic_memory = {
+        "topics_discussed": ["N/A"],
+        "emotional_state": "unknown",
+        "insights": "No insights extracted.",
+        }
+
+    # Add session metadata
+    episodic_memory["session_date"] = session_date
+    episodic_memory["session_id"] = session_id
+
+    return episodic_memory
+
+
+
+def extract_semantic_memory(latest_episodic_memory, existing_semantic_memory):
+    """
+    Updates semantic memory based on the latest episodic memory.
+    """
+    # Use the insights from episodic memory to infer stable traits and patterns
+    semantic_prompt = f"""
+    Please analyze the following session summary and infer long-term personality traits and stable characteristics:
+    Session Summary:
+    {json.dumps(latest_episodic_memory, indent=4)}
+
+    Update the user's semantic memory with:
+    1. Evolving personality traits (Big Five).
+    2. Core values and recurring motivations.
+    3. Behavioral patterns and consistent emotional responses.
+    4. Frequent topics and recurring themes.
+
+    Format response as JSON:
+    {{
+    "personality_traits": {{
+    "openness": 0.0,
+    "conscientiousness": 0.0,
+    "extraversion": 0.0,
+    "agreeableness": 0.0,
+    "neuroticism": 0.0
+    }},
+    "core_values": [],
+    "behavioral_patterns": [],
+    "recurring_themes": []
+    }}
+    """
+
+    response = get_llm_client().generate_text_simple(prompt=semantic_prompt, prompt_num=1, language='en')
+
+    # Safe JSON extraction using Regex
+    try:
+        # Find content between first { and last }
+        json_match = re.search(r'\{.*\}', response, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(0)
+            new_semantic_data = json.loads(json_str)
+        else:
+            raise ValueError("No JSON object found in response.")
+
+    except Exception as e:
+        print(f"⚠️ Failed to parse semantic memory response! Error: {e}")
+        print(f"LLM Response was:\n{response}") 
+        return existing_semantic_memory
+
+    # Merge new data into semantic memory
+    updated_semantic_memory = existing_semantic_memory if isinstance(existing_semantic_memory, dict) else {}
+
+    # Update or average personality traits
+    for trait, value in new_semantic_data.get("personality_traits", {}).items():
+        try:
+            val_float = float(value)
+        except (ValueError, TypeError):
+            val_float = 0.0
+
+        if trait in updated_semantic_memory.get("personality_traits", {}):
+            existing_value = float(updated_semantic_memory["personality_traits"].get(trait, 0.0))
+            updated_semantic_memory["personality_traits"][trait] = round((existing_value + val_float) / 2, 2)
+        else:
+            updated_semantic_memory.setdefault("personality_traits", {})[trait] = val_float
+
+    # Combine core values, avoiding duplicates
+    updated_semantic_memory["core_values"] = list(set(
+        updated_semantic_memory.get("core_values", []) + new_semantic_data.get("core_values", [])
+    ))
+
+    # Combine behavioral patterns
+    updated_semantic_memory["behavioral_patterns"] = list(set(
+        updated_semantic_memory.get("behavioral_patterns", []) + new_semantic_data.get("behavioral_patterns", [])
+    ))
+
+    # Combine recurring themes
+    updated_semantic_memory["recurring_themes"] = list(set(
+        updated_semantic_memory.get("recurring_themes", []) + new_semantic_data.get("recurring_themes", [])
+    ))
+
+    print("\n✅ Semantic memory updated successfully!")
+    return updated_semantic_memory
+
+    # Merge new data into semantic memory
+    updated_semantic_memory = existing_semantic_memory if isinstance(existing_semantic_memory, dict) else {}
+
+    # Update or average personality traits
+    for trait, value in new_semantic_data.get("personality_traits", {}).items():
+        try:
+            val_float = float(value)
+        except (ValueError, TypeError):
+            val_float = 0.0
+
+        if trait in updated_semantic_memory.get("personality_traits", {}):
+            existing_value = float(updated_semantic_memory["personality_traits"].get(trait, 0.0))
+            updated_semantic_memory["personality_traits"][trait] = round((existing_value + val_float) / 2, 2)
+        else:
+            updated_semantic_memory.setdefault("personality_traits", {})[trait] = val_float
+
+    # Combine core values, avoiding duplicates
+    updated_semantic_memory["core_values"] = list(set(
+        updated_semantic_memory.get("core_values", []) + new_semantic_data.get("core_values", [])
+    ))
+
+    # Combine behavioral patterns
+    updated_semantic_memory["behavioral_patterns"] = list(set(
+        updated_semantic_memory.get("behavioral_patterns", []) + new_semantic_data.get("behavioral_patterns", [])
+    ))
+
+    # Combine recurring themes
+    updated_semantic_memory["recurring_themes"] = list(set(
+        updated_semantic_memory.get("recurring_themes", []) + new_semantic_data.get("recurring_themes", [])
+    ))
+
+    print("\n✅ Semantic memory updated successfully!")
+    return updated_semantic_memory
+
+
+# --- Main Logic ---
+
+def summarize_memory(memory_dir, name=None, language='en'):
+    boot_name = 'AI'
+    gen_prompt_num = 1
+
+    with open(memory_dir, 'r', encoding='utf8') as f:
+        memory = json.loads(f.read())
+
+    for k, v in memory.items():
+        if name is not None and k != name:
+            continue
+
+        user_name = k
+        print(f'Updating memory for user {user_name}')
+
+        if v.get('history') is None:
+            continue
+
+        history = v['history']
+
+        if v.get('summary') is None:
+            memory[user_name]['summary'] = {}
+        if v.get('personality') is None:
+            memory[user_name]['personality'] = {}
+        if v.get('issues') is None:
+            memory[user_name]['issues'] = {}
+
+        for date, content in history.items():
+            # Check flags to see if update is needed
+            his_flag = False if (date in v['summary'].keys() and v['summary'][date]) else True
+            person_flag = False if (date in v['personality'].keys() and v['personality'][date]) else True
+            problem_flag = False if (date in v['issues'].keys() and v['issues'][date]) else True 
+
+            hisprompt = summarize_content_prompt(content, user_name, boot_name, language)
+            issues_prompt = summarize_user_issues_prompt(content, user_name, boot_name, language)
+            person_prompt = summarize_person_prompt(content, user_name, boot_name, language)
+
+            if his_flag:
+                his_summary = get_llm_client().generate_text_simple(prompt=hisprompt, prompt_num=gen_prompt_num, language=language)
+                memory[user_name]['summary'][date] = {'content': his_summary}
+
+            if problem_flag:
+                issues_summary = get_llm_client().generate_text_simple(prompt=issues_prompt, prompt_num=gen_prompt_num, language=language)
+                memory[user_name]['issues'][date] = {'content': issues_summary}
+                print(issues_summary)
+
+            if person_flag:
+                person_summary = get_llm_client().generate_text_simple(prompt=person_prompt, prompt_num=gen_prompt_num, language=language)
+                memory[user_name]['personality'][date] = person_summary
+
+        overall_his_prompt = summarize_overall_prompt(list(memory[user_name]['summary'].items()), language=language)
+        overall_person_prompt = summarize_overall_personality(list(memory[user_name]['personality'].items()), language=language)
+
+        memory[user_name]['overall_history'] = get_llm_client().generate_text_simple(prompt=overall_his_prompt, prompt_num=gen_prompt_num, language=language)
+        memory[user_name]['overall_personality'] = get_llm_client().generate_text_simple(prompt=overall_person_prompt, prompt_num=gen_prompt_num, language=language)
+     
+    with open(memory_dir, 'w', encoding='utf8') as f:
+        print(f'Successfully updated memory for {name}')
+        json.dump(memory, f, ensure_ascii=False)
+        
+    return memory
+
+
+if __name__ == '__main__':
+    target_memory = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'memories', 'eng_memory_cases.json')
+    summarize_memory(target_memory, language='en')

memory_bank/build_memory_index.py

@@ -1,9 +1,7 @@
+# -*- coding:utf-8 -*-
 #
 # Memory Indexing Utility for LlamaIndex
-#
-# This script provides functionalities to build and manage memory indices for a conversational AI
-# using the LlamaIndex framework. It supports creating structured memory documents from raw
-# conversation data and persisting them as vector indices for efficient retrieval.
+# Project: EMMA (Empathetic Memory-augmented Multi-layer Assistant)
 #
 
 # --- Standard Library Imports ---
@@ -15,50 +13,50 @@ import tiktoken
 from llama_index.core import (
     Document,
     PromptHelper,
-    Settings,           # Replaces the deprecated ServiceContext
+    Settings,
     StorageContext,
     VectorStoreIndex,
     load_index_from_storage,
 )
 from llama_index.llms.openai import OpenAI
-# If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
-# from llama_index.embeddings.openai import OpenAIEmbedding
-
+from llama_index.embeddings.openai import OpenAIEmbedding
 
-# --- Global Variables ---
-# A dictionary to hold the loaded or newly created indices in memory.
+# --- Global Variables & Paths ---
 index_set = {}
 
+BASE_DATA_DIR = os.getenv(
+    "EMMA_DATA_DIR", 
+    os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
+)
 _INDEX_BASE_DIR = os.getenv(
-    "EMMA_INDEX_DIR",
-    os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "memories", "memory_index", "llamaindex")
+    "EMMA_INDEX_DIR", 
+    os.path.join(BASE_DATA_DIR, "memories", "memory_index", "llamaindex")
 )
+GAPGPT_BASE_URL = os.getenv("GAPGPT_BASE_URL", "https://api.gapgpt.app/v1")
 
-# --- Core Functions ---
+
+# --- Core Configuration Functions ---
 
 def setup_global_settings():
     """
     Configures global settings for LlamaIndex (v0.10+).
-
-    This function replaces the deprecated `build_service_context` pattern by directly
-    configuring the global `Settings` object for the LLM, tokenizer, and other components.
+    Sets LLM and Embedding models to route through GapGPT.
     """
-    # 1. Set Tokenizer (to resolve potential tiktoken errors)
+    GAPGPTMASKTOKENwdfn4ezolb9X0X = os.getenv("GAPGPT_API_KEY") or os.getenv("OPENAI_API_KEY")
+
+    # 1. Set Tokenizer
     try:
-        # This encoding is commonly used by OpenAI models like GPT-4.
         Settings.tokenizer = tiktoken.get_encoding("cl100k_base").encode
     except Exception as e:
-        print(f"Warning: Could not set the tokenizer. This might lead to issues. Error: {e}")
+        print(f"Warning: Could not set tokenizer: {e}")
 
-    # 2. Configure the Language Model (LLM)
-    # Use the LlamaIndex-specific OpenAI class to avoid conflicts with other libraries.
-    # It automatically reads the API key from the "OPENAI_API_KEY" environment variable.
+    # 2. Configure LLM for GapGPT
     Settings.llm = OpenAI(
         model="gpt-4o",
         temperature=1.0,
         max_tokens=1024,
-        api_key=os.environ.get("GAPGPT_API_KEY"),
-        api_base="https://api.gapgpt.app/v1",
+        api_key=GAPGPTMASKTOKENwdfn4ezolb9X1X,
+        api_base=GAPGPT_BASE_URL,
         additional_kwargs={
             "top_p": 0.95,
             "frequency_penalty": 0.4,
@@ -66,37 +64,30 @@ def setup_global_settings():
         }
     )
 
-    # 3. Configure the Prompt Helper (with updated parameter names)
-    # This helps manage context window size, number of outputs, etc.
+    # 3. Configure Embedding Model for GapGPT
+    Settings.embed_model = OpenAIEmbedding(
+        model="text-embedding-ada-002",
+        api_key=GAPGPTMASKTOKENwdfn4ezolb9X2X,
+        api_base=GAPGPT_BASE_URL
+    )
+
+    # 4. Configure Prompt Helper
     Settings.prompt_helper = PromptHelper(
         context_window=4096,
         num_output=256,
         chunk_overlap_ratio=0.1
     )
 
-    # 4. Configure Chunk Size for document processing
+    # 5. Chunk Settings
     Settings.chunk_size = 512
     Settings.chunk_overlap = 20
 
-    # Optional: Configure the embedding model if needed.
-    # If not set, LlamaIndex defaults to a compatible model (e.g., OpenAI's text-embedding-ada-002).
-    # Settings.embed_model = OpenAIEmbedding(api_key=..., api_base=...)
 
+# --- Document Generation Functions ---
 
-def generate_memory_docs(data, language):
+def generate_memory_docs(data, language="en"):
     """
-    Generates structured LlamaIndex Document objects from a nested dictionary of user memories.
-
-    This function processes different types of memory (sessions, episodic, semantic) and
-    formats them into Document objects suitable for indexing.
-
-    Args:
-        data (dict): A dictionary where keys are user names and values contain their memories.
-        language (str): The language code (e.g., "en"), currently used for formatting.
-
-    Returns:
-        dict: A dictionary where keys are user names and values are dicts containing
-              lists of Document objects for each memory type.
+    Generates structured LlamaIndex Document objects from nested user memory dictionary.
     """
     all_user_memories = {}
 
@@ -107,11 +98,11 @@ def generate_memory_docs(data, language):
             "semantic_memory": [],
         }
 
-        # Process session history
-        if "sessions" in user_memory:
+        # 1. Process session history
+        if "sessions" in user_memory and user_memory["sessions"]:
             for session in user_memory["sessions"]:
-                date = session["date"]
-                content = session["conversation"]
+                date = session.get("date", "")
+                content = session.get("conversation", [])
 
                 memory_str = f"Session on {date}:\n"
                 for dialog in content:
@@ -125,18 +116,17 @@ def generate_memory_docs(data, language):
                     summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
                     memory_str += summary
 
-                # Use the 'text' argument for the Document object
                 all_user_memories[user_name]["sessions"].append(Document(text=memory_str))
 
-        # Process episodic memory
-        if "episodic_memory" in user_memory:
+        # 2. Process episodic memory
+        if "episodic_memory" in user_memory and user_memory["episodic_memory"]:
             episodic_str = "Recent experiences:\n"
             for event in user_memory["episodic_memory"]:
                 episodic_str += f"- {event}\n"
             all_user_memories[user_name]["episodic_memory"].append(Document(text=episodic_str))
 
-        # Process semantic memory (long-term facts and personality)
-        if "semantic_memory" in user_memory:
+        # 3. Process semantic memory
+        if "semantic_memory" in user_memory and user_memory["semantic_memory"]:
             semantic_str = "Long-term personality traits and facts:\n"
             if isinstance(user_memory["semantic_memory"], dict):
                 for trait, value in user_memory["semantic_memory"].items():
@@ -145,20 +135,21 @@ def generate_memory_docs(data, language):
                 for item in user_memory["semantic_memory"]:
                     semantic_str += f"- {item}\n"
             else:
-                print(f"WARNING: semantic_memory for user '{user_name}' has an unexpected format: {type(user_memory['semantic_memory'])}")
+                print(f"WARNING: semantic_memory for '{user_name}' has unexpected format: {type(user_memory['semantic_memory'])}")
 
             all_user_memories[user_name]["semantic_memory"].append(Document(text=semantic_str))
 
     return all_user_memories
 
 
-def build_memory_index(all_user_memories, data_args, name=None):
+# --- Indexing & Storage Functions ---
+
+def build_memory_index(all_user_memories, data_args=None, name=None):
     """
-    Build and persist memory indices for each user and memory type.
+    Build and persist memory indices for each user and memory layer.
     """
-    structured_docs = generate_memory_docs(
-        all_user_memories, data_args.language
-    )
+    language = getattr(data_args, 'language', 'en') if data_args else 'en'
+    structured_docs = generate_memory_docs(all_user_memories, language=language)
 
     setup_global_settings()
 
@@ -198,62 +189,38 @@ def build_memory_index(all_user_memories, data_args, name=None):
             index_set[f"{user_name}_{memory_type}"] = cur_index
 
 
-# --- Deprecated Functions (Kept for reference) ---
-
-def generate_memory_docs_old(data, language):
+def load_memory_index(user_name, memory_type):
     """
-    DEPRECATED: An older version of the document generation function.
-
-    This function had a simpler structure and contained Chinese string literals.
-    It is replaced by `generate_memory_docs`.
+    Loads a specific persisted VectorStoreIndex from disk.
     """
-    all_user_memories = {}
-    for user_name, user_memory in data.items():
-        all_user_memories[user_name] = []
-        if "history" not in user_memory:
-            continue
-        for date, content in user_memory["history"].items():
-            memory_str = f"Conversation on {date}:"
-
-            for dialog in content:
-                query = dialog.get("query", "")
-                response = dialog.get("response", "")
-                memory_str += f"\n{user_name}: {query.strip()}"
-                memory_str += f"\nAI: {response.strip()}"
-            memory_str += "\n"
-            if "summary" in user_memory and date in user_memory["summary"]:
-                summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
-                memory_str += summary
-
-            all_user_memories[user_name].append(Document(text=memory_str))
-    return all_user_memories
-
-
-def build_memory_index_old(all_user_memories, data_args, name=None):
-    """
-    DEPRECATED: An older version of the index building function.
-
-    This function used a flatter memory structure and is replaced by the more
-    comprehensive `build_memory_index`.
-    """
-    # The function name is a typo in the original code, corrected here for clarity.
-    all_user_memories_docs = generate_memory_docs(
-        all_user_memories, data_args.language
-    )
-
-    # Apply global settings
     setup_global_settings()
+    
+    key = f"{user_name}_{memory_type}"
+    if key in index_set:
+        return index_set[key]
 
-    for user_name, memories in all_user_memories_docs.items():
-        if name and user_name != name:
-            continue
-        print(f"Building index for user {user_name} (using old method)...")
+    index_dir = os.path.join(_INDEX_BASE_DIR, user_name, memory_type)
+    if not os.path.exists(index_dir):
+        return None
+
+    try:
+        storage_context = StorageContext.from_defaults(persist_dir=index_dir)
+        index = load_index_from_storage(storage_context)
+        index_set[key] = index
+        return index
+    except Exception as e:
+        print(f"Error loading index for {key} from {index_dir}: {e}")
+        return None
 
-        # The service_context argument is removed in newer versions.
-        cur_index = VectorStoreIndex.from_documents(memories)
-        index_set[user_name] = cur_index
 
-        # Corrected save path; the new persist method requires a directory.
-        save_dir = f"../memories/memory_index/llamaindex/{user_name}_store"
-        os.makedirs(save_dir, exist_ok=True)
-        cur_index.storage_context.persist(persist_dir=save_dir)
+def load_all_user_indices(user_name):
+    """
+    Loads all available memory layers (sessions, episodic, semantic) for a given user.
+    """
+    memory_types = ["sessions", "episodic_memory", "semantic_memory"]
+    loaded = {}
+    for m_type in memory_types:
+        idx = load_memory_index(user_name, m_type)
+        if idx:
+            loaded[m_type] = idx
+    return loaded

memory_bank/summarize_memory.py

@@ -11,8 +11,9 @@ from openai import OpenAI
 import re
 import openai
 
-# Add memory bank path
-sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'memory_bank'))
+# Add project root and memory bank path safely
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(os.path.join(PROJECT_ROOT, 'memory_bank'))
 
 # Configuration for OpenAI API
 CHATGPT_CONFIG = {
@@ -25,19 +26,18 @@ CHATGPT_CONFIG = {
     "stop": ["<|im_end|>", "¬Human"]
 }
 
-# Default memory directory
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-MEMORY_DIR = os.path.join(BASE_DIR, "memories", "update_memory_0512_eng.json")
+# Default memory directory using dynamic paths
+BASE_DATA_DIR = os.environ.get("EMMA_DATA_DIR", PROJECT_ROOT)
+MEMORY_DIR = os.path.join(BASE_DATA_DIR, "memories", "update_memory_0512_eng.json")
 
 class LLMClientSimple:
     def __init__(self, config, gen_config=None):
         self.config = config
         self.client = OpenAI(
-            api_key=self.config.get("api_key") or os.environ.get("GAPGPT_API_KEY"),
-            base_url=self.config.get("base_url") or os.environ.get(
-                "GAPGPT_BASE_URL", "https://api.gapgpt.app/v1"
-            ),
-        )
+            api_key=os.environ.get("GAPGPT_API_KEY", "YOUR_API_KEY_HERE"),
+            base_url=os.environ.get("GAPGPT_BASE_URL", "https://api.gapgpt.app/v1"),
+        ),
+        
         self.disable_tqdm = False
         self.gen_config = copy.deepcopy(
             gen_config if gen_config is not None else config
@@ -429,5 +429,9 @@ def summarize_memory(memory_dir, name=None, language='en'):
 
 
 if __name__ == '__main__':
-    target_memory = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'memories', 'eng_memory_cases.json')
+    # Use dynamic base directory for the target memory file
+    BASE_DATA_DIR = os.environ.get("EMMA_DATA_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+    target_memory = os.path.join(BASE_DATA_DIR, 'memories', 'eng_memory_cases.json')
+    
     summarize_memory(target_memory, language='en')
+