EMMA commited on
Commit
fa788b0
·
1 Parent(s): 40f5a32

update code

Browse files
Backup code/build_memory_index_23.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #
2
+ # Memory Indexing Utility for LlamaIndex
3
+ #
4
+ # This script provides functionalities to build and manage memory indices for a conversational AI
5
+ # using the LlamaIndex framework. It supports creating structured memory documents from raw
6
+ # conversation data and persisting them as vector indices for efficient retrieval.
7
+ #
8
+
9
+ # --- Standard Library Imports ---
10
+ import json
11
+ import os
12
+
13
+ # --- Third-Party Imports ---
14
+ import tiktoken
15
+ from llama_index.core import (
16
+ Document,
17
+ PromptHelper,
18
+ Settings, # Replaces the deprecated ServiceContext
19
+ StorageContext,
20
+ VectorStoreIndex,
21
+ load_index_from_storage,
22
+ )
23
+ from llama_index.llms.openai import OpenAI
24
+ # If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
25
+ # from llama_index.embeddings.openai import OpenAIEmbedding
26
+
27
+
28
+ # --- Global Variables ---
29
+ # A dictionary to hold the loaded or newly created indices in memory.
30
+ index_set = {}
31
+
32
+ # پشتیبانی هوشمند از Persistent Storage در Hugging Face (/data) یا مسیر محلی
33
+ BASE_DIR = "/data" if os.path.exists("/data") else os.path.dirname(os.path.abspath(__file__))
34
+
35
+ # تنظیم مسیر دقیق بر اساس ساختار ریپازیتوری (memory_index/llamaindex)
36
+ _INDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(BASE_DIR, "memory_index", "llamaindex"))
37
+
38
+ # اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
39
+ os.makedirs(_INDEX_BASE_DIR, exist_ok=True)
40
+
41
+
42
+ # --- Core Functions ---
43
+
44
+ def setup_global_settings():
45
+ """
46
+ Configures global settings for LlamaIndex (v0.10+).
47
+
48
+ This function replaces the deprecated `build_service_context` pattern by directly
49
+ configuring the global `Settings` object for the LLM, tokenizer, and other components.
50
+ """
51
+ # 1. Set Tokenizer (to resolve potential tiktoken errors)
52
+ try:
53
+ # This encoding is commonly used by OpenAI models like GPT-4.
54
+ Settings.tokenizer = tiktoken.get_encoding("cl100k_base").encode
55
+ except Exception as e:
56
+ print(f"Warning: Could not set the tokenizer. This might lead to issues. Error: {e}")
57
+
58
+ # 2. Configure the Language Model (LLM)
59
+ # Use the LlamaIndex-specific OpenAI class to avoid conflicts with other libraries.
60
+ # It automatically reads the API key from the "OPENAI_API_KEY" environment variable.
61
+ Settings.llm = OpenAI(
62
+ model="gpt-4o",
63
+ temperature=1.0,
64
+ max_tokens=1024,
65
+ api_key=os.environ.get("GAPGPT_API_KEY"),
66
+ api_base="https://api.gapgpt.app/v1",
67
+ additional_kwargs={
68
+ "top_p": 0.95,
69
+ "frequency_penalty": 0.4,
70
+ "presence_penalty": 0.2
71
+ }
72
+ )
73
+
74
+ # 3. Configure the Prompt Helper (with updated parameter names)
75
+ # This helps manage context window size, number of outputs, etc.
76
+ Settings.prompt_helper = PromptHelper(
77
+ context_window=4096,
78
+ num_output=256,
79
+ chunk_overlap_ratio=0.1
80
+ )
81
+
82
+ # 4. Configure Chunk Size for document processing
83
+ Settings.chunk_size = 512
84
+ Settings.chunk_overlap = 20
85
+
86
+ # Optional: Configure the embedding model if needed.
87
+ # If not set, LlamaIndex defaults to a compatible model (e.g., OpenAI's text-embedding-ada-002).
88
+ # Settings.embed_model = OpenAIEmbedding(api_key=GAPGPTMASKTOKENlgicihcumgiX0X, api_base=GAPGPTMASKTOKENlgicihcumgiX1X)
89
+
90
+
91
+ def generate_memory_docs(data, language):
92
+ """
93
+ Generates structured LlamaIndex Document objects from a nested dictionary of user memories.
94
+
95
+ This function processes different types of memory (sessions, episodic, semantic) and
96
+ formats them into Document objects suitable for indexing.
97
+
98
+ Args:
99
+ data (dict): A dictionary where keys are user names and values contain their memories.
100
+ language (str): The language code (e.g., "en"), currently used for formatting.
101
+
102
+ Returns:
103
+ dict: A dictionary where keys are user names and values are dicts containing
104
+ lists of Document objects for each memory type.
105
+ """
106
+ all_user_memories = {}
107
+
108
+ for user_name, user_memory in data.items():
109
+ all_user_memories[user_name] = {
110
+ "sessions": [],
111
+ "episodic_memory": [],
112
+ "semantic_memory": [],
113
+ }
114
+
115
+ # Process session history
116
+ if "sessions" in user_memory:
117
+ for session in user_memory["sessions"]:
118
+ date = session["date"]
119
+ content = session["conversation"]
120
+
121
+ memory_str = f"Session on {date}:\n"
122
+ for dialog in content:
123
+ query = dialog.get("query", "")
124
+ response = dialog.get("response", "")
125
+ memory_str += f"\n{user_name}: {query.strip()}"
126
+ memory_str += f"\nAI: {response.strip()}"
127
+ memory_str += "\n"
128
+
129
+ if "summary" in user_memory and date in user_memory["summary"]:
130
+ summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
131
+ memory_str += summary
132
+
133
+ # Use the 'text' argument for the Document object
134
+ all_user_memories[user_name]["sessions"].append(Document(text=memory_str))
135
+
136
+ # Process episodic memory
137
+ if "episodic_memory" in user_memory:
138
+ episodic_str = "Recent experiences:\n"
139
+ for event in user_memory["episodic_memory"]:
140
+ episodic_str += f"- {event}\n"
141
+ all_user_memories[user_name]["episodic_memory"].append(Document(text=episodic_str))
142
+
143
+ # Process semantic memory (long-term facts and personality)
144
+ if "semantic_memory" in user_memory:
145
+ semantic_str = "Long-term personality traits and facts:\n"
146
+ if isinstance(user_memory["semantic_memory"], dict):
147
+ for trait, value in user_memory["semantic_memory"].items():
148
+ semantic_str += f"- {trait}: {value}\n"
149
+ elif isinstance(user_memory["semantic_memory"], list):
150
+ for item in user_memory["semantic_memory"]:
151
+ semantic_str += f"- {item}\n"
152
+ else:
153
+ print(f"WARNING: semantic_memory for user '{user_name}' has an unexpected format: {type(user_memory['semantic_memory'])}")
154
+
155
+ all_user_memories[user_name]["semantic_memory"].append(Document(text=semantic_str))
156
+
157
+ return all_user_memories
158
+
159
+
160
+ def build_memory_index(all_user_memories, data_args, name=None):
161
+ """
162
+ Build and persist memory indices for each user and memory type.
163
+ """
164
+ structured_docs = generate_memory_docs(
165
+ all_user_memories, data_args.language
166
+ )
167
+
168
+ setup_global_settings()
169
+
170
+ for user_name, memories_by_type in structured_docs.items():
171
+ if name and user_name != name:
172
+ continue
173
+
174
+ print(f"Building indices for user '{user_name}'")
175
+
176
+ base_path = os.path.join(_INDEX_BASE_DIR, user_name)
177
+
178
+ path_map = {
179
+ "sessions": os.path.join(base_path, "sessions"),
180
+ "episodic_memory": os.path.join(base_path, "episodic_memory"),
181
+ "semantic_memory": os.path.join(base_path, "semantic_memory"),
182
+ }
183
+
184
+ for memory_type, docs in memories_by_type.items():
185
+ if not docs:
186
+ print(f" -> Skipping '{memory_type}' index (no documents found).")
187
+ continue
188
+
189
+ if memory_type not in path_map:
190
+ print(f" -> Skipping unknown memory type: '{memory_type}'")
191
+ continue
192
+
193
+ print(f" -> Building '{memory_type}' index...")
194
+
195
+ cur_index = VectorStoreIndex.from_documents(docs)
196
+
197
+ index_dir = path_map[memory_type]
198
+ os.makedirs(index_dir, exist_ok=True)
199
+
200
+ cur_index.storage_context.persist(persist_dir=index_dir)
201
+ print(f" + Saved '{memory_type}' index to: {index_dir}")
202
+
203
+ index_set[f"{user_name}_{memory_type}"] = cur_index
204
+
205
+
206
+ # --- Deprecated Functions (Kept for reference) ---
207
+
208
+ def generate_memory_docs_old(data, language):
209
+ """
210
+ DEPRECATED: An older version of the document generation function.
211
+
212
+ This function had a simpler structure and contained Chinese string literals.
213
+ It is replaced by `generate_memory_docs`.
214
+ """
215
+ all_user_memories = {}
216
+ for user_name, user_memory in data.items():
217
+ all_user_memories[user_name] = []
218
+ if "history" not in user_memory:
219
+ continue
220
+ for date, content in user_memory["history"].items():
221
+ memory_str = f"Conversation on {date}:"
222
+
223
+ for dialog in content:
224
+ query = dialog.get("query", "")
225
+ response = dialog.get("response", "")
226
+ memory_str += f"\n{user_name}: {query.strip()}"
227
+ memory_str += f"\nAI: {response.strip()}"
228
+ memory_str += "\n"
229
+ if "summary" in user_memory and date in user_memory["summary"]:
230
+ summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
231
+ memory_str += summary
232
+
233
+ all_user_memories[user_name].append(Document(text=memory_str))
234
+ return all_user_memories
235
+
236
+
237
+ def build_memory_index_old(all_user_memories, data_args, name=None):
238
+ """
239
+ DEPRECATED: An older version of the index building function.
240
+
241
+ This function used a flatter memory structure and is replaced by the more
242
+ comprehensive `build_memory_index`.
243
+ """
244
+ # The function name is a typo in the original code, corrected here for clarity.
245
+ all_user_memories_docs = generate_memory_docs(
246
+ all_user_memories, data_args.language
247
+ )
248
+
249
+ # Apply global settings
250
+ setup_global_settings()
251
+
252
+ for user_name, memories in all_user_memories_docs.items():
253
+ if name and user_name != name:
254
+ continue
255
+ print(f"Building index for user {user_name} (using old method)")
256
+
257
+ # The service_context argument is removed in newer versions.
258
+ cur_index = VectorStoreIndex.from_documents(memories)
259
+ index_set[user_name] = cur_index
260
+
261
+ # استفاده از مسیر داینامیک و امن
262
+ save_dir = os.path.join(_INDEX_BASE_DIR, f"{user_name}_store")
263
+ os.makedirs(save_dir, exist_ok=True)
264
+ cur_index.storage_context.persist(persist_dir=save_dir)
Backup code/{memory_utils.py → memory_utils_23.py} RENAMED
File without changes
Backup code/memory_utils_24.py ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import json
4
+ import time
5
+ import datetime
6
+ import shutil
7
+ import gradio as gr
8
+ from pprint import pprint
9
+ import traceback
10
+
11
+ # LlamaIndex Imports
12
+ from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
13
+
14
+ # Local Imports setup
15
+ # Assuming this file is in 'utils/', we step back to find 'memory_bank'
16
+ bank_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../memory_bank')
17
+ sys.path.append(bank_path)
18
+
19
+ # Import functions from sibling/child modules
20
+ # Ensure these files exist in the appended path
21
+ try:
22
+ from build_memory_index import build_memory_index
23
+ from summarize_memory import summarize_memory, extract_session_summary, extract_semantic_memory
24
+ except ImportError:
25
+ # Fallback or placeholder if running independently for testing
26
+ print("Warning: Could not import memory build/summary modules.")
27
+ def build_memory_index(*args, **kwargs): pass
28
+ def summarize_memory(*args, **kwargs): return {}
29
+ def extract_session_summary(*args, **kwargs): return {}
30
+ def extract_semantic_memory(*args, **kwargs): return {}
31
+
32
+
33
+ PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
34
+
35
+ # تشخیص خودکار حافظه دائمی ابری
36
+ BASE_DATA_DIR = (
37
+ "/data" if os.path.exists("/data") else os.path.join(PROJECT_ROOT, "data")
38
+ )
39
+
40
+ # مسیر فایل json
41
+ MEMORY_FILE_PATH = os.path.join(
42
+ BASE_DATA_DIR, "memories", "update_memory_0512_eng.json"
43
+ )
44
+
45
+ # مسیر ایندکس‌ها (هم‌سطح با پوشه memories)
46
+ INDEX_BASE_DIR = os.getenv(
47
+ "EMMA_INDEX_DIR", os.path.join(BASE_DATA_DIR, "memory_index", "llamaindex")
48
+ )
49
+
50
+
51
+ def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
52
+ """
53
+ Legacy/Basic function to load user memory and initialize vector store.
54
+ """
55
+ cur_date = datetime.date.today().strftime("%Y-%m-%d")
56
+ user_memory_index = None
57
+
58
+ # Handle Gradio States
59
+ if isinstance(data_args, gr.State): data_args = data_args.value
60
+ if isinstance(memory, gr.State): memory = memory.value
61
+ if isinstance(local_memory_qa, gr.State): local_memory_qa = local_memory_qa.value
62
+
63
+ memory_dir = MEMORY_FILE_PATH
64
+
65
+ if name in memory.keys():
66
+ user_memory = memory[name]
67
+ memory_index_path = os.path.join(data_args.memory_basic_dir, f'memory_index/{name}_index')
68
+ os.makedirs(os.path.dirname(memory_index_path), exist_ok=True)
69
+
70
+ if (not os.path.exists(memory_index_path)) or update_memory_index:
71
+ print(f'Initializing memory index {memory_index_path}...')
72
+
73
+ if os.path.exists(memory_index_path):
74
+ shutil.rmtree(memory_index_path)
75
+ # Initialize using the local QA object
76
+ memory_index_path, _ = local_memory_qa.init_memory_vector_store(
77
+ filepath=memory_dir,
78
+ vs_path=memory_index_path,
79
+ user_name=name,
80
+ cur_date=cur_date
81
+ )
82
+
83
+ user_memory_index = local_memory_qa.load_memory_index(memory_index_path) if memory_index_path else None
84
+ msg = f"Welcome back, {name}!"
85
+ return msg, user_memory, memory, name, user_memory_index
86
+ else:
87
+ memory[name] = {}
88
+ memory[name].update({"name": name})
89
+ msg = f"Welcome, new user {name}! I will remember your name, so next time we meet, I'll be able to call you by your name!"
90
+ return msg, memory[name], memory, name, user_memory_index
91
+
92
+
93
+ def enter_name_llamaindex(name, memory, data_args, update_memory_index=True):
94
+ """
95
+ Load the user's session, episodic, and semantic memory indices.
96
+ Compatible with LlamaIndex v0.10+.
97
+ """
98
+ sessions_memory = None
99
+ episodic_memory = None
100
+ semantic_memory = None
101
+
102
+ print(f"[DEBUG] enter_name_llamaindex called with name={name!r}")
103
+
104
+ if name not in memory:
105
+ print(f"[DEBUG] user {name!r} not found in memory")
106
+ return "User not found.", None, None, None, None
107
+
108
+ user_memory = memory[name]
109
+ # مسیر دقیق کاربر در پوشه llamaindex
110
+ base_path = os.path.join(INDEX_BASE_DIR, name)
111
+ sessions_path = os.path.join(base_path, "sessions")
112
+ episodic_path = os.path.join(base_path, "episodic_memory")
113
+ semantic_path = os.path.join(base_path, "semantic_memory")
114
+
115
+ print(f"[DEBUG] base_path: {base_path}")
116
+ print(f"[DEBUG] sessions_path: {sessions_path}")
117
+ print(f"[DEBUG] episodic_path: {episodic_path}")
118
+ print(f"[DEBUG] semantic_path: {semantic_path}")
119
+
120
+ indices_exist = (
121
+ os.path.isdir(sessions_path)
122
+ and os.path.isdir(episodic_path)
123
+ and os.path.isdir(semantic_path)
124
+ )
125
+
126
+ print(f"[DEBUG] indices_exist: {indices_exist}")
127
+ #print(f"[DEBUG] update_memory_index: {update_memory_index}")
128
+
129
+ if update_memory_index or not indices_exist:
130
+ print(f"[DEBUG] Initializing memory indices for {name}...")
131
+
132
+ # Important: build_memory_index must persist to these same absolute paths.
133
+ build_memory_index(memory, data_args, name=name)
134
+
135
+ print("[DEBUG] After build:")
136
+ print(f"[DEBUG] sessions_path exists: {os.path.isdir(sessions_path)}")
137
+ print(f"[DEBUG] episodic_path exists: {os.path.isdir(episodic_path)}")
138
+ print(f"[DEBUG] semantic_path exists: {os.path.isdir(semantic_path)}")
139
+
140
+ def load_index_safe(index_name, index_path):
141
+ if not os.path.isdir(index_path):
142
+ print(f"[DEBUG] {index_name} path does not exist: {index_path}")
143
+ return None
144
+
145
+ try:
146
+ storage_context = StorageContext.from_defaults(
147
+ persist_dir=index_path
148
+ )
149
+ index = load_index_from_storage(storage_context)
150
+ print(
151
+ f"[DEBUG] {index_name} loaded: "
152
+ f"{type(index).__name__}, is_none={index is None}"
153
+ )
154
+ return index
155
+ except Exception as exc:
156
+ print(f"[ERROR] Could not load {index_name}: {exc}")
157
+ traceback.print_exc()
158
+ return None
159
+
160
+ sessions_memory = load_index_safe("sessions_memory", sessions_path)
161
+ episodic_memory = load_index_safe("episodic_memory", episodic_path)
162
+ semantic_memory = load_index_safe("semantic_memory", semantic_path)
163
+
164
+ print("[DEBUG] RETURN VALUES:")
165
+ print(f" hello_msg: Welcome back, {name}!")
166
+ print(f" user_memory type: {type(user_memory).__name__}")
167
+ print(f" sessions_memory type: {type(sessions_memory).__name__}")
168
+ print(f" episodic_memory type: {type(episodic_memory).__name__}")
169
+ print(f" semantic_memory type: {type(semantic_memory).__name__}")
170
+
171
+ return (
172
+ f"Welcome back, {name}!",
173
+ user_memory,
174
+ sessions_memory,
175
+ episodic_memory,
176
+ semantic_memory,
177
+ )
178
+
179
+
180
+ def summarize_memory_event_personality(data_args, memory, user_name):
181
+ """
182
+ Summarizes the memory and returns the user-specific memory dict.
183
+ """
184
+ if isinstance(data_args, gr.State): data_args = data_args.value
185
+ if isinstance(memory, gr.State): memory = memory.value
186
+
187
+ memory_dir = MEMORY_FILE_PATH
188
+
189
+ # Note: Ensure summarize_memory handles the language argument correctly (passed 'en' or similar)
190
+ memory = summarize_memory(memory_dir, user_name, language=data_args.language)
191
+ user_memory = memory[user_name] if user_name in memory.keys() else {}
192
+ return user_memory
193
+
194
+
195
+ def save_local_memory(memory, history, user_name, data_args, new_conversation=False):
196
+ """
197
+ Saves user-model conversations into memory and adds episodic memory for each session.
198
+ Handles both list-of-lists (old Gradio) and list-of-dicts (new Gradio/OpenAI) formats.
199
+ """
200
+ if isinstance(data_args, gr.State): data_args = data_args.value
201
+ if isinstance(memory, gr.State): memory = memory.value
202
+
203
+ memory_dir = MEMORY_FILE_PATH
204
+
205
+ # 1. Initialize user memory with ALL required structures FIRST
206
+ if user_name not in memory:
207
+ memory[user_name] = {
208
+ "sessions": [],
209
+ "episodic_memory": [],
210
+ "semantic_memory": {}
211
+ }
212
+
213
+ # 2. Ensure all sub-structures exist and are correct type
214
+ memory[user_name].setdefault("sessions", [])
215
+ memory[user_name].setdefault("episodic_memory", [])
216
+ memory[user_name].setdefault("semantic_memory", {})
217
+
218
+ # 3. Now safely check types
219
+ if not isinstance(memory[user_name]["semantic_memory"], dict):
220
+ memory[user_name]["semantic_memory"] = {}
221
+
222
+ # Create new session or update existing one
223
+ if new_conversation or not memory[user_name]["sessions"]:
224
+ if new_conversation and memory[user_name]["sessions"]:
225
+ # Logic to summarize previous session before starting new one could go here
226
+ pass
227
+
228
+ # Create new session
229
+ session = {
230
+ "session_id": len(memory[user_name]["sessions"]),
231
+ "date": time.strftime("%Y-%m-%d", time.localtime()),
232
+ "conversation": []
233
+ }
234
+ memory[user_name]["sessions"].append(session)
235
+
236
+ current_session = memory[user_name]["sessions"][-1]
237
+
238
+ # --- Modified section to fix KeyError: 0 and handle History formats ---
239
+ if not new_conversation and history:
240
+ last_item = history[-1]
241
+
242
+ # Case 1: New Format (List of Dicts)
243
+ # In this format, history is linear. The last item is bot response, second to last is user query.
244
+ if isinstance(last_item, dict):
245
+ if len(history) >= 2:
246
+ user_query = history[-2].get('content', '')
247
+ bot_response = history[-1].get('content', '')
248
+
249
+ # Verify roles to ensure correct pairing
250
+ if history[-2].get('role') == 'user' and history[-1].get('role') == 'assistant':
251
+ current_session["conversation"].append({
252
+ 'query': user_query,
253
+ 'response': bot_response
254
+ })
255
+
256
+ # Case 2: Old Format (List of Lists/Tuples)
257
+ elif isinstance(last_item, (list, tuple)):
258
+ current_session["conversation"].append({
259
+ 'query': last_item[0],
260
+ 'response': last_item[1]
261
+ })
262
+ # ----------------------------------------------------------------------
263
+
264
+ # Optional: Update semantic memory in real-time
265
+ memory[user_name]["semantic_memory"] = extract_semantic_memory(
266
+ memory[user_name]["semantic_memory"],
267
+ current_session["conversation"]
268
+ )
269
+
270
+ semantic_memory_text = memory[user_name]["semantic_memory"]
271
+
272
+ # Save to file
273
+ # Ensure memory directory exists
274
+ os.makedirs(os.path.dirname(memory_dir), exist_ok=True)
275
+
276
+ with open(memory_dir, "w", encoding="utf-8") as f:
277
+ json.dump(memory, f, ensure_ascii=False, indent=4)
278
+
279
+ return memory, semantic_memory_text
memory_bank/build_memory_index.py CHANGED
@@ -29,14 +29,18 @@ from llama_index.llms.openai import OpenAI
29
  # A dictionary to hold the loaded or newly created indices in memory.
30
  index_set = {}
31
 
32
- # پشتیبانی هوشمند از Persistent Storage در Hugging Face (/data) یا مسیر محلی
33
- BASE_DIR = "/data" if os.path.exists("/data") else os.path.dirname(os.path.abspath(__file__))
 
 
 
34
 
35
- # تنظیم مسیر دقیق بر اساس ساختار ریپازیتوری (memory_index/llamaindex)
36
- _INDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(BASE_DIR, "memory_index", "llamaindex"))
37
 
38
  # اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
39
- os.makedirs(_INDEX_BASE_DIR, exist_ok=True)
 
40
 
41
 
42
  # --- Core Functions ---
@@ -173,7 +177,7 @@ def build_memory_index(all_user_memories, data_args, name=None):
173
 
174
  print(f"Building indices for user '{user_name}'")
175
 
176
- base_path = os.path.join(_INDEX_BASE_DIR, user_name)
177
 
178
  path_map = {
179
  "sessions": os.path.join(base_path, "sessions"),
@@ -258,7 +262,8 @@ def build_memory_index_old(all_user_memories, data_args, name=None):
258
  cur_index = VectorStoreIndex.from_documents(memories)
259
  index_set[user_name] = cur_index
260
 
261
- # استفاده از مسیر داینامیک و امن
262
- save_dir = os.path.join(_INDEX_BASE_DIR, f"{user_name}_store")
263
  os.makedirs(save_dir, exist_ok=True)
264
  cur_index.storage_context.persist(persist_dir=save_dir)
 
 
29
  # A dictionary to hold the loaded or newly created indices in memory.
30
  index_set = {}
31
 
32
+ # پیکربندی دایرکتوری‌ها بر اساس ساختار استاندارد درخواستی
33
+ CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
34
+ BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
35
+ MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
36
+ INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
37
 
38
+ # تنظیم مسیر دقیق برای llamaindex
39
+ _LLAMAINDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(INDEX_BASE_DIR, "llamaindex"))
40
 
41
  # اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
42
+ os.makedirs(_LLAMAINDEX_BASE_DIR, exist_ok=True)
43
+
44
 
45
 
46
  # --- Core Functions ---
 
177
 
178
  print(f"Building indices for user '{user_name}'")
179
 
180
+ base_path = os.path.join(_LLAMAINDEX_BASE_DIR, user_name)
181
 
182
  path_map = {
183
  "sessions": os.path.join(base_path, "sessions"),
 
262
  cur_index = VectorStoreIndex.from_documents(memories)
263
  index_set[user_name] = cur_index
264
 
265
+ # استفاده از مسیر داینامیک و امن
266
+ save_dir = os.path.join(_LLAMAINDEX_BASE_DIR, f"{user_name}_store")
267
  os.makedirs(save_dir, exist_ok=True)
268
  cur_index.storage_context.persist(persist_dir=save_dir)
269
+
utils/memory_utils.py CHANGED
@@ -11,9 +11,12 @@ import traceback
11
  # LlamaIndex Imports
12
  from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
13
 
 
14
  # Local Imports setup
15
  # Assuming this file is in 'utils/', we step back to find 'memory_bank'
16
- bank_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../memory_bank')
 
 
17
  sys.path.append(bank_path)
18
 
19
  # Import functions from sibling/child modules
@@ -30,22 +33,14 @@ except ImportError:
30
  def extract_semantic_memory(*args, **kwargs): return {}
31
 
32
 
33
- PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
34
-
35
- # تشخیص خودکار حافظه دائمی ابری
36
- BASE_DATA_DIR = (
37
- "/data" if os.path.exists("/data") else os.path.join(PROJECT_ROOT, "data")
38
- )
39
 
40
  # مسیر فایل json
41
  MEMORY_FILE_PATH = os.path.join(
42
- BASE_DATA_DIR, "memories", "update_memory_0512_eng.json"
43
- )
44
-
45
- # مسیر ایندکس‌ها (هم‌سطح با پوشه memories)
46
- INDEX_BASE_DIR = os.getenv(
47
- "EMMA_INDEX_DIR", os.path.join(BASE_DATA_DIR, "memory_index", "llamaindex")
48
- )
49
 
50
 
51
  def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
@@ -107,7 +102,7 @@ def enter_name_llamaindex(name, memory, data_args, update_memory_index=True):
107
 
108
  user_memory = memory[name]
109
  # مسیر دقیق کاربر در پوشه llamaindex
110
- base_path = os.path.join(INDEX_BASE_DIR, name)
111
  sessions_path = os.path.join(base_path, "sessions")
112
  episodic_path = os.path.join(base_path, "episodic_memory")
113
  semantic_path = os.path.join(base_path, "semantic_memory")
 
11
  # LlamaIndex Imports
12
  from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
13
 
14
+
15
  # Local Imports setup
16
  # Assuming this file is in 'utils/', we step back to find 'memory_bank'
17
+ CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
18
+ BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
19
+ bank_path = os.path.join(BASE_DIR, 'memory_bank')
20
  sys.path.append(bank_path)
21
 
22
  # Import functions from sibling/child modules
 
33
  def extract_semantic_memory(*args, **kwargs): return {}
34
 
35
 
36
+ # یکپارچه‌سازی و استانداردسازی مسیرها (منطبق با پیکربندی مصوب)
37
+ MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
38
+ INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
39
+ _LLAMAINDEX_BASE_DIR = os.path.join(INDEX_BASE_DIR, "llamaindex")
 
 
40
 
41
  # مسیر فایل json
42
  MEMORY_FILE_PATH = os.path.join(
43
+ MEMORIES_DIR, "update_memory_0512_eng.json")
 
 
 
 
 
 
44
 
45
 
46
  def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
 
102
 
103
  user_memory = memory[name]
104
  # مسیر دقیق کاربر در پوشه llamaindex
105
+ base_path = os.path.join(_LLAMAINDEX_BASE_DIR, name)
106
  sessions_path = os.path.join(base_path, "sessions")
107
  episodic_path = os.path.join(base_path, "episodic_memory")
108
  semantic_path = os.path.join(base_path, "semantic_memory")