Spaces:
Sleeping
Sleeping
EMMA commited on
Commit ·
fa788b0
1
Parent(s): 40f5a32
update code
Browse files
Backup code/build_memory_index_23.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#
|
| 2 |
+
# Memory Indexing Utility for LlamaIndex
|
| 3 |
+
#
|
| 4 |
+
# This script provides functionalities to build and manage memory indices for a conversational AI
|
| 5 |
+
# using the LlamaIndex framework. It supports creating structured memory documents from raw
|
| 6 |
+
# conversation data and persisting them as vector indices for efficient retrieval.
|
| 7 |
+
#
|
| 8 |
+
|
| 9 |
+
# --- Standard Library Imports ---
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
|
| 13 |
+
# --- Third-Party Imports ---
|
| 14 |
+
import tiktoken
|
| 15 |
+
from llama_index.core import (
|
| 16 |
+
Document,
|
| 17 |
+
PromptHelper,
|
| 18 |
+
Settings, # Replaces the deprecated ServiceContext
|
| 19 |
+
StorageContext,
|
| 20 |
+
VectorStoreIndex,
|
| 21 |
+
load_index_from_storage,
|
| 22 |
+
)
|
| 23 |
+
from llama_index.llms.openai import OpenAI
|
| 24 |
+
# If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
|
| 25 |
+
# from llama_index.embeddings.openai import OpenAIEmbedding
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# --- Global Variables ---
|
| 29 |
+
# A dictionary to hold the loaded or newly created indices in memory.
|
| 30 |
+
index_set = {}
|
| 31 |
+
|
| 32 |
+
# پشتیبانی هوشمند از Persistent Storage در Hugging Face (/data) یا مسیر محلی
|
| 33 |
+
BASE_DIR = "/data" if os.path.exists("/data") else os.path.dirname(os.path.abspath(__file__))
|
| 34 |
+
|
| 35 |
+
# تنظیم مسیر دقیق بر اساس ساختار ریپازیتوری (memory_index/llamaindex)
|
| 36 |
+
_INDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(BASE_DIR, "memory_index", "llamaindex"))
|
| 37 |
+
|
| 38 |
+
# اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
|
| 39 |
+
os.makedirs(_INDEX_BASE_DIR, exist_ok=True)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# --- Core Functions ---
|
| 43 |
+
|
| 44 |
+
def setup_global_settings():
|
| 45 |
+
"""
|
| 46 |
+
Configures global settings for LlamaIndex (v0.10+).
|
| 47 |
+
|
| 48 |
+
This function replaces the deprecated `build_service_context` pattern by directly
|
| 49 |
+
configuring the global `Settings` object for the LLM, tokenizer, and other components.
|
| 50 |
+
"""
|
| 51 |
+
# 1. Set Tokenizer (to resolve potential tiktoken errors)
|
| 52 |
+
try:
|
| 53 |
+
# This encoding is commonly used by OpenAI models like GPT-4.
|
| 54 |
+
Settings.tokenizer = tiktoken.get_encoding("cl100k_base").encode
|
| 55 |
+
except Exception as e:
|
| 56 |
+
print(f"Warning: Could not set the tokenizer. This might lead to issues. Error: {e}")
|
| 57 |
+
|
| 58 |
+
# 2. Configure the Language Model (LLM)
|
| 59 |
+
# Use the LlamaIndex-specific OpenAI class to avoid conflicts with other libraries.
|
| 60 |
+
# It automatically reads the API key from the "OPENAI_API_KEY" environment variable.
|
| 61 |
+
Settings.llm = OpenAI(
|
| 62 |
+
model="gpt-4o",
|
| 63 |
+
temperature=1.0,
|
| 64 |
+
max_tokens=1024,
|
| 65 |
+
api_key=os.environ.get("GAPGPT_API_KEY"),
|
| 66 |
+
api_base="https://api.gapgpt.app/v1",
|
| 67 |
+
additional_kwargs={
|
| 68 |
+
"top_p": 0.95,
|
| 69 |
+
"frequency_penalty": 0.4,
|
| 70 |
+
"presence_penalty": 0.2
|
| 71 |
+
}
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
# 3. Configure the Prompt Helper (with updated parameter names)
|
| 75 |
+
# This helps manage context window size, number of outputs, etc.
|
| 76 |
+
Settings.prompt_helper = PromptHelper(
|
| 77 |
+
context_window=4096,
|
| 78 |
+
num_output=256,
|
| 79 |
+
chunk_overlap_ratio=0.1
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
# 4. Configure Chunk Size for document processing
|
| 83 |
+
Settings.chunk_size = 512
|
| 84 |
+
Settings.chunk_overlap = 20
|
| 85 |
+
|
| 86 |
+
# Optional: Configure the embedding model if needed.
|
| 87 |
+
# If not set, LlamaIndex defaults to a compatible model (e.g., OpenAI's text-embedding-ada-002).
|
| 88 |
+
# Settings.embed_model = OpenAIEmbedding(api_key=GAPGPTMASKTOKENlgicihcumgiX0X, api_base=GAPGPTMASKTOKENlgicihcumgiX1X)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def generate_memory_docs(data, language):
|
| 92 |
+
"""
|
| 93 |
+
Generates structured LlamaIndex Document objects from a nested dictionary of user memories.
|
| 94 |
+
|
| 95 |
+
This function processes different types of memory (sessions, episodic, semantic) and
|
| 96 |
+
formats them into Document objects suitable for indexing.
|
| 97 |
+
|
| 98 |
+
Args:
|
| 99 |
+
data (dict): A dictionary where keys are user names and values contain their memories.
|
| 100 |
+
language (str): The language code (e.g., "en"), currently used for formatting.
|
| 101 |
+
|
| 102 |
+
Returns:
|
| 103 |
+
dict: A dictionary where keys are user names and values are dicts containing
|
| 104 |
+
lists of Document objects for each memory type.
|
| 105 |
+
"""
|
| 106 |
+
all_user_memories = {}
|
| 107 |
+
|
| 108 |
+
for user_name, user_memory in data.items():
|
| 109 |
+
all_user_memories[user_name] = {
|
| 110 |
+
"sessions": [],
|
| 111 |
+
"episodic_memory": [],
|
| 112 |
+
"semantic_memory": [],
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
# Process session history
|
| 116 |
+
if "sessions" in user_memory:
|
| 117 |
+
for session in user_memory["sessions"]:
|
| 118 |
+
date = session["date"]
|
| 119 |
+
content = session["conversation"]
|
| 120 |
+
|
| 121 |
+
memory_str = f"Session on {date}:\n"
|
| 122 |
+
for dialog in content:
|
| 123 |
+
query = dialog.get("query", "")
|
| 124 |
+
response = dialog.get("response", "")
|
| 125 |
+
memory_str += f"\n{user_name}: {query.strip()}"
|
| 126 |
+
memory_str += f"\nAI: {response.strip()}"
|
| 127 |
+
memory_str += "\n"
|
| 128 |
+
|
| 129 |
+
if "summary" in user_memory and date in user_memory["summary"]:
|
| 130 |
+
summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
|
| 131 |
+
memory_str += summary
|
| 132 |
+
|
| 133 |
+
# Use the 'text' argument for the Document object
|
| 134 |
+
all_user_memories[user_name]["sessions"].append(Document(text=memory_str))
|
| 135 |
+
|
| 136 |
+
# Process episodic memory
|
| 137 |
+
if "episodic_memory" in user_memory:
|
| 138 |
+
episodic_str = "Recent experiences:\n"
|
| 139 |
+
for event in user_memory["episodic_memory"]:
|
| 140 |
+
episodic_str += f"- {event}\n"
|
| 141 |
+
all_user_memories[user_name]["episodic_memory"].append(Document(text=episodic_str))
|
| 142 |
+
|
| 143 |
+
# Process semantic memory (long-term facts and personality)
|
| 144 |
+
if "semantic_memory" in user_memory:
|
| 145 |
+
semantic_str = "Long-term personality traits and facts:\n"
|
| 146 |
+
if isinstance(user_memory["semantic_memory"], dict):
|
| 147 |
+
for trait, value in user_memory["semantic_memory"].items():
|
| 148 |
+
semantic_str += f"- {trait}: {value}\n"
|
| 149 |
+
elif isinstance(user_memory["semantic_memory"], list):
|
| 150 |
+
for item in user_memory["semantic_memory"]:
|
| 151 |
+
semantic_str += f"- {item}\n"
|
| 152 |
+
else:
|
| 153 |
+
print(f"WARNING: semantic_memory for user '{user_name}' has an unexpected format: {type(user_memory['semantic_memory'])}")
|
| 154 |
+
|
| 155 |
+
all_user_memories[user_name]["semantic_memory"].append(Document(text=semantic_str))
|
| 156 |
+
|
| 157 |
+
return all_user_memories
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def build_memory_index(all_user_memories, data_args, name=None):
|
| 161 |
+
"""
|
| 162 |
+
Build and persist memory indices for each user and memory type.
|
| 163 |
+
"""
|
| 164 |
+
structured_docs = generate_memory_docs(
|
| 165 |
+
all_user_memories, data_args.language
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
setup_global_settings()
|
| 169 |
+
|
| 170 |
+
for user_name, memories_by_type in structured_docs.items():
|
| 171 |
+
if name and user_name != name:
|
| 172 |
+
continue
|
| 173 |
+
|
| 174 |
+
print(f"Building indices for user '{user_name}'")
|
| 175 |
+
|
| 176 |
+
base_path = os.path.join(_INDEX_BASE_DIR, user_name)
|
| 177 |
+
|
| 178 |
+
path_map = {
|
| 179 |
+
"sessions": os.path.join(base_path, "sessions"),
|
| 180 |
+
"episodic_memory": os.path.join(base_path, "episodic_memory"),
|
| 181 |
+
"semantic_memory": os.path.join(base_path, "semantic_memory"),
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
for memory_type, docs in memories_by_type.items():
|
| 185 |
+
if not docs:
|
| 186 |
+
print(f" -> Skipping '{memory_type}' index (no documents found).")
|
| 187 |
+
continue
|
| 188 |
+
|
| 189 |
+
if memory_type not in path_map:
|
| 190 |
+
print(f" -> Skipping unknown memory type: '{memory_type}'")
|
| 191 |
+
continue
|
| 192 |
+
|
| 193 |
+
print(f" -> Building '{memory_type}' index...")
|
| 194 |
+
|
| 195 |
+
cur_index = VectorStoreIndex.from_documents(docs)
|
| 196 |
+
|
| 197 |
+
index_dir = path_map[memory_type]
|
| 198 |
+
os.makedirs(index_dir, exist_ok=True)
|
| 199 |
+
|
| 200 |
+
cur_index.storage_context.persist(persist_dir=index_dir)
|
| 201 |
+
print(f" + Saved '{memory_type}' index to: {index_dir}")
|
| 202 |
+
|
| 203 |
+
index_set[f"{user_name}_{memory_type}"] = cur_index
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
# --- Deprecated Functions (Kept for reference) ---
|
| 207 |
+
|
| 208 |
+
def generate_memory_docs_old(data, language):
|
| 209 |
+
"""
|
| 210 |
+
DEPRECATED: An older version of the document generation function.
|
| 211 |
+
|
| 212 |
+
This function had a simpler structure and contained Chinese string literals.
|
| 213 |
+
It is replaced by `generate_memory_docs`.
|
| 214 |
+
"""
|
| 215 |
+
all_user_memories = {}
|
| 216 |
+
for user_name, user_memory in data.items():
|
| 217 |
+
all_user_memories[user_name] = []
|
| 218 |
+
if "history" not in user_memory:
|
| 219 |
+
continue
|
| 220 |
+
for date, content in user_memory["history"].items():
|
| 221 |
+
memory_str = f"Conversation on {date}:"
|
| 222 |
+
|
| 223 |
+
for dialog in content:
|
| 224 |
+
query = dialog.get("query", "")
|
| 225 |
+
response = dialog.get("response", "")
|
| 226 |
+
memory_str += f"\n{user_name}: {query.strip()}"
|
| 227 |
+
memory_str += f"\nAI: {response.strip()}"
|
| 228 |
+
memory_str += "\n"
|
| 229 |
+
if "summary" in user_memory and date in user_memory["summary"]:
|
| 230 |
+
summary = f'The summary of the conversation on {date} is: {user_memory["summary"][date]}'
|
| 231 |
+
memory_str += summary
|
| 232 |
+
|
| 233 |
+
all_user_memories[user_name].append(Document(text=memory_str))
|
| 234 |
+
return all_user_memories
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def build_memory_index_old(all_user_memories, data_args, name=None):
|
| 238 |
+
"""
|
| 239 |
+
DEPRECATED: An older version of the index building function.
|
| 240 |
+
|
| 241 |
+
This function used a flatter memory structure and is replaced by the more
|
| 242 |
+
comprehensive `build_memory_index`.
|
| 243 |
+
"""
|
| 244 |
+
# The function name is a typo in the original code, corrected here for clarity.
|
| 245 |
+
all_user_memories_docs = generate_memory_docs(
|
| 246 |
+
all_user_memories, data_args.language
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
# Apply global settings
|
| 250 |
+
setup_global_settings()
|
| 251 |
+
|
| 252 |
+
for user_name, memories in all_user_memories_docs.items():
|
| 253 |
+
if name and user_name != name:
|
| 254 |
+
continue
|
| 255 |
+
print(f"Building index for user {user_name} (using old method)")
|
| 256 |
+
|
| 257 |
+
# The service_context argument is removed in newer versions.
|
| 258 |
+
cur_index = VectorStoreIndex.from_documents(memories)
|
| 259 |
+
index_set[user_name] = cur_index
|
| 260 |
+
|
| 261 |
+
# استفاده از مسیر داینامیک و امن
|
| 262 |
+
save_dir = os.path.join(_INDEX_BASE_DIR, f"{user_name}_store")
|
| 263 |
+
os.makedirs(save_dir, exist_ok=True)
|
| 264 |
+
cur_index.storage_context.persist(persist_dir=save_dir)
|
Backup code/{memory_utils.py → memory_utils_23.py}
RENAMED
|
File without changes
|
Backup code/memory_utils_24.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import json
|
| 4 |
+
import time
|
| 5 |
+
import datetime
|
| 6 |
+
import shutil
|
| 7 |
+
import gradio as gr
|
| 8 |
+
from pprint import pprint
|
| 9 |
+
import traceback
|
| 10 |
+
|
| 11 |
+
# LlamaIndex Imports
|
| 12 |
+
from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
|
| 13 |
+
|
| 14 |
+
# Local Imports setup
|
| 15 |
+
# Assuming this file is in 'utils/', we step back to find 'memory_bank'
|
| 16 |
+
bank_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../memory_bank')
|
| 17 |
+
sys.path.append(bank_path)
|
| 18 |
+
|
| 19 |
+
# Import functions from sibling/child modules
|
| 20 |
+
# Ensure these files exist in the appended path
|
| 21 |
+
try:
|
| 22 |
+
from build_memory_index import build_memory_index
|
| 23 |
+
from summarize_memory import summarize_memory, extract_session_summary, extract_semantic_memory
|
| 24 |
+
except ImportError:
|
| 25 |
+
# Fallback or placeholder if running independently for testing
|
| 26 |
+
print("Warning: Could not import memory build/summary modules.")
|
| 27 |
+
def build_memory_index(*args, **kwargs): pass
|
| 28 |
+
def summarize_memory(*args, **kwargs): return {}
|
| 29 |
+
def extract_session_summary(*args, **kwargs): return {}
|
| 30 |
+
def extract_semantic_memory(*args, **kwargs): return {}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 34 |
+
|
| 35 |
+
# تشخیص خودکار حافظه دائمی ابری
|
| 36 |
+
BASE_DATA_DIR = (
|
| 37 |
+
"/data" if os.path.exists("/data") else os.path.join(PROJECT_ROOT, "data")
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
# مسیر فایل json
|
| 41 |
+
MEMORY_FILE_PATH = os.path.join(
|
| 42 |
+
BASE_DATA_DIR, "memories", "update_memory_0512_eng.json"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
# مسیر ایندکسها (همسطح با پوشه memories)
|
| 46 |
+
INDEX_BASE_DIR = os.getenv(
|
| 47 |
+
"EMMA_INDEX_DIR", os.path.join(BASE_DATA_DIR, "memory_index", "llamaindex")
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
|
| 52 |
+
"""
|
| 53 |
+
Legacy/Basic function to load user memory and initialize vector store.
|
| 54 |
+
"""
|
| 55 |
+
cur_date = datetime.date.today().strftime("%Y-%m-%d")
|
| 56 |
+
user_memory_index = None
|
| 57 |
+
|
| 58 |
+
# Handle Gradio States
|
| 59 |
+
if isinstance(data_args, gr.State): data_args = data_args.value
|
| 60 |
+
if isinstance(memory, gr.State): memory = memory.value
|
| 61 |
+
if isinstance(local_memory_qa, gr.State): local_memory_qa = local_memory_qa.value
|
| 62 |
+
|
| 63 |
+
memory_dir = MEMORY_FILE_PATH
|
| 64 |
+
|
| 65 |
+
if name in memory.keys():
|
| 66 |
+
user_memory = memory[name]
|
| 67 |
+
memory_index_path = os.path.join(data_args.memory_basic_dir, f'memory_index/{name}_index')
|
| 68 |
+
os.makedirs(os.path.dirname(memory_index_path), exist_ok=True)
|
| 69 |
+
|
| 70 |
+
if (not os.path.exists(memory_index_path)) or update_memory_index:
|
| 71 |
+
print(f'Initializing memory index {memory_index_path}...')
|
| 72 |
+
|
| 73 |
+
if os.path.exists(memory_index_path):
|
| 74 |
+
shutil.rmtree(memory_index_path)
|
| 75 |
+
# Initialize using the local QA object
|
| 76 |
+
memory_index_path, _ = local_memory_qa.init_memory_vector_store(
|
| 77 |
+
filepath=memory_dir,
|
| 78 |
+
vs_path=memory_index_path,
|
| 79 |
+
user_name=name,
|
| 80 |
+
cur_date=cur_date
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
user_memory_index = local_memory_qa.load_memory_index(memory_index_path) if memory_index_path else None
|
| 84 |
+
msg = f"Welcome back, {name}!"
|
| 85 |
+
return msg, user_memory, memory, name, user_memory_index
|
| 86 |
+
else:
|
| 87 |
+
memory[name] = {}
|
| 88 |
+
memory[name].update({"name": name})
|
| 89 |
+
msg = f"Welcome, new user {name}! I will remember your name, so next time we meet, I'll be able to call you by your name!"
|
| 90 |
+
return msg, memory[name], memory, name, user_memory_index
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def enter_name_llamaindex(name, memory, data_args, update_memory_index=True):
|
| 94 |
+
"""
|
| 95 |
+
Load the user's session, episodic, and semantic memory indices.
|
| 96 |
+
Compatible with LlamaIndex v0.10+.
|
| 97 |
+
"""
|
| 98 |
+
sessions_memory = None
|
| 99 |
+
episodic_memory = None
|
| 100 |
+
semantic_memory = None
|
| 101 |
+
|
| 102 |
+
print(f"[DEBUG] enter_name_llamaindex called with name={name!r}")
|
| 103 |
+
|
| 104 |
+
if name not in memory:
|
| 105 |
+
print(f"[DEBUG] user {name!r} not found in memory")
|
| 106 |
+
return "User not found.", None, None, None, None
|
| 107 |
+
|
| 108 |
+
user_memory = memory[name]
|
| 109 |
+
# مسیر دقیق کاربر در پوشه llamaindex
|
| 110 |
+
base_path = os.path.join(INDEX_BASE_DIR, name)
|
| 111 |
+
sessions_path = os.path.join(base_path, "sessions")
|
| 112 |
+
episodic_path = os.path.join(base_path, "episodic_memory")
|
| 113 |
+
semantic_path = os.path.join(base_path, "semantic_memory")
|
| 114 |
+
|
| 115 |
+
print(f"[DEBUG] base_path: {base_path}")
|
| 116 |
+
print(f"[DEBUG] sessions_path: {sessions_path}")
|
| 117 |
+
print(f"[DEBUG] episodic_path: {episodic_path}")
|
| 118 |
+
print(f"[DEBUG] semantic_path: {semantic_path}")
|
| 119 |
+
|
| 120 |
+
indices_exist = (
|
| 121 |
+
os.path.isdir(sessions_path)
|
| 122 |
+
and os.path.isdir(episodic_path)
|
| 123 |
+
and os.path.isdir(semantic_path)
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
print(f"[DEBUG] indices_exist: {indices_exist}")
|
| 127 |
+
#print(f"[DEBUG] update_memory_index: {update_memory_index}")
|
| 128 |
+
|
| 129 |
+
if update_memory_index or not indices_exist:
|
| 130 |
+
print(f"[DEBUG] Initializing memory indices for {name}...")
|
| 131 |
+
|
| 132 |
+
# Important: build_memory_index must persist to these same absolute paths.
|
| 133 |
+
build_memory_index(memory, data_args, name=name)
|
| 134 |
+
|
| 135 |
+
print("[DEBUG] After build:")
|
| 136 |
+
print(f"[DEBUG] sessions_path exists: {os.path.isdir(sessions_path)}")
|
| 137 |
+
print(f"[DEBUG] episodic_path exists: {os.path.isdir(episodic_path)}")
|
| 138 |
+
print(f"[DEBUG] semantic_path exists: {os.path.isdir(semantic_path)}")
|
| 139 |
+
|
| 140 |
+
def load_index_safe(index_name, index_path):
|
| 141 |
+
if not os.path.isdir(index_path):
|
| 142 |
+
print(f"[DEBUG] {index_name} path does not exist: {index_path}")
|
| 143 |
+
return None
|
| 144 |
+
|
| 145 |
+
try:
|
| 146 |
+
storage_context = StorageContext.from_defaults(
|
| 147 |
+
persist_dir=index_path
|
| 148 |
+
)
|
| 149 |
+
index = load_index_from_storage(storage_context)
|
| 150 |
+
print(
|
| 151 |
+
f"[DEBUG] {index_name} loaded: "
|
| 152 |
+
f"{type(index).__name__}, is_none={index is None}"
|
| 153 |
+
)
|
| 154 |
+
return index
|
| 155 |
+
except Exception as exc:
|
| 156 |
+
print(f"[ERROR] Could not load {index_name}: {exc}")
|
| 157 |
+
traceback.print_exc()
|
| 158 |
+
return None
|
| 159 |
+
|
| 160 |
+
sessions_memory = load_index_safe("sessions_memory", sessions_path)
|
| 161 |
+
episodic_memory = load_index_safe("episodic_memory", episodic_path)
|
| 162 |
+
semantic_memory = load_index_safe("semantic_memory", semantic_path)
|
| 163 |
+
|
| 164 |
+
print("[DEBUG] RETURN VALUES:")
|
| 165 |
+
print(f" hello_msg: Welcome back, {name}!")
|
| 166 |
+
print(f" user_memory type: {type(user_memory).__name__}")
|
| 167 |
+
print(f" sessions_memory type: {type(sessions_memory).__name__}")
|
| 168 |
+
print(f" episodic_memory type: {type(episodic_memory).__name__}")
|
| 169 |
+
print(f" semantic_memory type: {type(semantic_memory).__name__}")
|
| 170 |
+
|
| 171 |
+
return (
|
| 172 |
+
f"Welcome back, {name}!",
|
| 173 |
+
user_memory,
|
| 174 |
+
sessions_memory,
|
| 175 |
+
episodic_memory,
|
| 176 |
+
semantic_memory,
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def summarize_memory_event_personality(data_args, memory, user_name):
|
| 181 |
+
"""
|
| 182 |
+
Summarizes the memory and returns the user-specific memory dict.
|
| 183 |
+
"""
|
| 184 |
+
if isinstance(data_args, gr.State): data_args = data_args.value
|
| 185 |
+
if isinstance(memory, gr.State): memory = memory.value
|
| 186 |
+
|
| 187 |
+
memory_dir = MEMORY_FILE_PATH
|
| 188 |
+
|
| 189 |
+
# Note: Ensure summarize_memory handles the language argument correctly (passed 'en' or similar)
|
| 190 |
+
memory = summarize_memory(memory_dir, user_name, language=data_args.language)
|
| 191 |
+
user_memory = memory[user_name] if user_name in memory.keys() else {}
|
| 192 |
+
return user_memory
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def save_local_memory(memory, history, user_name, data_args, new_conversation=False):
|
| 196 |
+
"""
|
| 197 |
+
Saves user-model conversations into memory and adds episodic memory for each session.
|
| 198 |
+
Handles both list-of-lists (old Gradio) and list-of-dicts (new Gradio/OpenAI) formats.
|
| 199 |
+
"""
|
| 200 |
+
if isinstance(data_args, gr.State): data_args = data_args.value
|
| 201 |
+
if isinstance(memory, gr.State): memory = memory.value
|
| 202 |
+
|
| 203 |
+
memory_dir = MEMORY_FILE_PATH
|
| 204 |
+
|
| 205 |
+
# 1. Initialize user memory with ALL required structures FIRST
|
| 206 |
+
if user_name not in memory:
|
| 207 |
+
memory[user_name] = {
|
| 208 |
+
"sessions": [],
|
| 209 |
+
"episodic_memory": [],
|
| 210 |
+
"semantic_memory": {}
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
# 2. Ensure all sub-structures exist and are correct type
|
| 214 |
+
memory[user_name].setdefault("sessions", [])
|
| 215 |
+
memory[user_name].setdefault("episodic_memory", [])
|
| 216 |
+
memory[user_name].setdefault("semantic_memory", {})
|
| 217 |
+
|
| 218 |
+
# 3. Now safely check types
|
| 219 |
+
if not isinstance(memory[user_name]["semantic_memory"], dict):
|
| 220 |
+
memory[user_name]["semantic_memory"] = {}
|
| 221 |
+
|
| 222 |
+
# Create new session or update existing one
|
| 223 |
+
if new_conversation or not memory[user_name]["sessions"]:
|
| 224 |
+
if new_conversation and memory[user_name]["sessions"]:
|
| 225 |
+
# Logic to summarize previous session before starting new one could go here
|
| 226 |
+
pass
|
| 227 |
+
|
| 228 |
+
# Create new session
|
| 229 |
+
session = {
|
| 230 |
+
"session_id": len(memory[user_name]["sessions"]),
|
| 231 |
+
"date": time.strftime("%Y-%m-%d", time.localtime()),
|
| 232 |
+
"conversation": []
|
| 233 |
+
}
|
| 234 |
+
memory[user_name]["sessions"].append(session)
|
| 235 |
+
|
| 236 |
+
current_session = memory[user_name]["sessions"][-1]
|
| 237 |
+
|
| 238 |
+
# --- Modified section to fix KeyError: 0 and handle History formats ---
|
| 239 |
+
if not new_conversation and history:
|
| 240 |
+
last_item = history[-1]
|
| 241 |
+
|
| 242 |
+
# Case 1: New Format (List of Dicts)
|
| 243 |
+
# In this format, history is linear. The last item is bot response, second to last is user query.
|
| 244 |
+
if isinstance(last_item, dict):
|
| 245 |
+
if len(history) >= 2:
|
| 246 |
+
user_query = history[-2].get('content', '')
|
| 247 |
+
bot_response = history[-1].get('content', '')
|
| 248 |
+
|
| 249 |
+
# Verify roles to ensure correct pairing
|
| 250 |
+
if history[-2].get('role') == 'user' and history[-1].get('role') == 'assistant':
|
| 251 |
+
current_session["conversation"].append({
|
| 252 |
+
'query': user_query,
|
| 253 |
+
'response': bot_response
|
| 254 |
+
})
|
| 255 |
+
|
| 256 |
+
# Case 2: Old Format (List of Lists/Tuples)
|
| 257 |
+
elif isinstance(last_item, (list, tuple)):
|
| 258 |
+
current_session["conversation"].append({
|
| 259 |
+
'query': last_item[0],
|
| 260 |
+
'response': last_item[1]
|
| 261 |
+
})
|
| 262 |
+
# ----------------------------------------------------------------------
|
| 263 |
+
|
| 264 |
+
# Optional: Update semantic memory in real-time
|
| 265 |
+
memory[user_name]["semantic_memory"] = extract_semantic_memory(
|
| 266 |
+
memory[user_name]["semantic_memory"],
|
| 267 |
+
current_session["conversation"]
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
semantic_memory_text = memory[user_name]["semantic_memory"]
|
| 271 |
+
|
| 272 |
+
# Save to file
|
| 273 |
+
# Ensure memory directory exists
|
| 274 |
+
os.makedirs(os.path.dirname(memory_dir), exist_ok=True)
|
| 275 |
+
|
| 276 |
+
with open(memory_dir, "w", encoding="utf-8") as f:
|
| 277 |
+
json.dump(memory, f, ensure_ascii=False, indent=4)
|
| 278 |
+
|
| 279 |
+
return memory, semantic_memory_text
|
memory_bank/build_memory_index.py
CHANGED
|
@@ -29,14 +29,18 @@ from llama_index.llms.openai import OpenAI
|
|
| 29 |
# A dictionary to hold the loaded or newly created indices in memory.
|
| 30 |
index_set = {}
|
| 31 |
|
| 32 |
-
# پ
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
-
# تنظیم مسیر دقیق بر
|
| 36 |
-
|
| 37 |
|
| 38 |
# اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
|
| 39 |
-
os.makedirs(
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
# --- Core Functions ---
|
|
@@ -173,7 +177,7 @@ def build_memory_index(all_user_memories, data_args, name=None):
|
|
| 173 |
|
| 174 |
print(f"Building indices for user '{user_name}'")
|
| 175 |
|
| 176 |
-
base_path = os.path.join(
|
| 177 |
|
| 178 |
path_map = {
|
| 179 |
"sessions": os.path.join(base_path, "sessions"),
|
|
@@ -258,7 +262,8 @@ def build_memory_index_old(all_user_memories, data_args, name=None):
|
|
| 258 |
cur_index = VectorStoreIndex.from_documents(memories)
|
| 259 |
index_set[user_name] = cur_index
|
| 260 |
|
| 261 |
-
|
| 262 |
-
save_dir = os.path.join(
|
| 263 |
os.makedirs(save_dir, exist_ok=True)
|
| 264 |
cur_index.storage_context.persist(persist_dir=save_dir)
|
|
|
|
|
|
| 29 |
# A dictionary to hold the loaded or newly created indices in memory.
|
| 30 |
index_set = {}
|
| 31 |
|
| 32 |
+
# پیکربندی دایرکتوریها بر اساس ساختار استاندارد درخواستی
|
| 33 |
+
CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 34 |
+
BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
|
| 35 |
+
MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
|
| 36 |
+
INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
|
| 37 |
|
| 38 |
+
# تنظیم مسیر دقیق برای llamaindex
|
| 39 |
+
_LLAMAINDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(INDEX_BASE_DIR, "llamaindex"))
|
| 40 |
|
| 41 |
# اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
|
| 42 |
+
os.makedirs(_LLAMAINDEX_BASE_DIR, exist_ok=True)
|
| 43 |
+
|
| 44 |
|
| 45 |
|
| 46 |
# --- Core Functions ---
|
|
|
|
| 177 |
|
| 178 |
print(f"Building indices for user '{user_name}'")
|
| 179 |
|
| 180 |
+
base_path = os.path.join(_LLAMAINDEX_BASE_DIR, user_name)
|
| 181 |
|
| 182 |
path_map = {
|
| 183 |
"sessions": os.path.join(base_path, "sessions"),
|
|
|
|
| 262 |
cur_index = VectorStoreIndex.from_documents(memories)
|
| 263 |
index_set[user_name] = cur_index
|
| 264 |
|
| 265 |
+
# استفاده از مسیر داینامیک و امن
|
| 266 |
+
save_dir = os.path.join(_LLAMAINDEX_BASE_DIR, f"{user_name}_store")
|
| 267 |
os.makedirs(save_dir, exist_ok=True)
|
| 268 |
cur_index.storage_context.persist(persist_dir=save_dir)
|
| 269 |
+
|
utils/memory_utils.py
CHANGED
|
@@ -11,9 +11,12 @@ import traceback
|
|
| 11 |
# LlamaIndex Imports
|
| 12 |
from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
|
| 13 |
|
|
|
|
| 14 |
# Local Imports setup
|
| 15 |
# Assuming this file is in 'utils/', we step back to find 'memory_bank'
|
| 16 |
-
|
|
|
|
|
|
|
| 17 |
sys.path.append(bank_path)
|
| 18 |
|
| 19 |
# Import functions from sibling/child modules
|
|
@@ -30,22 +33,14 @@ except ImportError:
|
|
| 30 |
def extract_semantic_memory(*args, **kwargs): return {}
|
| 31 |
|
| 32 |
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
"/data" if os.path.exists("/data") else os.path.join(PROJECT_ROOT, "data")
|
| 38 |
-
)
|
| 39 |
|
| 40 |
# مسیر فایل json
|
| 41 |
MEMORY_FILE_PATH = os.path.join(
|
| 42 |
-
|
| 43 |
-
)
|
| 44 |
-
|
| 45 |
-
# مسیر ایندکسها (همسطح با پوشه memories)
|
| 46 |
-
INDEX_BASE_DIR = os.getenv(
|
| 47 |
-
"EMMA_INDEX_DIR", os.path.join(BASE_DATA_DIR, "memory_index", "llamaindex")
|
| 48 |
-
)
|
| 49 |
|
| 50 |
|
| 51 |
def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
|
|
@@ -107,7 +102,7 @@ def enter_name_llamaindex(name, memory, data_args, update_memory_index=True):
|
|
| 107 |
|
| 108 |
user_memory = memory[name]
|
| 109 |
# مسیر دقیق کاربر در پوشه llamaindex
|
| 110 |
-
base_path = os.path.join(
|
| 111 |
sessions_path = os.path.join(base_path, "sessions")
|
| 112 |
episodic_path = os.path.join(base_path, "episodic_memory")
|
| 113 |
semantic_path = os.path.join(base_path, "semantic_memory")
|
|
|
|
| 11 |
# LlamaIndex Imports
|
| 12 |
from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
|
| 13 |
|
| 14 |
+
|
| 15 |
# Local Imports setup
|
| 16 |
# Assuming this file is in 'utils/', we step back to find 'memory_bank'
|
| 17 |
+
CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 18 |
+
BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
|
| 19 |
+
bank_path = os.path.join(BASE_DIR, 'memory_bank')
|
| 20 |
sys.path.append(bank_path)
|
| 21 |
|
| 22 |
# Import functions from sibling/child modules
|
|
|
|
| 33 |
def extract_semantic_memory(*args, **kwargs): return {}
|
| 34 |
|
| 35 |
|
| 36 |
+
# یکپارچهسازی و استانداردسازی مسیرها (منطبق با پیکربندی مصوب)
|
| 37 |
+
MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
|
| 38 |
+
INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
|
| 39 |
+
_LLAMAINDEX_BASE_DIR = os.path.join(INDEX_BASE_DIR, "llamaindex")
|
|
|
|
|
|
|
| 40 |
|
| 41 |
# مسیر فایل json
|
| 42 |
MEMORY_FILE_PATH = os.path.join(
|
| 43 |
+
MEMORIES_DIR, "update_memory_0512_eng.json")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
def enter_name(name, memory, local_memory_qa, data_args, update_memory_index=True):
|
|
|
|
| 102 |
|
| 103 |
user_memory = memory[name]
|
| 104 |
# مسیر دقیق کاربر در پوشه llamaindex
|
| 105 |
+
base_path = os.path.join(_LLAMAINDEX_BASE_DIR, name)
|
| 106 |
sessions_path = os.path.join(base_path, "sessions")
|
| 107 |
episodic_path = os.path.join(base_path, "episodic_memory")
|
| 108 |
semantic_path = os.path.join(base_path, "semantic_memory")
|