Spaces:
Sleeping
Sleeping
EMMA commited on
Commit ·
75b96cc
1
Parent(s): 71508bc
update code
Browse files- memory_bank/build_memory_index.py +13 -12
- utils/memory_utils.py +15 -18
memory_bank/build_memory_index.py
CHANGED
|
@@ -23,25 +23,26 @@ from llama_index.core import (
|
|
| 23 |
from llama_index.llms.openai import OpenAI
|
| 24 |
# If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
|
| 25 |
# from llama_index.embeddings.openai import OpenAIEmbedding
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
# --- Global Variables ---
|
| 29 |
# A dictionary to hold the loaded or newly created indices in memory.
|
| 30 |
index_set = {}
|
| 31 |
|
| 32 |
-
# پیکربندی دایرکتوریها بر اساس ساختار استاندارد درخواستی
|
| 33 |
-
CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 34 |
-
BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
|
| 35 |
-
|
| 36 |
-
# <--- تغییر در اینجا: نام پوشه به Emma-memory-storage تغییر یافت --->
|
| 37 |
-
MEMORIES_DIR = os.path.join(BASE_DIR, "Emma-memory-storage")
|
| 38 |
-
INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
|
| 39 |
-
|
| 40 |
-
# تنظیم مسیر دقیق برای llamaindex
|
| 41 |
-
_LLAMAINDEX_BASE_DIR = os.getenv("EMMA_INDEX_DIR", os.path.join(INDEX_BASE_DIR, "llamaindex"))
|
| 42 |
|
| 43 |
-
# اطمینان از وجود دایرکتوری اصلی در صورت ساخت ایندکس جدید
|
| 44 |
-
os.makedirs(_LLAMAINDEX_BASE_DIR, exist_ok=True)
|
| 45 |
|
| 46 |
|
| 47 |
|
|
|
|
| 23 |
from llama_index.llms.openai import OpenAI
|
| 24 |
# If a specific embedding model is needed (e.g., from OpenAI), uncomment the line below
|
| 25 |
# from llama_index.embeddings.openai import OpenAIEmbedding
|
| 26 |
+
# --- PATH CONFIGURATION (CRITICAL UPDATE) ---
|
| 27 |
+
# پیدا کردن مسیر پایه بر اساس موقعیت فایل اسکریپت برای اطمینان از ذخیرهسازی صحیح
|
| 28 |
+
# فرض بر این است که فایل در /app/utils/ یا /app/memory_bank/ است
|
| 29 |
+
CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 30 |
+
BASE_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
|
| 31 |
|
| 32 |
+
# تنظیم مسیر دقیق برای ذخیره ایندکسها
|
| 33 |
+
# این مسیر باید دقیقاً همان مسیری باشد که memory_utils.py آن را آپلود میکند
|
| 34 |
+
MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
|
| 35 |
+
INDEX_BASE_DIR = os.path.join(MEMORIES_DIR, "memory_index")
|
| 36 |
+
|
| 37 |
+
print(f"📊 Index Builder Path Config:")
|
| 38 |
+
print(f" - Script Dir: {CURRENT_SCRIPT_DIR}")
|
| 39 |
+
print(f" - Index Target Dir: {INDEX_BASE_DIR}")
|
| 40 |
|
| 41 |
# --- Global Variables ---
|
| 42 |
# A dictionary to hold the loaded or newly created indices in memory.
|
| 43 |
index_set = {}
|
| 44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
|
|
|
|
|
|
| 46 |
|
| 47 |
|
| 48 |
|
utils/memory_utils.py
CHANGED
|
@@ -11,13 +11,7 @@ import traceback
|
|
| 11 |
# LlamaIndex Imports
|
| 12 |
from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
|
| 13 |
|
| 14 |
-
|
| 15 |
-
# Assuming this file is in 'utils/', we step back to find 'memory_bank'
|
| 16 |
-
CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 17 |
-
# تغییر نام BASE_DIR در اینجا برای جلوگیری از تداخل با BASE_DIR مسیرهای پایین
|
| 18 |
-
APP_ROOT_DIR = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, ".."))
|
| 19 |
-
bank_path = os.path.join(APP_ROOT_DIR, 'memory_bank')
|
| 20 |
-
sys.path.append(bank_path)
|
| 21 |
|
| 22 |
# Import functions from sibling/child modules
|
| 23 |
# Ensure these files exist in the appended path
|
|
@@ -33,32 +27,35 @@ except ImportError:
|
|
| 33 |
def extract_semantic_memory(*args, **kwargs): return {}
|
| 34 |
|
| 35 |
|
| 36 |
-
#
|
|
|
|
|
|
|
|
|
|
| 37 |
REPO_ID = "Keyvan1986/Emma-memory-storage"
|
| 38 |
REPO_TYPE = "dataset"
|
| 39 |
-
HF_TOKEN = os.environ.get("
|
| 40 |
|
| 41 |
-
# مسیرهای اصلی (Absolute Paths)
|
| 42 |
BASE_DIR = os.path.abspath(os.getcwd()) # معمولاً /app
|
| 43 |
MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
|
| 44 |
MEMORY_INDEX_DIR_NAME = "memory_index"
|
| 45 |
MEMORY_INDEX_PATH = os.path.join(MEMORIES_DIR, MEMORY_INDEX_DIR_NAME)
|
| 46 |
|
| 47 |
-
_LLAMAINDEX_BASE_DIR = os.path.join(MEMORY_INDEX_PATH, "llamaindex")
|
| 48 |
-
|
| 49 |
-
# مسیر فایل json
|
| 50 |
MEMORY_FILE_NAME = "update_memory_0512_eng.json"
|
| 51 |
MEMORY_FILE_PATH = os.path.join(MEMORIES_DIR, MEMORY_FILE_NAME)
|
| 52 |
|
| 53 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
os.makedirs(MEMORIES_DIR, exist_ok=True)
|
| 55 |
os.makedirs(MEMORY_INDEX_PATH, exist_ok=True)
|
| 56 |
-
os.makedirs(_LLAMAINDEX_BASE_DIR, exist_ok=True)
|
| 57 |
|
| 58 |
print(f"📂 Path Configuration:")
|
| 59 |
-
print(f" - App
|
| 60 |
-
print(f" -
|
| 61 |
-
|
| 62 |
# ------------------------------------------------------------------------
|
| 63 |
|
| 64 |
|
|
|
|
| 11 |
# LlamaIndex Imports
|
| 12 |
from llama_index.core import StorageContext, load_index_from_storage, VectorStoreIndex
|
| 13 |
|
| 14 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
# Import functions from sibling/child modules
|
| 17 |
# Ensure these files exist in the appended path
|
|
|
|
| 27 |
def extract_semantic_memory(*args, **kwargs): return {}
|
| 28 |
|
| 29 |
|
| 30 |
+
# ==========================================
|
| 31 |
+
# 1. تنظیمات مسیر و هاب (Cloud Config)
|
| 32 |
+
# ==========================================
|
| 33 |
+
|
| 34 |
REPO_ID = "Keyvan1986/Emma-memory-storage"
|
| 35 |
REPO_TYPE = "dataset"
|
| 36 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
| 37 |
|
| 38 |
+
# مسیرهای اصلی (Absolute Paths)
|
| 39 |
BASE_DIR = os.path.abspath(os.getcwd()) # معمولاً /app
|
| 40 |
MEMORIES_DIR = os.path.join(BASE_DIR, "memories")
|
| 41 |
MEMORY_INDEX_DIR_NAME = "memory_index"
|
| 42 |
MEMORY_INDEX_PATH = os.path.join(MEMORIES_DIR, MEMORY_INDEX_DIR_NAME)
|
| 43 |
|
|
|
|
|
|
|
|
|
|
| 44 |
MEMORY_FILE_NAME = "update_memory_0512_eng.json"
|
| 45 |
MEMORY_FILE_PATH = os.path.join(MEMORIES_DIR, MEMORY_FILE_NAME)
|
| 46 |
|
| 47 |
+
# --- مسیر "اشتباه" قدیمی که فایلها آنجا ساخته میشوند ---
|
| 48 |
+
# لاگ شما نشان داد فایلها اینجا میروند: ../memories
|
| 49 |
+
LEGACY_OUTSIDE_DIR = os.path.abspath(os.path.join(BASE_DIR, "..", "memories"))
|
| 50 |
+
LEGACY_INDEX_PATH = os.path.join(LEGACY_OUTSIDE_DIR, MEMORY_INDEX_DIR_NAME)
|
| 51 |
+
|
| 52 |
os.makedirs(MEMORIES_DIR, exist_ok=True)
|
| 53 |
os.makedirs(MEMORY_INDEX_PATH, exist_ok=True)
|
|
|
|
| 54 |
|
| 55 |
print(f"📂 Path Configuration:")
|
| 56 |
+
print(f" - App Dir (Target): {MEMORY_INDEX_PATH}")
|
| 57 |
+
print(f" - Builder Dir (Source): {LEGACY_INDEX_PATH}")
|
| 58 |
+
|
| 59 |
# ------------------------------------------------------------------------
|
| 60 |
|
| 61 |
|