Spaces:
Running
Running
Commit ·
a506cc3
1
Parent(s): 92742e1
feat(auth): enforce client ownership and data isolation across all endpoints and databases
Browse files- app/app.py +6 -0
- app/core/dependencies.py +20 -0
- app/core/persistent_db.py +160 -60
- app/core/postgres_db.py +160 -53
- app/core/storage.py +114 -72
- app/mcp_server/web_search_mcp.py +87 -84
- app/routers/chat.py +67 -29
- app/routers/documents.py +32 -18
- app/routers/memory.py +21 -16
- app/routers/studio.py +7 -5
- app/routers/tools.py +3 -2
- app/services/memory_service.py +29 -22
- pyproject.toml +11 -0
- tests/conftest.py +46 -0
- tests/test_client_isolation.py +160 -0
- uv.lock +59 -0
app/app.py
CHANGED
|
@@ -49,6 +49,12 @@ except Exception as mcp_err:
|
|
| 49 |
app_logger.warning(f"Could not mount FastMCP at /mcp: {mcp_err}")
|
| 50 |
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
@app.get("/health", tags=["Health"])
|
| 53 |
async def health_check():
|
| 54 |
"""Health check endpoint to verify service availability and database status."""
|
|
|
|
| 49 |
app_logger.warning(f"Could not mount FastMCP at /mcp: {mcp_err}")
|
| 50 |
|
| 51 |
|
| 52 |
+
@app.get("/", tags=["Health"])
|
| 53 |
+
async def root():
|
| 54 |
+
"""Public root endpoint."""
|
| 55 |
+
return {"message": "Atlas - Document Intelligence API", "docs": "/docs", "health": "/health"}
|
| 56 |
+
|
| 57 |
+
|
| 58 |
@app.get("/health", tags=["Health"])
|
| 59 |
async def health_check():
|
| 60 |
"""Health check endpoint to verify service availability and database status."""
|
app/core/dependencies.py
CHANGED
|
@@ -1,11 +1,31 @@
|
|
| 1 |
import os
|
|
|
|
|
|
|
| 2 |
from functools import lru_cache
|
|
|
|
| 3 |
from services.document_service import DocumentService
|
| 4 |
from services.llm_service import LLMService
|
| 5 |
from core.rag_pipeline import VectorlessRAG
|
| 6 |
from core.storage import storage, MemoryStorage
|
| 7 |
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
@lru_cache()
|
| 10 |
def get_document_service() -> DocumentService:
|
| 11 |
return DocumentService(api_key=os.getenv("PAGEINDEX_API_KEY"))
|
|
|
|
| 1 |
import os
|
| 2 |
+
import uuid
|
| 3 |
+
from typing import Optional
|
| 4 |
from functools import lru_cache
|
| 5 |
+
from fastapi import Header, HTTPException, status
|
| 6 |
from services.document_service import DocumentService
|
| 7 |
from services.llm_service import LLMService
|
| 8 |
from core.rag_pipeline import VectorlessRAG
|
| 9 |
from core.storage import storage, MemoryStorage
|
| 10 |
|
| 11 |
|
| 12 |
+
def get_client_id(x_client_id: Optional[str] = Header(None, alias="X-Client-ID")) -> str:
|
| 13 |
+
"""FastAPI dependency validating the X-Client-ID header as a valid UUIDv4."""
|
| 14 |
+
if not x_client_id:
|
| 15 |
+
raise HTTPException(
|
| 16 |
+
status_code=status.HTTP_400_BAD_REQUEST,
|
| 17 |
+
detail="Missing required header: X-Client-ID",
|
| 18 |
+
)
|
| 19 |
+
try:
|
| 20 |
+
val = uuid.UUID(x_client_id.strip(), version=4)
|
| 21 |
+
return str(val)
|
| 22 |
+
except (ValueError, AttributeError):
|
| 23 |
+
raise HTTPException(
|
| 24 |
+
status_code=status.HTTP_400_BAD_REQUEST,
|
| 25 |
+
detail="Invalid X-Client-ID format: must be a valid UUIDv4 string",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
@lru_cache()
|
| 30 |
def get_document_service() -> DocumentService:
|
| 31 |
return DocumentService(api_key=os.getenv("PAGEINDEX_API_KEY"))
|
app/core/persistent_db.py
CHANGED
|
@@ -37,6 +37,7 @@ CREATE TABLE IF NOT EXISTS chats (
|
|
| 37 |
title TEXT,
|
| 38 |
doc_id TEXT,
|
| 39 |
doc_name TEXT,
|
|
|
|
| 40 |
created_at REAL NOT NULL,
|
| 41 |
updated_at REAL NOT NULL
|
| 42 |
);
|
|
@@ -64,6 +65,7 @@ CREATE TABLE IF NOT EXISTS doc_meta (
|
|
| 64 |
error TEXT,
|
| 65 |
content_hash TEXT,
|
| 66 |
file_path TEXT,
|
|
|
|
| 67 |
created_at REAL NOT NULL,
|
| 68 |
updated_at REAL NOT NULL
|
| 69 |
);
|
|
@@ -111,6 +113,7 @@ CREATE TABLE IF NOT EXISTS knowledge_memory (
|
|
| 111 |
|
| 112 |
CREATE TABLE IF NOT EXISTS global_knowledge_summary (
|
| 113 |
id TEXT PRIMARY KEY,
|
|
|
|
| 114 |
collective_summary TEXT NOT NULL,
|
| 115 |
total_documents INTEGER DEFAULT 0,
|
| 116 |
key_insights TEXT,
|
|
@@ -158,9 +161,23 @@ class SQLitePersistentDB:
|
|
| 158 |
("indexing_method", "TEXT", "NULL"),
|
| 159 |
("duration", "REAL", "NULL"),
|
| 160 |
("error", "TEXT", "NULL"),
|
|
|
|
| 161 |
]:
|
| 162 |
if col not in existing_cols:
|
| 163 |
conn.execute(f"ALTER TABLE doc_meta ADD COLUMN {col} {col_type} DEFAULT {default_val}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
|
| 165 |
# Migrate existing chats.doc_id into chat_documents table
|
| 166 |
conn.execute("""
|
|
@@ -170,26 +187,36 @@ class SQLitePersistentDB:
|
|
| 170 |
""")
|
| 171 |
|
| 172 |
# ── chat CRUD ───────────────────────────────────────────────────────────
|
| 173 |
-
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None) -> str:
|
| 174 |
cid = chat_id or str(uuid.uuid4())
|
| 175 |
now = time.time()
|
| 176 |
with self._tx() as conn:
|
| 177 |
conn.execute(
|
| 178 |
-
"INSERT OR IGNORE INTO chats(chat_id, title, created_at, updated_at) VALUES(?,?,?,?)",
|
| 179 |
-
(cid, title, now, now),
|
| 180 |
)
|
| 181 |
return cid
|
| 182 |
|
| 183 |
-
def get_chat(self, chat_id: str) -> Optional[Dict[str, Any]]:
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
return dict(row) if row else None
|
| 188 |
|
| 189 |
-
def list_chats(self) -> List[Dict[str, Any]]:
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
return [dict(r) for r in rows]
|
| 194 |
|
| 195 |
def update_chat_title(self, chat_id: str, title: str) -> None:
|
|
@@ -213,17 +240,21 @@ class SQLitePersistentDB:
|
|
| 213 |
(time.time(), chat_id),
|
| 214 |
)
|
| 215 |
|
| 216 |
-
def delete_chat(self, chat_id: str) ->
|
| 217 |
with self._tx() as conn:
|
| 218 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
|
| 220 |
# ── message CRUD ────────────────────────────────────────────────────────
|
| 221 |
-
def add_message(self, chat_id: str, role: str, content: str) -> None:
|
| 222 |
now = time.time()
|
| 223 |
with self._tx() as conn:
|
| 224 |
conn.execute(
|
| 225 |
-
"INSERT OR IGNORE INTO chats(chat_id, title, created_at, updated_at) VALUES(?, 'New research', ?, ?)",
|
| 226 |
-
(chat_id, now, now),
|
| 227 |
)
|
| 228 |
conn.execute(
|
| 229 |
"INSERT INTO messages(chat_id, role, content, created_at) VALUES(?,?,?,?)",
|
|
@@ -263,6 +294,7 @@ class SQLitePersistentDB:
|
|
| 263 |
error: Optional[str] = None,
|
| 264 |
content_hash: Optional[str] = None,
|
| 265 |
file_path: Optional[str] = None,
|
|
|
|
| 266 |
) -> None:
|
| 267 |
now = time.time()
|
| 268 |
with self._tx() as conn:
|
|
@@ -271,9 +303,9 @@ class SQLitePersistentDB:
|
|
| 271 |
INSERT INTO doc_meta(
|
| 272 |
doc_id, filename, status, page_count, section_count,
|
| 273 |
progress, current_stage, indexing_method, duration,
|
| 274 |
-
error, content_hash, file_path, created_at, updated_at
|
| 275 |
)
|
| 276 |
-
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
| 277 |
ON CONFLICT(doc_id) DO UPDATE SET
|
| 278 |
filename=excluded.filename,
|
| 279 |
status=excluded.status,
|
|
@@ -286,36 +318,57 @@ class SQLitePersistentDB:
|
|
| 286 |
error=excluded.error,
|
| 287 |
content_hash=COALESCE(excluded.content_hash, doc_meta.content_hash),
|
| 288 |
file_path=COALESCE(excluded.file_path, doc_meta.file_path),
|
|
|
|
| 289 |
updated_at=excluded.updated_at
|
| 290 |
""",
|
| 291 |
(
|
| 292 |
doc_id, filename, status, page_count, section_count,
|
| 293 |
progress, current_stage, indexing_method, duration,
|
| 294 |
-
error, content_hash, file_path, now, now
|
| 295 |
),
|
| 296 |
)
|
| 297 |
|
| 298 |
-
def get_doc_meta(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
return dict(row) if row else None
|
| 303 |
|
| 304 |
-
def list_docs(self) -> List[Dict[str, Any]]:
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
return [dict(r) for r in rows]
|
| 309 |
|
| 310 |
-
def find_doc_by_hash(self, content_hash: str) -> Optional[Dict[str, Any]]:
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
return dict(row) if row else None
|
| 316 |
|
| 317 |
-
def delete_doc(self, doc_id: str) -> bool:
|
| 318 |
with self._tx() as conn:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
cur = conn.execute("DELETE FROM doc_meta WHERE doc_id=?", (doc_id,))
|
| 320 |
conn.execute("UPDATE chats SET doc_id=NULL, doc_name=NULL WHERE doc_id=?", (doc_id,))
|
| 321 |
conn.execute("DELETE FROM chat_documents WHERE doc_id=?", (doc_id,))
|
|
@@ -434,7 +487,11 @@ class SQLitePersistentDB:
|
|
| 434 |
(now, chat_id),
|
| 435 |
)
|
| 436 |
|
| 437 |
-
def get_chat_documents(self, chat_id: str) -> List[Dict[str, Any]]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
rows = self._conn().execute(
|
| 439 |
"""
|
| 440 |
SELECT cd.chat_id, cd.doc_id, cd.doc_name, cd.attached_at,
|
|
@@ -513,10 +570,20 @@ class SQLitePersistentDB:
|
|
| 513 |
),
|
| 514 |
)
|
| 515 |
|
| 516 |
-
def get_knowledge_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
if not row:
|
| 521 |
return None
|
| 522 |
res = dict(row)
|
|
@@ -528,10 +595,21 @@ class SQLitePersistentDB:
|
|
| 528 |
res[k] = []
|
| 529 |
return res
|
| 530 |
|
| 531 |
-
def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 535 |
result = []
|
| 536 |
for r in rows:
|
| 537 |
item = dict(r)
|
|
@@ -553,29 +631,51 @@ class SQLitePersistentDB:
|
|
| 553 |
collective_summary: str,
|
| 554 |
total_documents: int,
|
| 555 |
key_insights: Optional[List[str]] = None,
|
|
|
|
| 556 |
) -> None:
|
| 557 |
now = time.time()
|
| 558 |
-
gid = "global"
|
| 559 |
with self._tx() as conn:
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 564 |
)
|
| 565 |
-
VALUES(?,?,?,?,?)
|
| 566 |
-
ON CONFLICT(id) DO UPDATE SET
|
| 567 |
-
collective_summary=excluded.collective_summary,
|
| 568 |
-
total_documents=excluded.total_documents,
|
| 569 |
-
key_insights=excluded.key_insights,
|
| 570 |
-
updated_at=excluded.updated_at
|
| 571 |
-
""",
|
| 572 |
-
(gid, collective_summary, total_documents, json.dumps(key_insights or []), now),
|
| 573 |
-
)
|
| 574 |
|
| 575 |
-
def get_global_knowledge_summary(self) -> Optional[Dict[str, Any]]:
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
if not row:
|
| 580 |
return None
|
| 581 |
res = dict(row)
|
|
|
|
| 37 |
title TEXT,
|
| 38 |
doc_id TEXT,
|
| 39 |
doc_name TEXT,
|
| 40 |
+
owner_id TEXT,
|
| 41 |
created_at REAL NOT NULL,
|
| 42 |
updated_at REAL NOT NULL
|
| 43 |
);
|
|
|
|
| 65 |
error TEXT,
|
| 66 |
content_hash TEXT,
|
| 67 |
file_path TEXT,
|
| 68 |
+
owner_id TEXT,
|
| 69 |
created_at REAL NOT NULL,
|
| 70 |
updated_at REAL NOT NULL
|
| 71 |
);
|
|
|
|
| 113 |
|
| 114 |
CREATE TABLE IF NOT EXISTS global_knowledge_summary (
|
| 115 |
id TEXT PRIMARY KEY,
|
| 116 |
+
owner_id TEXT,
|
| 117 |
collective_summary TEXT NOT NULL,
|
| 118 |
total_documents INTEGER DEFAULT 0,
|
| 119 |
key_insights TEXT,
|
|
|
|
| 161 |
("indexing_method", "TEXT", "NULL"),
|
| 162 |
("duration", "REAL", "NULL"),
|
| 163 |
("error", "TEXT", "NULL"),
|
| 164 |
+
("owner_id", "TEXT", "NULL"),
|
| 165 |
]:
|
| 166 |
if col not in existing_cols:
|
| 167 |
conn.execute(f"ALTER TABLE doc_meta ADD COLUMN {col} {col_type} DEFAULT {default_val}")
|
| 168 |
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_sqlite_doc_meta_owner ON doc_meta(owner_id)")
|
| 169 |
+
|
| 170 |
+
cursor = conn.execute("PRAGMA table_info(chats)")
|
| 171 |
+
chat_cols = {row["name"] for row in cursor.fetchall()}
|
| 172 |
+
if "owner_id" not in chat_cols:
|
| 173 |
+
conn.execute("ALTER TABLE chats ADD COLUMN owner_id TEXT DEFAULT NULL")
|
| 174 |
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_sqlite_chats_owner ON chats(owner_id)")
|
| 175 |
+
|
| 176 |
+
cursor = conn.execute("PRAGMA table_info(global_knowledge_summary)")
|
| 177 |
+
gks_cols = {row["name"] for row in cursor.fetchall()}
|
| 178 |
+
if "owner_id" not in gks_cols:
|
| 179 |
+
conn.execute("ALTER TABLE global_knowledge_summary ADD COLUMN owner_id TEXT DEFAULT NULL")
|
| 180 |
+
conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_sqlite_gks_owner ON global_knowledge_summary(owner_id)")
|
| 181 |
|
| 182 |
# Migrate existing chats.doc_id into chat_documents table
|
| 183 |
conn.execute("""
|
|
|
|
| 187 |
""")
|
| 188 |
|
| 189 |
# ── chat CRUD ───────────────────────────────────────────────────────────
|
| 190 |
+
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None, owner_id: Optional[str] = None) -> str:
|
| 191 |
cid = chat_id or str(uuid.uuid4())
|
| 192 |
now = time.time()
|
| 193 |
with self._tx() as conn:
|
| 194 |
conn.execute(
|
| 195 |
+
"INSERT OR IGNORE INTO chats(chat_id, title, owner_id, created_at, updated_at) VALUES(?,?,?,?,?)",
|
| 196 |
+
(cid, title, owner_id, now, now),
|
| 197 |
)
|
| 198 |
return cid
|
| 199 |
|
| 200 |
+
def get_chat(self, chat_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 201 |
+
if owner_id:
|
| 202 |
+
row = self._conn().execute(
|
| 203 |
+
"SELECT * FROM chats WHERE chat_id=? AND owner_id=?", (chat_id, owner_id)
|
| 204 |
+
).fetchone()
|
| 205 |
+
else:
|
| 206 |
+
row = self._conn().execute(
|
| 207 |
+
"SELECT * FROM chats WHERE chat_id=?", (chat_id,)
|
| 208 |
+
).fetchone()
|
| 209 |
return dict(row) if row else None
|
| 210 |
|
| 211 |
+
def list_chats(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 212 |
+
if owner_id:
|
| 213 |
+
rows = self._conn().execute(
|
| 214 |
+
"SELECT * FROM chats WHERE owner_id=? ORDER BY updated_at DESC", (owner_id,)
|
| 215 |
+
).fetchall()
|
| 216 |
+
else:
|
| 217 |
+
rows = self._conn().execute(
|
| 218 |
+
"SELECT * FROM chats ORDER BY updated_at DESC"
|
| 219 |
+
).fetchall()
|
| 220 |
return [dict(r) for r in rows]
|
| 221 |
|
| 222 |
def update_chat_title(self, chat_id: str, title: str) -> None:
|
|
|
|
| 240 |
(time.time(), chat_id),
|
| 241 |
)
|
| 242 |
|
| 243 |
+
def delete_chat(self, chat_id: str, owner_id: Optional[str] = None) -> bool:
|
| 244 |
with self._tx() as conn:
|
| 245 |
+
if owner_id:
|
| 246 |
+
cur = conn.execute("DELETE FROM chats WHERE chat_id=? AND owner_id=?", (chat_id, owner_id))
|
| 247 |
+
else:
|
| 248 |
+
cur = conn.execute("DELETE FROM chats WHERE chat_id=?", (chat_id,))
|
| 249 |
+
return cur.rowcount > 0
|
| 250 |
|
| 251 |
# ── message CRUD ────────────────────────────────────────────────────────
|
| 252 |
+
def add_message(self, chat_id: str, role: str, content: str, owner_id: Optional[str] = None) -> None:
|
| 253 |
now = time.time()
|
| 254 |
with self._tx() as conn:
|
| 255 |
conn.execute(
|
| 256 |
+
"INSERT OR IGNORE INTO chats(chat_id, title, owner_id, created_at, updated_at) VALUES(?, 'New research', ?, ?, ?)",
|
| 257 |
+
(chat_id, owner_id, now, now),
|
| 258 |
)
|
| 259 |
conn.execute(
|
| 260 |
"INSERT INTO messages(chat_id, role, content, created_at) VALUES(?,?,?,?)",
|
|
|
|
| 294 |
error: Optional[str] = None,
|
| 295 |
content_hash: Optional[str] = None,
|
| 296 |
file_path: Optional[str] = None,
|
| 297 |
+
owner_id: Optional[str] = None,
|
| 298 |
) -> None:
|
| 299 |
now = time.time()
|
| 300 |
with self._tx() as conn:
|
|
|
|
| 303 |
INSERT INTO doc_meta(
|
| 304 |
doc_id, filename, status, page_count, section_count,
|
| 305 |
progress, current_stage, indexing_method, duration,
|
| 306 |
+
error, content_hash, file_path, owner_id, created_at, updated_at
|
| 307 |
)
|
| 308 |
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
| 309 |
ON CONFLICT(doc_id) DO UPDATE SET
|
| 310 |
filename=excluded.filename,
|
| 311 |
status=excluded.status,
|
|
|
|
| 318 |
error=excluded.error,
|
| 319 |
content_hash=COALESCE(excluded.content_hash, doc_meta.content_hash),
|
| 320 |
file_path=COALESCE(excluded.file_path, doc_meta.file_path),
|
| 321 |
+
owner_id=COALESCE(excluded.owner_id, doc_meta.owner_id),
|
| 322 |
updated_at=excluded.updated_at
|
| 323 |
""",
|
| 324 |
(
|
| 325 |
doc_id, filename, status, page_count, section_count,
|
| 326 |
progress, current_stage, indexing_method, duration,
|
| 327 |
+
error, content_hash, file_path, owner_id, now, now
|
| 328 |
),
|
| 329 |
)
|
| 330 |
|
| 331 |
+
def get_doc_meta(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 332 |
+
if owner_id:
|
| 333 |
+
row = self._conn().execute(
|
| 334 |
+
"SELECT * FROM doc_meta WHERE doc_id=? AND owner_id=?", (doc_id, owner_id)
|
| 335 |
+
).fetchone()
|
| 336 |
+
else:
|
| 337 |
+
row = self._conn().execute(
|
| 338 |
+
"SELECT * FROM doc_meta WHERE doc_id=?", (doc_id,)
|
| 339 |
+
).fetchone()
|
| 340 |
return dict(row) if row else None
|
| 341 |
|
| 342 |
+
def list_docs(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 343 |
+
if owner_id:
|
| 344 |
+
rows = self._conn().execute(
|
| 345 |
+
"SELECT * FROM doc_meta WHERE owner_id=? ORDER BY created_at DESC", (owner_id,)
|
| 346 |
+
).fetchall()
|
| 347 |
+
else:
|
| 348 |
+
rows = self._conn().execute(
|
| 349 |
+
"SELECT * FROM doc_meta ORDER BY created_at DESC"
|
| 350 |
+
).fetchall()
|
| 351 |
return [dict(r) for r in rows]
|
| 352 |
|
| 353 |
+
def find_doc_by_hash(self, content_hash: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 354 |
+
if owner_id:
|
| 355 |
+
row = self._conn().execute(
|
| 356 |
+
"SELECT * FROM doc_meta WHERE content_hash=? AND owner_id=? ORDER BY updated_at DESC LIMIT 1",
|
| 357 |
+
(content_hash, owner_id),
|
| 358 |
+
).fetchone()
|
| 359 |
+
else:
|
| 360 |
+
row = self._conn().execute(
|
| 361 |
+
"SELECT * FROM doc_meta WHERE content_hash=? ORDER BY updated_at DESC LIMIT 1",
|
| 362 |
+
(content_hash,),
|
| 363 |
+
).fetchone()
|
| 364 |
return dict(row) if row else None
|
| 365 |
|
| 366 |
+
def delete_doc(self, doc_id: str, owner_id: Optional[str] = None) -> bool:
|
| 367 |
with self._tx() as conn:
|
| 368 |
+
if owner_id:
|
| 369 |
+
doc = self.get_doc_meta(doc_id, owner_id=owner_id)
|
| 370 |
+
if not doc:
|
| 371 |
+
return False
|
| 372 |
cur = conn.execute("DELETE FROM doc_meta WHERE doc_id=?", (doc_id,))
|
| 373 |
conn.execute("UPDATE chats SET doc_id=NULL, doc_name=NULL WHERE doc_id=?", (doc_id,))
|
| 374 |
conn.execute("DELETE FROM chat_documents WHERE doc_id=?", (doc_id,))
|
|
|
|
| 487 |
(now, chat_id),
|
| 488 |
)
|
| 489 |
|
| 490 |
+
def get_chat_documents(self, chat_id: str, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 491 |
+
if owner_id:
|
| 492 |
+
chat = self.get_chat(chat_id, owner_id=owner_id)
|
| 493 |
+
if not chat:
|
| 494 |
+
return []
|
| 495 |
rows = self._conn().execute(
|
| 496 |
"""
|
| 497 |
SELECT cd.chat_id, cd.doc_id, cd.doc_name, cd.attached_at,
|
|
|
|
| 570 |
),
|
| 571 |
)
|
| 572 |
|
| 573 |
+
def get_knowledge_dossier(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 574 |
+
if owner_id:
|
| 575 |
+
row = self._conn().execute(
|
| 576 |
+
"""
|
| 577 |
+
SELECT km.* FROM knowledge_memory km
|
| 578 |
+
JOIN doc_meta dm ON km.doc_id = dm.doc_id
|
| 579 |
+
WHERE km.doc_id=? AND dm.owner_id=?
|
| 580 |
+
""",
|
| 581 |
+
(doc_id, owner_id),
|
| 582 |
+
).fetchone()
|
| 583 |
+
else:
|
| 584 |
+
row = self._conn().execute(
|
| 585 |
+
"SELECT * FROM knowledge_memory WHERE doc_id=?", (doc_id,)
|
| 586 |
+
).fetchone()
|
| 587 |
if not row:
|
| 588 |
return None
|
| 589 |
res = dict(row)
|
|
|
|
| 595 |
res[k] = []
|
| 596 |
return res
|
| 597 |
|
| 598 |
+
def list_knowledge_dossiers(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 599 |
+
if owner_id:
|
| 600 |
+
rows = self._conn().execute(
|
| 601 |
+
"""
|
| 602 |
+
SELECT km.* FROM knowledge_memory km
|
| 603 |
+
JOIN doc_meta dm ON km.doc_id = dm.doc_id
|
| 604 |
+
WHERE dm.owner_id=?
|
| 605 |
+
ORDER BY km.updated_at DESC
|
| 606 |
+
""",
|
| 607 |
+
(owner_id,),
|
| 608 |
+
).fetchall()
|
| 609 |
+
else:
|
| 610 |
+
rows = self._conn().execute(
|
| 611 |
+
"SELECT * FROM knowledge_memory ORDER BY updated_at DESC"
|
| 612 |
+
).fetchall()
|
| 613 |
result = []
|
| 614 |
for r in rows:
|
| 615 |
item = dict(r)
|
|
|
|
| 631 |
collective_summary: str,
|
| 632 |
total_documents: int,
|
| 633 |
key_insights: Optional[List[str]] = None,
|
| 634 |
+
owner_id: Optional[str] = None,
|
| 635 |
) -> None:
|
| 636 |
now = time.time()
|
| 637 |
+
gid = owner_id or "global"
|
| 638 |
with self._tx() as conn:
|
| 639 |
+
if owner_id:
|
| 640 |
+
conn.execute(
|
| 641 |
+
"""
|
| 642 |
+
INSERT INTO global_knowledge_summary(
|
| 643 |
+
id, owner_id, collective_summary, total_documents, key_insights, updated_at
|
| 644 |
+
)
|
| 645 |
+
VALUES(?,?,?,?,?,?)
|
| 646 |
+
ON CONFLICT(owner_id) DO UPDATE SET
|
| 647 |
+
collective_summary=excluded.collective_summary,
|
| 648 |
+
total_documents=excluded.total_documents,
|
| 649 |
+
key_insights=excluded.key_insights,
|
| 650 |
+
updated_at=excluded.updated_at
|
| 651 |
+
""",
|
| 652 |
+
(gid, owner_id, collective_summary, total_documents, json.dumps(key_insights or []), now),
|
| 653 |
+
)
|
| 654 |
+
else:
|
| 655 |
+
conn.execute(
|
| 656 |
+
"""
|
| 657 |
+
INSERT INTO global_knowledge_summary(
|
| 658 |
+
id, collective_summary, total_documents, key_insights, updated_at
|
| 659 |
+
)
|
| 660 |
+
VALUES(?,?,?,?,?)
|
| 661 |
+
ON CONFLICT(id) DO UPDATE SET
|
| 662 |
+
collective_summary=excluded.collective_summary,
|
| 663 |
+
total_documents=excluded.total_documents,
|
| 664 |
+
key_insights=excluded.key_insights,
|
| 665 |
+
updated_at=excluded.updated_at
|
| 666 |
+
""",
|
| 667 |
+
(gid, collective_summary, total_documents, json.dumps(key_insights or []), now),
|
| 668 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 669 |
|
| 670 |
+
def get_global_knowledge_summary(self, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 671 |
+
if owner_id:
|
| 672 |
+
row = self._conn().execute(
|
| 673 |
+
"SELECT * FROM global_knowledge_summary WHERE owner_id=?", (owner_id,)
|
| 674 |
+
).fetchone()
|
| 675 |
+
else:
|
| 676 |
+
row = self._conn().execute(
|
| 677 |
+
"SELECT * FROM global_knowledge_summary WHERE id='global'"
|
| 678 |
+
).fetchone()
|
| 679 |
if not row:
|
| 680 |
return None
|
| 681 |
res = dict(row)
|
app/core/postgres_db.py
CHANGED
|
@@ -27,10 +27,13 @@ CREATE TABLE IF NOT EXISTS chats (
|
|
| 27 |
title TEXT,
|
| 28 |
doc_id TEXT,
|
| 29 |
doc_name TEXT,
|
|
|
|
| 30 |
created_at DOUBLE PRECISION NOT NULL,
|
| 31 |
updated_at DOUBLE PRECISION NOT NULL
|
| 32 |
);
|
| 33 |
|
|
|
|
|
|
|
| 34 |
CREATE TABLE IF NOT EXISTS messages (
|
| 35 |
id BIGSERIAL PRIMARY KEY,
|
| 36 |
chat_id TEXT NOT NULL REFERENCES chats(chat_id) ON DELETE CASCADE,
|
|
@@ -54,11 +57,13 @@ CREATE TABLE IF NOT EXISTS doc_meta (
|
|
| 54 |
error TEXT,
|
| 55 |
content_hash TEXT,
|
| 56 |
file_path TEXT,
|
|
|
|
| 57 |
created_at DOUBLE PRECISION NOT NULL,
|
| 58 |
updated_at DOUBLE PRECISION NOT NULL
|
| 59 |
);
|
| 60 |
|
| 61 |
CREATE INDEX IF NOT EXISTS idx_pg_doc_meta_hash ON doc_meta(content_hash);
|
|
|
|
| 62 |
|
| 63 |
CREATE TABLE IF NOT EXISTS doc_indices (
|
| 64 |
doc_id TEXT PRIMARY KEY REFERENCES doc_meta(doc_id) ON DELETE CASCADE,
|
|
@@ -102,11 +107,14 @@ CREATE TABLE IF NOT EXISTS knowledge_memory (
|
|
| 102 |
|
| 103 |
CREATE TABLE IF NOT EXISTS global_knowledge_summary (
|
| 104 |
id TEXT PRIMARY KEY,
|
|
|
|
| 105 |
collective_summary TEXT NOT NULL,
|
| 106 |
total_documents INTEGER DEFAULT 0,
|
| 107 |
key_insights JSONB,
|
| 108 |
updated_at DOUBLE PRECISION NOT NULL
|
| 109 |
);
|
|
|
|
|
|
|
| 110 |
"""
|
| 111 |
|
| 112 |
|
|
@@ -172,35 +180,59 @@ class PostgresDB:
|
|
| 172 |
with self._conn() as conn:
|
| 173 |
with conn.cursor() as cur:
|
| 174 |
cur.execute(_PG_DDL)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
app_logger.info("[PostgresDB] Schema initialized / verified successfully.")
|
| 176 |
|
| 177 |
# ── chat CRUD ───────────────────────────────────────────────────────────
|
| 178 |
-
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None) -> str:
|
| 179 |
cid = chat_id or str(uuid.uuid4())
|
| 180 |
now = time.time()
|
| 181 |
with self._conn() as conn:
|
| 182 |
with conn.cursor() as cur:
|
| 183 |
cur.execute(
|
| 184 |
"""
|
| 185 |
-
INSERT INTO chats(chat_id, title, created_at, updated_at)
|
| 186 |
-
VALUES (%s, %s, %s, %s)
|
| 187 |
ON CONFLICT (chat_id) DO NOTHING
|
| 188 |
""",
|
| 189 |
-
(cid, title, now, now),
|
| 190 |
)
|
| 191 |
return cid
|
| 192 |
|
| 193 |
-
def get_chat(self, chat_id: str) -> Optional[Dict[str, Any]]:
|
| 194 |
with self._conn() as conn:
|
| 195 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
| 197 |
row = cur.fetchone()
|
| 198 |
return dict(row) if row else None
|
| 199 |
|
| 200 |
-
def list_chats(self) -> List[Dict[str, Any]]:
|
| 201 |
with self._conn() as conn:
|
| 202 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 203 |
-
|
|
|
|
|
|
|
|
|
|
| 204 |
rows = cur.fetchall()
|
| 205 |
return [dict(r) for r in rows]
|
| 206 |
|
|
@@ -228,23 +260,27 @@ class PostgresDB:
|
|
| 228 |
(time.time(), chat_id),
|
| 229 |
)
|
| 230 |
|
| 231 |
-
def delete_chat(self, chat_id: str) ->
|
| 232 |
with self._conn() as conn:
|
| 233 |
with conn.cursor() as cur:
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
|
| 236 |
# ── message CRUD ────────────────────────────────────────────────────────
|
| 237 |
-
def add_message(self, chat_id: str, role: str, content: str) -> None:
|
| 238 |
now = time.time()
|
| 239 |
with self._conn() as conn:
|
| 240 |
with conn.cursor() as cur:
|
| 241 |
cur.execute(
|
| 242 |
"""
|
| 243 |
-
INSERT INTO chats(chat_id, title, created_at, updated_at)
|
| 244 |
-
VALUES (%s, 'New research', %s, %s)
|
| 245 |
ON CONFLICT (chat_id) DO NOTHING
|
| 246 |
""",
|
| 247 |
-
(chat_id, now, now),
|
| 248 |
)
|
| 249 |
cur.execute(
|
| 250 |
"INSERT INTO messages(chat_id, role, content, created_at) VALUES (%s, %s, %s, %s)",
|
|
@@ -292,6 +328,7 @@ class PostgresDB:
|
|
| 292 |
error: Optional[str] = None,
|
| 293 |
content_hash: Optional[str] = None,
|
| 294 |
file_path: Optional[str] = None,
|
|
|
|
| 295 |
) -> None:
|
| 296 |
now = time.time()
|
| 297 |
with self._conn() as conn:
|
|
@@ -301,9 +338,9 @@ class PostgresDB:
|
|
| 301 |
INSERT INTO doc_meta(
|
| 302 |
doc_id, filename, status, page_count, section_count,
|
| 303 |
progress, current_stage, indexing_method, duration,
|
| 304 |
-
error, content_hash, file_path, created_at, updated_at
|
| 305 |
)
|
| 306 |
-
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
| 307 |
ON CONFLICT (doc_id) DO UPDATE SET
|
| 308 |
filename = EXCLUDED.filename,
|
| 309 |
status = EXCLUDED.status,
|
|
@@ -316,6 +353,7 @@ class PostgresDB:
|
|
| 316 |
error = EXCLUDED.error,
|
| 317 |
content_hash = COALESCE(EXCLUDED.content_hash, doc_meta.content_hash),
|
| 318 |
file_path = COALESCE(EXCLUDED.file_path, doc_meta.file_path),
|
|
|
|
| 319 |
updated_at = EXCLUDED.updated_at
|
| 320 |
""",
|
| 321 |
(
|
|
@@ -331,38 +369,55 @@ class PostgresDB:
|
|
| 331 |
error,
|
| 332 |
content_hash,
|
| 333 |
file_path,
|
|
|
|
| 334 |
now,
|
| 335 |
now,
|
| 336 |
),
|
| 337 |
)
|
| 338 |
|
| 339 |
-
def get_doc_meta(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 340 |
with self._conn() as conn:
|
| 341 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 342 |
-
|
|
|
|
|
|
|
|
|
|
| 343 |
row = cur.fetchone()
|
| 344 |
return dict(row) if row else None
|
| 345 |
|
| 346 |
-
def list_docs(self) -> List[Dict[str, Any]]:
|
| 347 |
with self._conn() as conn:
|
| 348 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
| 350 |
rows = cur.fetchall()
|
| 351 |
return [dict(r) for r in rows]
|
| 352 |
|
| 353 |
-
def find_doc_by_hash(self, content_hash: str) -> Optional[Dict[str, Any]]:
|
| 354 |
with self._conn() as conn:
|
| 355 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
row = cur.fetchone()
|
| 361 |
return dict(row) if row else None
|
| 362 |
|
| 363 |
-
def delete_doc(self, doc_id: str) -> bool:
|
| 364 |
with self._conn() as conn:
|
| 365 |
with conn.cursor() as cur:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
cur.execute("DELETE FROM doc_meta WHERE doc_id = %s", (doc_id,))
|
| 367 |
rc = cur.rowcount
|
| 368 |
cur.execute("UPDATE chats SET doc_id = NULL, doc_name = NULL WHERE doc_id = %s", (doc_id,))
|
|
@@ -474,9 +529,13 @@ class PostgresDB:
|
|
| 474 |
(now, chat_id),
|
| 475 |
)
|
| 476 |
|
| 477 |
-
def get_chat_documents(self, chat_id: str) -> List[Dict[str, Any]]:
|
| 478 |
with self._conn() as conn:
|
| 479 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
cur.execute(
|
| 481 |
"""
|
| 482 |
SELECT cd.chat_id, cd.doc_id, cd.doc_name, cd.attached_at,
|
|
@@ -563,10 +622,20 @@ class PostgresDB:
|
|
| 563 |
),
|
| 564 |
)
|
| 565 |
|
| 566 |
-
def get_knowledge_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 567 |
with self._conn() as conn:
|
| 568 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 569 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 570 |
row = cur.fetchone()
|
| 571 |
if not row:
|
| 572 |
return None
|
|
@@ -576,10 +645,21 @@ class PostgresDB:
|
|
| 576 |
res[k] = []
|
| 577 |
return res
|
| 578 |
|
| 579 |
-
def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
|
| 580 |
with self._conn() as conn:
|
| 581 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 582 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 583 |
rows = cur.fetchall()
|
| 584 |
result = []
|
| 585 |
for r in rows:
|
|
@@ -600,36 +680,63 @@ class PostgresDB:
|
|
| 600 |
collective_summary: str,
|
| 601 |
total_documents: int,
|
| 602 |
key_insights: Optional[List[str]] = None,
|
|
|
|
| 603 |
) -> None:
|
| 604 |
now = time.time()
|
| 605 |
-
gid = "global"
|
| 606 |
with self._conn() as conn:
|
| 607 |
with conn.cursor() as cur:
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 612 |
)
|
| 613 |
-
VALUES (%s, %s, %s, %s, %s)
|
| 614 |
-
ON CONFLICT (id) DO UPDATE SET
|
| 615 |
-
collective_summary = EXCLUDED.collective_summary,
|
| 616 |
-
total_documents = EXCLUDED.total_documents,
|
| 617 |
-
key_insights = EXCLUDED.key_insights,
|
| 618 |
-
updated_at = EXCLUDED.updated_at
|
| 619 |
-
""",
|
| 620 |
-
(
|
| 621 |
-
gid,
|
| 622 |
-
collective_summary,
|
| 623 |
-
total_documents,
|
| 624 |
-
psycopg2.extras.Json(key_insights or []),
|
| 625 |
-
now,
|
| 626 |
-
),
|
| 627 |
-
)
|
| 628 |
|
| 629 |
-
def get_global_knowledge_summary(self) -> Optional[Dict[str, Any]]:
|
| 630 |
with self._conn() as conn:
|
| 631 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 632 |
-
|
|
|
|
|
|
|
|
|
|
| 633 |
row = cur.fetchone()
|
| 634 |
if not row:
|
| 635 |
return None
|
|
|
|
| 27 |
title TEXT,
|
| 28 |
doc_id TEXT,
|
| 29 |
doc_name TEXT,
|
| 30 |
+
owner_id TEXT,
|
| 31 |
created_at DOUBLE PRECISION NOT NULL,
|
| 32 |
updated_at DOUBLE PRECISION NOT NULL
|
| 33 |
);
|
| 34 |
|
| 35 |
+
CREATE INDEX IF NOT EXISTS idx_pg_chats_owner ON chats(owner_id);
|
| 36 |
+
|
| 37 |
CREATE TABLE IF NOT EXISTS messages (
|
| 38 |
id BIGSERIAL PRIMARY KEY,
|
| 39 |
chat_id TEXT NOT NULL REFERENCES chats(chat_id) ON DELETE CASCADE,
|
|
|
|
| 57 |
error TEXT,
|
| 58 |
content_hash TEXT,
|
| 59 |
file_path TEXT,
|
| 60 |
+
owner_id TEXT,
|
| 61 |
created_at DOUBLE PRECISION NOT NULL,
|
| 62 |
updated_at DOUBLE PRECISION NOT NULL
|
| 63 |
);
|
| 64 |
|
| 65 |
CREATE INDEX IF NOT EXISTS idx_pg_doc_meta_hash ON doc_meta(content_hash);
|
| 66 |
+
CREATE INDEX IF NOT EXISTS idx_pg_doc_meta_owner ON doc_meta(owner_id);
|
| 67 |
|
| 68 |
CREATE TABLE IF NOT EXISTS doc_indices (
|
| 69 |
doc_id TEXT PRIMARY KEY REFERENCES doc_meta(doc_id) ON DELETE CASCADE,
|
|
|
|
| 107 |
|
| 108 |
CREATE TABLE IF NOT EXISTS global_knowledge_summary (
|
| 109 |
id TEXT PRIMARY KEY,
|
| 110 |
+
owner_id TEXT,
|
| 111 |
collective_summary TEXT NOT NULL,
|
| 112 |
total_documents INTEGER DEFAULT 0,
|
| 113 |
key_insights JSONB,
|
| 114 |
updated_at DOUBLE PRECISION NOT NULL
|
| 115 |
);
|
| 116 |
+
|
| 117 |
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_pg_gks_owner ON global_knowledge_summary(owner_id);
|
| 118 |
"""
|
| 119 |
|
| 120 |
|
|
|
|
| 180 |
with self._conn() as conn:
|
| 181 |
with conn.cursor() as cur:
|
| 182 |
cur.execute(_PG_DDL)
|
| 183 |
+
for tbl, idx in [
|
| 184 |
+
("chats", "idx_pg_chats_owner"),
|
| 185 |
+
("doc_meta", "idx_pg_doc_meta_owner"),
|
| 186 |
+
("global_knowledge_summary", "idx_pg_gks_owner"),
|
| 187 |
+
]:
|
| 188 |
+
cur.execute(
|
| 189 |
+
"""
|
| 190 |
+
SELECT column_name FROM information_schema.columns
|
| 191 |
+
WHERE table_name = %s AND column_name = 'owner_id';
|
| 192 |
+
""",
|
| 193 |
+
(tbl,),
|
| 194 |
+
)
|
| 195 |
+
if not cur.fetchone():
|
| 196 |
+
cur.execute(f"ALTER TABLE {tbl} ADD COLUMN owner_id TEXT;")
|
| 197 |
+
if tbl == "global_knowledge_summary":
|
| 198 |
+
cur.execute(f"CREATE UNIQUE INDEX IF NOT EXISTS {idx} ON {tbl}(owner_id);")
|
| 199 |
+
else:
|
| 200 |
+
cur.execute(f"CREATE INDEX IF NOT EXISTS {idx} ON {tbl}(owner_id);")
|
| 201 |
app_logger.info("[PostgresDB] Schema initialized / verified successfully.")
|
| 202 |
|
| 203 |
# ── chat CRUD ───────────────────────────────────────────────────────────
|
| 204 |
+
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None, owner_id: Optional[str] = None) -> str:
|
| 205 |
cid = chat_id or str(uuid.uuid4())
|
| 206 |
now = time.time()
|
| 207 |
with self._conn() as conn:
|
| 208 |
with conn.cursor() as cur:
|
| 209 |
cur.execute(
|
| 210 |
"""
|
| 211 |
+
INSERT INTO chats(chat_id, title, owner_id, created_at, updated_at)
|
| 212 |
+
VALUES (%s, %s, %s, %s, %s)
|
| 213 |
ON CONFLICT (chat_id) DO NOTHING
|
| 214 |
""",
|
| 215 |
+
(cid, title, owner_id, now, now),
|
| 216 |
)
|
| 217 |
return cid
|
| 218 |
|
| 219 |
+
def get_chat(self, chat_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 220 |
with self._conn() as conn:
|
| 221 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 222 |
+
if owner_id:
|
| 223 |
+
cur.execute("SELECT * FROM chats WHERE chat_id = %s AND owner_id = %s", (chat_id, owner_id))
|
| 224 |
+
else:
|
| 225 |
+
cur.execute("SELECT * FROM chats WHERE chat_id = %s", (chat_id,))
|
| 226 |
row = cur.fetchone()
|
| 227 |
return dict(row) if row else None
|
| 228 |
|
| 229 |
+
def list_chats(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 230 |
with self._conn() as conn:
|
| 231 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 232 |
+
if owner_id:
|
| 233 |
+
cur.execute("SELECT * FROM chats WHERE owner_id = %s ORDER BY updated_at DESC", (owner_id,))
|
| 234 |
+
else:
|
| 235 |
+
cur.execute("SELECT * FROM chats ORDER BY updated_at DESC")
|
| 236 |
rows = cur.fetchall()
|
| 237 |
return [dict(r) for r in rows]
|
| 238 |
|
|
|
|
| 260 |
(time.time(), chat_id),
|
| 261 |
)
|
| 262 |
|
| 263 |
+
def delete_chat(self, chat_id: str, owner_id: Optional[str] = None) -> bool:
|
| 264 |
with self._conn() as conn:
|
| 265 |
with conn.cursor() as cur:
|
| 266 |
+
if owner_id:
|
| 267 |
+
cur.execute("DELETE FROM chats WHERE chat_id = %s AND owner_id = %s", (chat_id, owner_id))
|
| 268 |
+
else:
|
| 269 |
+
cur.execute("DELETE FROM chats WHERE chat_id = %s", (chat_id,))
|
| 270 |
+
return cur.rowcount > 0
|
| 271 |
|
| 272 |
# ── message CRUD ────────────────────────────────────────────────────────
|
| 273 |
+
def add_message(self, chat_id: str, role: str, content: str, owner_id: Optional[str] = None) -> None:
|
| 274 |
now = time.time()
|
| 275 |
with self._conn() as conn:
|
| 276 |
with conn.cursor() as cur:
|
| 277 |
cur.execute(
|
| 278 |
"""
|
| 279 |
+
INSERT INTO chats(chat_id, title, owner_id, created_at, updated_at)
|
| 280 |
+
VALUES (%s, 'New research', %s, %s, %s)
|
| 281 |
ON CONFLICT (chat_id) DO NOTHING
|
| 282 |
""",
|
| 283 |
+
(chat_id, owner_id, now, now),
|
| 284 |
)
|
| 285 |
cur.execute(
|
| 286 |
"INSERT INTO messages(chat_id, role, content, created_at) VALUES (%s, %s, %s, %s)",
|
|
|
|
| 328 |
error: Optional[str] = None,
|
| 329 |
content_hash: Optional[str] = None,
|
| 330 |
file_path: Optional[str] = None,
|
| 331 |
+
owner_id: Optional[str] = None,
|
| 332 |
) -> None:
|
| 333 |
now = time.time()
|
| 334 |
with self._conn() as conn:
|
|
|
|
| 338 |
INSERT INTO doc_meta(
|
| 339 |
doc_id, filename, status, page_count, section_count,
|
| 340 |
progress, current_stage, indexing_method, duration,
|
| 341 |
+
error, content_hash, file_path, owner_id, created_at, updated_at
|
| 342 |
)
|
| 343 |
+
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
| 344 |
ON CONFLICT (doc_id) DO UPDATE SET
|
| 345 |
filename = EXCLUDED.filename,
|
| 346 |
status = EXCLUDED.status,
|
|
|
|
| 353 |
error = EXCLUDED.error,
|
| 354 |
content_hash = COALESCE(EXCLUDED.content_hash, doc_meta.content_hash),
|
| 355 |
file_path = COALESCE(EXCLUDED.file_path, doc_meta.file_path),
|
| 356 |
+
owner_id = COALESCE(EXCLUDED.owner_id, doc_meta.owner_id),
|
| 357 |
updated_at = EXCLUDED.updated_at
|
| 358 |
""",
|
| 359 |
(
|
|
|
|
| 369 |
error,
|
| 370 |
content_hash,
|
| 371 |
file_path,
|
| 372 |
+
owner_id,
|
| 373 |
now,
|
| 374 |
now,
|
| 375 |
),
|
| 376 |
)
|
| 377 |
|
| 378 |
+
def get_doc_meta(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 379 |
with self._conn() as conn:
|
| 380 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 381 |
+
if owner_id:
|
| 382 |
+
cur.execute("SELECT * FROM doc_meta WHERE doc_id = %s AND owner_id = %s", (doc_id, owner_id))
|
| 383 |
+
else:
|
| 384 |
+
cur.execute("SELECT * FROM doc_meta WHERE doc_id = %s", (doc_id,))
|
| 385 |
row = cur.fetchone()
|
| 386 |
return dict(row) if row else None
|
| 387 |
|
| 388 |
+
def list_docs(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 389 |
with self._conn() as conn:
|
| 390 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 391 |
+
if owner_id:
|
| 392 |
+
cur.execute("SELECT * FROM doc_meta WHERE owner_id = %s ORDER BY created_at DESC", (owner_id,))
|
| 393 |
+
else:
|
| 394 |
+
cur.execute("SELECT * FROM doc_meta ORDER BY created_at DESC")
|
| 395 |
rows = cur.fetchall()
|
| 396 |
return [dict(r) for r in rows]
|
| 397 |
|
| 398 |
+
def find_doc_by_hash(self, content_hash: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 399 |
with self._conn() as conn:
|
| 400 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 401 |
+
if owner_id:
|
| 402 |
+
cur.execute(
|
| 403 |
+
"SELECT * FROM doc_meta WHERE content_hash = %s AND owner_id = %s ORDER BY updated_at DESC LIMIT 1",
|
| 404 |
+
(content_hash, owner_id),
|
| 405 |
+
)
|
| 406 |
+
else:
|
| 407 |
+
cur.execute(
|
| 408 |
+
"SELECT * FROM doc_meta WHERE content_hash = %s ORDER BY updated_at DESC LIMIT 1",
|
| 409 |
+
(content_hash,),
|
| 410 |
+
)
|
| 411 |
row = cur.fetchone()
|
| 412 |
return dict(row) if row else None
|
| 413 |
|
| 414 |
+
def delete_doc(self, doc_id: str, owner_id: Optional[str] = None) -> bool:
|
| 415 |
with self._conn() as conn:
|
| 416 |
with conn.cursor() as cur:
|
| 417 |
+
if owner_id:
|
| 418 |
+
cur.execute("SELECT doc_id FROM doc_meta WHERE doc_id = %s AND owner_id = %s", (doc_id, owner_id))
|
| 419 |
+
if not cur.fetchone():
|
| 420 |
+
return False
|
| 421 |
cur.execute("DELETE FROM doc_meta WHERE doc_id = %s", (doc_id,))
|
| 422 |
rc = cur.rowcount
|
| 423 |
cur.execute("UPDATE chats SET doc_id = NULL, doc_name = NULL WHERE doc_id = %s", (doc_id,))
|
|
|
|
| 529 |
(now, chat_id),
|
| 530 |
)
|
| 531 |
|
| 532 |
+
def get_chat_documents(self, chat_id: str, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 533 |
with self._conn() as conn:
|
| 534 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 535 |
+
if owner_id:
|
| 536 |
+
cur.execute("SELECT chat_id FROM chats WHERE chat_id = %s AND owner_id = %s", (chat_id, owner_id))
|
| 537 |
+
if not cur.fetchone():
|
| 538 |
+
return []
|
| 539 |
cur.execute(
|
| 540 |
"""
|
| 541 |
SELECT cd.chat_id, cd.doc_id, cd.doc_name, cd.attached_at,
|
|
|
|
| 622 |
),
|
| 623 |
)
|
| 624 |
|
| 625 |
+
def get_knowledge_dossier(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 626 |
with self._conn() as conn:
|
| 627 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 628 |
+
if owner_id:
|
| 629 |
+
cur.execute(
|
| 630 |
+
"""
|
| 631 |
+
SELECT km.* FROM knowledge_memory km
|
| 632 |
+
JOIN doc_meta dm ON km.doc_id = dm.doc_id
|
| 633 |
+
WHERE km.doc_id = %s AND dm.owner_id = %s
|
| 634 |
+
""",
|
| 635 |
+
(doc_id, owner_id),
|
| 636 |
+
)
|
| 637 |
+
else:
|
| 638 |
+
cur.execute("SELECT * FROM knowledge_memory WHERE doc_id = %s", (doc_id,))
|
| 639 |
row = cur.fetchone()
|
| 640 |
if not row:
|
| 641 |
return None
|
|
|
|
| 645 |
res[k] = []
|
| 646 |
return res
|
| 647 |
|
| 648 |
+
def list_knowledge_dossiers(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 649 |
with self._conn() as conn:
|
| 650 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 651 |
+
if owner_id:
|
| 652 |
+
cur.execute(
|
| 653 |
+
"""
|
| 654 |
+
SELECT km.* FROM knowledge_memory km
|
| 655 |
+
JOIN doc_meta dm ON km.doc_id = dm.doc_id
|
| 656 |
+
WHERE dm.owner_id = %s
|
| 657 |
+
ORDER BY km.updated_at DESC
|
| 658 |
+
""",
|
| 659 |
+
(owner_id,),
|
| 660 |
+
)
|
| 661 |
+
else:
|
| 662 |
+
cur.execute("SELECT * FROM knowledge_memory ORDER BY updated_at DESC")
|
| 663 |
rows = cur.fetchall()
|
| 664 |
result = []
|
| 665 |
for r in rows:
|
|
|
|
| 680 |
collective_summary: str,
|
| 681 |
total_documents: int,
|
| 682 |
key_insights: Optional[List[str]] = None,
|
| 683 |
+
owner_id: Optional[str] = None,
|
| 684 |
) -> None:
|
| 685 |
now = time.time()
|
| 686 |
+
gid = owner_id or "global"
|
| 687 |
with self._conn() as conn:
|
| 688 |
with conn.cursor() as cur:
|
| 689 |
+
if owner_id:
|
| 690 |
+
cur.execute(
|
| 691 |
+
"""
|
| 692 |
+
INSERT INTO global_knowledge_summary(
|
| 693 |
+
id, owner_id, collective_summary, total_documents, key_insights, updated_at
|
| 694 |
+
)
|
| 695 |
+
VALUES (%s, %s, %s, %s, %s, %s)
|
| 696 |
+
ON CONFLICT (owner_id) DO UPDATE SET
|
| 697 |
+
collective_summary = EXCLUDED.collective_summary,
|
| 698 |
+
total_documents = EXCLUDED.total_documents,
|
| 699 |
+
key_insights = EXCLUDED.key_insights,
|
| 700 |
+
updated_at = EXCLUDED.updated_at
|
| 701 |
+
""",
|
| 702 |
+
(
|
| 703 |
+
gid,
|
| 704 |
+
owner_id,
|
| 705 |
+
collective_summary,
|
| 706 |
+
total_documents,
|
| 707 |
+
psycopg2.extras.Json(key_insights or []),
|
| 708 |
+
now,
|
| 709 |
+
),
|
| 710 |
+
)
|
| 711 |
+
else:
|
| 712 |
+
cur.execute(
|
| 713 |
+
"""
|
| 714 |
+
INSERT INTO global_knowledge_summary(
|
| 715 |
+
id, collective_summary, total_documents, key_insights, updated_at
|
| 716 |
+
)
|
| 717 |
+
VALUES (%s, %s, %s, %s, %s)
|
| 718 |
+
ON CONFLICT (id) DO UPDATE SET
|
| 719 |
+
collective_summary = EXCLUDED.collective_summary,
|
| 720 |
+
total_documents = EXCLUDED.total_documents,
|
| 721 |
+
key_insights = EXCLUDED.key_insights,
|
| 722 |
+
updated_at = EXCLUDED.updated_at
|
| 723 |
+
""",
|
| 724 |
+
(
|
| 725 |
+
gid,
|
| 726 |
+
collective_summary,
|
| 727 |
+
total_documents,
|
| 728 |
+
psycopg2.extras.Json(key_insights or []),
|
| 729 |
+
now,
|
| 730 |
+
),
|
| 731 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
|
| 733 |
+
def get_global_knowledge_summary(self, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 734 |
with self._conn() as conn:
|
| 735 |
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
| 736 |
+
if owner_id:
|
| 737 |
+
cur.execute("SELECT * FROM global_knowledge_summary WHERE owner_id = %s", (owner_id,))
|
| 738 |
+
else:
|
| 739 |
+
cur.execute("SELECT * FROM global_knowledge_summary WHERE id = 'global'")
|
| 740 |
row = cur.fetchone()
|
| 741 |
if not row:
|
| 742 |
return None
|
app/core/storage.py
CHANGED
|
@@ -48,18 +48,27 @@ class MemoryStorage:
|
|
| 48 |
# ──────────────────────────────────────────────────────────────────────
|
| 49 |
# Document hash / cache helpers
|
| 50 |
# ──────────────────────────────────────────────────────────────────────
|
| 51 |
-
def get_doc_by_hash(self, content_hash: str) -> Optional[Dict[str, Any]]:
|
| 52 |
# 1. In-memory hot cache
|
| 53 |
doc_id = self._hash_to_doc_id.get(content_hash)
|
| 54 |
if doc_id and doc_id in self._documents:
|
| 55 |
doc = self._documents[doc_id]
|
| 56 |
-
if
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
if db_meta:
|
| 64 |
doc_id = db_meta["doc_id"]
|
| 65 |
if db_meta.get("status") == "indexing":
|
|
@@ -74,6 +83,7 @@ class MemoryStorage:
|
|
| 74 |
"section_count": 0,
|
| 75 |
"content_hash": content_hash,
|
| 76 |
"file_path": db_meta.get("file_path"),
|
|
|
|
| 77 |
"error": None,
|
| 78 |
}
|
| 79 |
self._documents[doc_id] = doc_obj
|
|
@@ -119,6 +129,7 @@ class MemoryStorage:
|
|
| 119 |
"duration": db_meta.get("duration"),
|
| 120 |
"error": None,
|
| 121 |
"content_hash": content_hash,
|
|
|
|
| 122 |
"created_at": cached.get("created_at", time.time()),
|
| 123 |
"updated_at": time.time(),
|
| 124 |
}
|
|
@@ -194,6 +205,7 @@ class MemoryStorage:
|
|
| 194 |
file_path: Optional[str] = None,
|
| 195 |
current_stage: str = "Processing PDF",
|
| 196 |
progress: int = 10,
|
|
|
|
| 197 |
) -> str:
|
| 198 |
document_id = doc_id or str(uuid.uuid4())
|
| 199 |
now = time.time()
|
|
@@ -212,6 +224,7 @@ class MemoryStorage:
|
|
| 212 |
"error": None,
|
| 213 |
"content_hash": content_hash,
|
| 214 |
"file_path": file_path,
|
|
|
|
| 215 |
"created_at": now,
|
| 216 |
"updated_at": now,
|
| 217 |
}
|
|
@@ -227,6 +240,7 @@ class MemoryStorage:
|
|
| 227 |
current_stage=current_stage,
|
| 228 |
content_hash=content_hash,
|
| 229 |
file_path=file_path,
|
|
|
|
| 230 |
)
|
| 231 |
return document_id
|
| 232 |
|
|
@@ -404,15 +418,25 @@ class MemoryStorage:
|
|
| 404 |
file_path=doc.get("file_path"),
|
| 405 |
)
|
| 406 |
|
| 407 |
-
def get_document(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 408 |
if not doc_id:
|
| 409 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 410 |
doc = self._documents.get(doc_id)
|
| 411 |
-
if doc
|
| 412 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 413 |
|
| 414 |
-
# Check Persistent DB
|
| 415 |
-
db_meta = persistent_db.get_doc_meta(doc_id)
|
| 416 |
if db_meta:
|
| 417 |
doc_obj = {
|
| 418 |
"doc_id": doc_id,
|
|
@@ -428,6 +452,7 @@ class MemoryStorage:
|
|
| 428 |
"error": db_meta.get("error"),
|
| 429 |
"content_hash": db_meta.get("content_hash"),
|
| 430 |
"file_path": db_meta.get("file_path"),
|
|
|
|
| 431 |
"created_at": db_meta.get("created_at", time.time()),
|
| 432 |
"updated_at": db_meta.get("updated_at", time.time()),
|
| 433 |
}
|
|
@@ -498,24 +523,25 @@ class MemoryStorage:
|
|
| 498 |
# ──────────────────────────────────────────────────────────────────────
|
| 499 |
# Legacy / convenience tree helpers
|
| 500 |
# ──────────────────────────────────────────────────────────────────────
|
| 501 |
-
def save_tree(self, tree: Any, doc_id: Optional[str] = None, filename: str = "document.pdf") -> str:
|
| 502 |
document_id = doc_id or str(uuid.uuid4())
|
| 503 |
self.set_document_ready(document_id, tree=tree, filename=filename)
|
|
|
|
|
|
|
| 504 |
return document_id
|
| 505 |
|
| 506 |
-
def get_tree(self, doc_id: str) -> Optional[Any]:
|
| 507 |
-
doc = self.
|
| 508 |
if doc and doc.get("tree") is not None:
|
| 509 |
return doc["tree"]
|
| 510 |
-
return self._trees.get(doc_id)
|
| 511 |
|
| 512 |
-
def tree_exists(self, doc_id: str) -> bool:
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
return doc_id in self._trees
|
| 516 |
|
| 517 |
-
def list_documents(self) -> List[Dict[str, Any]]:
|
| 518 |
-
db_docs = persistent_db.list_docs()
|
| 519 |
results = []
|
| 520 |
for d in db_docs:
|
| 521 |
doc_id = d["doc_id"]
|
|
@@ -527,98 +553,112 @@ class MemoryStorage:
|
|
| 527 |
results.append(d)
|
| 528 |
return results
|
| 529 |
|
| 530 |
-
def get_document_tree(self, doc_id: str) -> Optional[Any]:
|
| 531 |
-
doc = self.get_document(doc_id)
|
| 532 |
if doc and doc.get("tree") is not None:
|
| 533 |
return doc["tree"]
|
| 534 |
-
return self._trees.get(doc_id)
|
| 535 |
|
| 536 |
-
def delete_document(self, doc_id: str) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 537 |
self._documents.pop(doc_id, None)
|
| 538 |
self._trees.pop(doc_id, None)
|
| 539 |
self._node_maps.pop(doc_id, None)
|
| 540 |
self._flat_sections.pop(doc_id, None)
|
| 541 |
self._active_tasks.pop(doc_id, None)
|
| 542 |
persistent_db.delete_doc_index(doc_id)
|
| 543 |
-
return persistent_db.delete_doc(doc_id)
|
| 544 |
|
| 545 |
|
| 546 |
# ──────────────────────────────────────────────────────────────────────
|
| 547 |
-
# Chat operations (
|
| 548 |
# ──────────────────────────────────────────────────────────────────────
|
| 549 |
-
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None) -> str:
|
| 550 |
cid = chat_id or str(uuid.uuid4())
|
| 551 |
-
persistent_db.create_chat(chat_id=cid, title=title)
|
| 552 |
return cid
|
| 553 |
|
| 554 |
-
def
|
| 555 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
persistent_db.create_chat(chat_id=chat_id)
|
| 557 |
return []
|
| 558 |
return persistent_db.get_messages(chat_id)
|
| 559 |
|
| 560 |
-
def chat_exists(self, chat_id: str) -> bool:
|
| 561 |
-
return persistent_db.get_chat(chat_id) is not None
|
| 562 |
|
| 563 |
-
def ensure_chat(self, chat_id: str) -> List[Dict[str, str]]:
|
| 564 |
-
if not self.chat_exists(chat_id):
|
| 565 |
-
persistent_db.create_chat(chat_id=chat_id)
|
| 566 |
return persistent_db.get_messages(chat_id)
|
| 567 |
|
| 568 |
-
def add_message(self, chat_id: str, role: str, content: str) -> None:
|
| 569 |
-
persistent_db.add_message(chat_id=chat_id, role=role, content=content)
|
| 570 |
|
| 571 |
-
def update_chat_history(self, chat_id: str, history: List[Dict[str, str]]) -> None:
|
|
|
|
|
|
|
| 572 |
persistent_db.clear_messages(chat_id)
|
| 573 |
for msg in history:
|
| 574 |
-
persistent_db.add_message(chat_id=chat_id, role=msg["role"], content=msg["content"])
|
| 575 |
|
| 576 |
-
def set_chat_doc(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None) -> None:
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
persistent_db.create_chat(chat_id=chat_id)
|
| 580 |
-
# look up doc name if not supplied
|
| 581 |
if doc_name is None:
|
| 582 |
-
doc = self.
|
| 583 |
doc_name = doc.get("filename") if doc else None
|
| 584 |
persistent_db.set_chat_doc(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 585 |
persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 586 |
|
| 587 |
-
def add_chat_doc(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None) -> None:
|
| 588 |
-
if not self.chat_exists(chat_id):
|
| 589 |
-
persistent_db.create_chat(chat_id=chat_id)
|
| 590 |
if doc_name is None:
|
| 591 |
-
doc = self.
|
| 592 |
doc_name = doc.get("filename") if doc else None
|
| 593 |
persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 594 |
|
| 595 |
-
def get_chat_doc(self, chat_id: str) -> Optional[str]:
|
| 596 |
-
|
| 597 |
-
meta = persistent_db.get_chat(chat_id)
|
| 598 |
if meta and meta.get("doc_id"):
|
| 599 |
return meta["doc_id"]
|
| 600 |
-
docs = persistent_db.get_chat_documents(chat_id)
|
| 601 |
return docs[0]["doc_id"] if docs else None
|
| 602 |
|
| 603 |
-
def get_chat_docs(self, chat_id: str) -> List[Dict[str, Any]]:
|
| 604 |
-
return persistent_db.get_chat_documents(chat_id)
|
| 605 |
|
| 606 |
-
def remove_chat_doc(self, chat_id: str, doc_id: Optional[str] = None) -> None:
|
|
|
|
|
|
|
| 607 |
if doc_id:
|
| 608 |
persistent_db.remove_chat_document(chat_id, doc_id)
|
| 609 |
else:
|
| 610 |
persistent_db.remove_chat_doc(chat_id)
|
| 611 |
|
| 612 |
-
def clear_chat_docs(self, chat_id: str) -> None:
|
|
|
|
|
|
|
| 613 |
persistent_db.clear_chat_documents(chat_id)
|
| 614 |
|
| 615 |
-
def delete_chat(self, chat_id: str) ->
|
| 616 |
-
persistent_db.delete_chat(chat_id)
|
| 617 |
|
| 618 |
-
def list_chats(self) -> List[Dict[str, Any]]:
|
| 619 |
-
return persistent_db.list_chats()
|
| 620 |
|
| 621 |
-
def update_chat_title(self, chat_id: str, title: str) -> None:
|
|
|
|
|
|
|
| 622 |
persistent_db.update_chat_title(chat_id=chat_id, title=title)
|
| 623 |
|
| 624 |
# ── session long memory & rolling summaries ─────────────────────────────
|
|
@@ -628,7 +668,6 @@ class MemoryStorage:
|
|
| 628 |
def get_chat_summary(self, chat_id: str) -> Optional[Dict[str, Any]]:
|
| 629 |
return persistent_db.get_chat_summary(chat_id=chat_id)
|
| 630 |
|
| 631 |
-
# ── global knowledge memory & document dossiers ────────────────────────
|
| 632 |
def save_knowledge_dossier(
|
| 633 |
self,
|
| 634 |
doc_id: str,
|
|
@@ -637,6 +676,7 @@ class MemoryStorage:
|
|
| 637 |
key_entities: Optional[List[str]] = None,
|
| 638 |
key_findings: Optional[List[str]] = None,
|
| 639 |
topics: Optional[List[str]] = None,
|
|
|
|
| 640 |
) -> None:
|
| 641 |
persistent_db.save_knowledge_dossier(
|
| 642 |
doc_id=doc_id,
|
|
@@ -647,11 +687,11 @@ class MemoryStorage:
|
|
| 647 |
topics=topics,
|
| 648 |
)
|
| 649 |
|
| 650 |
-
def get_knowledge_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 651 |
-
return persistent_db.get_knowledge_dossier(doc_id)
|
| 652 |
|
| 653 |
-
def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
|
| 654 |
-
return persistent_db.list_knowledge_dossiers()
|
| 655 |
|
| 656 |
def delete_knowledge_dossier(self, doc_id: str) -> None:
|
| 657 |
persistent_db.delete_knowledge_dossier(doc_id)
|
|
@@ -661,15 +701,17 @@ class MemoryStorage:
|
|
| 661 |
collective_summary: str,
|
| 662 |
total_documents: int,
|
| 663 |
key_insights: Optional[List[str]] = None,
|
|
|
|
| 664 |
) -> None:
|
| 665 |
persistent_db.save_global_knowledge_summary(
|
| 666 |
collective_summary=collective_summary,
|
| 667 |
total_documents=total_documents,
|
| 668 |
key_insights=key_insights,
|
|
|
|
| 669 |
)
|
| 670 |
|
| 671 |
-
def get_global_knowledge_summary(self) -> Optional[Dict[str, Any]]:
|
| 672 |
-
return persistent_db.get_global_knowledge_summary()
|
| 673 |
|
| 674 |
|
| 675 |
# Global singleton storage instance
|
|
|
|
| 48 |
# ──────────────────────────────────────────────────────────────────────
|
| 49 |
# Document hash / cache helpers
|
| 50 |
# ──────────────────────────────────────────────────────────────────────
|
| 51 |
+
def get_doc_by_hash(self, content_hash: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 52 |
# 1. In-memory hot cache
|
| 53 |
doc_id = self._hash_to_doc_id.get(content_hash)
|
| 54 |
if doc_id and doc_id in self._documents:
|
| 55 |
doc = self._documents[doc_id]
|
| 56 |
+
if owner_id is not None:
|
| 57 |
+
doc_owner = doc.get("owner_id")
|
| 58 |
+
if doc_owner is None:
|
| 59 |
+
db_meta = persistent_db.get_doc_meta(doc_id)
|
| 60 |
+
doc_owner = db_meta.get("owner_id") if db_meta else None
|
| 61 |
+
doc["owner_id"] = doc_owner
|
| 62 |
+
if doc_owner != owner_id:
|
| 63 |
+
doc = None
|
| 64 |
+
if doc:
|
| 65 |
+
if doc.get("status") == "ready" and doc.get("tree"):
|
| 66 |
+
return doc
|
| 67 |
+
if doc.get("status") == "indexing":
|
| 68 |
+
return doc
|
| 69 |
+
|
| 70 |
+
# 2. Persistent metadata
|
| 71 |
+
db_meta = persistent_db.find_doc_by_hash(content_hash, owner_id=owner_id)
|
| 72 |
if db_meta:
|
| 73 |
doc_id = db_meta["doc_id"]
|
| 74 |
if db_meta.get("status") == "indexing":
|
|
|
|
| 83 |
"section_count": 0,
|
| 84 |
"content_hash": content_hash,
|
| 85 |
"file_path": db_meta.get("file_path"),
|
| 86 |
+
"owner_id": db_meta.get("owner_id"),
|
| 87 |
"error": None,
|
| 88 |
}
|
| 89 |
self._documents[doc_id] = doc_obj
|
|
|
|
| 129 |
"duration": db_meta.get("duration"),
|
| 130 |
"error": None,
|
| 131 |
"content_hash": content_hash,
|
| 132 |
+
"owner_id": db_meta.get("owner_id"),
|
| 133 |
"created_at": cached.get("created_at", time.time()),
|
| 134 |
"updated_at": time.time(),
|
| 135 |
}
|
|
|
|
| 205 |
file_path: Optional[str] = None,
|
| 206 |
current_stage: str = "Processing PDF",
|
| 207 |
progress: int = 10,
|
| 208 |
+
owner_id: Optional[str] = None,
|
| 209 |
) -> str:
|
| 210 |
document_id = doc_id or str(uuid.uuid4())
|
| 211 |
now = time.time()
|
|
|
|
| 224 |
"error": None,
|
| 225 |
"content_hash": content_hash,
|
| 226 |
"file_path": file_path,
|
| 227 |
+
"owner_id": owner_id,
|
| 228 |
"created_at": now,
|
| 229 |
"updated_at": now,
|
| 230 |
}
|
|
|
|
| 240 |
current_stage=current_stage,
|
| 241 |
content_hash=content_hash,
|
| 242 |
file_path=file_path,
|
| 243 |
+
owner_id=owner_id,
|
| 244 |
)
|
| 245 |
return document_id
|
| 246 |
|
|
|
|
| 418 |
file_path=doc.get("file_path"),
|
| 419 |
)
|
| 420 |
|
| 421 |
+
def get_document(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 422 |
if not doc_id:
|
| 423 |
return None
|
| 424 |
+
|
| 425 |
+
# Check Persistent DB first if owner_id is specified to enforce isolation
|
| 426 |
+
db_meta = persistent_db.get_doc_meta(doc_id, owner_id=owner_id) if owner_id is not None else persistent_db.get_doc_meta(doc_id)
|
| 427 |
+
if owner_id is not None and not db_meta:
|
| 428 |
+
return None
|
| 429 |
+
|
| 430 |
doc = self._documents.get(doc_id)
|
| 431 |
+
if doc:
|
| 432 |
+
if owner_id is not None:
|
| 433 |
+
if doc.get("owner_id") is None and db_meta:
|
| 434 |
+
doc["owner_id"] = db_meta.get("owner_id")
|
| 435 |
+
if doc.get("owner_id") != owner_id:
|
| 436 |
+
return None
|
| 437 |
+
if doc.get("status") == "ready" and doc.get("tree"):
|
| 438 |
+
return doc
|
| 439 |
|
|
|
|
|
|
|
| 440 |
if db_meta:
|
| 441 |
doc_obj = {
|
| 442 |
"doc_id": doc_id,
|
|
|
|
| 452 |
"error": db_meta.get("error"),
|
| 453 |
"content_hash": db_meta.get("content_hash"),
|
| 454 |
"file_path": db_meta.get("file_path"),
|
| 455 |
+
"owner_id": db_meta.get("owner_id"),
|
| 456 |
"created_at": db_meta.get("created_at", time.time()),
|
| 457 |
"updated_at": db_meta.get("updated_at", time.time()),
|
| 458 |
}
|
|
|
|
| 523 |
# ──────────────────────────────────────────────────────────────────────
|
| 524 |
# Legacy / convenience tree helpers
|
| 525 |
# ──────────────────────────────────────────────────────────────────────
|
| 526 |
+
def save_tree(self, tree: Any, doc_id: Optional[str] = None, filename: str = "document.pdf", owner_id: Optional[str] = None) -> str:
|
| 527 |
document_id = doc_id or str(uuid.uuid4())
|
| 528 |
self.set_document_ready(document_id, tree=tree, filename=filename)
|
| 529 |
+
if owner_id:
|
| 530 |
+
persistent_db.upsert_doc_meta(doc_id=document_id, filename=filename, status="ready", owner_id=owner_id)
|
| 531 |
return document_id
|
| 532 |
|
| 533 |
+
def get_tree(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Any]:
|
| 534 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 535 |
if doc and doc.get("tree") is not None:
|
| 536 |
return doc["tree"]
|
| 537 |
+
return self._trees.get(doc_id) if owner_id is None else None
|
| 538 |
|
| 539 |
+
def tree_exists(self, doc_id: str, owner_id: Optional[str] = None) -> bool:
|
| 540 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 541 |
+
return doc is not None and doc.get("status") == "ready" and doc.get("tree") is not None
|
|
|
|
| 542 |
|
| 543 |
+
def list_documents(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 544 |
+
db_docs = persistent_db.list_docs(owner_id=owner_id)
|
| 545 |
results = []
|
| 546 |
for d in db_docs:
|
| 547 |
doc_id = d["doc_id"]
|
|
|
|
| 553 |
results.append(d)
|
| 554 |
return results
|
| 555 |
|
| 556 |
+
def get_document_tree(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Any]:
|
| 557 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 558 |
if doc and doc.get("tree") is not None:
|
| 559 |
return doc["tree"]
|
| 560 |
+
return self._trees.get(doc_id) if owner_id is None else None
|
| 561 |
|
| 562 |
+
def delete_document(self, doc_id: str, owner_id: Optional[str] = None) -> bool:
|
| 563 |
+
if owner_id is not None:
|
| 564 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 565 |
+
if not doc:
|
| 566 |
+
return False
|
| 567 |
self._documents.pop(doc_id, None)
|
| 568 |
self._trees.pop(doc_id, None)
|
| 569 |
self._node_maps.pop(doc_id, None)
|
| 570 |
self._flat_sections.pop(doc_id, None)
|
| 571 |
self._active_tasks.pop(doc_id, None)
|
| 572 |
persistent_db.delete_doc_index(doc_id)
|
| 573 |
+
return persistent_db.delete_doc(doc_id, owner_id=owner_id)
|
| 574 |
|
| 575 |
|
| 576 |
# ──────────────────────────────────────────────────────────────────────
|
| 577 |
+
# Chat operations (backed by persistent DB)
|
| 578 |
# ──────────────────────────────────────────────────────────────────────
|
| 579 |
+
def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None, owner_id: Optional[str] = None) -> str:
|
| 580 |
cid = chat_id or str(uuid.uuid4())
|
| 581 |
+
persistent_db.create_chat(chat_id=cid, title=title, owner_id=owner_id)
|
| 582 |
return cid
|
| 583 |
|
| 584 |
+
def get_chat(self, chat_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 585 |
+
return persistent_db.get_chat(chat_id, owner_id=owner_id)
|
| 586 |
+
|
| 587 |
+
def get_chat_history(self, chat_id: str, owner_id: Optional[str] = None) -> List[Dict[str, str]]:
|
| 588 |
+
if not self.chat_exists(chat_id, owner_id=owner_id):
|
| 589 |
+
if owner_id is not None:
|
| 590 |
+
return []
|
| 591 |
persistent_db.create_chat(chat_id=chat_id)
|
| 592 |
return []
|
| 593 |
return persistent_db.get_messages(chat_id)
|
| 594 |
|
| 595 |
+
def chat_exists(self, chat_id: str, owner_id: Optional[str] = None) -> bool:
|
| 596 |
+
return persistent_db.get_chat(chat_id, owner_id=owner_id) is not None
|
| 597 |
|
| 598 |
+
def ensure_chat(self, chat_id: str, owner_id: Optional[str] = None) -> List[Dict[str, str]]:
|
| 599 |
+
if not self.chat_exists(chat_id, owner_id=owner_id):
|
| 600 |
+
persistent_db.create_chat(chat_id=chat_id, owner_id=owner_id)
|
| 601 |
return persistent_db.get_messages(chat_id)
|
| 602 |
|
| 603 |
+
def add_message(self, chat_id: str, role: str, content: str, owner_id: Optional[str] = None) -> None:
|
| 604 |
+
persistent_db.add_message(chat_id=chat_id, role=role, content=content, owner_id=owner_id)
|
| 605 |
|
| 606 |
+
def update_chat_history(self, chat_id: str, history: List[Dict[str, str]], owner_id: Optional[str] = None) -> None:
|
| 607 |
+
if owner_id and not self.chat_exists(chat_id, owner_id=owner_id):
|
| 608 |
+
return
|
| 609 |
persistent_db.clear_messages(chat_id)
|
| 610 |
for msg in history:
|
| 611 |
+
persistent_db.add_message(chat_id=chat_id, role=msg["role"], content=msg["content"], owner_id=owner_id)
|
| 612 |
|
| 613 |
+
def set_chat_doc(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None, owner_id: Optional[str] = None) -> None:
|
| 614 |
+
if not self.chat_exists(chat_id, owner_id=owner_id):
|
| 615 |
+
persistent_db.create_chat(chat_id=chat_id, owner_id=owner_id)
|
|
|
|
|
|
|
| 616 |
if doc_name is None:
|
| 617 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 618 |
doc_name = doc.get("filename") if doc else None
|
| 619 |
persistent_db.set_chat_doc(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 620 |
persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 621 |
|
| 622 |
+
def add_chat_doc(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None, owner_id: Optional[str] = None) -> None:
|
| 623 |
+
if not self.chat_exists(chat_id, owner_id=owner_id):
|
| 624 |
+
persistent_db.create_chat(chat_id=chat_id, owner_id=owner_id)
|
| 625 |
if doc_name is None:
|
| 626 |
+
doc = self.get_document(doc_id, owner_id=owner_id)
|
| 627 |
doc_name = doc.get("filename") if doc else None
|
| 628 |
persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
|
| 629 |
|
| 630 |
+
def get_chat_doc(self, chat_id: str, owner_id: Optional[str] = None) -> Optional[str]:
|
| 631 |
+
meta = persistent_db.get_chat(chat_id, owner_id=owner_id)
|
|
|
|
| 632 |
if meta and meta.get("doc_id"):
|
| 633 |
return meta["doc_id"]
|
| 634 |
+
docs = persistent_db.get_chat_documents(chat_id, owner_id=owner_id)
|
| 635 |
return docs[0]["doc_id"] if docs else None
|
| 636 |
|
| 637 |
+
def get_chat_docs(self, chat_id: str, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 638 |
+
return persistent_db.get_chat_documents(chat_id, owner_id=owner_id)
|
| 639 |
|
| 640 |
+
def remove_chat_doc(self, chat_id: str, doc_id: Optional[str] = None, owner_id: Optional[str] = None) -> None:
|
| 641 |
+
if owner_id and not self.chat_exists(chat_id, owner_id=owner_id):
|
| 642 |
+
return
|
| 643 |
if doc_id:
|
| 644 |
persistent_db.remove_chat_document(chat_id, doc_id)
|
| 645 |
else:
|
| 646 |
persistent_db.remove_chat_doc(chat_id)
|
| 647 |
|
| 648 |
+
def clear_chat_docs(self, chat_id: str, owner_id: Optional[str] = None) -> None:
|
| 649 |
+
if owner_id and not self.chat_exists(chat_id, owner_id=owner_id):
|
| 650 |
+
return
|
| 651 |
persistent_db.clear_chat_documents(chat_id)
|
| 652 |
|
| 653 |
+
def delete_chat(self, chat_id: str, owner_id: Optional[str] = None) -> bool:
|
| 654 |
+
return persistent_db.delete_chat(chat_id, owner_id=owner_id)
|
| 655 |
|
| 656 |
+
def list_chats(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 657 |
+
return persistent_db.list_chats(owner_id=owner_id)
|
| 658 |
|
| 659 |
+
def update_chat_title(self, chat_id: str, title: str, owner_id: Optional[str] = None) -> None:
|
| 660 |
+
if owner_id and not self.chat_exists(chat_id, owner_id=owner_id):
|
| 661 |
+
return
|
| 662 |
persistent_db.update_chat_title(chat_id=chat_id, title=title)
|
| 663 |
|
| 664 |
# ── session long memory & rolling summaries ─────────────────────────────
|
|
|
|
| 668 |
def get_chat_summary(self, chat_id: str) -> Optional[Dict[str, Any]]:
|
| 669 |
return persistent_db.get_chat_summary(chat_id=chat_id)
|
| 670 |
|
|
|
|
| 671 |
def save_knowledge_dossier(
|
| 672 |
self,
|
| 673 |
doc_id: str,
|
|
|
|
| 676 |
key_entities: Optional[List[str]] = None,
|
| 677 |
key_findings: Optional[List[str]] = None,
|
| 678 |
topics: Optional[List[str]] = None,
|
| 679 |
+
owner_id: Optional[str] = None,
|
| 680 |
) -> None:
|
| 681 |
persistent_db.save_knowledge_dossier(
|
| 682 |
doc_id=doc_id,
|
|
|
|
| 687 |
topics=topics,
|
| 688 |
)
|
| 689 |
|
| 690 |
+
def get_knowledge_dossier(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 691 |
+
return persistent_db.get_knowledge_dossier(doc_id, owner_id=owner_id)
|
| 692 |
|
| 693 |
+
def list_knowledge_dossiers(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 694 |
+
return persistent_db.list_knowledge_dossiers(owner_id=owner_id)
|
| 695 |
|
| 696 |
def delete_knowledge_dossier(self, doc_id: str) -> None:
|
| 697 |
persistent_db.delete_knowledge_dossier(doc_id)
|
|
|
|
| 701 |
collective_summary: str,
|
| 702 |
total_documents: int,
|
| 703 |
key_insights: Optional[List[str]] = None,
|
| 704 |
+
owner_id: Optional[str] = None,
|
| 705 |
) -> None:
|
| 706 |
persistent_db.save_global_knowledge_summary(
|
| 707 |
collective_summary=collective_summary,
|
| 708 |
total_documents=total_documents,
|
| 709 |
key_insights=key_insights,
|
| 710 |
+
owner_id=owner_id,
|
| 711 |
)
|
| 712 |
|
| 713 |
+
def get_global_knowledge_summary(self, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 714 |
+
return persistent_db.get_global_knowledge_summary(owner_id=owner_id)
|
| 715 |
|
| 716 |
|
| 717 |
# Global singleton storage instance
|
app/mcp_server/web_search_mcp.py
CHANGED
|
@@ -10,6 +10,7 @@ and natively importable across Atlas services and external MCP clients (Claude D
|
|
| 10 |
6. Internal Atlas Document Intelligence Tools (`atlas_list_documents`, `atlas_query_document`, `atlas_get_knowledge_dossier`).
|
| 11 |
"""
|
| 12 |
|
|
|
|
| 13 |
import asyncio
|
| 14 |
import json
|
| 15 |
import re
|
|
@@ -261,92 +262,94 @@ def mcp_evaluate_math_expression(expression: str) -> str:
|
|
| 261 |
return f"Math Evaluation Error: {res.get('error')}"
|
| 262 |
|
| 263 |
|
| 264 |
-
@mcp.tool(
|
| 265 |
-
name="atlas_list_documents",
|
| 266 |
-
description="List all ingested documents in Atlas knowledge base with document IDs, filenames, page counts, and statuses.",
|
| 267 |
-
)
|
| 268 |
-
def mcp_atlas_list_documents() -> str:
|
| 269 |
-
"""List all indexed documents stored in Atlas."""
|
| 270 |
-
docs = atlas_list_documents_impl()
|
| 271 |
-
if not docs:
|
| 272 |
-
return "No documents currently indexed in the Atlas knowledge base."
|
| 273 |
-
|
| 274 |
-
lines = ["### Atlas Ingested Documents\n"]
|
| 275 |
-
for idx, d in enumerate(docs, 1):
|
| 276 |
-
lines.append(
|
| 277 |
-
f"{idx}. **{d['filename']}** (ID: `{d['doc_id']}`)\n"
|
| 278 |
-
f" - Status: `{d['status']}` | Pages: {d['page_count']} | Sections: {d['section_count']} | Method: {d['indexing_method']}"
|
| 279 |
-
)
|
| 280 |
-
return "\n".join(lines)
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
@mcp.tool(
|
| 284 |
-
name="atlas_query_document",
|
| 285 |
-
description="Query a specific document tree in Atlas and retrieve grounded passages with exact (Section, Page) citations.",
|
| 286 |
-
)
|
| 287 |
-
def mcp_atlas_query_document(doc_id: str, query: str) -> str:
|
| 288 |
-
"""Run vectorless hierarchical retrieval on a specific indexed document."""
|
| 289 |
-
res = atlas_query_document_impl(doc_id=doc_id, query=query)
|
| 290 |
-
if res.get("status") != "success":
|
| 291 |
-
return f"Error querying document {doc_id}: {res.get('error')}"
|
| 292 |
-
|
| 293 |
-
passages = res.get("passages", [])
|
| 294 |
-
if not passages:
|
| 295 |
-
return f"No matching sections found for query '{query}' in document {res.get('filename')}."
|
| 296 |
-
|
| 297 |
-
lines = [f"### Grounded Retrieval Passages for '{query}' in {res.get('filename')}\n"]
|
| 298 |
-
for idx, p in enumerate(passages, 1):
|
| 299 |
-
lines.append(f"{idx}. **[{p['section_title']} - Page {p['page']}]** (Citation: `{p['citation']}`)")
|
| 300 |
-
lines.append(f"{p['text']}\n")
|
| 301 |
-
return "\n".join(lines).strip()
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
@mcp.tool(
|
| 305 |
-
name="atlas_get_knowledge_dossier",
|
| 306 |
-
description="Retrieve the executive knowledge dossier (summary, key entities, findings, topics) and workspace global summary for a document.",
|
| 307 |
-
)
|
| 308 |
-
def mcp_atlas_get_knowledge_dossier(doc_id: str) -> str:
|
| 309 |
-
"""Retrieve structured document dossier and workspace knowledge synthesis."""
|
| 310 |
-
res = atlas_get_knowledge_dossier_impl(doc_id=doc_id)
|
| 311 |
-
if res.get("status") != "success":
|
| 312 |
-
return f"Knowledge Dossier for {doc_id}: {res.get('message', res.get('error', 'Not found'))}"
|
| 313 |
-
|
| 314 |
-
lines = [
|
| 315 |
-
f"### Knowledge Dossier: {res.get('filename')} (ID: `{doc_id}`)\n",
|
| 316 |
-
f"**Executive Summary**:\n{res.get('executive_summary')}\n",
|
| 317 |
-
f"**Key Entities**: {', '.join(res.get('key_entities', []))}\n",
|
| 318 |
-
"**Key Findings**:",
|
| 319 |
-
]
|
| 320 |
-
for kf in res.get("key_findings", []):
|
| 321 |
-
lines.append(f"- {kf}")
|
| 322 |
-
lines.append(f"\n**Topics**: {', '.join(res.get('topics', []))}\n")
|
| 323 |
-
|
| 324 |
-
if res.get("global_workspace_summary"):
|
| 325 |
-
lines.append(f"**Workspace Global Summary**:\n{res.get('global_workspace_summary')}")
|
| 326 |
-
|
| 327 |
-
return "\n".join(lines).strip()
|
| 328 |
-
|
| 329 |
-
|
| 330 |
# ==============================================================================
|
| 331 |
-
#
|
| 332 |
# ==============================================================================
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
|
| 351 |
|
| 352 |
if __name__ == "__main__":
|
|
|
|
| 10 |
6. Internal Atlas Document Intelligence Tools (`atlas_list_documents`, `atlas_query_document`, `atlas_get_knowledge_dossier`).
|
| 11 |
"""
|
| 12 |
|
| 13 |
+
import os
|
| 14 |
import asyncio
|
| 15 |
import json
|
| 16 |
import re
|
|
|
|
| 262 |
return f"Math Evaluation Error: {res.get('error')}"
|
| 263 |
|
| 264 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
# ==============================================================================
|
| 266 |
+
# INTERNAL ATLAS DOCUMENT INTELLIGENCE TOOLS (Gated by ATLAS_MCP_INTERNAL_TOOLS)
|
| 267 |
# ==============================================================================
|
| 268 |
+
ENABLE_INTERNAL_TOOLS = os.getenv("ATLAS_MCP_INTERNAL_TOOLS", "false").lower() in ("true", "1", "yes")
|
| 269 |
+
|
| 270 |
+
if ENABLE_INTERNAL_TOOLS:
|
| 271 |
+
@mcp.tool(
|
| 272 |
+
name="atlas_list_documents",
|
| 273 |
+
description="List all ingested documents in Atlas knowledge base with document IDs, filenames, page counts, and statuses.",
|
| 274 |
+
)
|
| 275 |
+
def mcp_atlas_list_documents() -> str:
|
| 276 |
+
"""List all indexed documents stored in Atlas."""
|
| 277 |
+
docs = atlas_list_documents_impl()
|
| 278 |
+
if not docs:
|
| 279 |
+
return "No documents currently indexed in the Atlas knowledge base."
|
| 280 |
+
|
| 281 |
+
lines = ["### Atlas Ingested Documents\n"]
|
| 282 |
+
for idx, d in enumerate(docs, 1):
|
| 283 |
+
lines.append(
|
| 284 |
+
f"{idx}. **{d['filename']}** (ID: `{d['doc_id']}`)\n"
|
| 285 |
+
f" - Status: `{d['status']}` | Pages: {d['page_count']} | Sections: {d['section_count']} | Method: {d['indexing_method']}"
|
| 286 |
+
)
|
| 287 |
+
return "\n".join(lines)
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
@mcp.tool(
|
| 291 |
+
name="atlas_query_document",
|
| 292 |
+
description="Query a specific document tree in Atlas and retrieve grounded passages with exact (Section, Page) citations.",
|
| 293 |
+
)
|
| 294 |
+
def mcp_atlas_query_document(doc_id: str, query: str) -> str:
|
| 295 |
+
"""Run vectorless hierarchical retrieval on a specific indexed document."""
|
| 296 |
+
res = atlas_query_document_impl(doc_id=doc_id, query=query)
|
| 297 |
+
if res.get("status") != "success":
|
| 298 |
+
return f"Error querying document {doc_id}: {res.get('error')}"
|
| 299 |
+
|
| 300 |
+
passages = res.get("passages", [])
|
| 301 |
+
if not passages:
|
| 302 |
+
return f"No matching sections found for query '{query}' in document {res.get('filename')}."
|
| 303 |
+
|
| 304 |
+
lines = [f"### Grounded Retrieval Passages for '{query}' in {res.get('filename')}\n"]
|
| 305 |
+
for idx, p in enumerate(passages, 1):
|
| 306 |
+
lines.append(f"{idx}. **[{p['section_title']} - Page {p['page']}]** (Citation: `{p['citation']}`)")
|
| 307 |
+
lines.append(f"{p['text']}\n")
|
| 308 |
+
return "\n".join(lines).strip()
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
@mcp.tool(
|
| 312 |
+
name="atlas_get_knowledge_dossier",
|
| 313 |
+
description="Retrieve the executive knowledge dossier (summary, key entities, findings, topics) and workspace global summary for a document.",
|
| 314 |
+
)
|
| 315 |
+
def mcp_atlas_get_knowledge_dossier(doc_id: str) -> str:
|
| 316 |
+
"""Retrieve structured document dossier and workspace knowledge synthesis."""
|
| 317 |
+
res = atlas_get_knowledge_dossier_impl(doc_id=doc_id)
|
| 318 |
+
if res.get("status") != "success":
|
| 319 |
+
return f"Knowledge Dossier for {doc_id}: {res.get('message', res.get('error', 'Not found'))}"
|
| 320 |
+
|
| 321 |
+
lines = [
|
| 322 |
+
f"### Knowledge Dossier: {res.get('filename')} (ID: `{doc_id}`)\n",
|
| 323 |
+
f"**Executive Summary**:\n{res.get('executive_summary')}\n",
|
| 324 |
+
f"**Key Entities**: {', '.join(res.get('key_entities', []))}\n",
|
| 325 |
+
"**Key Findings**:",
|
| 326 |
+
]
|
| 327 |
+
for kf in res.get("key_findings", []):
|
| 328 |
+
lines.append(f"- {kf}")
|
| 329 |
+
lines.append(f"\n**Topics**: {', '.join(res.get('topics', []))}\n")
|
| 330 |
+
|
| 331 |
+
if res.get("global_workspace_summary"):
|
| 332 |
+
lines.append(f"**Workspace Global Summary**:\n{res.get('global_workspace_summary')}")
|
| 333 |
+
|
| 334 |
+
return "\n".join(lines).strip()
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
@mcp.resource("atlas://documents")
|
| 338 |
+
def resource_atlas_documents() -> str:
|
| 339 |
+
"""Resource returning JSON list of all documents in the knowledge base."""
|
| 340 |
+
docs = atlas_list_documents_impl()
|
| 341 |
+
return json.dumps(docs, indent=2)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
@mcp.resource("atlas://knowledge-summary")
|
| 345 |
+
def resource_atlas_knowledge_summary() -> str:
|
| 346 |
+
"""Resource returning the collective master knowledge summary of the workspace."""
|
| 347 |
+
try:
|
| 348 |
+
from core.storage import storage
|
| 349 |
+
summary = storage.get_global_knowledge_summary() or {}
|
| 350 |
+
return json.dumps(summary, indent=2)
|
| 351 |
+
except Exception as e:
|
| 352 |
+
return json.dumps({"error": str(e)})
|
| 353 |
|
| 354 |
|
| 355 |
if __name__ == "__main__":
|
app/routers/chat.py
CHANGED
|
@@ -18,6 +18,7 @@ from core.dependencies import (
|
|
| 18 |
get_rag_pipeline,
|
| 19 |
get_storage,
|
| 20 |
get_memory_service,
|
|
|
|
| 21 |
)
|
| 22 |
from core.rag_pipeline import VectorlessRAG
|
| 23 |
from services.llm_service import LLMService
|
|
@@ -28,9 +29,12 @@ router = APIRouter(tags=["Chat"])
|
|
| 28 |
|
| 29 |
|
| 30 |
@router.post("/chat/new", response_model=NewChatResponse)
|
| 31 |
-
async def create_chat(
|
| 32 |
-
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
| 34 |
return NewChatResponse(chat_id=chat_id)
|
| 35 |
|
| 36 |
|
|
@@ -39,11 +43,14 @@ async def attach_document_to_chat(
|
|
| 39 |
chat_id: str,
|
| 40 |
body: AttachDocRequest,
|
| 41 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 42 |
):
|
| 43 |
"""Attach an indexed document to an active conversation session."""
|
| 44 |
-
if not storage.
|
|
|
|
|
|
|
| 45 |
raise HTTPException(status_code=404, detail="Document ID not found in memory")
|
| 46 |
-
storage.add_chat_doc(chat_id, body.doc_id, doc_name=body.doc_name)
|
| 47 |
app_logger.info(f"Attached document {body.doc_id} to chat session {chat_id}")
|
| 48 |
return {"message": "Document attached successfully", "chat_id": chat_id, "doc_id": body.doc_id}
|
| 49 |
|
|
@@ -53,12 +60,15 @@ async def attach_multiple_documents_to_chat(
|
|
| 53 |
chat_id: str,
|
| 54 |
body: AttachMultipleDocsRequest,
|
| 55 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 56 |
):
|
| 57 |
"""Attach multiple indexed documents to an active conversation session."""
|
|
|
|
|
|
|
| 58 |
attached = []
|
| 59 |
for doc_id in body.doc_ids:
|
| 60 |
-
if storage.tree_exists(doc_id):
|
| 61 |
-
storage.add_chat_doc(chat_id, doc_id)
|
| 62 |
attached.append(doc_id)
|
| 63 |
app_logger.info(f"Attached {len(attached)} documents to chat session {chat_id}")
|
| 64 |
return {"message": f"Attached {len(attached)} documents successfully", "chat_id": chat_id, "doc_ids": attached}
|
|
@@ -68,12 +78,15 @@ async def attach_multiple_documents_to_chat(
|
|
| 68 |
async def get_chat_documents(
|
| 69 |
chat_id: str,
|
| 70 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 71 |
):
|
| 72 |
"""Return all attached documents for this session with status and page counts."""
|
| 73 |
-
|
|
|
|
|
|
|
| 74 |
detailed_docs = []
|
| 75 |
for d in docs:
|
| 76 |
-
doc_info = storage.get_document(d["doc_id"])
|
| 77 |
if doc_info:
|
| 78 |
detailed_docs.append({
|
| 79 |
"doc_id": d["doc_id"],
|
|
@@ -92,9 +105,12 @@ async def detach_specific_document_from_chat(
|
|
| 92 |
chat_id: str,
|
| 93 |
doc_id: str,
|
| 94 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 95 |
):
|
| 96 |
"""Detach a specific document from the active chat session."""
|
| 97 |
-
storage.
|
|
|
|
|
|
|
| 98 |
app_logger.info(f"Detached document {doc_id} from chat session {chat_id}")
|
| 99 |
return {"message": "Document detached successfully", "chat_id": chat_id, "doc_id": doc_id}
|
| 100 |
|
|
@@ -103,10 +119,13 @@ async def detach_specific_document_from_chat(
|
|
| 103 |
async def detach_document_from_chat(
|
| 104 |
chat_id: str,
|
| 105 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 106 |
):
|
| 107 |
"""Detach all documents from active chat session."""
|
| 108 |
-
storage.
|
| 109 |
-
|
|
|
|
|
|
|
| 110 |
app_logger.info(f"Cleared all documents from chat session {chat_id}")
|
| 111 |
return {"message": "All documents detached successfully", "chat_id": chat_id}
|
| 112 |
|
|
@@ -118,9 +137,14 @@ async def ask_question(
|
|
| 118 |
llm_service: LLMService = Depends(get_llm_service),
|
| 119 |
rag_pipeline: VectorlessRAG = Depends(get_rag_pipeline),
|
| 120 |
memory_service: MemoryService = Depends(get_memory_service),
|
|
|
|
| 121 |
):
|
| 122 |
-
# Ensure chat session exists in persistent storage
|
| 123 |
-
storage.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
|
| 125 |
# 1. Resolve target document IDs for this query session
|
| 126 |
target_doc_ids: List[str] = []
|
|
@@ -130,16 +154,18 @@ async def ask_question(
|
|
| 130 |
target_doc_ids.append(request.doc_id)
|
| 131 |
|
| 132 |
if target_doc_ids:
|
| 133 |
-
#
|
| 134 |
for did in target_doc_ids:
|
| 135 |
-
storage.
|
|
|
|
|
|
|
| 136 |
else:
|
| 137 |
# Check existing docs attached to this session
|
| 138 |
-
session_docs = storage.get_chat_docs(request.chat_id)
|
| 139 |
if session_docs:
|
| 140 |
target_doc_ids = [d["doc_id"] for d in session_docs]
|
| 141 |
else:
|
| 142 |
-
primary_id = storage.get_chat_doc(request.chat_id)
|
| 143 |
if primary_id:
|
| 144 |
target_doc_ids = [primary_id]
|
| 145 |
|
|
@@ -152,7 +178,7 @@ async def ask_question(
|
|
| 152 |
error_docs = []
|
| 153 |
|
| 154 |
for did in target_doc_ids:
|
| 155 |
-
doc_state = storage.get_document(did)
|
| 156 |
if not doc_state:
|
| 157 |
error_docs.append({"doc_id": did, "filename": did, "error": "Document not found or expired"})
|
| 158 |
elif doc_state.get("status") == "ready" and doc_state.get("tree"):
|
|
@@ -193,12 +219,12 @@ async def ask_question(
|
|
| 193 |
)
|
| 194 |
|
| 195 |
# 4. Retrieve session memory (isolated short-term window + long-term rolling summary)
|
| 196 |
-
mem_context = memory_service.get_session_context(request.chat_id)
|
| 197 |
recent_turns = mem_context.get("recent_turns", [])
|
| 198 |
session_summary = mem_context.get("session_summary")
|
| 199 |
|
| 200 |
-
# 5. Retrieve global workspace knowledge summary
|
| 201 |
-
global_knowledge = memory_service.get_global_knowledge()
|
| 202 |
|
| 203 |
is_arabic = any("\u0600" <= c <= "\u06ff" for c in request.query)
|
| 204 |
|
|
@@ -288,9 +314,9 @@ async def ask_question(
|
|
| 288 |
finally:
|
| 289 |
if full_answer:
|
| 290 |
clean_answer = re.sub(r'<!--STAGE:.*?-->\n?', '', full_answer).strip()
|
| 291 |
-
storage.add_message(request.chat_id, role="user", content=request.query)
|
| 292 |
storage.add_message(
|
| 293 |
-
request.chat_id, role="assistant", content=clean_answer or full_answer
|
| 294 |
)
|
| 295 |
# Asynchronously update long-term rolling summary if conversation threshold reached
|
| 296 |
try:
|
|
@@ -307,17 +333,23 @@ async def ask_question(
|
|
| 307 |
async def get_chat_history(
|
| 308 |
chat_id: str,
|
| 309 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 310 |
):
|
| 311 |
-
|
|
|
|
|
|
|
| 312 |
return ChatHistoryResponse(history=history or [])
|
| 313 |
|
| 314 |
|
| 315 |
# ── Long Memory & Conversation Management Endpoints ────────────────────────
|
| 316 |
|
| 317 |
@router.get("/chats")
|
| 318 |
-
async def list_all_chats(
|
|
|
|
|
|
|
|
|
|
| 319 |
"""Return all persisted chat sessions ordered by most recent."""
|
| 320 |
-
chats = storage.list_chats()
|
| 321 |
return {"chats": chats}
|
| 322 |
|
| 323 |
|
|
@@ -325,9 +357,12 @@ async def list_all_chats(storage: MemoryStorage = Depends(get_storage)):
|
|
| 325 |
async def delete_chat(
|
| 326 |
chat_id: str,
|
| 327 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 328 |
):
|
| 329 |
"""Permanently delete a chat session and all its messages."""
|
| 330 |
-
storage.delete_chat(chat_id)
|
|
|
|
|
|
|
| 331 |
app_logger.info(f"Deleted chat session: {chat_id}")
|
| 332 |
return {"message": "Chat deleted", "chat_id": chat_id}
|
| 333 |
|
|
@@ -341,7 +376,10 @@ async def rename_chat(
|
|
| 341 |
chat_id: str,
|
| 342 |
body: RenameChatRequest,
|
| 343 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 344 |
):
|
| 345 |
"""Update the display title of a chat session."""
|
| 346 |
-
storage.
|
|
|
|
|
|
|
| 347 |
return {"message": "Title updated", "chat_id": chat_id, "title": body.title}
|
|
|
|
| 18 |
get_rag_pipeline,
|
| 19 |
get_storage,
|
| 20 |
get_memory_service,
|
| 21 |
+
get_client_id,
|
| 22 |
)
|
| 23 |
from core.rag_pipeline import VectorlessRAG
|
| 24 |
from services.llm_service import LLMService
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
@router.post("/chat/new", response_model=NewChatResponse)
|
| 32 |
+
async def create_chat(
|
| 33 |
+
storage: MemoryStorage = Depends(get_storage),
|
| 34 |
+
client_id: str = Depends(get_client_id),
|
| 35 |
+
):
|
| 36 |
+
chat_id = storage.create_chat(owner_id=client_id)
|
| 37 |
+
app_logger.info(f"Created new chat session: {chat_id} for client: {client_id}")
|
| 38 |
return NewChatResponse(chat_id=chat_id)
|
| 39 |
|
| 40 |
|
|
|
|
| 43 |
chat_id: str,
|
| 44 |
body: AttachDocRequest,
|
| 45 |
storage: MemoryStorage = Depends(get_storage),
|
| 46 |
+
client_id: str = Depends(get_client_id),
|
| 47 |
):
|
| 48 |
"""Attach an indexed document to an active conversation session."""
|
| 49 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 50 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 51 |
+
if not storage.tree_exists(body.doc_id, owner_id=client_id):
|
| 52 |
raise HTTPException(status_code=404, detail="Document ID not found in memory")
|
| 53 |
+
storage.add_chat_doc(chat_id, body.doc_id, doc_name=body.doc_name, owner_id=client_id)
|
| 54 |
app_logger.info(f"Attached document {body.doc_id} to chat session {chat_id}")
|
| 55 |
return {"message": "Document attached successfully", "chat_id": chat_id, "doc_id": body.doc_id}
|
| 56 |
|
|
|
|
| 60 |
chat_id: str,
|
| 61 |
body: AttachMultipleDocsRequest,
|
| 62 |
storage: MemoryStorage = Depends(get_storage),
|
| 63 |
+
client_id: str = Depends(get_client_id),
|
| 64 |
):
|
| 65 |
"""Attach multiple indexed documents to an active conversation session."""
|
| 66 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 67 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 68 |
attached = []
|
| 69 |
for doc_id in body.doc_ids:
|
| 70 |
+
if storage.tree_exists(doc_id, owner_id=client_id):
|
| 71 |
+
storage.add_chat_doc(chat_id, doc_id, owner_id=client_id)
|
| 72 |
attached.append(doc_id)
|
| 73 |
app_logger.info(f"Attached {len(attached)} documents to chat session {chat_id}")
|
| 74 |
return {"message": f"Attached {len(attached)} documents successfully", "chat_id": chat_id, "doc_ids": attached}
|
|
|
|
| 78 |
async def get_chat_documents(
|
| 79 |
chat_id: str,
|
| 80 |
storage: MemoryStorage = Depends(get_storage),
|
| 81 |
+
client_id: str = Depends(get_client_id),
|
| 82 |
):
|
| 83 |
"""Return all attached documents for this session with status and page counts."""
|
| 84 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 85 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 86 |
+
docs = storage.get_chat_docs(chat_id, owner_id=client_id)
|
| 87 |
detailed_docs = []
|
| 88 |
for d in docs:
|
| 89 |
+
doc_info = storage.get_document(d["doc_id"], owner_id=client_id)
|
| 90 |
if doc_info:
|
| 91 |
detailed_docs.append({
|
| 92 |
"doc_id": d["doc_id"],
|
|
|
|
| 105 |
chat_id: str,
|
| 106 |
doc_id: str,
|
| 107 |
storage: MemoryStorage = Depends(get_storage),
|
| 108 |
+
client_id: str = Depends(get_client_id),
|
| 109 |
):
|
| 110 |
"""Detach a specific document from the active chat session."""
|
| 111 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 112 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 113 |
+
storage.remove_chat_doc(chat_id, doc_id=doc_id, owner_id=client_id)
|
| 114 |
app_logger.info(f"Detached document {doc_id} from chat session {chat_id}")
|
| 115 |
return {"message": "Document detached successfully", "chat_id": chat_id, "doc_id": doc_id}
|
| 116 |
|
|
|
|
| 119 |
async def detach_document_from_chat(
|
| 120 |
chat_id: str,
|
| 121 |
storage: MemoryStorage = Depends(get_storage),
|
| 122 |
+
client_id: str = Depends(get_client_id),
|
| 123 |
):
|
| 124 |
"""Detach all documents from active chat session."""
|
| 125 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 126 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 127 |
+
storage.clear_chat_docs(chat_id, owner_id=client_id)
|
| 128 |
+
storage.remove_chat_doc(chat_id, owner_id=client_id)
|
| 129 |
app_logger.info(f"Cleared all documents from chat session {chat_id}")
|
| 130 |
return {"message": "All documents detached successfully", "chat_id": chat_id}
|
| 131 |
|
|
|
|
| 137 |
llm_service: LLMService = Depends(get_llm_service),
|
| 138 |
rag_pipeline: VectorlessRAG = Depends(get_rag_pipeline),
|
| 139 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 140 |
+
client_id: str = Depends(get_client_id),
|
| 141 |
):
|
| 142 |
+
# Ensure chat session exists in persistent storage and is owned by client_id
|
| 143 |
+
if storage.chat_exists(request.chat_id):
|
| 144 |
+
if not storage.chat_exists(request.chat_id, owner_id=client_id):
|
| 145 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 146 |
+
else:
|
| 147 |
+
storage.create_chat(request.chat_id, owner_id=client_id)
|
| 148 |
|
| 149 |
# 1. Resolve target document IDs for this query session
|
| 150 |
target_doc_ids: List[str] = []
|
|
|
|
| 154 |
target_doc_ids.append(request.doc_id)
|
| 155 |
|
| 156 |
if target_doc_ids:
|
| 157 |
+
# Validate ownership of each explicitly requested document
|
| 158 |
for did in target_doc_ids:
|
| 159 |
+
if not storage.get_document(did, owner_id=client_id):
|
| 160 |
+
raise HTTPException(status_code=404, detail=f"Document {did} not found")
|
| 161 |
+
storage.add_chat_doc(request.chat_id, did, owner_id=client_id)
|
| 162 |
else:
|
| 163 |
# Check existing docs attached to this session
|
| 164 |
+
session_docs = storage.get_chat_docs(request.chat_id, owner_id=client_id)
|
| 165 |
if session_docs:
|
| 166 |
target_doc_ids = [d["doc_id"] for d in session_docs]
|
| 167 |
else:
|
| 168 |
+
primary_id = storage.get_chat_doc(request.chat_id, owner_id=client_id)
|
| 169 |
if primary_id:
|
| 170 |
target_doc_ids = [primary_id]
|
| 171 |
|
|
|
|
| 178 |
error_docs = []
|
| 179 |
|
| 180 |
for did in target_doc_ids:
|
| 181 |
+
doc_state = storage.get_document(did, owner_id=client_id)
|
| 182 |
if not doc_state:
|
| 183 |
error_docs.append({"doc_id": did, "filename": did, "error": "Document not found or expired"})
|
| 184 |
elif doc_state.get("status") == "ready" and doc_state.get("tree"):
|
|
|
|
| 219 |
)
|
| 220 |
|
| 221 |
# 4. Retrieve session memory (isolated short-term window + long-term rolling summary)
|
| 222 |
+
mem_context = memory_service.get_session_context(request.chat_id, owner_id=client_id)
|
| 223 |
recent_turns = mem_context.get("recent_turns", [])
|
| 224 |
session_summary = mem_context.get("session_summary")
|
| 225 |
|
| 226 |
+
# 5. Retrieve global workspace knowledge summary scoped to owner
|
| 227 |
+
global_knowledge = memory_service.get_global_knowledge(owner_id=client_id)
|
| 228 |
|
| 229 |
is_arabic = any("\u0600" <= c <= "\u06ff" for c in request.query)
|
| 230 |
|
|
|
|
| 314 |
finally:
|
| 315 |
if full_answer:
|
| 316 |
clean_answer = re.sub(r'<!--STAGE:.*?-->\n?', '', full_answer).strip()
|
| 317 |
+
storage.add_message(request.chat_id, role="user", content=request.query, owner_id=client_id)
|
| 318 |
storage.add_message(
|
| 319 |
+
request.chat_id, role="assistant", content=clean_answer or full_answer, owner_id=client_id
|
| 320 |
)
|
| 321 |
# Asynchronously update long-term rolling summary if conversation threshold reached
|
| 322 |
try:
|
|
|
|
| 333 |
async def get_chat_history(
|
| 334 |
chat_id: str,
|
| 335 |
storage: MemoryStorage = Depends(get_storage),
|
| 336 |
+
client_id: str = Depends(get_client_id),
|
| 337 |
):
|
| 338 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 339 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 340 |
+
history = storage.get_chat_history(chat_id, owner_id=client_id)
|
| 341 |
return ChatHistoryResponse(history=history or [])
|
| 342 |
|
| 343 |
|
| 344 |
# ── Long Memory & Conversation Management Endpoints ────────────────────────
|
| 345 |
|
| 346 |
@router.get("/chats")
|
| 347 |
+
async def list_all_chats(
|
| 348 |
+
storage: MemoryStorage = Depends(get_storage),
|
| 349 |
+
client_id: str = Depends(get_client_id),
|
| 350 |
+
):
|
| 351 |
"""Return all persisted chat sessions ordered by most recent."""
|
| 352 |
+
chats = storage.list_chats(owner_id=client_id)
|
| 353 |
return {"chats": chats}
|
| 354 |
|
| 355 |
|
|
|
|
| 357 |
async def delete_chat(
|
| 358 |
chat_id: str,
|
| 359 |
storage: MemoryStorage = Depends(get_storage),
|
| 360 |
+
client_id: str = Depends(get_client_id),
|
| 361 |
):
|
| 362 |
"""Permanently delete a chat session and all its messages."""
|
| 363 |
+
deleted = storage.delete_chat(chat_id, owner_id=client_id)
|
| 364 |
+
if not deleted:
|
| 365 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 366 |
app_logger.info(f"Deleted chat session: {chat_id}")
|
| 367 |
return {"message": "Chat deleted", "chat_id": chat_id}
|
| 368 |
|
|
|
|
| 376 |
chat_id: str,
|
| 377 |
body: RenameChatRequest,
|
| 378 |
storage: MemoryStorage = Depends(get_storage),
|
| 379 |
+
client_id: str = Depends(get_client_id),
|
| 380 |
):
|
| 381 |
"""Update the display title of a chat session."""
|
| 382 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 383 |
+
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 384 |
+
storage.update_chat_title(chat_id, body.title, owner_id=client_id)
|
| 385 |
return {"message": "Title updated", "chat_id": chat_id, "title": body.title}
|
app/routers/documents.py
CHANGED
|
@@ -8,7 +8,7 @@ from fastapi import APIRouter, UploadFile, File, Form, HTTPException, Depends, R
|
|
| 8 |
from services.document_service import DocumentService
|
| 9 |
from services.llm_service import LLMService
|
| 10 |
from core.storage import MemoryStorage
|
| 11 |
-
from core.dependencies import get_document_service, get_llm_service, get_storage
|
| 12 |
from schemas.schemas import UploadResponse, DocumentStatusResponse
|
| 13 |
from utils.logger import app_logger
|
| 14 |
from core.persistent_db import DATA_DIR
|
|
@@ -27,6 +27,7 @@ async def run_background_indexing(
|
|
| 27 |
doc_service: DocumentService,
|
| 28 |
llm_service: LLMService,
|
| 29 |
storage: MemoryStorage,
|
|
|
|
| 30 |
):
|
| 31 |
"""Background task for long-running PDF parsing, PageIndex polling, and fallback extraction."""
|
| 32 |
t_start = time.monotonic()
|
|
@@ -96,6 +97,7 @@ async def run_background_indexing(
|
|
| 96 |
filename=filename,
|
| 97 |
tree=tree,
|
| 98 |
flat_sections=flat_sections,
|
|
|
|
| 99 |
)
|
| 100 |
)
|
| 101 |
except Exception as mex:
|
|
@@ -126,6 +128,7 @@ async def upload_pdf(
|
|
| 126 |
doc_service: DocumentService = Depends(get_document_service),
|
| 127 |
llm_service: LLMService = Depends(get_llm_service),
|
| 128 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 129 |
):
|
| 130 |
"""
|
| 131 |
Asynchronous PDF upload and indexing endpoint.
|
|
@@ -147,14 +150,14 @@ async def upload_pdf(
|
|
| 147 |
|
| 148 |
content_hash = hashlib.sha256(contents).hexdigest()
|
| 149 |
|
| 150 |
-
# Deduplication check: Check if same document is already ready or currently indexing
|
| 151 |
-
existing_doc = storage.get_doc_by_hash(content_hash)
|
| 152 |
if existing_doc:
|
| 153 |
doc_id = existing_doc["doc_id"]
|
| 154 |
doc_status = existing_doc.get("status")
|
| 155 |
|
| 156 |
if chat_id:
|
| 157 |
-
storage.set_chat_doc(chat_id, doc_id)
|
| 158 |
|
| 159 |
if doc_status == "ready":
|
| 160 |
app_logger.info(
|
|
@@ -194,10 +197,10 @@ async def upload_pdf(
|
|
| 194 |
|
| 195 |
app_logger.info(
|
| 196 |
f"[INDEX] document_id={doc_id} stage=upload filename='{file.filename}' "
|
| 197 |
-
f"size={len(contents)} hash={content_hash[:12]}"
|
| 198 |
)
|
| 199 |
|
| 200 |
-
# Register document in storage
|
| 201 |
storage.create_document(
|
| 202 |
doc_id=doc_id,
|
| 203 |
filename=file.filename,
|
|
@@ -206,10 +209,11 @@ async def upload_pdf(
|
|
| 206 |
file_path=file_path,
|
| 207 |
current_stage="Processing PDF",
|
| 208 |
progress=10,
|
|
|
|
| 209 |
)
|
| 210 |
|
| 211 |
if chat_id:
|
| 212 |
-
storage.set_chat_doc(chat_id, doc_id)
|
| 213 |
app_logger.info(f"Associated document {doc_id} ('{file.filename}') with chat {chat_id}")
|
| 214 |
|
| 215 |
# Launch detached asynchronous background task
|
|
@@ -222,6 +226,7 @@ async def upload_pdf(
|
|
| 222 |
doc_service=doc_service,
|
| 223 |
llm_service=llm_service,
|
| 224 |
storage=storage,
|
|
|
|
| 225 |
)
|
| 226 |
)
|
| 227 |
storage.register_active_task(doc_id, task)
|
|
@@ -241,9 +246,10 @@ async def upload_pdf(
|
|
| 241 |
async def get_document_status(
|
| 242 |
document_id: str,
|
| 243 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 244 |
):
|
| 245 |
"""Retrieve indexing/ready/failed status and operational metadata for a document."""
|
| 246 |
-
doc = storage.get_document(document_id)
|
| 247 |
if not doc:
|
| 248 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 249 |
|
|
@@ -279,9 +285,10 @@ async def retry_document_indexing(
|
|
| 279 |
doc_service: DocumentService = Depends(get_document_service),
|
| 280 |
llm_service: LLMService = Depends(get_llm_service),
|
| 281 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 282 |
):
|
| 283 |
"""Explicitly retry indexing for a failed document."""
|
| 284 |
-
doc = storage.get_document(document_id)
|
| 285 |
if not doc:
|
| 286 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 287 |
|
|
@@ -319,6 +326,7 @@ async def retry_document_indexing(
|
|
| 319 |
doc_service=doc_service,
|
| 320 |
llm_service=llm_service,
|
| 321 |
storage=storage,
|
|
|
|
| 322 |
)
|
| 323 |
)
|
| 324 |
storage.register_active_task(document_id, task)
|
|
@@ -334,9 +342,12 @@ async def retry_document_indexing(
|
|
| 334 |
|
| 335 |
|
| 336 |
@router.get("/documents")
|
| 337 |
-
async def list_documents(
|
| 338 |
-
|
| 339 |
-
|
|
|
|
|
|
|
|
|
|
| 340 |
return {"documents": docs}
|
| 341 |
|
| 342 |
|
|
@@ -344,9 +355,10 @@ async def list_documents(storage: MemoryStorage = Depends(get_storage)):
|
|
| 344 |
async def get_document_details(
|
| 345 |
document_id: str,
|
| 346 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 347 |
):
|
| 348 |
"""Retrieve details and operational metadata for a specific document."""
|
| 349 |
-
doc = storage.get_document(document_id)
|
| 350 |
if not doc:
|
| 351 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 352 |
return {
|
|
@@ -371,13 +383,14 @@ async def get_document_tree(
|
|
| 371 |
storage: MemoryStorage = Depends(get_storage),
|
| 372 |
doc_service: DocumentService = Depends(get_document_service),
|
| 373 |
llm_service: LLMService = Depends(get_llm_service),
|
|
|
|
| 374 |
):
|
| 375 |
"""Retrieve the hierarchical section tree for a document."""
|
| 376 |
-
doc = storage.get_document(document_id)
|
| 377 |
if not doc:
|
| 378 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 379 |
|
| 380 |
-
tree = storage.get_document_tree(document_id)
|
| 381 |
if tree is None and doc.get("file_path") and os.path.exists(doc["file_path"]):
|
| 382 |
# Reconstruct tree using local parser and cache
|
| 383 |
tree = doc_service._local_pdf_parse(doc["file_path"])
|
|
@@ -390,6 +403,7 @@ async def get_document_tree(
|
|
| 390 |
indexing_method=doc.get("indexing_method") or "pymupdf",
|
| 391 |
filename=doc.get("filename"),
|
| 392 |
file_path=doc.get("file_path"),
|
|
|
|
| 393 |
)
|
| 394 |
|
| 395 |
if tree is None:
|
|
@@ -405,14 +419,14 @@ async def get_document_tree(
|
|
| 405 |
}
|
| 406 |
|
| 407 |
|
| 408 |
-
|
| 409 |
@router.delete("/documents/{document_id}")
|
| 410 |
async def delete_document(
|
| 411 |
document_id: str,
|
| 412 |
storage: MemoryStorage = Depends(get_storage),
|
|
|
|
| 413 |
):
|
| 414 |
"""Permanently delete a document, its cached index, and local file."""
|
| 415 |
-
doc = storage.get_document(document_id)
|
| 416 |
if not doc:
|
| 417 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 418 |
|
|
@@ -423,7 +437,7 @@ async def delete_document(
|
|
| 423 |
except Exception as e:
|
| 424 |
app_logger.warning(f"Could not remove document file {file_path}: {e}")
|
| 425 |
|
| 426 |
-
storage.delete_document(document_id)
|
| 427 |
app_logger.info(f"Deleted document {document_id}")
|
| 428 |
return {"message": "Document deleted successfully", "doc_id": document_id}
|
| 429 |
|
|
|
|
| 8 |
from services.document_service import DocumentService
|
| 9 |
from services.llm_service import LLMService
|
| 10 |
from core.storage import MemoryStorage
|
| 11 |
+
from core.dependencies import get_document_service, get_llm_service, get_storage, get_client_id
|
| 12 |
from schemas.schemas import UploadResponse, DocumentStatusResponse
|
| 13 |
from utils.logger import app_logger
|
| 14 |
from core.persistent_db import DATA_DIR
|
|
|
|
| 27 |
doc_service: DocumentService,
|
| 28 |
llm_service: LLMService,
|
| 29 |
storage: MemoryStorage,
|
| 30 |
+
owner_id: Optional[str] = None,
|
| 31 |
):
|
| 32 |
"""Background task for long-running PDF parsing, PageIndex polling, and fallback extraction."""
|
| 33 |
t_start = time.monotonic()
|
|
|
|
| 97 |
filename=filename,
|
| 98 |
tree=tree,
|
| 99 |
flat_sections=flat_sections,
|
| 100 |
+
owner_id=owner_id,
|
| 101 |
)
|
| 102 |
)
|
| 103 |
except Exception as mex:
|
|
|
|
| 128 |
doc_service: DocumentService = Depends(get_document_service),
|
| 129 |
llm_service: LLMService = Depends(get_llm_service),
|
| 130 |
storage: MemoryStorage = Depends(get_storage),
|
| 131 |
+
client_id: str = Depends(get_client_id),
|
| 132 |
):
|
| 133 |
"""
|
| 134 |
Asynchronous PDF upload and indexing endpoint.
|
|
|
|
| 150 |
|
| 151 |
content_hash = hashlib.sha256(contents).hexdigest()
|
| 152 |
|
| 153 |
+
# Deduplication check: Check if same document is already ready or currently indexing for this owner
|
| 154 |
+
existing_doc = storage.get_doc_by_hash(content_hash, owner_id=client_id)
|
| 155 |
if existing_doc:
|
| 156 |
doc_id = existing_doc["doc_id"]
|
| 157 |
doc_status = existing_doc.get("status")
|
| 158 |
|
| 159 |
if chat_id:
|
| 160 |
+
storage.set_chat_doc(chat_id, doc_id, owner_id=client_id)
|
| 161 |
|
| 162 |
if doc_status == "ready":
|
| 163 |
app_logger.info(
|
|
|
|
| 197 |
|
| 198 |
app_logger.info(
|
| 199 |
f"[INDEX] document_id={doc_id} stage=upload filename='{file.filename}' "
|
| 200 |
+
f"size={len(contents)} hash={content_hash[:12]} owner={client_id}"
|
| 201 |
)
|
| 202 |
|
| 203 |
+
# Register document in storage with owner_id
|
| 204 |
storage.create_document(
|
| 205 |
doc_id=doc_id,
|
| 206 |
filename=file.filename,
|
|
|
|
| 209 |
file_path=file_path,
|
| 210 |
current_stage="Processing PDF",
|
| 211 |
progress=10,
|
| 212 |
+
owner_id=client_id,
|
| 213 |
)
|
| 214 |
|
| 215 |
if chat_id:
|
| 216 |
+
storage.set_chat_doc(chat_id, doc_id, owner_id=client_id)
|
| 217 |
app_logger.info(f"Associated document {doc_id} ('{file.filename}') with chat {chat_id}")
|
| 218 |
|
| 219 |
# Launch detached asynchronous background task
|
|
|
|
| 226 |
doc_service=doc_service,
|
| 227 |
llm_service=llm_service,
|
| 228 |
storage=storage,
|
| 229 |
+
owner_id=client_id,
|
| 230 |
)
|
| 231 |
)
|
| 232 |
storage.register_active_task(doc_id, task)
|
|
|
|
| 246 |
async def get_document_status(
|
| 247 |
document_id: str,
|
| 248 |
storage: MemoryStorage = Depends(get_storage),
|
| 249 |
+
client_id: str = Depends(get_client_id),
|
| 250 |
):
|
| 251 |
"""Retrieve indexing/ready/failed status and operational metadata for a document."""
|
| 252 |
+
doc = storage.get_document(document_id, owner_id=client_id)
|
| 253 |
if not doc:
|
| 254 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 255 |
|
|
|
|
| 285 |
doc_service: DocumentService = Depends(get_document_service),
|
| 286 |
llm_service: LLMService = Depends(get_llm_service),
|
| 287 |
storage: MemoryStorage = Depends(get_storage),
|
| 288 |
+
client_id: str = Depends(get_client_id),
|
| 289 |
):
|
| 290 |
"""Explicitly retry indexing for a failed document."""
|
| 291 |
+
doc = storage.get_document(document_id, owner_id=client_id)
|
| 292 |
if not doc:
|
| 293 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 294 |
|
|
|
|
| 326 |
doc_service=doc_service,
|
| 327 |
llm_service=llm_service,
|
| 328 |
storage=storage,
|
| 329 |
+
owner_id=client_id,
|
| 330 |
)
|
| 331 |
)
|
| 332 |
storage.register_active_task(document_id, task)
|
|
|
|
| 342 |
|
| 343 |
|
| 344 |
@router.get("/documents")
|
| 345 |
+
async def list_documents(
|
| 346 |
+
storage: MemoryStorage = Depends(get_storage),
|
| 347 |
+
client_id: str = Depends(get_client_id),
|
| 348 |
+
):
|
| 349 |
+
"""List all ingested documents with status and metadata for current client."""
|
| 350 |
+
docs = storage.list_documents(owner_id=client_id)
|
| 351 |
return {"documents": docs}
|
| 352 |
|
| 353 |
|
|
|
|
| 355 |
async def get_document_details(
|
| 356 |
document_id: str,
|
| 357 |
storage: MemoryStorage = Depends(get_storage),
|
| 358 |
+
client_id: str = Depends(get_client_id),
|
| 359 |
):
|
| 360 |
"""Retrieve details and operational metadata for a specific document."""
|
| 361 |
+
doc = storage.get_document(document_id, owner_id=client_id)
|
| 362 |
if not doc:
|
| 363 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 364 |
return {
|
|
|
|
| 383 |
storage: MemoryStorage = Depends(get_storage),
|
| 384 |
doc_service: DocumentService = Depends(get_document_service),
|
| 385 |
llm_service: LLMService = Depends(get_llm_service),
|
| 386 |
+
client_id: str = Depends(get_client_id),
|
| 387 |
):
|
| 388 |
"""Retrieve the hierarchical section tree for a document."""
|
| 389 |
+
doc = storage.get_document(document_id, owner_id=client_id)
|
| 390 |
if not doc:
|
| 391 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 392 |
|
| 393 |
+
tree = storage.get_document_tree(document_id, owner_id=client_id)
|
| 394 |
if tree is None and doc.get("file_path") and os.path.exists(doc["file_path"]):
|
| 395 |
# Reconstruct tree using local parser and cache
|
| 396 |
tree = doc_service._local_pdf_parse(doc["file_path"])
|
|
|
|
| 403 |
indexing_method=doc.get("indexing_method") or "pymupdf",
|
| 404 |
filename=doc.get("filename"),
|
| 405 |
file_path=doc.get("file_path"),
|
| 406 |
+
owner_id=client_id,
|
| 407 |
)
|
| 408 |
|
| 409 |
if tree is None:
|
|
|
|
| 419 |
}
|
| 420 |
|
| 421 |
|
|
|
|
| 422 |
@router.delete("/documents/{document_id}")
|
| 423 |
async def delete_document(
|
| 424 |
document_id: str,
|
| 425 |
storage: MemoryStorage = Depends(get_storage),
|
| 426 |
+
client_id: str = Depends(get_client_id),
|
| 427 |
):
|
| 428 |
"""Permanently delete a document, its cached index, and local file."""
|
| 429 |
+
doc = storage.get_document(document_id, owner_id=client_id)
|
| 430 |
if not doc:
|
| 431 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 432 |
|
|
|
|
| 437 |
except Exception as e:
|
| 438 |
app_logger.warning(f"Could not remove document file {file_path}: {e}")
|
| 439 |
|
| 440 |
+
storage.delete_document(document_id, owner_id=client_id)
|
| 441 |
app_logger.info(f"Deleted document {document_id}")
|
| 442 |
return {"message": "Document deleted successfully", "doc_id": document_id}
|
| 443 |
|
app/routers/memory.py
CHANGED
|
@@ -3,7 +3,7 @@ from fastapi import APIRouter, HTTPException, Depends
|
|
| 3 |
from typing import Optional, List, Dict, Any
|
| 4 |
from pydantic import BaseModel
|
| 5 |
|
| 6 |
-
from core.dependencies import get_memory_service, get_storage
|
| 7 |
from core.storage import MemoryStorage
|
| 8 |
from services.memory_service import MemoryService
|
| 9 |
from utils.logger import app_logger
|
|
@@ -13,25 +13,27 @@ router = APIRouter(prefix="/memory", tags=["Memory & Knowledge"])
|
|
| 13 |
|
| 14 |
@router.get("/knowledge")
|
| 15 |
async def get_global_knowledge(
|
|
|
|
| 16 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 17 |
):
|
| 18 |
"""
|
| 19 |
-
Retrieve workspace collective knowledge summary and
|
| 20 |
Provides cross-document intelligence and long-term memory across sessions.
|
| 21 |
"""
|
| 22 |
-
return memory_service.get_global_knowledge()
|
| 23 |
|
| 24 |
|
| 25 |
@router.post("/knowledge/refresh")
|
| 26 |
async def refresh_global_knowledge(
|
|
|
|
| 27 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 28 |
):
|
| 29 |
"""
|
| 30 |
-
Synthesize and rebuild the global workspace knowledge memory across all indexed documents.
|
| 31 |
"""
|
| 32 |
-
app_logger.info("Triggered manual refresh of global knowledge memory")
|
| 33 |
-
result = await memory_service.synthesize_global_knowledge()
|
| 34 |
-
dossiers = memory_service.list_knowledge_dossiers()
|
| 35 |
return {
|
| 36 |
"message": "Global knowledge memory refreshed successfully",
|
| 37 |
"collective_summary": result.get("collective_summary"),
|
|
@@ -44,12 +46,13 @@ async def refresh_global_knowledge(
|
|
| 44 |
@router.get("/dossier/{doc_id}")
|
| 45 |
async def get_document_dossier(
|
| 46 |
doc_id: str,
|
|
|
|
| 47 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 48 |
):
|
| 49 |
"""
|
| 50 |
-
Retrieve structured knowledge dossier for an individual document.
|
| 51 |
"""
|
| 52 |
-
dossier = memory_service.get_document_dossier(doc_id)
|
| 53 |
if not dossier:
|
| 54 |
raise HTTPException(status_code=404, detail="Knowledge dossier not found for this document")
|
| 55 |
return dossier
|
|
@@ -58,17 +61,18 @@ async def get_document_dossier(
|
|
| 58 |
@router.get("/chat/{chat_id}")
|
| 59 |
async def get_chat_memory_context(
|
| 60 |
chat_id: str,
|
|
|
|
| 61 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 62 |
storage: MemoryStorage = Depends(get_storage),
|
| 63 |
):
|
| 64 |
"""
|
| 65 |
-
Inspect the short-term and long-term isolated memory for a specific chat session.
|
| 66 |
"""
|
| 67 |
-
if not storage.chat_exists(chat_id):
|
| 68 |
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 69 |
|
| 70 |
-
context = memory_service.get_session_context(chat_id)
|
| 71 |
-
attached_docs = storage.get_chat_docs(chat_id)
|
| 72 |
return {
|
| 73 |
"chat_id": chat_id,
|
| 74 |
"session_summary": context.get("session_summary"),
|
|
@@ -81,16 +85,17 @@ async def get_chat_memory_context(
|
|
| 81 |
@router.post("/chat/{chat_id}/summarize")
|
| 82 |
async def force_summarize_chat(
|
| 83 |
chat_id: str,
|
|
|
|
| 84 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 85 |
storage: MemoryStorage = Depends(get_storage),
|
| 86 |
):
|
| 87 |
"""
|
| 88 |
-
Force-generate or update the rolling long-term memory summary for this chat session.
|
| 89 |
"""
|
| 90 |
-
if not storage.chat_exists(chat_id):
|
| 91 |
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 92 |
|
| 93 |
-
summary = await memory_service.update_session_summary_if_needed(chat_id, force=True)
|
| 94 |
return {
|
| 95 |
"message": "Chat session summary updated",
|
| 96 |
"chat_id": chat_id,
|
|
|
|
| 3 |
from typing import Optional, List, Dict, Any
|
| 4 |
from pydantic import BaseModel
|
| 5 |
|
| 6 |
+
from core.dependencies import get_memory_service, get_storage, get_client_id
|
| 7 |
from core.storage import MemoryStorage
|
| 8 |
from services.memory_service import MemoryService
|
| 9 |
from utils.logger import app_logger
|
|
|
|
| 13 |
|
| 14 |
@router.get("/knowledge")
|
| 15 |
async def get_global_knowledge(
|
| 16 |
+
client_id: str = Depends(get_client_id),
|
| 17 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 18 |
):
|
| 19 |
"""
|
| 20 |
+
Retrieve workspace collective knowledge summary and per-document knowledge dossiers for this client.
|
| 21 |
Provides cross-document intelligence and long-term memory across sessions.
|
| 22 |
"""
|
| 23 |
+
return memory_service.get_global_knowledge(owner_id=client_id)
|
| 24 |
|
| 25 |
|
| 26 |
@router.post("/knowledge/refresh")
|
| 27 |
async def refresh_global_knowledge(
|
| 28 |
+
client_id: str = Depends(get_client_id),
|
| 29 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 30 |
):
|
| 31 |
"""
|
| 32 |
+
Synthesize and rebuild the global workspace knowledge memory across all indexed documents for this client.
|
| 33 |
"""
|
| 34 |
+
app_logger.info(f"Triggered manual refresh of global knowledge memory for client {client_id}")
|
| 35 |
+
result = await memory_service.synthesize_global_knowledge(owner_id=client_id)
|
| 36 |
+
dossiers = memory_service.list_knowledge_dossiers(owner_id=client_id)
|
| 37 |
return {
|
| 38 |
"message": "Global knowledge memory refreshed successfully",
|
| 39 |
"collective_summary": result.get("collective_summary"),
|
|
|
|
| 46 |
@router.get("/dossier/{doc_id}")
|
| 47 |
async def get_document_dossier(
|
| 48 |
doc_id: str,
|
| 49 |
+
client_id: str = Depends(get_client_id),
|
| 50 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 51 |
):
|
| 52 |
"""
|
| 53 |
+
Retrieve structured knowledge dossier for an individual document belonging to this client.
|
| 54 |
"""
|
| 55 |
+
dossier = memory_service.get_document_dossier(doc_id, owner_id=client_id)
|
| 56 |
if not dossier:
|
| 57 |
raise HTTPException(status_code=404, detail="Knowledge dossier not found for this document")
|
| 58 |
return dossier
|
|
|
|
| 61 |
@router.get("/chat/{chat_id}")
|
| 62 |
async def get_chat_memory_context(
|
| 63 |
chat_id: str,
|
| 64 |
+
client_id: str = Depends(get_client_id),
|
| 65 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 66 |
storage: MemoryStorage = Depends(get_storage),
|
| 67 |
):
|
| 68 |
"""
|
| 69 |
+
Inspect the short-term and long-term isolated memory for a specific chat session belonging to this client.
|
| 70 |
"""
|
| 71 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 72 |
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 73 |
|
| 74 |
+
context = memory_service.get_session_context(chat_id, owner_id=client_id)
|
| 75 |
+
attached_docs = storage.get_chat_docs(chat_id, owner_id=client_id)
|
| 76 |
return {
|
| 77 |
"chat_id": chat_id,
|
| 78 |
"session_summary": context.get("session_summary"),
|
|
|
|
| 85 |
@router.post("/chat/{chat_id}/summarize")
|
| 86 |
async def force_summarize_chat(
|
| 87 |
chat_id: str,
|
| 88 |
+
client_id: str = Depends(get_client_id),
|
| 89 |
memory_service: MemoryService = Depends(get_memory_service),
|
| 90 |
storage: MemoryStorage = Depends(get_storage),
|
| 91 |
):
|
| 92 |
"""
|
| 93 |
+
Force-generate or update the rolling long-term memory summary for this chat session belonging to this client.
|
| 94 |
"""
|
| 95 |
+
if not storage.chat_exists(chat_id, owner_id=client_id):
|
| 96 |
raise HTTPException(status_code=404, detail="Chat session not found")
|
| 97 |
|
| 98 |
+
summary = await memory_service.update_session_summary_if_needed(chat_id, owner_id=client_id, force=True)
|
| 99 |
return {
|
| 100 |
"message": "Chat session summary updated",
|
| 101 |
"chat_id": chat_id,
|
app/routers/studio.py
CHANGED
|
@@ -4,7 +4,7 @@ from typing import List, Dict, Any, Optional
|
|
| 4 |
from fastapi import APIRouter, Depends, HTTPException, status
|
| 5 |
from pydantic import BaseModel
|
| 6 |
|
| 7 |
-
from core.dependencies import get_storage, get_document_service, get_llm_service, get_memory_service
|
| 8 |
from core.storage import MemoryStorage
|
| 9 |
from services.document_service import DocumentService
|
| 10 |
from services.llm_service import LLMService
|
|
@@ -23,6 +23,7 @@ class QuizGenerateRequest(BaseModel):
|
|
| 23 |
@router.get("/{doc_id}/mindmap")
|
| 24 |
async def get_document_mindmap(
|
| 25 |
doc_id: str,
|
|
|
|
| 26 |
storage: MemoryStorage = Depends(get_storage),
|
| 27 |
doc_service: DocumentService = Depends(get_document_service),
|
| 28 |
llm_service: LLMService = Depends(get_llm_service),
|
|
@@ -32,7 +33,7 @@ async def get_document_mindmap(
|
|
| 32 |
into a structured node-edge graph for interactive visualization.
|
| 33 |
Each node includes: id, label, section_title, page_number, type, summary, level.
|
| 34 |
"""
|
| 35 |
-
doc = storage.get_document(doc_id)
|
| 36 |
if not doc:
|
| 37 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 38 |
|
|
@@ -50,7 +51,7 @@ async def get_document_mindmap(
|
|
| 50 |
tree = []
|
| 51 |
|
| 52 |
# Retrieve dossier if available
|
| 53 |
-
dossier = storage.get_knowledge_dossier(doc_id) or {}
|
| 54 |
topics = dossier.get("topics") or []
|
| 55 |
key_findings = dossier.get("key_findings") or []
|
| 56 |
|
|
@@ -165,6 +166,7 @@ async def get_document_mindmap(
|
|
| 165 |
async def generate_document_quiz(
|
| 166 |
doc_id: str,
|
| 167 |
payload: Optional[QuizGenerateRequest] = None,
|
|
|
|
| 168 |
storage: MemoryStorage = Depends(get_storage),
|
| 169 |
doc_service: DocumentService = Depends(get_document_service),
|
| 170 |
llm_service: LLMService = Depends(get_llm_service),
|
|
@@ -175,7 +177,7 @@ async def generate_document_quiz(
|
|
| 175 |
Each item includes question, options, answer, explanation, and exact citation (Section, Page).
|
| 176 |
"""
|
| 177 |
t0 = time.monotonic()
|
| 178 |
-
doc = storage.get_document(doc_id)
|
| 179 |
if not doc:
|
| 180 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 181 |
|
|
@@ -186,7 +188,7 @@ async def generate_document_quiz(
|
|
| 186 |
tree = llm_service.normalize_tree(tree)
|
| 187 |
storage.set_document_tree(doc_id, tree)
|
| 188 |
|
| 189 |
-
dossier = storage.get_knowledge_dossier(doc_id)
|
| 190 |
|
| 191 |
app_logger.info(f"[STUDIO-QUIZ] Generating quiz and flashcards for doc_id={doc_id} ('{filename}')...")
|
| 192 |
result = await llm_service.generate_quiz_and_flashcards(
|
|
|
|
| 4 |
from fastapi import APIRouter, Depends, HTTPException, status
|
| 5 |
from pydantic import BaseModel
|
| 6 |
|
| 7 |
+
from core.dependencies import get_storage, get_document_service, get_llm_service, get_memory_service, get_client_id
|
| 8 |
from core.storage import MemoryStorage
|
| 9 |
from services.document_service import DocumentService
|
| 10 |
from services.llm_service import LLMService
|
|
|
|
| 23 |
@router.get("/{doc_id}/mindmap")
|
| 24 |
async def get_document_mindmap(
|
| 25 |
doc_id: str,
|
| 26 |
+
client_id: str = Depends(get_client_id),
|
| 27 |
storage: MemoryStorage = Depends(get_storage),
|
| 28 |
doc_service: DocumentService = Depends(get_document_service),
|
| 29 |
llm_service: LLMService = Depends(get_llm_service),
|
|
|
|
| 33 |
into a structured node-edge graph for interactive visualization.
|
| 34 |
Each node includes: id, label, section_title, page_number, type, summary, level.
|
| 35 |
"""
|
| 36 |
+
doc = storage.get_document(doc_id, owner_id=client_id)
|
| 37 |
if not doc:
|
| 38 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 39 |
|
|
|
|
| 51 |
tree = []
|
| 52 |
|
| 53 |
# Retrieve dossier if available
|
| 54 |
+
dossier = storage.get_knowledge_dossier(doc_id, owner_id=client_id) or {}
|
| 55 |
topics = dossier.get("topics") or []
|
| 56 |
key_findings = dossier.get("key_findings") or []
|
| 57 |
|
|
|
|
| 166 |
async def generate_document_quiz(
|
| 167 |
doc_id: str,
|
| 168 |
payload: Optional[QuizGenerateRequest] = None,
|
| 169 |
+
client_id: str = Depends(get_client_id),
|
| 170 |
storage: MemoryStorage = Depends(get_storage),
|
| 171 |
doc_service: DocumentService = Depends(get_document_service),
|
| 172 |
llm_service: LLMService = Depends(get_llm_service),
|
|
|
|
| 177 |
Each item includes question, options, answer, explanation, and exact citation (Section, Page).
|
| 178 |
"""
|
| 179 |
t0 = time.monotonic()
|
| 180 |
+
doc = storage.get_document(doc_id, owner_id=client_id)
|
| 181 |
if not doc:
|
| 182 |
raise HTTPException(status_code=404, detail="Document ID not found.")
|
| 183 |
|
|
|
|
| 188 |
tree = llm_service.normalize_tree(tree)
|
| 189 |
storage.set_document_tree(doc_id, tree)
|
| 190 |
|
| 191 |
+
dossier = storage.get_knowledge_dossier(doc_id, owner_id=client_id)
|
| 192 |
|
| 193 |
app_logger.info(f"[STUDIO-QUIZ] Generating quiz and flashcards for doc_id={doc_id} ('{filename}')...")
|
| 194 |
result = await llm_service.generate_quiz_and_flashcards(
|
app/routers/tools.py
CHANGED
|
@@ -9,9 +9,10 @@ Exposes direct REST endpoints for MCP tools:
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from typing import Any, Dict, Optional
|
| 12 |
-
from fastapi import APIRouter, HTTPException
|
| 13 |
from pydantic import BaseModel, Field
|
| 14 |
|
|
|
|
| 15 |
from mcp_server.web_search_mcp import execute_web_search
|
| 16 |
from mcp_server.research_tools import (
|
| 17 |
fetch_webpage_markdown,
|
|
@@ -20,7 +21,7 @@ from mcp_server.research_tools import (
|
|
| 20 |
evaluate_math_expression_safe,
|
| 21 |
)
|
| 22 |
|
| 23 |
-
router = APIRouter(prefix="/tools", tags=["Tools & MCP"])
|
| 24 |
|
| 25 |
|
| 26 |
class WebSearchRequest(BaseModel):
|
|
|
|
| 9 |
"""
|
| 10 |
|
| 11 |
from typing import Any, Dict, Optional
|
| 12 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 13 |
from pydantic import BaseModel, Field
|
| 14 |
|
| 15 |
+
from core.dependencies import get_client_id
|
| 16 |
from mcp_server.web_search_mcp import execute_web_search
|
| 17 |
from mcp_server.research_tools import (
|
| 18 |
fetch_webpage_markdown,
|
|
|
|
| 21 |
evaluate_math_expression_safe,
|
| 22 |
)
|
| 23 |
|
| 24 |
+
router = APIRouter(prefix="/tools", tags=["Tools & MCP"], dependencies=[Depends(get_client_id)])
|
| 25 |
|
| 26 |
|
| 27 |
class WebSearchRequest(BaseModel):
|
app/services/memory_service.py
CHANGED
|
@@ -22,14 +22,14 @@ class MemoryService:
|
|
| 22 |
|
| 23 |
# ── 1. Short-Term & Long-Term Session Memory ────────────────────────────
|
| 24 |
def get_session_context(
|
| 25 |
-
self, chat_id: str, max_recent_turns: int = 6
|
| 26 |
) -> Dict[str, Any]:
|
| 27 |
"""
|
| 28 |
Retrieves strictly isolated session memory:
|
| 29 |
- Recent turns (short-term memory window)
|
| 30 |
- Rolling session summary (long-term memory)
|
| 31 |
"""
|
| 32 |
-
full_history = storage.get_chat_history(chat_id) or []
|
| 33 |
summary_record = storage.get_chat_summary(chat_id)
|
| 34 |
session_summary = summary_record.get("summary") if summary_record else None
|
| 35 |
|
|
@@ -42,27 +42,34 @@ class MemoryService:
|
|
| 42 |
"total_messages": len(full_history),
|
| 43 |
}
|
| 44 |
|
| 45 |
-
async def update_session_summary_if_needed(
|
| 46 |
-
|
|
|
|
|
|
|
| 47 |
try:
|
| 48 |
-
full_history = storage.get_chat_history(chat_id) or []
|
| 49 |
-
if
|
| 50 |
-
return
|
|
|
|
|
|
|
| 51 |
|
| 52 |
last_summary = storage.get_chat_summary(chat_id)
|
| 53 |
last_count = last_summary.get("turn_count", 0) if last_summary else 0
|
| 54 |
|
| 55 |
-
# Only re-summarize if at least 4 new messages occurred
|
| 56 |
-
if (len(full_history) - last_count) < 4:
|
| 57 |
-
return
|
| 58 |
|
| 59 |
app_logger.info(f"[MEMORY] Updating rolling summary for chat_id={chat_id}")
|
| 60 |
summary = await self.llm.summarize_conversation(full_history)
|
| 61 |
if summary:
|
| 62 |
storage.set_chat_summary(chat_id, summary, turn_count=len(full_history))
|
| 63 |
app_logger.info(f"[MEMORY] Successfully updated summary for chat_id={chat_id}")
|
|
|
|
|
|
|
| 64 |
except Exception as e:
|
| 65 |
app_logger.error(f"[MEMORY] Error updating session summary: {e}")
|
|
|
|
| 66 |
|
| 67 |
# ── 2. Document Knowledge Dossier ──────────────────────────────────────
|
| 68 |
async def generate_document_dossier(
|
|
@@ -166,28 +173,28 @@ Return ONLY valid JSON matching this schema:
|
|
| 166 |
}
|
| 167 |
except Exception as e:
|
| 168 |
app_logger.error(f"[MEMORY] Failed to generate document dossier for {doc_id}: {e}", exc_info=True)
|
| 169 |
-
def get_document_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
|
| 170 |
"""Retrieve structured knowledge dossier for an individual document."""
|
| 171 |
-
return storage.get_knowledge_dossier(doc_id)
|
| 172 |
|
| 173 |
-
def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
|
| 174 |
"""Retrieve all knowledge dossiers across indexed documents."""
|
| 175 |
-
return storage.list_knowledge_dossiers()
|
| 176 |
|
| 177 |
# ── 3. Global Knowledge Summary Across All Data ────────────────────────
|
| 178 |
-
async def synthesize_global_knowledge(self) -> Dict[str, Any]:
|
| 179 |
"""
|
| 180 |
Synthesizes a collective master knowledge summary of all indexed documents in the workspace.
|
| 181 |
"""
|
| 182 |
-
dossiers = storage.list_knowledge_dossiers()
|
| 183 |
total_docs = len(dossiers)
|
| 184 |
|
| 185 |
if not dossiers:
|
| 186 |
# Check if any documents exist in doc_meta without dossiers yet
|
| 187 |
-
all_docs = storage.list_documents()
|
| 188 |
if not all_docs:
|
| 189 |
default_summary = "No documents currently indexed in the knowledge base."
|
| 190 |
-
storage.save_global_knowledge_summary(default_summary, 0, [])
|
| 191 |
return {"collective_summary": default_summary, "total_documents": 0, "key_insights": []}
|
| 192 |
|
| 193 |
# Build combined summary representation of all dossiers
|
|
@@ -252,7 +259,7 @@ Return ONLY valid JSON with this exact schema:
|
|
| 252 |
collective = data.get("collective_summary") or "Atlas Global Knowledge Base synthesis complete."
|
| 253 |
insights = data.get("key_insights") or []
|
| 254 |
|
| 255 |
-
storage.save_global_knowledge_summary(collective, total_docs, insights)
|
| 256 |
app_logger.info(f"[MEMORY] Successfully synthesized global knowledge across {total_docs} documents.")
|
| 257 |
|
| 258 |
return {
|
|
@@ -262,10 +269,10 @@ Return ONLY valid JSON with this exact schema:
|
|
| 262 |
"updated_at": time.time(),
|
| 263 |
}
|
| 264 |
|
| 265 |
-
def get_global_knowledge(self) -> Dict[str, Any]:
|
| 266 |
"""Fetch current global knowledge summary and document dossiers."""
|
| 267 |
-
global_rec = storage.get_global_knowledge_summary() or {}
|
| 268 |
-
dossiers = storage.list_knowledge_dossiers()
|
| 269 |
return {
|
| 270 |
"collective_summary": global_rec.get(
|
| 271 |
"collective_summary", "No collective summary available yet. Ingest documents to build knowledge."
|
|
|
|
| 22 |
|
| 23 |
# ── 1. Short-Term & Long-Term Session Memory ────────────────────────────
|
| 24 |
def get_session_context(
|
| 25 |
+
self, chat_id: str, max_recent_turns: int = 6, owner_id: Optional[str] = None
|
| 26 |
) -> Dict[str, Any]:
|
| 27 |
"""
|
| 28 |
Retrieves strictly isolated session memory:
|
| 29 |
- Recent turns (short-term memory window)
|
| 30 |
- Rolling session summary (long-term memory)
|
| 31 |
"""
|
| 32 |
+
full_history = storage.get_chat_history(chat_id, owner_id=owner_id) or []
|
| 33 |
summary_record = storage.get_chat_summary(chat_id)
|
| 34 |
session_summary = summary_record.get("summary") if summary_record else None
|
| 35 |
|
|
|
|
| 42 |
"total_messages": len(full_history),
|
| 43 |
}
|
| 44 |
|
| 45 |
+
async def update_session_summary_if_needed(
|
| 46 |
+
self, chat_id: str, owner_id: Optional[str] = None, force: bool = False
|
| 47 |
+
) -> Optional[str]:
|
| 48 |
+
"""Trigger rolling summarization if conversation length exceeds threshold or if force=True."""
|
| 49 |
try:
|
| 50 |
+
full_history = storage.get_chat_history(chat_id, owner_id=owner_id) or []
|
| 51 |
+
if not full_history:
|
| 52 |
+
return None
|
| 53 |
+
if not force and len(full_history) < 8:
|
| 54 |
+
return None
|
| 55 |
|
| 56 |
last_summary = storage.get_chat_summary(chat_id)
|
| 57 |
last_count = last_summary.get("turn_count", 0) if last_summary else 0
|
| 58 |
|
| 59 |
+
# Only re-summarize if force=True or at least 4 new messages occurred
|
| 60 |
+
if not force and (len(full_history) - last_count) < 4:
|
| 61 |
+
return last_summary.get("summary") if last_summary else None
|
| 62 |
|
| 63 |
app_logger.info(f"[MEMORY] Updating rolling summary for chat_id={chat_id}")
|
| 64 |
summary = await self.llm.summarize_conversation(full_history)
|
| 65 |
if summary:
|
| 66 |
storage.set_chat_summary(chat_id, summary, turn_count=len(full_history))
|
| 67 |
app_logger.info(f"[MEMORY] Successfully updated summary for chat_id={chat_id}")
|
| 68 |
+
return summary
|
| 69 |
+
return None
|
| 70 |
except Exception as e:
|
| 71 |
app_logger.error(f"[MEMORY] Error updating session summary: {e}")
|
| 72 |
+
return None
|
| 73 |
|
| 74 |
# ── 2. Document Knowledge Dossier ──────────────────────────────────────
|
| 75 |
async def generate_document_dossier(
|
|
|
|
| 173 |
}
|
| 174 |
except Exception as e:
|
| 175 |
app_logger.error(f"[MEMORY] Failed to generate document dossier for {doc_id}: {e}", exc_info=True)
|
| 176 |
+
def get_document_dossier(self, doc_id: str, owner_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
| 177 |
"""Retrieve structured knowledge dossier for an individual document."""
|
| 178 |
+
return storage.get_knowledge_dossier(doc_id, owner_id=owner_id)
|
| 179 |
|
| 180 |
+
def list_knowledge_dossiers(self, owner_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
| 181 |
"""Retrieve all knowledge dossiers across indexed documents."""
|
| 182 |
+
return storage.list_knowledge_dossiers(owner_id=owner_id)
|
| 183 |
|
| 184 |
# ── 3. Global Knowledge Summary Across All Data ────────────────────────
|
| 185 |
+
async def synthesize_global_knowledge(self, owner_id: Optional[str] = None) -> Dict[str, Any]:
|
| 186 |
"""
|
| 187 |
Synthesizes a collective master knowledge summary of all indexed documents in the workspace.
|
| 188 |
"""
|
| 189 |
+
dossiers = storage.list_knowledge_dossiers(owner_id=owner_id)
|
| 190 |
total_docs = len(dossiers)
|
| 191 |
|
| 192 |
if not dossiers:
|
| 193 |
# Check if any documents exist in doc_meta without dossiers yet
|
| 194 |
+
all_docs = storage.list_documents(owner_id=owner_id)
|
| 195 |
if not all_docs:
|
| 196 |
default_summary = "No documents currently indexed in the knowledge base."
|
| 197 |
+
storage.save_global_knowledge_summary(default_summary, 0, [], owner_id=owner_id)
|
| 198 |
return {"collective_summary": default_summary, "total_documents": 0, "key_insights": []}
|
| 199 |
|
| 200 |
# Build combined summary representation of all dossiers
|
|
|
|
| 259 |
collective = data.get("collective_summary") or "Atlas Global Knowledge Base synthesis complete."
|
| 260 |
insights = data.get("key_insights") or []
|
| 261 |
|
| 262 |
+
storage.save_global_knowledge_summary(collective, total_docs, insights, owner_id=owner_id)
|
| 263 |
app_logger.info(f"[MEMORY] Successfully synthesized global knowledge across {total_docs} documents.")
|
| 264 |
|
| 265 |
return {
|
|
|
|
| 269 |
"updated_at": time.time(),
|
| 270 |
}
|
| 271 |
|
| 272 |
+
def get_global_knowledge(self, owner_id: Optional[str] = None) -> Dict[str, Any]:
|
| 273 |
"""Fetch current global knowledge summary and document dossiers."""
|
| 274 |
+
global_rec = storage.get_global_knowledge_summary(owner_id=owner_id) or {}
|
| 275 |
+
dossiers = storage.list_knowledge_dossiers(owner_id=owner_id)
|
| 276 |
return {
|
| 277 |
"collective_summary": global_rec.get(
|
| 278 |
"collective_summary", "No collective summary available yet. Ingest documents to build knowledge."
|
pyproject.toml
CHANGED
|
@@ -22,3 +22,14 @@ dependencies = [
|
|
| 22 |
"uvicorn>=0.46.0",
|
| 23 |
"youtube-transcript-api>=1.2.4",
|
| 24 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
"uvicorn>=0.46.0",
|
| 23 |
"youtube-transcript-api>=1.2.4",
|
| 24 |
]
|
| 25 |
+
|
| 26 |
+
[dependency-groups]
|
| 27 |
+
dev = [
|
| 28 |
+
"pytest>=9.1.1",
|
| 29 |
+
"pytest-asyncio>=1.4.1",
|
| 30 |
+
]
|
| 31 |
+
|
| 32 |
+
[tool.pytest.ini_options]
|
| 33 |
+
asyncio_mode = "auto"
|
| 34 |
+
testpaths = ["tests"]
|
| 35 |
+
pythonpath = ["app", "."]
|
tests/conftest.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import tempfile
|
| 4 |
+
import pytest
|
| 5 |
+
|
| 6 |
+
# Ensure app is discoverable on python path
|
| 7 |
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 8 |
+
APP_DIR = os.path.join(BASE_DIR, "app")
|
| 9 |
+
if APP_DIR not in sys.path:
|
| 10 |
+
sys.path.insert(0, APP_DIR)
|
| 11 |
+
if BASE_DIR not in sys.path:
|
| 12 |
+
sys.path.insert(0, BASE_DIR)
|
| 13 |
+
|
| 14 |
+
# Strictly isolate test execution from any live database and default directories BEFORE imports
|
| 15 |
+
_SESSION_TMP = tempfile.mkdtemp(prefix="atlas_test_isolation_")
|
| 16 |
+
_SESSION_DATA = os.path.join(_SESSION_TMP, "data")
|
| 17 |
+
_SESSION_UPLOADS = os.path.join(_SESSION_TMP, "uploads")
|
| 18 |
+
_SESSION_STATIC = os.path.join(_SESSION_TMP, "static")
|
| 19 |
+
os.makedirs(_SESSION_DATA, exist_ok=True)
|
| 20 |
+
os.makedirs(_SESSION_UPLOADS, exist_ok=True)
|
| 21 |
+
os.makedirs(_SESSION_STATIC, exist_ok=True)
|
| 22 |
+
|
| 23 |
+
os.environ["DATABASE_URL"] = ""
|
| 24 |
+
os.environ["POSTGRES_URL"] = ""
|
| 25 |
+
os.environ["POSTGRESQL_URL"] = ""
|
| 26 |
+
os.environ["ATLAS_DATA_DIR"] = _SESSION_DATA
|
| 27 |
+
os.environ["ATLAS_UPLOAD_DIR"] = _SESSION_UPLOADS
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@pytest.fixture(autouse=True)
|
| 31 |
+
def setup_isolated_test_env(tmp_path, monkeypatch):
|
| 32 |
+
"""Enforces completely isolated local directories and SQLite database for each test."""
|
| 33 |
+
data_dir = str(tmp_path / "data")
|
| 34 |
+
upload_dir = str(tmp_path / "uploads")
|
| 35 |
+
static_dir = str(tmp_path / "static")
|
| 36 |
+
os.makedirs(data_dir, exist_ok=True)
|
| 37 |
+
os.makedirs(upload_dir, exist_ok=True)
|
| 38 |
+
os.makedirs(static_dir, exist_ok=True)
|
| 39 |
+
|
| 40 |
+
monkeypatch.setenv("DATABASE_URL", "")
|
| 41 |
+
monkeypatch.setenv("POSTGRES_URL", "")
|
| 42 |
+
monkeypatch.setenv("POSTGRESQL_URL", "")
|
| 43 |
+
monkeypatch.setenv("ATLAS_DATA_DIR", data_dir)
|
| 44 |
+
monkeypatch.setenv("ATLAS_UPLOAD_DIR", upload_dir)
|
| 45 |
+
|
| 46 |
+
yield
|
tests/test_client_isolation.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
+
import pytest
|
| 3 |
+
from fastapi.testclient import TestClient
|
| 4 |
+
|
| 5 |
+
from app import app
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@pytest.fixture
|
| 9 |
+
def client():
|
| 10 |
+
return TestClient(app)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def test_public_endpoints_accessible_without_client_id(client):
|
| 14 |
+
"""Verifies that public endpoints (/, /health, /docs, /openapi.json) require no X-Client-ID."""
|
| 15 |
+
r_root = client.get("/")
|
| 16 |
+
assert r_root.status_code == 200
|
| 17 |
+
assert "message" in r_root.json()
|
| 18 |
+
|
| 19 |
+
r_health = client.get("/health")
|
| 20 |
+
assert r_health.status_code == 200
|
| 21 |
+
assert r_health.json()["status"] == "ok"
|
| 22 |
+
|
| 23 |
+
r_docs = client.get("/docs")
|
| 24 |
+
assert r_docs.status_code == 200
|
| 25 |
+
|
| 26 |
+
r_openapi = client.get("/openapi.json")
|
| 27 |
+
assert r_openapi.status_code == 200
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def test_protected_endpoints_reject_missing_client_id(client):
|
| 31 |
+
"""Protected endpoints must return 400 if X-Client-ID is missing."""
|
| 32 |
+
r = client.get("/chats")
|
| 33 |
+
assert r.status_code == 400
|
| 34 |
+
assert "Missing required header: X-Client-ID" in r.json()["detail"]
|
| 35 |
+
|
| 36 |
+
r_docs = client.get("/documents")
|
| 37 |
+
assert r_docs.status_code == 400
|
| 38 |
+
|
| 39 |
+
r_tools = client.post("/tools/calculate-math", json={"expression": "1 + 1"})
|
| 40 |
+
assert r_tools.status_code == 400
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_protected_endpoints_reject_invalid_uuid(client):
|
| 44 |
+
"""Protected endpoints must return 400 if X-Client-ID is not a valid UUIDv4."""
|
| 45 |
+
bad_headers = {"X-Client-ID": "not-a-valid-uuid-string"}
|
| 46 |
+
r = client.get("/chats", headers=bad_headers)
|
| 47 |
+
assert r.status_code == 400
|
| 48 |
+
assert "valid UUID" in r.json()["detail"]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def test_chat_isolation_between_clients(client):
|
| 52 |
+
"""Client A and Client B must have complete data isolation for chats and messages."""
|
| 53 |
+
client_a_id = str(uuid.uuid4())
|
| 54 |
+
client_b_id = str(uuid.uuid4())
|
| 55 |
+
|
| 56 |
+
headers_a = {"X-Client-ID": client_a_id}
|
| 57 |
+
headers_b = {"X-Client-ID": client_b_id}
|
| 58 |
+
|
| 59 |
+
# 1. Client A creates a chat
|
| 60 |
+
create_res = client.post("/chat/new", headers=headers_a)
|
| 61 |
+
assert create_res.status_code == 200
|
| 62 |
+
chat_a_id = create_res.json()["chat_id"]
|
| 63 |
+
|
| 64 |
+
# 2. Client A lists chats -> finds chat_a_id
|
| 65 |
+
list_a = client.get("/chats", headers=headers_a).json()["chats"]
|
| 66 |
+
assert any(c["chat_id"] == chat_a_id for c in list_a)
|
| 67 |
+
|
| 68 |
+
# 3. Client B lists chats -> must NOT find chat_a_id
|
| 69 |
+
list_b = client.get("/chats", headers=headers_b).json()["chats"]
|
| 70 |
+
assert not any(c["chat_id"] == chat_a_id for c in list_b)
|
| 71 |
+
|
| 72 |
+
# 4. Client B attempts to fetch Client A's chat history -> 404
|
| 73 |
+
hist_b = client.get(f"/chat/{chat_a_id}/history", headers=headers_b)
|
| 74 |
+
assert hist_b.status_code == 404
|
| 75 |
+
|
| 76 |
+
# 5. Client B attempts to delete Client A's chat -> 404
|
| 77 |
+
del_res = client.delete(f"/chat/{chat_a_id}", headers=headers_b)
|
| 78 |
+
assert del_res.status_code == 404
|
| 79 |
+
|
| 80 |
+
# 6. Client A renames chat -> succeeds
|
| 81 |
+
rename_res = client.patch(f"/chat/{chat_a_id}/title", headers=headers_a, json={"title": "Client A Topic"})
|
| 82 |
+
assert rename_res.status_code == 200
|
| 83 |
+
|
| 84 |
+
# 7. Client B attempts to rename Client A's chat -> 404
|
| 85 |
+
rename_b = client.patch(f"/chat/{chat_a_id}/title", headers=headers_b, json={"title": "Hacked Title"})
|
| 86 |
+
assert rename_b.status_code == 404
|
| 87 |
+
|
| 88 |
+
# 8. Client A deletes chat -> succeeds
|
| 89 |
+
del_res_a = client.delete(f"/chat/{chat_a_id}", headers=headers_a)
|
| 90 |
+
assert del_res_a.status_code == 200
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def test_tools_endpoint_accessible_with_valid_client_id(client):
|
| 94 |
+
"""Tools endpoint succeeds with valid client_id."""
|
| 95 |
+
client_id = str(uuid.uuid4())
|
| 96 |
+
headers = {"X-Client-ID": client_id}
|
| 97 |
+
res = client.post("/tools/calculate-math", headers=headers, json={"expression": "10 * 5"})
|
| 98 |
+
assert res.status_code == 200
|
| 99 |
+
assert res.json()["result"] == 50
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def test_document_and_memory_isolation_between_clients(client):
|
| 103 |
+
"""Client A's documents, dossiers, and summaries must remain invisible to Client B."""
|
| 104 |
+
from core.dependencies import get_storage
|
| 105 |
+
storage = get_storage()
|
| 106 |
+
|
| 107 |
+
client_a = str(uuid.uuid4())
|
| 108 |
+
client_b = str(uuid.uuid4())
|
| 109 |
+
|
| 110 |
+
doc_a_id = f"doc_test_{uuid.uuid4().hex[:8]}"
|
| 111 |
+
|
| 112 |
+
# Seed document metadata for Client A directly into storage
|
| 113 |
+
storage.create_document(
|
| 114 |
+
doc_id=doc_a_id,
|
| 115 |
+
filename="client_a_contract.pdf",
|
| 116 |
+
content_hash="hash_a_12345",
|
| 117 |
+
file_path="/tmp/fake_contract.pdf",
|
| 118 |
+
owner_id=client_a,
|
| 119 |
+
)
|
| 120 |
+
storage.save_knowledge_dossier(
|
| 121 |
+
doc_id=doc_a_id,
|
| 122 |
+
filename="client_a_contract.pdf",
|
| 123 |
+
executive_summary="Confidential contract for Client A",
|
| 124 |
+
key_entities=["Client A Corp"],
|
| 125 |
+
key_findings=["Revenue target $10M"],
|
| 126 |
+
topics=["Confidential", "Finance"],
|
| 127 |
+
owner_id=client_a,
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
# Client A lists documents -> sees doc_a_id
|
| 131 |
+
headers_a = {"X-Client-ID": client_a}
|
| 132 |
+
docs_a = client.get("/documents", headers=headers_a).json()["documents"]
|
| 133 |
+
assert any(d["doc_id"] == doc_a_id for d in docs_a)
|
| 134 |
+
|
| 135 |
+
# Client B lists documents -> does NOT see doc_a_id
|
| 136 |
+
headers_b = {"X-Client-ID": client_b}
|
| 137 |
+
docs_b = client.get("/documents", headers=headers_b).json()["documents"]
|
| 138 |
+
assert not any(d["doc_id"] == doc_a_id for d in docs_b)
|
| 139 |
+
|
| 140 |
+
# Client B attempts to fetch Client A's document details -> 404
|
| 141 |
+
get_doc_b = client.get(f"/documents/{doc_a_id}", headers=headers_b)
|
| 142 |
+
assert get_doc_b.status_code == 404
|
| 143 |
+
|
| 144 |
+
# Client B attempts to fetch Client A's dossier -> 404
|
| 145 |
+
dossier_b = client.get(f"/memory/dossier/{doc_a_id}", headers=headers_b)
|
| 146 |
+
assert dossier_b.status_code == 404
|
| 147 |
+
|
| 148 |
+
# Client A fetches dossier -> 200
|
| 149 |
+
dossier_a = client.get(f"/memory/dossier/{doc_a_id}", headers=headers_a)
|
| 150 |
+
assert dossier_a.status_code == 200
|
| 151 |
+
assert dossier_a.json()["executive_summary"] == "Confidential contract for Client A"
|
| 152 |
+
|
| 153 |
+
# Client B attempts to delete Client A's document -> 404
|
| 154 |
+
del_b = client.delete(f"/documents/{doc_a_id}", headers=headers_b)
|
| 155 |
+
assert del_b.status_code == 404
|
| 156 |
+
|
| 157 |
+
# Client A deletes document -> 200
|
| 158 |
+
del_a = client.delete(f"/documents/{doc_a_id}", headers=headers_a)
|
| 159 |
+
assert del_a.status_code == 200
|
| 160 |
+
|
uv.lock
CHANGED
|
@@ -1101,6 +1101,15 @@ wheels = [
|
|
| 1101 |
{ url = "https://files.pythonhosted.org/packages/b3/55/ecca97ae19075f1fac62def77731e7f535e6c1fb8f92ff08160c5e6dade8/importlib_metadata-9.0.1-py3-none-any.whl", hash = "sha256:bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0", size = 27920, upload-time = "2026-08-28T15:30:33.433Z" },
|
| 1102 |
]
|
| 1103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
[[package]]
|
| 1105 |
name = "jaraco-classes"
|
| 1106 |
version = "3.4.0"
|
|
@@ -1763,6 +1772,12 @@ dependencies = [
|
|
| 1763 |
{ name = "youtube-transcript-api" },
|
| 1764 |
]
|
| 1765 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1766 |
[package.metadata]
|
| 1767 |
requires-dist = [
|
| 1768 |
{ name = "beautifulsoup4", specifier = ">=4.15.0" },
|
|
@@ -1783,6 +1798,12 @@ requires-dist = [
|
|
| 1783 |
{ name = "youtube-transcript-api", specifier = ">=1.2.4" },
|
| 1784 |
]
|
| 1785 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1786 |
[[package]]
|
| 1787 |
name = "pathable"
|
| 1788 |
version = "0.6.0"
|
|
@@ -1801,6 +1822,15 @@ wheels = [
|
|
| 1801 |
{ url = "https://files.pythonhosted.org/packages/58/51/5614d0537dd830ca64d95a2f254bd66bdf3ac61d41cffcccaec62121fcfd/platformdirs-4.13.4-py3-none-any.whl", hash = "sha256:502ce08e9d36cf90579ee44e1e764838842fa658e4975ef47344fd053011d2c8", size = 32782, upload-time = "2026-10-17T19:31:52.293Z" },
|
| 1802 |
]
|
| 1803 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1804 |
[[package]]
|
| 1805 |
name = "primp"
|
| 1806 |
version = "2.0.1"
|
|
@@ -2249,6 +2279,35 @@ wheels = [
|
|
| 2249 |
{ url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" },
|
| 2250 |
]
|
| 2251 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2252 |
[[package]]
|
| 2253 |
name = "python-dotenv"
|
| 2254 |
version = "1.2.2"
|
|
|
|
| 1101 |
{ url = "https://files.pythonhosted.org/packages/b3/55/ecca97ae19075f1fac62def77731e7f535e6c1fb8f92ff08160c5e6dade8/importlib_metadata-9.0.1-py3-none-any.whl", hash = "sha256:bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0", size = 27920, upload-time = "2026-08-28T15:30:33.433Z" },
|
| 1102 |
]
|
| 1103 |
|
| 1104 |
+
[[package]]
|
| 1105 |
+
name = "iniconfig"
|
| 1106 |
+
version = "2.3.1"
|
| 1107 |
+
source = { registry = "https://pypi.org/simple" }
|
| 1108 |
+
sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" }
|
| 1109 |
+
wheels = [
|
| 1110 |
+
{ url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" },
|
| 1111 |
+
]
|
| 1112 |
+
|
| 1113 |
[[package]]
|
| 1114 |
name = "jaraco-classes"
|
| 1115 |
version = "3.4.0"
|
|
|
|
| 1772 |
{ name = "youtube-transcript-api" },
|
| 1773 |
]
|
| 1774 |
|
| 1775 |
+
[package.dev-dependencies]
|
| 1776 |
+
dev = [
|
| 1777 |
+
{ name = "pytest" },
|
| 1778 |
+
{ name = "pytest-asyncio" },
|
| 1779 |
+
]
|
| 1780 |
+
|
| 1781 |
[package.metadata]
|
| 1782 |
requires-dist = [
|
| 1783 |
{ name = "beautifulsoup4", specifier = ">=4.15.0" },
|
|
|
|
| 1798 |
{ name = "youtube-transcript-api", specifier = ">=1.2.4" },
|
| 1799 |
]
|
| 1800 |
|
| 1801 |
+
[package.metadata.requires-dev]
|
| 1802 |
+
dev = [
|
| 1803 |
+
{ name = "pytest", specifier = ">=9.1.1" },
|
| 1804 |
+
{ name = "pytest-asyncio", specifier = ">=1.4.1" },
|
| 1805 |
+
]
|
| 1806 |
+
|
| 1807 |
[[package]]
|
| 1808 |
name = "pathable"
|
| 1809 |
version = "0.6.0"
|
|
|
|
| 1822 |
{ url = "https://files.pythonhosted.org/packages/58/51/5614d0537dd830ca64d95a2f254bd66bdf3ac61d41cffcccaec62121fcfd/platformdirs-4.13.4-py3-none-any.whl", hash = "sha256:502ce08e9d36cf90579ee44e1e764838842fa658e4975ef47344fd053011d2c8", size = 32782, upload-time = "2026-10-17T19:31:52.293Z" },
|
| 1823 |
]
|
| 1824 |
|
| 1825 |
+
[[package]]
|
| 1826 |
+
name = "pluggy"
|
| 1827 |
+
version = "1.7.0"
|
| 1828 |
+
source = { registry = "https://pypi.org/simple" }
|
| 1829 |
+
sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" }
|
| 1830 |
+
wheels = [
|
| 1831 |
+
{ url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" },
|
| 1832 |
+
]
|
| 1833 |
+
|
| 1834 |
[[package]]
|
| 1835 |
name = "primp"
|
| 1836 |
version = "2.0.1"
|
|
|
|
| 2279 |
{ url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" },
|
| 2280 |
]
|
| 2281 |
|
| 2282 |
+
[[package]]
|
| 2283 |
+
name = "pytest"
|
| 2284 |
+
version = "9.1.1"
|
| 2285 |
+
source = { registry = "https://pypi.org/simple" }
|
| 2286 |
+
dependencies = [
|
| 2287 |
+
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
| 2288 |
+
{ name = "iniconfig" },
|
| 2289 |
+
{ name = "packaging" },
|
| 2290 |
+
{ name = "pluggy" },
|
| 2291 |
+
{ name = "pygments" },
|
| 2292 |
+
]
|
| 2293 |
+
sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
|
| 2294 |
+
wheels = [
|
| 2295 |
+
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
| 2296 |
+
]
|
| 2297 |
+
|
| 2298 |
+
[[package]]
|
| 2299 |
+
name = "pytest-asyncio"
|
| 2300 |
+
version = "1.4.1"
|
| 2301 |
+
source = { registry = "https://pypi.org/simple" }
|
| 2302 |
+
dependencies = [
|
| 2303 |
+
{ name = "pytest" },
|
| 2304 |
+
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
| 2305 |
+
]
|
| 2306 |
+
sdist = { url = "https://files.pythonhosted.org/packages/f8/9a/887535e477847f5159377ab35303923644840b13c0ca03c0a467f84d4680/pytest_asyncio-1.4.1.tar.gz", hash = "sha256:e8266eb094e5ee9ffdb2068c9836f33e209ea77a970eb846933d416b5602669b", size = 145237, upload-time = "2026-10-25T21:05:37.135Z" }
|
| 2307 |
+
wheels = [
|
| 2308 |
+
{ url = "https://files.pythonhosted.org/packages/46/d7/cd9e2708afdbf059658adaf08bdb0792ca52489871051d6ce7d498b51277/pytest_asyncio-1.4.1-py3-none-any.whl", hash = "sha256:507afa78ec77c6661164ef1a2368bc7820383fd1a14b2a7f813de8b5a92949cc", size = 17061, upload-time = "2026-10-25T21:05:35.473Z" },
|
| 2309 |
+
]
|
| 2310 |
+
|
| 2311 |
[[package]]
|
| 2312 |
name = "python-dotenv"
|
| 2313 |
version = "1.2.2"
|