Fuugi-API / app /memory.py
2Btainment's picture
Upgrade Fuugi chat, search and anonymous research
45b7965 verified
Raw History Blame Contribute Delete
10.2 kB
from __future__ import annotations
import asyncio
import hashlib
import hmac
import os
import re
import sqlite3
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
MEMORY_COMMAND = re.compile(
r"^\s*(?:merke\s+dir|remember)\s*:\s*(.+?)\s*$", re.IGNORECASE | re.DOTALL
)
SENSITIVE_MEMORY = re.compile(
r"\b(?:passwort|password|api[-_ ]?key|secret|token|cvv|kreditkarten?|credit\s*card|iban|pin)\b",
re.IGNORECASE,
)
class InvalidMemorySession(ValueError):
pass
class MemoryConsentRequired(ValueError):
pass
class MemoryLimitReached(ValueError):
pass
class SensitiveMemoryRejected(ValueError):
pass
@dataclass(frozen=True, slots=True)
class MemoryRecord:
id: str
content: str
created_at: int
expires_at: int
def extract_explicit_memory(message: str) -> str | None:
match = MEMORY_COMMAND.fullmatch(message)
return match.group(1).strip() if match else None
class MemoryStore:
def __init__(
self,
database_path: str,
pepper: str,
retention_days: int = 90,
max_items: int = 50,
max_item_characters: int = 500,
max_context_characters: int = 4_000,
) -> None:
if len(pepper) < 32:
raise ValueError("Memory session pepper must contain at least 32 characters.")
self.database_path = Path(database_path).expanduser().resolve()
self.pepper = pepper.encode("utf-8")
self.retention_seconds = retention_days * 86_400
self.max_items = max_items
self.max_item_characters = max_item_characters
self.max_context_characters = max_context_characters
self.database_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
self._initialize()
os.chmod(self.database_path, 0o600)
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.database_path, timeout=5)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=5000")
connection.execute("PRAGMA secure_delete=ON")
return connection
def _initialize(self) -> None:
with self._connect() as connection:
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS memory_sessions (
session_hash TEXT PRIMARY KEY,
consent_version TEXT NOT NULL,
consented_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS memory_nodes (
id TEXT PRIMARY KEY,
session_hash TEXT NOT NULL
REFERENCES memory_sessions(session_hash) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind = 'explicit'),
content TEXT NOT NULL,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
UNIQUE(session_hash, content)
);
CREATE INDEX IF NOT EXISTS memory_nodes_owner_created
ON memory_nodes(session_hash, created_at DESC);
"""
)
def _digest(self, session_token: str) -> str:
try:
parsed = uuid.UUID(session_token)
except (ValueError, AttributeError) as error:
raise InvalidMemorySession("Invalid memory session token.") from error
if parsed.version != 4 or str(parsed) != session_token.lower():
raise InvalidMemorySession("Invalid memory session token.")
return hmac.new(self.pepper, parsed.bytes, hashlib.sha256).hexdigest()
@staticmethod
def _purge(connection: sqlite3.Connection, now: int) -> None:
connection.execute("DELETE FROM memory_nodes WHERE expires_at <= ?", (now,))
connection.execute("DELETE FROM memory_sessions WHERE expires_at <= ?", (now,))
async def set_consent(self, session_token: str, enabled: bool) -> bool:
return await asyncio.to_thread(self._set_consent, session_token, enabled)
def _set_consent(self, session_token: str, enabled: bool) -> bool:
session_hash = self._digest(session_token)
now = int(time.time())
with self._connect() as connection:
self._purge(connection, now)
if not enabled:
connection.execute(
"DELETE FROM memory_sessions WHERE session_hash = ?", (session_hash,)
)
return False
connection.execute(
"""
INSERT INTO memory_sessions(session_hash, consent_version, consented_at, expires_at)
VALUES (?, 'explicit-v1', ?, ?)
ON CONFLICT(session_hash) DO UPDATE SET
consent_version = excluded.consent_version,
consented_at = excluded.consented_at,
expires_at = excluded.expires_at
""",
(session_hash, now, now + self.retention_seconds),
)
return True
async def list(self, session_token: str) -> tuple[bool, list[MemoryRecord]]:
return await asyncio.to_thread(self._list, session_token)
def _list(self, session_token: str) -> tuple[bool, list[MemoryRecord]]:
session_hash = self._digest(session_token)
now = int(time.time())
with self._connect() as connection:
self._purge(connection, now)
consent = connection.execute(
"SELECT 1 FROM memory_sessions WHERE session_hash = ?", (session_hash,)
).fetchone()
if consent is None:
return False, []
rows = connection.execute(
"""
SELECT id, content, created_at, expires_at
FROM memory_nodes
WHERE session_hash = ? AND expires_at > ?
ORDER BY created_at DESC, id DESC
""",
(session_hash, now),
).fetchall()
return True, [MemoryRecord(**dict(row)) for row in rows]
async def context(self, session_token: str) -> list[str]:
enabled, records = await self.list(session_token)
if not enabled:
return []
selected: list[str] = []
size = 0
for record in records:
next_size = size + len(record.content)
if next_size > self.max_context_characters:
break
selected.append(record.content)
size = next_size
return selected
async def add(self, session_token: str, content: str) -> MemoryRecord:
return await asyncio.to_thread(self._add, session_token, content)
def _add(self, session_token: str, content: str) -> MemoryRecord:
cleaned = " ".join(content.split())
if not cleaned or len(cleaned) > self.max_item_characters:
raise MemoryLimitReached("Memory item is empty or too long.")
if SENSITIVE_MEMORY.search(cleaned):
raise SensitiveMemoryRejected("Sensitive memory cannot be stored.")
session_hash = self._digest(session_token)
now = int(time.time())
expires_at = now + self.retention_seconds
with self._connect() as connection:
self._purge(connection, now)
consent = connection.execute(
"SELECT 1 FROM memory_sessions WHERE session_hash = ?", (session_hash,)
).fetchone()
if consent is None:
raise MemoryConsentRequired("Memory consent is required.")
existing = connection.execute(
"SELECT id, content, created_at FROM memory_nodes "
"WHERE session_hash = ? AND content = ?",
(session_hash, cleaned),
).fetchone()
if existing is not None:
connection.execute(
"UPDATE memory_nodes SET expires_at = ? WHERE id = ?",
(expires_at, existing["id"]),
)
return MemoryRecord(
id=existing["id"],
content=existing["content"],
created_at=existing["created_at"],
expires_at=expires_at,
)
count = connection.execute(
"SELECT COUNT(*) FROM memory_nodes WHERE session_hash = ?", (session_hash,)
).fetchone()[0]
if count >= self.max_items:
raise MemoryLimitReached("Memory item limit reached.")
memory_id = str(uuid.uuid4())
connection.execute(
"""
INSERT INTO memory_nodes(id, session_hash, kind, content, created_at, expires_at)
VALUES (?, ?, 'explicit', ?, ?, ?)
""",
(memory_id, session_hash, cleaned, now, expires_at),
)
return MemoryRecord(memory_id, cleaned, now, expires_at)
async def delete(self, session_token: str, memory_id: str) -> bool:
return await asyncio.to_thread(self._delete, session_token, memory_id)
def _delete(self, session_token: str, memory_id: str) -> bool:
session_hash = self._digest(session_token)
try:
normalized_id = str(uuid.UUID(memory_id))
except ValueError as error:
raise InvalidMemorySession("Invalid memory id.") from error
with self._connect() as connection:
cursor = connection.execute(
"DELETE FROM memory_nodes WHERE id = ? AND session_hash = ?",
(normalized_id, session_hash),
)
return cursor.rowcount > 0
async def clear(self, session_token: str) -> int:
return await asyncio.to_thread(self._clear, session_token)
def _clear(self, session_token: str) -> int:
session_hash = self._digest(session_token)
with self._connect() as connection:
cursor = connection.execute(
"DELETE FROM memory_nodes WHERE session_hash = ?", (session_hash,)
)
return cursor.rowcount