Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /avinash-mall_CognitiveMemoryLayer /src /storage /postgres.py
| """PostgreSQL memory store with pgvector.""" | |
| import hashlib | |
| from datetime import UTC, datetime | |
| from typing import Any, cast | |
| from uuid import UUID | |
| from sqlalchemy import and_, delete, func, or_, select, text, update | |
| from ..core.enums import MemorySource, MemoryStatus, MemoryType | |
| from ..core.schemas import EntityMention, MemoryRecord, MemoryRecordCreate, Provenance, Relation | |
| from ..utils.logging_config import get_logger | |
| from .base import MemoryStoreBase | |
| from .models import MemoryRecordModel | |
| from .utils import naive_utc as _naive_utc | |
| _logger = get_logger(__name__) | |
| _DATETIME_KEYS = frozenset( | |
| {"last_accessed_at", "timestamp", "written_at", "valid_from", "valid_to"} | |
| ) | |
| def _time_column(filters: dict[str, Any] | None): | |
| """The column ``since``/``until`` compare against. | |
| Defaults to ``timestamp`` — turn time — which is what every caller meant before | |
| ``event_date`` was queryable, and what temporal contiguity still means: it anchors a | |
| window on a seed's own ``record.timestamp`` to pull neighbouring turns, so resolving | |
| that window against event time would drop exactly the neighbours whose turn recalls | |
| a distant event. | |
| ``time_basis="event"`` — set by the planner, whose filters come from a *question* | |
| about when something happened — coalesces to event time where it is known. Coalesce | |
| rather than a plain compare because only 4.3% of stored records carry an | |
| ``event_date``; filtering on the bare column would hide the other 95.7%. | |
| """ | |
| if filters and filters.get("time_basis") == "event": | |
| return func.coalesce(MemoryRecordModel.event_date, MemoryRecordModel.timestamp) | |
| return MemoryRecordModel.timestamp | |
| def _parse_event_date(metadata: dict[str, Any] | None) -> datetime | None: | |
| """Read ``metadata['event_date']`` into a column value. | |
| The extractor writes ISO strings, date-only or full. Anything unparseable is | |
| dropped rather than raised on: a bad event_date must not fail a write, because the | |
| turn text is the thing worth keeping and the metadata copy survives regardless. | |
| """ | |
| if not metadata: | |
| return None | |
| raw = metadata.get("event_date") | |
| if isinstance(raw, datetime): | |
| return _naive_utc(raw) | |
| if not isinstance(raw, str) or not raw.strip(): | |
| return None | |
| try: | |
| return _naive_utc(datetime.fromisoformat(raw.strip())) | |
| except ValueError: | |
| return None | |
| def _unconsolidated_clause(): | |
| """Records the consolidation migrator has not already absorbed into a gist. | |
| ``consolidated`` lives in the metadata JSON, so this is a ``->>`` text compare | |
| rather than a column predicate. Absent key and explicit false both count as | |
| un-consolidated. | |
| """ | |
| marker = MemoryRecordModel.meta.op("->>")("consolidated") | |
| return or_(marker.is_(None), marker != "true") | |
| class PostgresMemoryStore(MemoryStoreBase): | |
| """PostgreSQL-based memory store with pgvector.""" | |
| def __init__(self, session_factory: Any) -> None: | |
| self.session_factory = session_factory | |
| async def upsert(self, record: MemoryRecordCreate) -> MemoryRecord: | |
| content_hash = self._hash_content(record.text, record.tenant_id) | |
| async with self.session_factory() as session: | |
| # Single SELECT with OR clause — avoids a second round-trip vs the old | |
| # sequential key-lookup + hash-lookup pattern (BUG-06 fix preserved). | |
| existing_record = None | |
| if record.key: | |
| existing_result = await session.execute( | |
| select(MemoryRecordModel) | |
| .where( | |
| and_( | |
| MemoryRecordModel.tenant_id == record.tenant_id, | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| or_( | |
| MemoryRecordModel.key == record.key, | |
| MemoryRecordModel.content_hash == content_hash, | |
| ), | |
| ) | |
| ) | |
| .limit(1) | |
| ) | |
| existing_record = existing_result.scalar_one_or_none() | |
| else: | |
| existing_hash = await session.execute( | |
| select(MemoryRecordModel).where( | |
| and_( | |
| MemoryRecordModel.tenant_id == record.tenant_id, | |
| MemoryRecordModel.content_hash == content_hash, | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| ) | |
| ) | |
| ) | |
| existing_record = existing_hash.scalar_one_or_none() | |
| if existing_record: | |
| # Update existing record | |
| existing_record.access_count += 1 | |
| existing_record.last_accessed_at = _naive_utc(datetime.now(UTC)) | |
| existing_record.confidence = max(existing_record.confidence, record.confidence) | |
| # If we found it by key but content changed, update content & hash | |
| if ( | |
| record.key | |
| and existing_record.key == record.key | |
| and existing_record.content_hash != content_hash | |
| ): | |
| existing_record.text = record.text | |
| existing_record.content_hash = content_hash | |
| existing_record.embedding = record.embedding | |
| # Update other fields that might have changed | |
| existing_record.meta = record.metadata | |
| await session.commit() | |
| return cast("MemoryRecord", self._to_schema(existing_record)) | |
| ts = _naive_utc(record.timestamp or datetime.now(UTC)) | |
| now_naive = _naive_utc(datetime.now(UTC)) | |
| model = MemoryRecordModel( | |
| tenant_id=record.tenant_id, | |
| agent_id=record.agent_id, | |
| context_tags=record.context_tags or [], | |
| source_session_id=record.source_session_id, | |
| namespace=record.namespace, | |
| type=record.type.value, | |
| text=record.text, | |
| key=record.key, | |
| embedding=record.embedding if record.embedding else None, | |
| entities=[e.model_dump() for e in record.entities], | |
| relations=[r.model_dump() for r in record.relations], | |
| meta=record.metadata, | |
| timestamp=ts, | |
| written_at=now_naive, | |
| event_date=_parse_event_date(record.metadata), | |
| valid_from=_naive_utc(record.valid_from), | |
| valid_to=_naive_utc(record.valid_to), | |
| confidence=record.confidence, | |
| importance=record.importance, | |
| decay_rate=record.decay_rate if record.decay_rate is not None else 0.01, | |
| provenance=record.provenance.model_dump(), | |
| content_hash=content_hash, | |
| ) | |
| session.add(model) | |
| await session.commit() | |
| return cast("MemoryRecord", self._to_schema(model)) | |
| async def get_by_id(self, record_id: UUID) -> MemoryRecord | None: | |
| async with self.session_factory() as session: | |
| r = await session.execute( | |
| select(MemoryRecordModel).where(MemoryRecordModel.id == record_id) | |
| ) | |
| model = r.scalar_one_or_none() | |
| return self._to_schema(model) if model else None | |
| async def get_by_ids_batch(self, record_ids: list[UUID]) -> list[MemoryRecord]: | |
| """Get multiple records by IDs in a single query.""" | |
| if not record_ids: | |
| return [] | |
| async with self.session_factory() as session: | |
| r = await session.execute( | |
| select(MemoryRecordModel).where(MemoryRecordModel.id.in_(record_ids)) | |
| ) | |
| models = r.scalars().all() | |
| return [rec for m in models if (rec := self._to_schema(m)) is not None] | |
| async def get_by_key( | |
| self, | |
| tenant_id: str, | |
| key: str, | |
| context_filter: list[str] | None = None, | |
| ) -> MemoryRecord | None: | |
| async with self.session_factory() as session: | |
| q = select(MemoryRecordModel).where( | |
| and_( | |
| MemoryRecordModel.tenant_id == tenant_id, | |
| MemoryRecordModel.key == key, | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| ) | |
| ) | |
| if context_filter: | |
| q = q.where(MemoryRecordModel.context_tags.overlap(context_filter)) | |
| r = await session.execute(q) | |
| model = r.scalar_one_or_none() | |
| return self._to_schema(model) if model else None | |
| async def delete(self, record_id: UUID, hard: bool = False) -> bool: | |
| async with self.session_factory() as session: | |
| if hard: | |
| r = await session.execute( | |
| delete(MemoryRecordModel).where(MemoryRecordModel.id == record_id) | |
| ) | |
| else: | |
| r = await session.execute( | |
| update(MemoryRecordModel) | |
| .where(MemoryRecordModel.id == record_id) | |
| .values(status=MemoryStatus.DELETED.value) | |
| ) | |
| await session.commit() | |
| return r.rowcount > 0 | |
| async def update( | |
| self, | |
| record_id: UUID, | |
| patch: dict[str, Any], | |
| increment_version: bool = True, | |
| ) -> MemoryRecord | None: | |
| async with self.session_factory() as session: | |
| r = await session.execute( | |
| select(MemoryRecordModel).where(MemoryRecordModel.id == record_id) | |
| ) | |
| model = r.scalar_one_or_none() | |
| if not model: | |
| return None | |
| key_allow = { | |
| "access_count", | |
| "last_accessed_at", | |
| "confidence", | |
| "importance", | |
| "status", | |
| "meta", | |
| "text", | |
| "embedding", | |
| "valid_to", | |
| "entities", | |
| "relations", | |
| "context_tags", | |
| "source_session_id", | |
| } | |
| for key, value in patch.items(): | |
| if key == "metadata": | |
| model.meta = value | |
| elif key in key_allow and hasattr(model, key): | |
| if key in _DATETIME_KEYS and isinstance(value, datetime): | |
| value = _naive_utc(value) | |
| setattr(model, key, value) | |
| if increment_version: | |
| model.version += 1 | |
| await session.commit() | |
| await session.refresh(model) | |
| return self._to_schema(model) | |
| async def vector_search( | |
| self, | |
| tenant_id: str, | |
| embedding: list[float], | |
| top_k: int = 10, | |
| context_filter: list[str] | None = None, | |
| filters: dict[str, Any] | None = None, | |
| min_similarity: float = -1.0, | |
| ) -> list[MemoryRecord]: | |
| async with self.session_factory() as session: | |
| # Phase 6.1: Set HNSW ef_search for this query transaction | |
| try: | |
| from ..core.config import get_settings | |
| settings = get_settings() | |
| if settings.features.hnsw_ef_search_tuning: | |
| ef_search = max(settings.retrieval.hnsw_ef_search, top_k) | |
| await session.execute(text(f"SET LOCAL hnsw.ef_search = {ef_search}")) | |
| except Exception as exc: | |
| _logger.debug("hnsw_ef_search_set_failed", error=str(exc)) | |
| base = and_( | |
| MemoryRecordModel.tenant_id == tenant_id, | |
| MemoryRecordModel.embedding.isnot(None), | |
| ) | |
| if context_filter: | |
| base = and_(base, MemoryRecordModel.context_tags.overlap(context_filter)) | |
| q = select( | |
| MemoryRecordModel, | |
| (1 - MemoryRecordModel.embedding.cosine_distance(embedding)).label("similarity"), | |
| ).where( | |
| and_( | |
| base, | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| ) | |
| ) | |
| if filters: | |
| if "type" in filters: | |
| t = filters["type"] | |
| if isinstance(t, list): | |
| q = q.where(MemoryRecordModel.type.in_(t)) | |
| else: | |
| q = q.where(MemoryRecordModel.type == t) | |
| if "source_session_id" in filters: | |
| source_session_id = filters["source_session_id"] | |
| if isinstance(source_session_id, list): | |
| q = q.where(MemoryRecordModel.source_session_id.in_(source_session_id)) | |
| else: | |
| q = q.where(MemoryRecordModel.source_session_id == source_session_id) | |
| time_col = _time_column(filters) | |
| if "since" in filters: | |
| since = _naive_utc(filters["since"]) | |
| if since is not None: | |
| q = q.where(time_col >= since) | |
| if "until" in filters: | |
| until = _naive_utc(filters["until"]) | |
| if until is not None: | |
| q = q.where(time_col <= until) | |
| if "min_confidence" in filters: | |
| q = q.where(MemoryRecordModel.confidence >= filters["min_confidence"]) | |
| if filters.get("exclude_expired"): | |
| now = datetime.now(UTC).replace(tzinfo=None) | |
| q = q.where( | |
| or_( | |
| MemoryRecordModel.valid_to.is_(None), | |
| MemoryRecordModel.valid_to >= now, | |
| ) | |
| ) | |
| q = q.order_by(MemoryRecordModel.embedding.cosine_distance(embedding)).limit(top_k) | |
| result = await session.execute(q) | |
| records = [] | |
| for row in result: | |
| model, similarity = row[0], row[1] | |
| # Cosine similarity is valid on [-1, 1], so the "no threshold" default | |
| # must be -1.0. It used to be 0.0, which silently dropped every | |
| # negatively-correlated row: invisible with embedding models whose vectors | |
| # are effectively non-negative (nomic), but with hashed/mock embeddings — | |
| # what CI runs — legitimate results just vanished from the result set. | |
| if similarity >= min_similarity: | |
| rec = self._to_schema(model) | |
| if rec is not None: | |
| rec.metadata["_similarity"] = similarity | |
| records.append(rec) | |
| return records | |
| async def scan( | |
| self, | |
| tenant_id: str, | |
| filters: dict[str, Any] | None = None, | |
| order_by: str | None = None, | |
| limit: int = 100, | |
| offset: int = 0, | |
| ) -> list[MemoryRecord]: | |
| async with self.session_factory() as session: | |
| q = select(MemoryRecordModel).where(MemoryRecordModel.tenant_id == tenant_id) | |
| if filters and "context_tags" in filters: | |
| q = q.where(MemoryRecordModel.context_tags.overlap(filters["context_tags"])) | |
| if filters and "source_session_id" in filters: | |
| source_session_id = filters["source_session_id"] | |
| if isinstance(source_session_id, list): | |
| q = q.where(MemoryRecordModel.source_session_id.in_(source_session_id)) | |
| else: | |
| q = q.where(MemoryRecordModel.source_session_id == source_session_id) | |
| if filters: | |
| if "status" in filters: | |
| q = q.where(MemoryRecordModel.status == filters["status"]) | |
| if "type" in filters: | |
| t = filters["type"] | |
| if isinstance(t, list): | |
| q = q.where(MemoryRecordModel.type.in_(t)) | |
| else: | |
| q = q.where(MemoryRecordModel.type == t) | |
| time_col = _time_column(filters) | |
| if "since" in filters: | |
| since = _naive_utc(filters["since"]) | |
| if since is not None: | |
| q = q.where(time_col >= since) | |
| if "until" in filters: | |
| until = _naive_utc(filters["until"]) | |
| if until is not None: | |
| q = q.where(time_col <= until) | |
| if filters.get("unconsolidated"): | |
| q = q.where(_unconsolidated_clause()) | |
| if order_by: | |
| col_name = order_by.lstrip("-") | |
| col = getattr(MemoryRecordModel, col_name, None) | |
| if col is not None: | |
| q = q.order_by(col.desc() if order_by.startswith("-") else col) | |
| q = q.offset(offset).limit(limit) | |
| r = await session.execute(q) | |
| return [rec for m in r.scalars().all() if (rec := self._to_schema(m)) is not None] | |
| async def scan_texts_for_gate( | |
| self, | |
| tenant_id: str, | |
| limit: int = 50, | |
| ) -> list[str]: | |
| """Fetch only text column for write-gate novelty check. | |
| Avoids transferring 768-dim embedding vectors (150KB per 50 records) | |
| that the gate discards immediately. | |
| Prospective-index rows are excluded. They are LLM-written paraphrases of a real | |
| memory, and with several per chunk they would quickly fill this 10-row window — | |
| the gate would then measure the next genuine turn's novelty against machine | |
| paraphrases of itself, which are high-similarity by construction, and start | |
| skipping real input. | |
| The ``or_`` is load-bearing: most episodic rows have ``key IS NULL``, and | |
| ``NULL NOT LIKE 'x'`` evaluates to NULL, which SQL treats as false. A bare | |
| ``notlike`` would therefore drop nearly every row and blind the gate entirely. | |
| """ | |
| async with self.session_factory() as session: | |
| q = ( | |
| select(MemoryRecordModel.text) | |
| .where( | |
| MemoryRecordModel.tenant_id == tenant_id, | |
| MemoryRecordModel.status == "active", | |
| or_( | |
| MemoryRecordModel.key.is_(None), | |
| ~MemoryRecordModel.key.like("prospective:%"), | |
| ), | |
| ) | |
| .order_by(MemoryRecordModel.timestamp.desc()) | |
| .limit(limit) | |
| ) | |
| r = await session.execute(q) | |
| return [row[0] for row in r.all()] | |
| async def count( | |
| self, | |
| tenant_id: str, | |
| filters: dict[str, Any] | None = None, | |
| ) -> int: | |
| async with self.session_factory() as session: | |
| q = select(func.count(MemoryRecordModel.id)).where( | |
| MemoryRecordModel.tenant_id == tenant_id | |
| ) | |
| if filters and "context_tags" in filters: | |
| q = q.where(MemoryRecordModel.context_tags.overlap(filters["context_tags"])) | |
| if filters and "source_session_id" in filters: | |
| source_session_id = filters["source_session_id"] | |
| if isinstance(source_session_id, list): | |
| q = q.where(MemoryRecordModel.source_session_id.in_(source_session_id)) | |
| else: | |
| q = q.where(MemoryRecordModel.source_session_id == source_session_id) | |
| if filters and "status" in filters: | |
| q = q.where(MemoryRecordModel.status == filters["status"]) | |
| if filters and "type" in filters: | |
| t = filters["type"] | |
| if isinstance(t, list): | |
| q = q.where(MemoryRecordModel.type.in_(t)) | |
| else: | |
| q = q.where(MemoryRecordModel.type == t) | |
| time_col = _time_column(filters) | |
| if filters and "since" in filters: | |
| since = _naive_utc(filters["since"]) | |
| if since is not None: | |
| q = q.where(time_col >= since) | |
| if filters and "until" in filters: | |
| until = _naive_utc(filters["until"]) | |
| if until is not None: | |
| q = q.where(time_col <= until) | |
| if filters and filters.get("unconsolidated"): | |
| q = q.where(_unconsolidated_clause()) | |
| r = await session.execute(q) | |
| return r.scalar() or 0 | |
| async def unconsolidated_counts_by_tenant( | |
| self, | |
| min_count: int, | |
| types: list[str] | None = None, | |
| ) -> list[tuple[str, int]]: | |
| """Tenants holding at least ``min_count`` un-consolidated active records. | |
| One GROUP BY rather than a count per tenant: the sweep runs on an interval | |
| against every tenant in the database, and N round-trips would scale with | |
| tenant count for a question a single aggregate answers. | |
| """ | |
| async with self.session_factory() as session: | |
| q = ( | |
| select(MemoryRecordModel.tenant_id, func.count(MemoryRecordModel.id).label("n")) | |
| .where( | |
| and_( | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| _unconsolidated_clause(), | |
| ) | |
| ) | |
| .group_by(MemoryRecordModel.tenant_id) | |
| .having(func.count(MemoryRecordModel.id) >= min_count) | |
| .order_by(func.count(MemoryRecordModel.id).desc()) | |
| ) | |
| if types: | |
| q = q.where(MemoryRecordModel.type.in_(types)) | |
| r = await session.execute(q) | |
| return [(row[0], int(row[1])) for row in r.all()] | |
| async def delete_by_filter( | |
| self, | |
| tenant_id: str, | |
| filters: dict[str, Any], | |
| ) -> int: | |
| """Delete records matching filters. Used for efficient ScratchPad clearing (BUG-07).""" | |
| async with self.session_factory() as session: | |
| q = delete(MemoryRecordModel).where(MemoryRecordModel.tenant_id == tenant_id) | |
| if "context_tags" in filters: | |
| q = q.where(MemoryRecordModel.context_tags.overlap(filters["context_tags"])) | |
| if "source_session_id" in filters: | |
| source_session_id = filters["source_session_id"] | |
| if isinstance(source_session_id, list): | |
| q = q.where(MemoryRecordModel.source_session_id.in_(source_session_id)) | |
| else: | |
| q = q.where(MemoryRecordModel.source_session_id == source_session_id) | |
| if "status" in filters: | |
| q = q.where(MemoryRecordModel.status == filters["status"]) | |
| if "type" in filters: | |
| t = filters["type"] | |
| if isinstance(t, list): | |
| q = q.where(MemoryRecordModel.type.in_(t)) | |
| else: | |
| q = q.where(MemoryRecordModel.type == t) | |
| has_specific_filter = any( | |
| k in filters for k in ("context_tags", "source_session_id", "status", "type") | |
| ) | |
| if not has_specific_filter: | |
| _logger.warning( | |
| "delete_by_filter_blocked_no_specific_filter", | |
| extra={"tenant_id": tenant_id, "filters": list(filters.keys())}, | |
| ) | |
| return 0 | |
| r = await session.execute(q) | |
| await session.commit() | |
| return r.rowcount | |
| async def count_references_to(self, record_id: UUID) -> int: | |
| """ | |
| Count how many other memory records reference this one (supersedes_id or evidence_refs). | |
| Used to block delete when dependencies exist. | |
| """ | |
| async with self.session_factory() as session: | |
| # supersedes_id or metadata.evidence_refs array contains record_id | |
| refs_contains = text( | |
| "(metadata::jsonb)->'evidence_refs' IS NOT NULL AND EXISTS (" | |
| "SELECT 1 FROM jsonb_array_elements_text((metadata::jsonb)->'evidence_refs') AS elem " | |
| "WHERE elem = :rid)" | |
| ).bindparams(rid=str(record_id)) | |
| q = select(func.count(MemoryRecordModel.id)).where( | |
| or_( | |
| MemoryRecordModel.supersedes_id == record_id, | |
| refs_contains, | |
| ) | |
| ) | |
| r = await session.execute(q) | |
| return r.scalar() or 0 | |
| async def increment_access_counts( | |
| self, record_ids: list[UUID], last_accessed_at: datetime | None = None | |
| ) -> None: | |
| """Atomic increment of access_count for given records (BUG-02: avoid lost update).""" | |
| if not record_ids: | |
| return | |
| now = _naive_utc(last_accessed_at or datetime.now(UTC)) | |
| async with self.session_factory() as session: | |
| await session.execute( | |
| update(MemoryRecordModel) | |
| .where(MemoryRecordModel.id.in_(record_ids)) | |
| .values( | |
| access_count=MemoryRecordModel.access_count + 1, | |
| last_accessed_at=now, | |
| ) | |
| ) | |
| await session.commit() | |
| async def deactivate_constraints_by_key( | |
| self, | |
| tenant_id: str, | |
| constraint_key: str, | |
| superseded_by_key: str | None = None, | |
| ) -> int: | |
| """Deactivate episodic CONSTRAINT records with the given fact key. | |
| When ``superseded_by_key`` is provided, append lightweight lineage | |
| metadata so retrieval and audits can trace which newer constraint | |
| superseded the old one. | |
| """ | |
| async with self.session_factory() as session: | |
| result = await session.execute( | |
| select(MemoryRecordModel).where( | |
| and_( | |
| MemoryRecordModel.tenant_id == tenant_id, | |
| MemoryRecordModel.type == MemoryType.CONSTRAINT.value, | |
| MemoryRecordModel.key == constraint_key, | |
| MemoryRecordModel.status == MemoryStatus.ACTIVE.value, | |
| ) | |
| ) | |
| ) | |
| rows = result.scalars().all() | |
| if not rows: | |
| return 0 | |
| now = _naive_utc(datetime.now(UTC)) | |
| assert now is not None | |
| for row in rows: | |
| row.status = MemoryStatus.SILENT.value | |
| row.valid_to = now | |
| if superseded_by_key: | |
| meta = dict(getattr(row, "meta", {}) or {}) | |
| lineage = meta.get("supersession_lineage", []) | |
| if not isinstance(lineage, list): | |
| lineage = [] | |
| lineage.append( | |
| { | |
| "superseded_by_key": superseded_by_key, | |
| "superseded_at": now.isoformat(), | |
| } | |
| ) | |
| meta["supersession_lineage"] = lineage[-5:] | |
| row.meta = meta | |
| await session.commit() | |
| return len(rows) | |
| def _hash_content(self, text: str, tenant_id: str) -> str: | |
| content = f"{tenant_id}:{text.lower().strip()}" | |
| return hashlib.sha256(content.encode()).hexdigest() | |
| def _to_schema(self, model: MemoryRecordModel | None) -> MemoryRecord | None: | |
| if model is None: | |
| return None | |
| try: | |
| mem_type = MemoryType(cast("str", model.type)) | |
| except ValueError: | |
| _logger.warning( | |
| "unknown_memory_type_in_db record_id=%s value=%s", | |
| model.id, | |
| model.type, | |
| ) | |
| mem_type = MemoryType.EPISODIC_EVENT | |
| try: | |
| status = MemoryStatus(cast("str", model.status)) | |
| except ValueError: | |
| _logger.warning( | |
| "unknown_memory_status_in_db record_id=%s value=%s", | |
| model.id, | |
| model.status, | |
| ) | |
| status = MemoryStatus.ACTIVE | |
| try: | |
| provenance = Provenance(**(cast("dict", model.provenance) or {})) | |
| except (TypeError, ValueError): | |
| provenance = Provenance(source=MemorySource.AGENT_INFERRED) | |
| context_tags = getattr(model, "context_tags", None) or [] | |
| source_session_id = getattr(model, "source_session_id", None) | |
| return MemoryRecord( | |
| id=cast("UUID", model.id), | |
| tenant_id=cast("str", model.tenant_id), | |
| context_tags=list(context_tags), | |
| source_session_id=source_session_id, | |
| agent_id=cast("str | None", model.agent_id), | |
| namespace=getattr(model, "namespace", None), | |
| type=mem_type, | |
| text=cast("str", model.text), | |
| key=cast("str | None", model.key), | |
| embedding=list(model.embedding) if model.embedding is not None else None, | |
| entities=[EntityMention(**e) for e in (cast("list", model.entities) or [])], | |
| relations=[Relation(**r) for r in (cast("list", model.relations) or [])], | |
| metadata=cast("dict", model.meta) or {}, | |
| timestamp=cast("datetime", model.timestamp), | |
| written_at=cast("datetime", model.written_at), | |
| valid_from=cast("datetime | None", model.valid_from), | |
| valid_to=cast("datetime | None", model.valid_to), | |
| confidence=cast("float", model.confidence), | |
| importance=cast("float", model.importance), | |
| access_count=cast("int", model.access_count), | |
| last_accessed_at=cast("datetime | None", model.last_accessed_at), | |
| decay_rate=cast("float", model.decay_rate), | |
| status=status, | |
| labile=cast("bool", model.labile), | |
| provenance=provenance, | |
| version=cast("int", model.version), | |
| supersedes_id=cast("UUID | None", model.supersedes_id), | |
| content_hash=cast("str | None", model.content_hash), | |
| ) | |
| # ── Phase 4.1: Bulk dependency counts ────────────────────────── | |
| async def bulk_dependency_counts( | |
| self, | |
| tenant_id: str, | |
| memory_ids: list[str], | |
| ) -> dict[str, int]: | |
| """Count references to each memory ID in a single DB query. | |
| Counts two reference types: | |
| 1. ``supersedes_id`` — direct version chains. | |
| 2. ``metadata.evidence_refs`` — JSON array back-references. | |
| Returns ``{memory_id: reference_count}``. | |
| """ | |
| if not memory_ids: | |
| return {} | |
| query_str = """ | |
| WITH target_ids AS ( | |
| SELECT unnest(:ids ::text[]) AS target_id | |
| ), | |
| supersede_counts AS ( | |
| SELECT | |
| CAST(supersedes_id AS text) AS target_id, | |
| COUNT(*) AS cnt | |
| FROM memory_records | |
| WHERE tenant_id = :tenant_id | |
| AND supersedes_id IS NOT NULL | |
| AND CAST(supersedes_id AS text) = ANY(:ids) | |
| GROUP BY supersedes_id | |
| ), | |
| evidence_counts AS ( | |
| SELECT | |
| ref.value #>> '{}' AS target_id, | |
| COUNT(*) AS cnt | |
| FROM memory_records, | |
| jsonb_array_elements( | |
| COALESCE((metadata::jsonb) -> 'evidence_refs', '[]'::jsonb) | |
| ) AS ref(value) | |
| WHERE tenant_id = :tenant_id | |
| AND ref.value #>> '{}' = ANY(:ids) | |
| GROUP BY ref.value #>> '{}' | |
| ) | |
| SELECT | |
| t.target_id, | |
| COALESCE(s.cnt, 0) + COALESCE(e.cnt, 0) AS total_refs | |
| FROM target_ids t | |
| LEFT JOIN supersede_counts s ON t.target_id = s.target_id | |
| LEFT JOIN evidence_counts e ON t.target_id = e.target_id | |
| """ | |
| async with self.session_factory() as session: | |
| result = await session.execute( | |
| text(query_str), | |
| {"tenant_id": tenant_id, "ids": memory_ids}, | |
| ) | |
| counts: dict[str, int] = {} | |
| for row in result: | |
| counts[row.target_id] = int(row.total_refs or 0) | |
| # Ensure every requested ID has an entry | |
| for mid in memory_ids: | |
| counts.setdefault(mid, 0) | |
| return counts | |
Xet Storage Details
- Size:
- 32.5 kB
- Xet hash:
- 7ea5f31a701b739db9156ec15c9e15d51296939fd575ff008f0afa68906ec762
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.