Buckets:

download
raw
4.4 kB
"""Interference detection between memories (duplicates, overlap)."""
from dataclasses import dataclass
from ..core.schemas import MemoryRecord
from ..utils.similarity import cosine_similarity, jaccard
@dataclass
class InterferenceResult:
"""Result of interference detection."""
memory_id: str
interfering_memory_id: str
similarity: float
interference_type: str # "duplicate", "conflicting", "overlapping"
recommendation: str # "merge", "keep_newer", "keep_higher_confidence"
keep_id: str | None = None # id of record to keep when deleting the other
class InterferenceDetector:
"""
Detects interference between memories (duplicates, overlapping content).
"""
def __init__(
self,
embedding_client: object | None = None,
similarity_threshold: float = 0.9,
conflict_threshold: float = 0.7,
) -> None:
self.embeddings = embedding_client
self.similarity_threshold = similarity_threshold
self.conflict_threshold = conflict_threshold
def detect_duplicates(
self,
records: list[MemoryRecord],
) -> list[InterferenceResult]:
"""Detect near-duplicate memories using embeddings."""
results: list[InterferenceResult] = []
with_embedding = [(r, r.embedding) for r in records if r.embedding]
ids = [str(r.id) for r, _ in with_embedding]
vecs = {str(r.id): emb for r, emb in with_embedding}
for i, id1 in enumerate(ids):
for id2 in ids[i + 1 :]:
sim = cosine_similarity(vecs[id1], vecs[id2])
if sim >= self.similarity_threshold:
r1 = next(r for r in records if str(r.id) == id1)
r2 = next(r for r in records if str(r.id) == id2)
rec = self._recommend_resolution(r1, r2)
keep_id = self._resolve_keep_id(r1, r2, rec)
results.append(
InterferenceResult(
memory_id=id1,
interfering_memory_id=id2,
similarity=sim,
interference_type="duplicate",
recommendation=rec,
keep_id=keep_id,
)
)
return results
def detect_overlapping(
self,
records: list[MemoryRecord],
text_overlap_threshold: float = 0.7,
) -> list[InterferenceResult]:
"""Detect memories with significant text overlap."""
results: list[InterferenceResult] = []
for i, r1 in enumerate(records):
for r2 in records[i + 1 :]:
overlap = jaccard(r1.text, r2.text)
if overlap >= text_overlap_threshold:
rec = self._recommend_resolution(r1, r2)
keep_id = self._resolve_keep_id(r1, r2, rec)
results.append(
InterferenceResult(
memory_id=str(r1.id),
interfering_memory_id=str(r2.id),
similarity=overlap,
interference_type="overlapping",
recommendation=rec,
keep_id=keep_id,
)
)
return results
def _recommend_resolution(
self,
r1: MemoryRecord,
r2: MemoryRecord,
) -> str:
"""Recommend how to resolve interference."""
if abs(r1.confidence - r2.confidence) > 0.2:
return "keep_higher_confidence"
if r1.timestamp != r2.timestamp:
return "keep_newer"
return "merge"
def _resolve_keep_id(
self, r1: MemoryRecord, r2: MemoryRecord, recommendation: str
) -> str | None:
"""Return the id of the record to keep; the other will be deleted."""
if recommendation == "merge":
return None
if recommendation == "keep_higher_confidence":
return str(r1.id) if r1.confidence >= r2.confidence else str(r2.id)
if recommendation == "keep_newer":
return (
str(r1.id)
if (r1.timestamp or r1.written_at) >= (r2.timestamp or r2.written_at)
else str(r2.id)
)
return None

Xet Storage Details

Size:
4.4 kB
·
Xet hash:
288f026b01167678b196eb3974056c65d0bfc7b0109285e9560b42f5c28806fc

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.