Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /avinash-mall_CognitiveMemoryLayer /src /forgetting /interference.py
| """Interference detection between memories (duplicates, overlap).""" | |
| from dataclasses import dataclass | |
| from ..core.schemas import MemoryRecord | |
| from ..utils.similarity import cosine_similarity, jaccard | |
| class InterferenceResult: | |
| """Result of interference detection.""" | |
| memory_id: str | |
| interfering_memory_id: str | |
| similarity: float | |
| interference_type: str # "duplicate", "conflicting", "overlapping" | |
| recommendation: str # "merge", "keep_newer", "keep_higher_confidence" | |
| keep_id: str | None = None # id of record to keep when deleting the other | |
| class InterferenceDetector: | |
| """ | |
| Detects interference between memories (duplicates, overlapping content). | |
| """ | |
| def __init__( | |
| self, | |
| embedding_client: object | None = None, | |
| similarity_threshold: float = 0.9, | |
| conflict_threshold: float = 0.7, | |
| ) -> None: | |
| self.embeddings = embedding_client | |
| self.similarity_threshold = similarity_threshold | |
| self.conflict_threshold = conflict_threshold | |
| def detect_duplicates( | |
| self, | |
| records: list[MemoryRecord], | |
| ) -> list[InterferenceResult]: | |
| """Detect near-duplicate memories using embeddings.""" | |
| results: list[InterferenceResult] = [] | |
| with_embedding = [(r, r.embedding) for r in records if r.embedding] | |
| ids = [str(r.id) for r, _ in with_embedding] | |
| vecs = {str(r.id): emb for r, emb in with_embedding} | |
| for i, id1 in enumerate(ids): | |
| for id2 in ids[i + 1 :]: | |
| sim = cosine_similarity(vecs[id1], vecs[id2]) | |
| if sim >= self.similarity_threshold: | |
| r1 = next(r for r in records if str(r.id) == id1) | |
| r2 = next(r for r in records if str(r.id) == id2) | |
| rec = self._recommend_resolution(r1, r2) | |
| keep_id = self._resolve_keep_id(r1, r2, rec) | |
| results.append( | |
| InterferenceResult( | |
| memory_id=id1, | |
| interfering_memory_id=id2, | |
| similarity=sim, | |
| interference_type="duplicate", | |
| recommendation=rec, | |
| keep_id=keep_id, | |
| ) | |
| ) | |
| return results | |
| def detect_overlapping( | |
| self, | |
| records: list[MemoryRecord], | |
| text_overlap_threshold: float = 0.7, | |
| ) -> list[InterferenceResult]: | |
| """Detect memories with significant text overlap.""" | |
| results: list[InterferenceResult] = [] | |
| for i, r1 in enumerate(records): | |
| for r2 in records[i + 1 :]: | |
| overlap = jaccard(r1.text, r2.text) | |
| if overlap >= text_overlap_threshold: | |
| rec = self._recommend_resolution(r1, r2) | |
| keep_id = self._resolve_keep_id(r1, r2, rec) | |
| results.append( | |
| InterferenceResult( | |
| memory_id=str(r1.id), | |
| interfering_memory_id=str(r2.id), | |
| similarity=overlap, | |
| interference_type="overlapping", | |
| recommendation=rec, | |
| keep_id=keep_id, | |
| ) | |
| ) | |
| return results | |
| def _recommend_resolution( | |
| self, | |
| r1: MemoryRecord, | |
| r2: MemoryRecord, | |
| ) -> str: | |
| """Recommend how to resolve interference.""" | |
| if abs(r1.confidence - r2.confidence) > 0.2: | |
| return "keep_higher_confidence" | |
| if r1.timestamp != r2.timestamp: | |
| return "keep_newer" | |
| return "merge" | |
| def _resolve_keep_id( | |
| self, r1: MemoryRecord, r2: MemoryRecord, recommendation: str | |
| ) -> str | None: | |
| """Return the id of the record to keep; the other will be deleted.""" | |
| if recommendation == "merge": | |
| return None | |
| if recommendation == "keep_higher_confidence": | |
| return str(r1.id) if r1.confidence >= r2.confidence else str(r2.id) | |
| if recommendation == "keep_newer": | |
| return ( | |
| str(r1.id) | |
| if (r1.timestamp or r1.written_at) >= (r2.timestamp or r2.written_at) | |
| else str(r2.id) | |
| ) | |
| return None | |
Xet Storage Details
- Size:
- 4.4 kB
- Xet hash:
- 288f026b01167678b196eb3974056c65d0bfc7b0109285e9560b42f5c28806fc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.