Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /DansiDanutz_MyWork-AI /tools /brain_semantic.py
| #!/usr/bin/env python3 | |
| """ | |
| Brain Semantic Search — Phase 9: Brain Intelligence | |
| ==================================================== | |
| Adds embedding-based semantic search, deduplication, and quality scoring | |
| to the Brain knowledge vault. | |
| Uses TF-IDF + cosine similarity for zero-dependency semantic search. | |
| No API keys needed — runs fully offline. | |
| Usage: | |
| python brain_semantic.py search <query> # Semantic search | |
| python brain_semantic.py dedupe [--dry-run] # Find duplicates | |
| python brain_semantic.py score # Quality score all entries | |
| python brain_semantic.py provenance <id> # Show entry provenance | |
| python brain_semantic.py reindex # Rebuild search index | |
| """ | |
| import json | |
| import math | |
| import os | |
| import re | |
| import sys | |
| import time | |
| from collections import Counter, defaultdict | |
| from dataclasses import dataclass, field | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Tuple | |
| # ─── Configuration ─────────────────────────────────────────────── | |
| try: | |
| from config import get_mywork_root | |
| except ImportError: | |
| def get_mywork_root() -> Path: | |
| if env_root := os.environ.get("MYWORK_ROOT"): | |
| return Path(env_root) | |
| script_dir = Path(__file__).resolve().parent | |
| return script_dir.parent if script_dir.name == "tools" else Path.home() / "MyWork" | |
| MYWORK_ROOT = get_mywork_root().resolve() | |
| BRAIN_JSON = MYWORK_ROOT / ".planning" / "brain_data.json" | |
| INDEX_FILE = MYWORK_ROOT / ".planning" / "brain_index.json" | |
| PROVENANCE_FILE = MYWORK_ROOT / ".planning" / "brain_provenance.json" | |
| # ─── Stop words (common English) ──────────────────────────────── | |
| STOP_WORDS = frozenset( | |
| "a an the is are was were be been being have has had do does did " | |
| "will would shall should can could may might must to of in for on " | |
| "with at by from as into through during before after above below " | |
| "between out off over under again further then once here there when " | |
| "where why how all both each few more most other some such no nor " | |
| "not only own same so than too very it its and but or if that this " | |
| "these those which what who whom".split() | |
| ) | |
| # ─── TF-IDF Engine ────────────────────────────────────────────── | |
| def tokenize(text: str) -> List[str]: | |
| """Tokenize text into lowercase words, removing stop words.""" | |
| words = re.findall(r'[a-z0-9]+', text.lower()) | |
| return [w for w in words if w not in STOP_WORDS and len(w) > 1] | |
| def compute_tf(tokens: List[str]) -> Dict[str, float]: | |
| """Term frequency (normalized).""" | |
| counts = Counter(tokens) | |
| total = len(tokens) or 1 | |
| return {word: count / total for word, count in counts.items()} | |
| def compute_idf(documents: List[List[str]]) -> Dict[str, float]: | |
| """Inverse document frequency.""" | |
| n_docs = len(documents) or 1 | |
| df = Counter() | |
| for doc in documents: | |
| df.update(set(doc)) | |
| return {word: math.log(n_docs / (count + 1)) + 1 for word, count in df.items()} | |
| def tfidf_vector(tf: Dict[str, float], idf: Dict[str, float]) -> Dict[str, float]: | |
| """TF-IDF vector for a document.""" | |
| return {word: tf_val * idf.get(word, 1.0) for word, tf_val in tf.items()} | |
| def cosine_similarity(v1: Dict[str, float], v2: Dict[str, float]) -> float: | |
| """Cosine similarity between two sparse vectors.""" | |
| common = set(v1.keys()) & set(v2.keys()) | |
| if not common: | |
| return 0.0 | |
| dot = sum(v1[k] * v2[k] for k in common) | |
| mag1 = math.sqrt(sum(v ** 2 for v in v1.values())) | |
| mag2 = math.sqrt(sum(v ** 2 for v in v2.values())) | |
| if mag1 == 0 or mag2 == 0: | |
| return 0.0 | |
| return dot / (mag1 * mag2) | |
| # ─── Brain Loader ──────────────────────────────────────────────── | |
| def load_brain_entries() -> List[dict]: | |
| """Load all brain entries from JSON.""" | |
| if not BRAIN_JSON.exists(): | |
| return [] | |
| try: | |
| data = json.loads(BRAIN_JSON.read_text()) | |
| if isinstance(data, list): | |
| return data | |
| if isinstance(data, dict) and "entries" in data: | |
| return data["entries"] | |
| return [] | |
| except (json.JSONDecodeError, KeyError): | |
| return [] | |
| def entry_text(entry: dict) -> str: | |
| """Combine entry fields into searchable text.""" | |
| parts = [ | |
| entry.get("content", ""), | |
| entry.get("context", ""), | |
| entry.get("type", ""), | |
| " ".join(entry.get("tags", [])), | |
| ] | |
| return " ".join(p for p in parts if p) | |
| # ─── Search Index ──────────────────────────────────────────────── | |
| class SearchIndex: | |
| """TF-IDF search index for brain entries.""" | |
| entries: List[dict] = field(default_factory=list) | |
| tokens: List[List[str]] = field(default_factory=list) | |
| idf: Dict[str, float] = field(default_factory=dict) | |
| vectors: List[Dict[str, float]] = field(default_factory=list) | |
| built_at: str = "" | |
| def build(self, entries: List[dict]): | |
| """Build index from entries.""" | |
| self.entries = entries | |
| self.tokens = [tokenize(entry_text(e)) for e in entries] | |
| self.idf = compute_idf(self.tokens) | |
| self.vectors = [ | |
| tfidf_vector(compute_tf(tok), self.idf) for tok in self.tokens | |
| ] | |
| self.built_at = datetime.now().isoformat() | |
| def search(self, query: str, top_k: int = 10, min_score: float = 0.05) -> List[Tuple[dict, float]]: | |
| """Search entries by semantic similarity.""" | |
| q_tokens = tokenize(query) | |
| q_tf = compute_tf(q_tokens) | |
| q_vec = tfidf_vector(q_tf, self.idf) | |
| results = [] | |
| for i, vec in enumerate(self.vectors): | |
| score = cosine_similarity(q_vec, vec) | |
| if score >= min_score: | |
| results.append((self.entries[i], score)) | |
| results.sort(key=lambda x: x[1], reverse=True) | |
| return results[:top_k] | |
| def find_duplicates(self, threshold: float = 0.85) -> List[Tuple[dict, dict, float]]: | |
| """Find duplicate/near-duplicate entries.""" | |
| dupes = [] | |
| n = len(self.vectors) | |
| for i in range(n): | |
| for j in range(i + 1, n): | |
| sim = cosine_similarity(self.vectors[i], self.vectors[j]) | |
| if sim >= threshold: | |
| dupes.append((self.entries[i], self.entries[j], sim)) | |
| dupes.sort(key=lambda x: x[2], reverse=True) | |
| return dupes | |
| def save(self, path: Path = INDEX_FILE): | |
| """Save index metadata (not vectors, those are rebuilt).""" | |
| meta = { | |
| "built_at": self.built_at, | |
| "entry_count": len(self.entries), | |
| "vocab_size": len(self.idf), | |
| } | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text(json.dumps(meta, indent=2)) | |
| # ─── Quality Scoring ──────────────────────────────────────────── | |
| def quality_score(entry: dict) -> dict: | |
| """Score an entry's quality on multiple dimensions (0-100).""" | |
| scores = {} | |
| # Completeness: has content, context, tags | |
| content = entry.get("content", "") | |
| context = entry.get("context", "") | |
| tags = entry.get("tags", []) | |
| completeness = 0 | |
| if content: | |
| completeness += 40 | |
| if context: | |
| completeness += 30 | |
| if tags: | |
| completeness += 20 | |
| if len(content) > 50: | |
| completeness += 10 | |
| scores["completeness"] = min(completeness, 100) | |
| # Freshness: how recent | |
| date_str = entry.get("date_updated", entry.get("date_added", "")) | |
| if date_str: | |
| try: | |
| updated = datetime.strptime(date_str[:10], "%Y-%m-%d") | |
| days_old = (datetime.now() - updated).days | |
| if days_old < 7: | |
| scores["freshness"] = 100 | |
| elif days_old < 30: | |
| scores["freshness"] = 80 | |
| elif days_old < 90: | |
| scores["freshness"] = 60 | |
| elif days_old < 365: | |
| scores["freshness"] = 40 | |
| else: | |
| scores["freshness"] = 20 | |
| except ValueError: | |
| scores["freshness"] = 50 | |
| else: | |
| scores["freshness"] = 50 | |
| # Usage: reference count | |
| refs = entry.get("references", 0) | |
| if refs >= 10: | |
| scores["usage"] = 100 | |
| elif refs >= 5: | |
| scores["usage"] = 80 | |
| elif refs >= 2: | |
| scores["usage"] = 60 | |
| elif refs >= 1: | |
| scores["usage"] = 40 | |
| else: | |
| scores["usage"] = 20 | |
| # Status bonus | |
| status = entry.get("status", "").upper() | |
| if status == "TESTED": | |
| scores["reliability"] = 90 | |
| elif status == "EXPERIMENTAL": | |
| scores["reliability"] = 50 | |
| elif status == "DEPRECATED": | |
| scores["reliability"] = 10 | |
| else: | |
| scores["reliability"] = 50 | |
| # Overall weighted score | |
| weights = {"completeness": 0.3, "freshness": 0.2, "usage": 0.25, "reliability": 0.25} | |
| scores["overall"] = sum(scores[k] * weights[k] for k in weights) | |
| return scores | |
| # ─── Provenance Tracking ──────────────────────────────────────── | |
| def load_provenance() -> dict: | |
| """Load provenance data.""" | |
| if PROVENANCE_FILE.exists(): | |
| try: | |
| return json.loads(PROVENANCE_FILE.read_text()) | |
| except json.JSONDecodeError: | |
| return {} | |
| return {} | |
| def save_provenance(data: dict): | |
| """Save provenance data.""" | |
| PROVENANCE_FILE.parent.mkdir(parents=True, exist_ok=True) | |
| PROVENANCE_FILE.write_text(json.dumps(data, indent=2)) | |
| def track_provenance(entry_id: str, action: str, actor: str = "system", details: str = ""): | |
| """Record a provenance event for an entry.""" | |
| prov = load_provenance() | |
| if entry_id not in prov: | |
| prov[entry_id] = {"events": []} | |
| prov[entry_id]["events"].append({ | |
| "action": action, | |
| "actor": actor, | |
| "timestamp": datetime.now().isoformat(), | |
| "details": details, | |
| }) | |
| save_provenance(prov) | |
| def get_provenance(entry_id: str) -> List[dict]: | |
| """Get provenance history for an entry.""" | |
| prov = load_provenance() | |
| return prov.get(entry_id, {}).get("events", []) | |
| # ─── CLI ───────────────────────────────────────────────────────── | |
| def main(): | |
| if len(sys.argv) < 2: | |
| print(__doc__) | |
| return | |
| cmd = sys.argv[1] | |
| entries = load_brain_entries() | |
| if cmd == "search": | |
| if len(sys.argv) < 3: | |
| print("Usage: brain_semantic.py search <query>") | |
| return | |
| query = " ".join(sys.argv[2:]) | |
| idx = SearchIndex() | |
| idx.build(entries) | |
| results = idx.search(query) | |
| if not results: | |
| print("No results found.") | |
| return | |
| print(f"\n🔍 Semantic search: '{query}' ({len(results)} results)\n") | |
| for entry, score in results: | |
| eid = entry.get("id", "?") | |
| etype = entry.get("type", "?") | |
| content = entry.get("content", "")[:80] | |
| print(f" [{score:.2f}] {eid} ({etype}): {content}") | |
| elif cmd == "dedupe": | |
| dry_run = "--dry-run" in sys.argv | |
| idx = SearchIndex() | |
| idx.build(entries) | |
| dupes = idx.find_duplicates() | |
| if not dupes: | |
| print("✅ No duplicates found.") | |
| return | |
| print(f"\n⚠️ Found {len(dupes)} potential duplicate pairs:\n") | |
| for e1, e2, sim in dupes: | |
| print(f" [{sim:.2f}] {e1.get('id','?')} ↔ {e2.get('id','?')}") | |
| print(f" A: {e1.get('content','')[:60]}") | |
| print(f" B: {e2.get('content','')[:60]}") | |
| print() | |
| if dry_run: | |
| print("(dry run — no changes made)") | |
| elif cmd == "score": | |
| if not entries: | |
| print("No brain entries found.") | |
| return | |
| scored = [(e, quality_score(e)) for e in entries] | |
| scored.sort(key=lambda x: x[1]["overall"], reverse=True) | |
| print(f"\n📊 Quality Scores ({len(scored)} entries):\n") | |
| print(f" {'ID':<15} {'Type':<13} {'Overall':>7} {'Complete':>8} {'Fresh':>5} {'Usage':>5} {'Reliable':>8}") | |
| print(" " + "─" * 75) | |
| for entry, sc in scored[:20]: | |
| eid = entry.get("id", "?")[:14] | |
| etype = entry.get("type", "?")[:12] | |
| print(f" {eid:<15} {etype:<13} {sc['overall']:>6.1f} {sc['completeness']:>7} {sc['freshness']:>5} {sc['usage']:>5} {sc['reliability']:>7}") | |
| avg = sum(s["overall"] for _, s in scored) / len(scored) | |
| print(f"\n Average quality: {avg:.1f}/100") | |
| low_quality = [e for e, s in scored if s["overall"] < 40] | |
| if low_quality: | |
| print(f" ⚠️ {len(low_quality)} entries below quality threshold (40)") | |
| elif cmd == "provenance": | |
| if len(sys.argv) < 3: | |
| print("Usage: brain_semantic.py provenance <entry_id>") | |
| return | |
| entry_id = sys.argv[2] | |
| events = get_provenance(entry_id) | |
| if not events: | |
| print(f"No provenance data for {entry_id}") | |
| return | |
| print(f"\n📋 Provenance for {entry_id}:\n") | |
| for ev in events: | |
| print(f" {ev['timestamp'][:19]} {ev['action']:<12} by {ev['actor']} {ev.get('details','')}") | |
| elif cmd == "reindex": | |
| idx = SearchIndex() | |
| idx.build(entries) | |
| idx.save() | |
| print(f"✅ Index rebuilt: {len(entries)} entries, {len(idx.idf)} terms") | |
| else: | |
| print(f"Unknown command: {cmd}") | |
| print(__doc__) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.1 kB
- Xet hash:
- 8f698fa548278412bc1aab6146fcf27c12986e75f887302d6690a540cbd3c583
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.