feat: enhance knowledge graph visualization and add new features

- Updated package.json and package-lock.json to include "three-spritetext" for improved text rendering in the 3D graph.
- Modified KnowledgeGraph component to integrate SpriteText for always-visible node labels and relationship tags.
- Introduced new scripts for benchmarking memory retrieval latency and debugging direct interactions with the knowledge extraction system.
- Added tests for belief revision, context extraction, plural reconciliation, and semantic aliasing to ensure robust knowledge handling.
- Implemented a script to strip co-author information from commit messages.

app/api/endpoints.py

@@ -454,7 +454,16 @@ async def process_query_stream(request: QueryRequest, current_user: str = Depend
                                 "phase": "graph",
                                 "message": "Extracting relationships for semantic memory."
                             })
-                            triples = await asyncio.to_thread(extract_and_store_knowledge, request.text, current_user)
+                            # Short replies ("umm tomato") only carry meaning
+                            # alongside the question Soma asked, so pass the
+                            # last assistant message as extraction context.
+                            last_soma_msg = next(
+                                (m["content"] for m in reversed(history) if m["role"] == "assistant"),
+                                ""
+                            )
+                            triples = await asyncio.to_thread(
+                                extract_and_store_knowledge, request.text, current_user, last_soma_msg
+                            )
                             yield sse_event("brain_trace", build_brain_event(
                                 "graph",
                                 71,

app/db/session.py

@@ -201,12 +201,12 @@ def get_recent_messages(session_id: str, exchanges: int = 5):
         cur = _cursor(conn)
         if _db_backend == "postgres":
             cur.execute(
-                'SELECT role, content FROM messages WHERE session_id = %s ORDER BY timestamp DESC LIMIT %s',
+                'SELECT role, content FROM messages WHERE session_id = %s ORDER BY timestamp DESC, id DESC LIMIT %s',
                 (session_id, exchanges * 2)
             )
         else:
             cur.execute(
-                'SELECT role, content FROM messages WHERE session_id = ? ORDER BY timestamp DESC LIMIT ?',
+                'SELECT role, content FROM messages WHERE session_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
                 (session_id, exchanges * 2)
             )
         rows = cur.fetchall()
@@ -248,14 +248,14 @@ def prune_old_messages(session_id: str, keep_recent: int = 10):
             cur.execute('''
                 DELETE FROM messages WHERE session_id = %s AND id NOT IN (
                     SELECT id FROM messages WHERE session_id = %s
-                    ORDER BY timestamp DESC LIMIT %s
+                    ORDER BY timestamp DESC, id DESC LIMIT %s
                 )
             ''', (session_id, session_id, keep_recent))
         else:
             cur.execute('''
                 DELETE FROM messages WHERE session_id = ? AND id NOT IN (
                     SELECT id FROM messages WHERE session_id = ?
-                    ORDER BY timestamp DESC LIMIT ?
+                    ORDER BY timestamp DESC, id DESC LIMIT ?
                 )
             ''', (session_id, session_id, keep_recent))
     return get_message_count(session_id)
@@ -295,12 +295,12 @@ def get_recent_sparks(user_id: str = "default_user", limit: int = 5):
         cur = _cursor(conn)
         if _db_backend == "postgres":
             cur.execute(
-                'SELECT content, entities, timestamp FROM neural_sparks WHERE user_id = %s ORDER BY timestamp DESC LIMIT %s',
+                'SELECT content, entities, timestamp FROM neural_sparks WHERE user_id = %s ORDER BY timestamp DESC, id DESC LIMIT %s',
                 (user_id, limit)
             )
         else:
             cur.execute(
-                'SELECT content, entities, timestamp FROM neural_sparks WHERE user_id = ? ORDER BY timestamp DESC LIMIT ?',
+                'SELECT content, entities, timestamp FROM neural_sparks WHERE user_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
                 (user_id, limit)
             )
         rows = cur.fetchall()

app/services/neocortex.py

@@ -7,6 +7,29 @@ from langchain_core.messages import HumanMessage
 from app.core.config import settings
 from app.db.neo4j_driver import neo4j_db
 
+# ── Models ──
+# Extraction runs on every message, so it gets the fast small model.
+# Judgment calls (same-entity matching, contradiction detection) need
+# precision — the 8B model over-merges and over-deletes — so they get the
+# bigger free-tier model, with automatic fallback if it's unavailable.
+FAST_MODEL = "llama-3.1-8b-instant"
+JUDGE_MODEL = "llama-3.3-70b-versatile"
+
+
+def _llm_complete(prompt: str, model: str) -> str:
+    api_key = settings.GROQ_API_KEY if settings.GROQ_API_KEY else "dummy_key"
+    llm = ChatGroq(model=model, api_key=api_key)
+    return llm.invoke([HumanMessage(content=prompt)]).content
+
+
+def _judge_complete(prompt: str) -> str:
+    try:
+        return _llm_complete(prompt, JUDGE_MODEL)
+    except Exception as e:
+        print(f"Neocortex: judge model unavailable ({e}), falling back to {FAST_MODEL}.")
+        return _llm_complete(prompt, FAST_MODEL)
+
+
 # ── Blocked meta-nodes that should never become graph entities ──
 BLOCKED_NODES = {
     "USER", "SOMA", "AI", "ASSISTANT", "BOT", "HUMAN", "SYSTEM",
@@ -74,6 +97,23 @@ def _is_valid_node(name: str) -> bool:
     return True
 
 
+def _extract_json(content: str, required_key: str) -> dict:
+    """
+    Pull the first valid JSON object containing `required_key` out of an LLM
+    response. Small models often wrap JSON in prose or emit several blocks,
+    so a bare regex-then-parse is not reliable.
+    """
+    decoder = json.JSONDecoder()
+    for m in re.finditer(r'\{', content):
+        try:
+            obj, _ = decoder.raw_decode(content[m.start():])
+        except Exception:
+            continue
+        if isinstance(obj, dict) and required_key in obj:
+            return obj
+    return {}
+
+
 def _sanitize_relation(rel: str) -> str:
     """Clean a relation name for Neo4j compatibility."""
     rel = rel.upper().strip()
@@ -82,29 +122,250 @@ def _sanitize_relation(rel: str) -> str:
     return rel or "RELATED_TO"
 
 
-def extract_and_store_knowledge(text: str, user_id: str = "default_user"):
+# ── Belief reconciliation ──
+# New facts are never written blindly: they are first compared against what
+# the graph already believes, so contradictions replace old beliefs instead
+# of piling up next to them.
+
+def _normalize_name(name: str) -> str:
+    """
+    Spelling-insensitive matching form: 'READING_BOOKS' and 'READING BOOKS'
+    match, and so do singular/plural variants like 'LILY' and 'LILIES'.
+    Only used for comparing names — never stored.
+    """
+    n = re.sub(r'[^A-Z0-9]', '', name.upper())
+    if n.endswith('IES') and len(n) > 4:
+        n = n[:-3] + 'Y'
+    elif n.endswith('S') and not n.endswith('SS') and len(n) > 3:
+        n = n[:-1]
+    return n
+
+
+def _canonicalize_nodes(triples: list, user_id: str) -> list:
+    """
+    Entity resolution: if a new node name is just a different spelling of an
+    entity already in the graph, reuse the existing spelling so duplicates
+    like READING_BOOKS next to READING BOOKS are never created.
+    """
+    rows = neo4j_db.query(
+        "MATCH (n:Entity) WHERE n.user_id = $u RETURN n.name AS name LIMIT 300",
+        {"u": user_id},
+    ) or []
+    by_norm = {}
+    for r in rows:
+        by_norm.setdefault(_normalize_name(r["name"]), r["name"])
+    for t in triples:
+        t["subject"] = by_norm.get(_normalize_name(t["subject"]), t["subject"])
+        t["object"] = by_norm.get(_normalize_name(t["object"]), t["object"])
+    return triples
+
+
+def _verify_alias(name_a: str, name_b: str) -> bool:
+    """
+    Yes/no double-check before merging two entity names. The matcher can
+    over-merge, and a wrong merge corrupts two facts at once — so every
+    match must survive this gate. Denies on any error.
+    """
+    q = (
+        f'Answer with exactly one word, YES or NO. Are "{name_a}" and "{name_b}" '
+        f'two names for the exact same thing? (Two different foods, cities, '
+        f'sports, people or hobbies are NO, even if similar.)'
+    )
+    try:
+        return _judge_complete(q).strip().upper().startswith("YES")
+    except Exception:
+        return False
+
+
+def _can_coexist(new_fact: dict, edge: dict) -> bool:
+    """
+    Final safety gate before deleting a memory. Deleting is destructive,
+    so the judge must explicitly confirm the two facts cannot both be true.
+    Any doubt or error keeps the old belief.
+    """
+    q = (
+        f'Fact A: "{edge["s"]} {edge["rel"]} {edge["o"]}". '
+        f'Fact B: "{new_fact["subject"]} {new_fact["relation"]} {new_fact["object"]}". '
+        f'Can both facts be true at the same time? Answer with exactly one word: YES or NO. '
+        f'(Opposite feelings about the same thing, or two current home cities, cannot both be true. '
+        f'Facts about different topics can.)'
+    )
+    try:
+        return _judge_complete(q).strip().upper().startswith("YES")
+    except Exception:
+        return True
+
+
+def _semantic_canonicalize(triples: list, user_id: str) -> list:
+    """
+    Meaning-based entity resolution: map a new node name onto an existing
+    entity when both refer to the SAME real-world thing in different words
+    (FOOTBALL -> SOCCER, MOM -> MOTHER, NYC -> NEW YORK).
+
+    An LLM does the matching instead of string or embedding similarity,
+    because similarity confuses "related" with "same" — DELHI and MUMBAI
+    are highly similar but must never merge. Falls back to no mapping on
+    any failure, which just means a new node is created (old behavior).
+    """
+    rows = neo4j_db.query(
+        "MATCH (n:Entity) WHERE n.user_id = $u RETURN n.name AS name LIMIT 150",
+        {"u": user_id},
+    ) or []
+    existing = [r["name"] for r in rows]
+    if not existing:
+        return triples
+
+    existing_set = set(existing)
+    new_names = sorted({
+        name for t in triples for name in (t["subject"], t["object"])
+        if name not in existing_set
+    })
+    if not new_names:
+        return triples
+
+    prompt = f"""You maintain a person's memory graph. These entities already exist in it:
+{", ".join(existing)}
+
+These NEW entity names just appeared:
+{", ".join(new_names)}
+
+For each NEW name, check if it means THE SAME real-world thing as ONE existing entity — a synonym, alias, nickname, abbreviation or translation. Examples: FOOTBALL = SOCCER, MOM = MOTHER, NYC = NEW YORK.
+
+STRICT RULE: related is NOT the same. Different cities (DELHI vs MUMBAI), different foods (PIZZA vs PASTA), different sports (CRICKET vs BADMINTON) are NOT matches — two names merely in the same category are different things. When unsure, do NOT match.
+
+Return ONLY JSON: {{"matches": {{"NEW_NAME": "EXISTING_NAME"}}}}. If nothing matches, return {{"matches": {{}}}}."""
+
+    try:
+        data = _extract_json(_judge_complete(prompt), "matches")
+        raw = data.get("matches", {}) or {}
+
+        mapping = {}
+        for new_name, old_name in raw.items():
+            new_name, old_name = str(new_name).upper(), str(old_name).upper()
+            # Only trust mappings between names we actually presented,
+            # and only after they survive the yes/no verification gate.
+            if new_name in new_names and old_name in existing_set and _verify_alias(new_name, old_name):
+                mapping[new_name] = old_name
+
+        if mapping:
+            print(f"Neocortex: semantic aliases resolved: {mapping}")
+        for t in triples:
+            t["subject"] = mapping.get(t["subject"], t["subject"])
+            t["object"] = mapping.get(t["object"], t["object"])
+        return triples
+    except Exception as e:
+        print(f"Neocortex: semantic canonicalization failed ({e}), keeping new names.")
+        return triples
+
+
+def _fetch_related_beliefs(names: list, user_id: str, limit: int = 30) -> list:
+    """Existing edges that touch any entity mentioned in the new facts."""
+    rows = neo4j_db.query(
+        """
+        MATCH (n:Entity)-[r]->(m:Entity)
+        WHERE n.user_id = $u AND m.user_id = $u
+          AND (n.name IN $names OR m.name IN $names)
+        RETURN n.name AS s, type(r) AS rel, m.name AS o
+        LIMIT $limit
+        """,
+        {"u": user_id, "names": names, "limit": limit},
+    ) or []
+    return [{"s": r["s"], "rel": r["rel"], "o": r["o"]} for r in rows]
+
+
+def _reconcile(new_triples: list, existing: list) -> tuple:
+    """
+    One LLM call decides, for each new fact: ADD (genuinely new), SKIP
+    (already known), or REPLACE (contradicts existing beliefs, which get
+    deleted). Falls back to storing everything if the call fails, so a
+    flaky LLM can never lose new information.
+    Returns (triples_to_store, deletions) where each deletion pairs the
+    edge with the new fact that contradicts it: {"edge": ..., "new": ...}.
+    """
+    e_lines = "\n".join(f"E{i+1}: {e['s']} {e['rel']} {e['o']}" for i, e in enumerate(existing))
+    n_lines = "\n".join(f"N{i+1}: {t['subject']} {t['relation']} {t['object']}" for i, t in enumerate(new_triples))
+
+    prompt = f"""You maintain a person's memory. Compare each NEW fact against the EXISTING beliefs.
+
+EXISTING beliefs:
+{e_lines}
+
+NEW facts:
+{n_lines}
+
+For each NEW fact choose exactly ONE action:
+- "ADD": new information not covered by any existing belief.
+- "SKIP": same meaning as an existing belief (already known).
+- "REPLACE": contradicts or updates existing belief(s). Put those belief ids in "deletes".
+
+Two facts conflict when they cannot both be true at once: opposite feelings about the same thing (LOVES vs HATES), a new home city, a changed job or status. Facts about different topics never conflict.
+Treat singular/plural or spelling variants of a name as the SAME thing: "LILY" and "LILIES" are one concept, so LIKES LILY conflicts with HATES LILIES.
+
+Return ONLY JSON in this exact format:
+{{"decisions": [{{"fact": "N1", "action": "REPLACE", "deletes": ["E2"]}}, {{"fact": "N2", "action": "ADD", "deletes": []}}]}}"""
+
+    to_store, to_delete = [], []
+    try:
+        data = _extract_json(_judge_complete(prompt), "decisions")
+
+        decisions = {}
+        for d in data.get("decisions", []):
+            fact_id = str(d.get("fact", "")).upper().replace("N", "")
+            if fact_id.isdigit():
+                decisions[int(fact_id) - 1] = d
+
+        for i, t in enumerate(new_triples):
+            d = decisions.get(i, {})
+            action = str(d.get("action", "ADD")).upper()
+            if action == "SKIP":
+                continue
+            if action == "REPLACE":
+                for e_id in d.get("deletes", []) or []:
+                    e_idx = str(e_id).upper().replace("E", "")
+                    if e_idx.isdigit() and 0 < int(e_idx) <= len(existing):
+                        to_delete.append({"edge": existing[int(e_idx) - 1], "new": t})
+            to_store.append(t)
+        return to_store, to_delete
+    except Exception as e:
+        print(f"Neocortex: reconciliation failed ({e}), storing all new facts.")
+        return new_triples, []
+
+
+def extract_and_store_knowledge(text: str, user_id: str = "default_user", context: str = ""):
     """
     Child-brain knowledge extraction with 100% structurally guaranteed JSON output.
-    
+
     Reads a conversation and extracts simple, clean concept associations —
     the way a child's brain naturally builds connections between ideas.
+
+    `context` is the assistant message the user was replying to. With it,
+    short answers like "umm tomato" become extractable facts, because the
+    question ("What's your favorite topping?") supplies the missing meaning.
     """
     if not neo4j_db.driver:
         print("Knowledge Graph disabled (No DB connection).")
         return 0
-        
-    api_key = settings.GROQ_API_KEY if settings.GROQ_API_KEY else "dummy_key"
-    llm = ChatGroq(model="llama-3.1-8b-instant", api_key=api_key)
-    
+
     clean = _clean_text(text)
+    ctx = _clean_text(context)[-500:] if context else ""
 
-    # Need at least 3 words to extract a relationship (e.g., "I like apples")
-    if len(clean.split()) < 3:
+    # Without context, a fact needs at least 3 words ("I like apples").
+    # With context, even a one-word reply can carry a fact.
+    min_words = 1 if ctx else 3
+    if len(clean.split()) < min_words:
         print(f"Neocortex: Input too short ({len(clean.split())} words), skipping.")
         return 0
 
     owner = user_id.upper()
 
+    if ctx:
+        text_block = f'Soma asked: "{ctx}"\n{owner} replied: "{clean}"'
+        context_rule = f"""
+6. The reply may be a short answer to Soma's question. Combine the question and the reply to infer the fact. Example: if Soma asked "What's your favorite topping?" and the reply is "umm tomato", extract {owner} --LIKES--> TOMATO."""
+    else:
+        text_block = clean
+        context_rule = ""
+
     prompt = f"""You are a child's brain learning about the world. Read the text and pick out SIMPLE facts as connections between concepts.
 
 Think like a child drawing a mind-map:
@@ -113,14 +374,14 @@ Think like a child drawing a mind-map:
 - Focus on: people, places, things, hobbies, foods, animals, feelings, skills, jobs
 
 RULES:
-1. Nodes must be 1-3 word concept names, ALL CAPS. Example: "CRICKET", "DELHI", "MOM", "CODING"
+1. Nodes must be 1-3 word concept names, ALL CAPS, in SINGULAR form. Example: "CRICKET", "DELHI", "MOM", "LILY" (not "LILIES"), "BOOK" (not "BOOKS")
 2. Relations must be simple verbs: LIKES, IS_A, LIVES_IN, PLAYS, WORKS_AT, HAS, KNOWS, STUDIES, etc.
 3. "I" or "my" in the text refers to "{owner}" — always use "{owner}" as the node name for the speaker.
 4. DO NOT create nodes named "USER", "SOMA", "AI", "ASSISTANT", or any chat/bot terms.
-5. If the text is just greetings or small talk with zero factual content, return an empty triples list.
+5. If the text is just greetings or small talk with zero factual content, return an empty triples list.{context_rule}
 
 Text:
-{clean}
+{text_block}
 
 Return the extracted facts ONLY as a valid JSON block in this exact format:
 {{
@@ -131,49 +392,87 @@ Return the extracted facts ONLY as a valid JSON block in this exact format:
 Do not write any other explanation or thoughts outside the JSON block. If there are no facts, return: {{"triples": []}}"""
 
     try:
-        response = llm.invoke([HumanMessage(content=prompt)])
-        content = response.content.strip()
-        
-        # Robustly extract the JSON block
-        triples_data = []
-        match = re.search(r'\{.*\}', content, re.DOTALL)
-        if match:
-            json_str = match.group(0)
-            try:
-                data = json.loads(json_str)
-                triples_data = data.get("triples", [])
-            except Exception as e:
-                print(f"Neocortex: Failed to parse JSON block: {e}")
-                return 0
-        else:
+        content = _llm_complete(prompt, FAST_MODEL).strip()
+
+        data = _extract_json(content, "triples")
+        if not data:
             print("Neocortex: No JSON block found in LLM response.")
             return 0
+        triples_data = data.get("triples", [])
             
         if not triples_data:
             print("Neocortex: No triples extracted.")
             return 0
             
-        stored_count = 0
-        
+        # ── Validate and clean the extracted triples ──
+        new_triples = []
+        seen = set()
         for t in triples_data:
             subj = str(t.get("subject", "")).strip().upper()
             rel  = _sanitize_relation(str(t.get("relation", "")))
             obj  = str(t.get("object", "")).strip().upper()
-            
-            # Validate both nodes
+
             if not _is_valid_node(subj) or not _is_valid_node(obj):
                 continue
             if subj == obj:  # Self-loops are meaningless
                 continue
-            
+            if (subj, rel, obj) in seen:
+                continue
+            seen.add((subj, rel, obj))
+            new_triples.append({"subject": subj, "relation": rel, "object": obj})
+
+        if not new_triples:
+            return 0
+
+        # ── Reconcile with existing beliefs before writing ──
+        new_triples = _canonicalize_nodes(new_triples, user_id)     # spelling/plural variants (free)
+        new_triples = _semantic_canonicalize(new_triples, user_id)  # same meaning, different words
+        names = list({n for t in new_triples for n in (t["subject"], t["object"])})
+        existing = _fetch_related_beliefs(names, user_id)
+        if existing:
+            to_store, to_delete = _reconcile(new_triples, existing)
+        else:
+            to_store, to_delete = new_triples, []
+
+        deleted = set()
+        for pair in to_delete:
+            edge, new_fact = pair["edge"], pair["new"]
+            key = (edge["s"], edge["rel"], edge["o"])
+            if key in deleted:
+                continue
+            if _can_coexist(new_fact, edge):
+                print(f"Neocortex: kept '{edge['s']} {edge['rel']} {edge['o']}' — no real conflict with new fact.")
+                continue
+            neo4j_db.query(
+                """
+                MATCH (s:Entity {name: $s, user_id: $u})-[r]->(o:Entity {name: $o, user_id: $u})
+                WHERE type(r) = $rel
+                DELETE r
+                """,
+                {"s": edge["s"], "o": edge["o"], "rel": edge["rel"], "u": user_id},
+            )
+            deleted.add(key)
+        if deleted:
+            print(f"Neocortex: belief revision removed {len(deleted)} outdated relation(s).")
+
+        stored_count = 0
+        for t in to_store:
             cypher = f"""
             MERGE (s:Entity {{name: $subject, user_id: $user_id}})
             MERGE (o:Entity {{name: $object, user_id: $user_id}})
-            MERGE (s)-[r:`{rel}`]->(o)
+            MERGE (s)-[r:`{t["relation"]}`]->(o)
             """
-            neo4j_db.query(cypher, {"subject": subj, "object": obj, "user_id": user_id})
+            neo4j_db.query(cypher, {"subject": t["subject"], "object": t["object"], "user_id": user_id})
             stored_count += 1
-                
+
+        # Replaced edges can leave nodes with no connections — remove them so
+        # the mesh doesn't show floating orphans.
+        if deleted:
+            neo4j_db.query(
+                "MATCH (n:Entity) WHERE n.user_id = $u AND NOT (n)--() DELETE n",
+                {"u": user_id},
+            )
+
         return stored_count
     except Exception as e:
         print(f"Neocortex extraction error: {e}")

frontend/package-lock.json

@@ -11,7 +11,8 @@
         "react": "^18.3.1",
         "react-dom": "^18.3.1",
         "react-force-graph-2d": "^1.29.1",
-        "react-force-graph-3d": "^1.29.1"
+        "react-force-graph-3d": "^1.29.1",
+        "three-spritetext": "^1.10.0"
       },
       "devDependencies": {
         "@eslint/js": "^9.15.0",
@@ -3273,6 +3274,18 @@
         "three": ">=0.179"
       }
     },
+    "node_modules/three-spritetext": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/three-spritetext/-/three-spritetext-1.10.0.tgz",
+      "integrity": "sha512-t08iP1FCU1lQh8T5MmCpdijKgas8GDHJE0LqMGBuVu3xqMMpFnEZhTlih7FlxLPQizHIGoumUSpfOlY1GO/Tgg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "peerDependencies": {
+        "three": ">=0.86.0"
+      }
+    },
     "node_modules/tinycolor2": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz",

frontend/package.json

@@ -13,7 +13,8 @@
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-force-graph-2d": "^1.29.1",
-    "react-force-graph-3d": "^1.29.1"
+    "react-force-graph-3d": "^1.29.1",
+    "three-spritetext": "^1.10.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.15.0",

frontend/src/components/KnowledgeGraph.jsx

@@ -1,6 +1,7 @@
 import { useEffect, useState, useRef } from 'react';
 import ForceGraph3D from 'react-force-graph-3d';
 import * as THREE from 'three';
+import SpriteText from 'three-spritetext';
 import { apiFetch } from '../api';
 import './KnowledgeGraph.css';
 
@@ -278,22 +279,45 @@ function KnowledgeGraph({ refreshTick }) {
               emissive: getNodeColor(node),
               emissiveIntensity: 0.45
             });
-            return new THREE.Mesh(geom, mat);
+            const sphere = new THREE.Mesh(geom, mat);
+
+            // Always-visible name tag floating above the sphere
+            const tag = new SpriteText((node.label || node.id).toUpperCase());
+            tag.color = getNodeColor(node);
+            tag.textHeight = 3.2;
+            tag.fontWeight = '700';
+            tag.backgroundColor = 'rgba(10, 10, 10, 0.55)';
+            tag.padding = 1.2;
+            tag.borderRadius = 2;
+            tag.position.y = size + 4;
+            tag.material.depthWrite = false; // keep tag readable through overlapping geometry
+
+            const group = new THREE.Group();
+            group.add(sphere);
+            group.add(tag);
+            return group;
           }}
           
-          // Outlined holographic label rendered natively floating beside cursor on hover (100% stable!)
-          nodeLabel={node => `
-            <span style="color: ${getNodeColor(node)}; font-size: 12px; font-weight: 700; letter-spacing: 0.05em; text-transform: uppercase;">
-              ${node.label || node.id}
-            </span>
-          `}
-          
-          // Outlined relationship tag rendered on link hover (100% stable!)
-          linkLabel={link => `
-            <span style="color: #ff6b35; font-size: 10px; font-weight: 700; letter-spacing: 0.05em; text-transform: uppercase;">
-              ${link.label || 'RELATED_TO'}
-            </span>
-          `}
+          // Always-visible relationship tag floating at the middle of each link
+          linkThreeObjectExtend={true}
+          linkThreeObject={link => {
+            const tag = new SpriteText((link.label || 'RELATED_TO').toUpperCase());
+            tag.color = '#ff9c7a';
+            tag.textHeight = 2.2;
+            tag.fontWeight = '700';
+            tag.backgroundColor = 'rgba(10, 10, 10, 0.45)';
+            tag.padding = 1;
+            tag.borderRadius = 2;
+            tag.material.depthWrite = false;
+            return tag;
+          }}
+          linkPositionUpdate={(sprite, { start, end }) => {
+            sprite.position.set(
+              start.x + (end.x - start.x) / 2,
+              start.y + (end.y - start.y) / 2,
+              start.z + (end.z - start.z) / 2
+            );
+          }}
           
           nodeRelSize={3}
           linkColor={() => 'rgba(255, 107, 53, 0.25)'} // Soft, clean axon link fibers

scratch/bench_retrieval_latency.py

@@ -0,0 +1,38 @@
+"""Benchmark memory retrieval latency — the source of the resume numbers.
+
+Run from repo root:  python scratch/bench_retrieval_latency.py
+Measures warm-cache latency; first call is excluded as warmup.
+"""
+import time
+import statistics
+
+from app.services.memory import retrieve_context, get_embeddings
+from app.services.neocortex import retrieve_graph_context
+
+QUERIES = [
+    "what do I like",
+    "tell me about cricket",
+    "who is komal",
+    "what did we discuss",
+    "my favorite food",
+]
+
+get_embeddings()
+retrieve_context("warmup", "default_user")
+vector_times = []
+for q in QUERIES:
+    t0 = time.perf_counter()
+    retrieve_context(q, "default_user")
+    vector_times.append((time.perf_counter() - t0) * 1000)
+
+retrieve_graph_context("warmup", "default_user")
+graph_times = []
+for q in QUERIES:
+    t0 = time.perf_counter()
+    retrieve_graph_context(q, "default_user")
+    graph_times.append((time.perf_counter() - t0) * 1000)
+
+print(f"Vector (ChromaDB, local): median {statistics.median(vector_times):.1f} ms "
+      f"| runs: {[f'{t:.0f}' for t in vector_times]}")
+print(f"Graph (Neo4j Aura, cloud round-trip): median {statistics.median(graph_times):.1f} ms "
+      f"| runs: {[f'{t:.0f}' for t in graph_times]}")

scratch/run_direct_debug.py

@@ -0,0 +1,62 @@
+import sys
+import os
+import json
+import re
+from langchain_groq import ChatGroq
+from langchain_core.messages import HumanMessage
+
+# Add project root to path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.core.config import settings
+from app.services.neocortex import _is_valid_node, _sanitize_relation
+
+def extract_and_debug(text: str, user_id: str = "default_user"):
+    api_key = settings.GROQ_API_KEY if settings.GROQ_API_KEY else "dummy_key"
+    llm = ChatGroq(model="llama-3.1-8b-instant", api_key=api_key)
+    
+    owner = user_id.upper()
+    prompt = f"""You are a child's brain learning about the world. Read the text and pick out SIMPLE facts as connections between concepts.
+
+Think like a child drawing a mind-map:
+- "{owner}" is the person speaking. If they say "I like X" → {owner} --LIKES--> X
+- Extract only SHORT concept names (1-3 words). Never use full sentences as names.
+- Focus on: people, places, things, hobbies, foods, animals, feelings, skills, jobs
+
+RULES:
+1. Nodes must be 1-3 word concept names, ALL CAPS. Example: "CRICKET", "DELHI", "MOM", "CODING"
+2. Relations must be simple verbs: LIKES, IS_A, LIVES_IN, PLAYS, WORKS_AT, HAS, KNOWS, STUDIES, etc.
+3. "I" or "my" in the text refers to "{owner}" — always use "{owner}" as the node name for the speaker.
+4. DO NOT create nodes named "USER", "SOMA", "AI", "ASSISTANT", or any chat/bot terms.
+5. If the text is just greetings or small talk with zero factual content, return an empty triples list.
+
+Text:
+{text}
+
+Return the extracted facts ONLY as a valid JSON block in this exact format:
+{{
+  "triples": [
+    {{"subject": "SUBJECT", "relation": "RELATION", "object": "OBJECT"}}
+  ]
+}}
+Do not write any other explanation or thoughts outside the JSON block. If there are no facts, return: {{"triples": []}}"""
+
+    response = llm.invoke([HumanMessage(content=prompt)])
+    content = response.content.strip()
+    
+    print("RAW RESPONSE:")
+    print(content)
+    
+    match = re.search(r'\{.*\}', content, re.DOTALL)
+    if match:
+        data = json.loads(match.group(0))
+        for t in data.get("triples", []):
+            subj = str(t.get("subject", "")).strip().upper()
+            rel = _sanitize_relation(str(t.get("relation", "")))
+            obj = str(t.get("object", "")).strip().upper()
+            print(f"\nTriple: {subj} --{rel}--> {obj}")
+            print(f"  Valid Subject ({subj}): {_is_valid_node(subj)}")
+            print(f"  Valid Object ({obj}): {_is_valid_node(obj)}")
+            print(f"  Equal: {subj == obj}")
+
+extract_and_debug("Dr. Aris is the creator of the Soma Project. The Soma Project is an advanced cognitive architecture.")

scratch/strip_coauthors.py

@@ -0,0 +1,13 @@
+import sys
+import re
+
+if __name__ == "__main__":
+    msg = sys.stdin.read()
+    # Remove lines that contain 'Co-Authored-By: Claude'
+    cleaned_lines = []
+    for line in msg.split('\n'):
+        if not re.search(r'Co-Authored-By: Claude', line, re.IGNORECASE):
+            cleaned_lines.append(line)
+    
+    # Print the cleaned message
+    sys.stdout.write('\n'.join(cleaned_lines))

scratch/test_belief_revision.py

@@ -0,0 +1,44 @@
+"""Test that a new sentiment (HATES) replaces the old one (LOVES) instead of coexisting."""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge
+from app.db.neo4j_driver import neo4j_db
+
+TEST_USER = "test_belief_rev"
+
+
+def show_graph(label):
+    res = neo4j_db.query(
+        "MATCH (n:Entity)-[r]->(m:Entity) WHERE n.user_id = $u AND m.user_id = $u "
+        "RETURN n.name AS s, type(r) AS rel, m.name AS o",
+        {"u": TEST_USER},
+    )
+    print(f"{label}:")
+    for row in res or []:
+        print(f"  {row['s']} --{row['rel']}--> {row['o']}")
+    print()
+
+
+def main():
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    print("--- Step 1: 'I love pizza' ---")
+    extract_and_store_knowledge("I love pizza", TEST_USER)
+    show_graph("Graph after step 1")
+
+    print("--- Step 2: 'I hate pizza' (should REPLACE the loves edge) ---")
+    extract_and_store_knowledge("I hate pizza", TEST_USER)
+    show_graph("Graph after step 2")
+
+    # Clean up test nodes
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+    print("Test nodes cleaned up.")
+
+
+if __name__ == "__main__":
+    main()

scratch/test_context_extraction.py

@@ -0,0 +1,43 @@
+"""Test that short replies get extracted when the previous Soma question is passed as context."""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge
+from app.db.neo4j_driver import neo4j_db
+
+TEST_USER = "test_topping_ctx"
+
+
+def main():
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    soma_question = "That's awesome! I'm glad you love pizza. What's your favorite topping?"
+
+    print("--- Case 1: 'umm tomato' WITHOUT context (old behavior, should skip) ---")
+    count_no_ctx = extract_and_store_knowledge("umm tomato", TEST_USER)
+    print(f"Triples stored: {count_no_ctx}\n")
+
+    print("--- Case 2: 'umm tomato' WITH context (new behavior, should extract) ---")
+    count_ctx = extract_and_store_knowledge("umm tomato", TEST_USER, context=soma_question)
+    print(f"Triples stored: {count_ctx}\n")
+
+    print(f"Graph contents for user '{TEST_USER}':")
+    res = neo4j_db.query(
+        "MATCH (n:Entity)-[r]->(m:Entity) WHERE n.user_id = $u AND m.user_id = $u "
+        "RETURN n.name AS s, type(r) AS rel, m.name AS o",
+        {"u": TEST_USER},
+    )
+    for row in res or []:
+        print(f"  {row['s']} --{row['rel']}--> {row['o']}")
+
+    # Clean up test nodes so they don't pollute the real graph
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+    print("\nTest nodes cleaned up.")
+
+
+if __name__ == "__main__":
+    main()

scratch/test_komal_fact.py

@@ -0,0 +1,30 @@
+import sys
+import os
+import asyncio
+
+# Add project root to path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge
+from app.db.neo4j_driver import neo4j_db
+
+async def main():
+    print("Testing neocortex on Komal fact...")
+    
+    # Check driver
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    fact = "Komal likes playing cricket."
+    triples = extract_and_store_knowledge(fact, "komal")
+    print(f"Extracted count: {triples}")
+    
+    if neo4j_db.driver:
+        print("\nCurrent DB Nodes for User 'komal':")
+        res = neo4j_db.query("MATCH (n) WHERE n.user_id = 'komal' RETURN n.name as name LIMIT 10")
+        for r in res:
+            print(r)
+
+if __name__ == "__main__":
+    asyncio.run(main())

scratch/test_plural_reconcile.py

@@ -0,0 +1,52 @@
+"""Test that singular/plural variants resolve to one node and contradictions still replace."""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge, _normalize_name
+from app.db.neo4j_driver import neo4j_db
+
+TEST_USER = "test_lily"
+
+
+def show_graph(label):
+    res = neo4j_db.query(
+        "MATCH (n:Entity)-[r]->(m:Entity) WHERE n.user_id = $u AND m.user_id = $u "
+        "RETURN n.name AS s, type(r) AS rel, m.name AS o",
+        {"u": TEST_USER},
+    )
+    print(f"{label}:")
+    for row in res or []:
+        print(f"  {row['s']} --{row['rel']}--> {row['o']}")
+    print()
+
+
+def main():
+    print("Unit check: normalize('LILIES') == normalize('LILY') ->",
+          _normalize_name("LILIES") == _normalize_name("LILY"))
+    print("Unit check: normalize('BOOKS') == normalize('BOOK')   ->",
+          _normalize_name("BOOKS") == _normalize_name("BOOK"))
+    print("Unit check: 'GLASS' unharmed ->", _normalize_name("GLASS") == "GLASS")
+    print()
+
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+
+    print("=== Step 1: 'I love lily' ===")
+    extract_and_store_knowledge("I love lily", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 2: 'I hate lilies' (plural! should still REPLACE, one node) ===")
+    extract_and_store_knowledge("I hate lilies", TEST_USER)
+    show_graph("Graph")
+
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+    print("Test nodes cleaned up.")
+
+
+if __name__ == "__main__":
+    main()

scratch/test_reconciliation.py

@@ -0,0 +1,61 @@
+"""Test the belief reconciliation pipeline end-to-end:
+ADD new facts, SKIP known ones, REPLACE contradicted ones, canonicalize spelling variants."""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge, _canonicalize_nodes
+from app.db.neo4j_driver import neo4j_db
+
+TEST_USER = "test_reconcile"
+
+
+def show_graph(label):
+    res = neo4j_db.query(
+        "MATCH (n:Entity)-[r]->(m:Entity) WHERE n.user_id = $u AND m.user_id = $u "
+        "RETURN n.name AS s, type(r) AS rel, m.name AS o",
+        {"u": TEST_USER},
+    )
+    print(f"{label}:")
+    for row in res or []:
+        print(f"  {row['s']} --{row['rel']}--> {row['o']}")
+    print()
+
+
+def main():
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    # Start clean
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+
+    print("=== Step 1: 'I love pizza and I live in Delhi' (all new -> ADD) ===")
+    extract_and_store_knowledge("I love pizza and I live in Delhi", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 2: 'I hate pizza' (contradiction -> REPLACE) ===")
+    extract_and_store_knowledge("I hate pizza", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 3: 'I really hate pizza' (already known -> SKIP, no duplicates) ===")
+    extract_and_store_knowledge("I really hate pizza", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 4: 'I moved to Mumbai' (updates home city -> REPLACE + orphan cleanup) ===")
+    extract_and_store_knowledge("I moved to Mumbai", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 5: entity resolution unit check ===")
+    fake = [{"subject": "TEST_RECONCILE", "relation": "ENJOYS", "object": "MUM_BAI"}]
+    fixed = _canonicalize_nodes(fake, TEST_USER)
+    print(f"  'MUM_BAI' canonicalized to: {fixed[0]['object']} (expected MUMBAI)")
+
+    # Clean up
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+    print("\nTest nodes cleaned up.")
+
+
+if __name__ == "__main__":
+    main()

scratch/test_robust_extraction.py

@@ -0,0 +1,90 @@
+import sys
+import os
+import json
+import re
+from langchain_groq import ChatGroq
+from langchain_core.messages import HumanMessage
+from pydantic import BaseModel, Field
+from typing import List
+
+# Add project root to path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.core.config import settings
+
+class RelationshipTriple(BaseModel):
+    subject: str
+    relation: str
+    object: str
+
+class KnowledgeGraphExtraction(BaseModel):
+    triples: List[RelationshipTriple]
+
+def extract_knowledge_robust(text: str, user_id: str = "default_user"):
+    api_key = settings.GROQ_API_KEY if settings.GROQ_API_KEY else "dummy_key"
+    llm = ChatGroq(model="llama-3.1-8b-instant", api_key=api_key)
+    
+    owner = user_id.upper()
+    
+    prompt = f"""You are a child's brain learning about the world. Read the text and pick out SIMPLE facts as connections between concepts.
+
+Think like a child drawing a mind-map:
+- "{owner}" is the person speaking. If they say "I like X" → {owner} --LIKES--> X
+- Extract only SHORT concept names (1-3 words). Never use full sentences as names.
+- Focus on: people, places, things, hobbies, foods, animals, feelings, skills, jobs
+
+RULES:
+1. Nodes must be 1-3 word concept names, ALL CAPS. Example: "CRICKET", "DELHI", "MOM", "CODING"
+2. Relations must be simple verbs: LIKES, IS_A, LIVES_IN, PLAYS, WORKS_AT, HAS, KNOWS, STUDIES, etc.
+3. "I" or "my" in the text refers to "{owner}" — always use "{owner}" as the node name for the speaker.
+4. DO NOT create nodes named "USER", "SOMA", "AI", "ASSISTANT", or any chat/bot terms.
+5. If the text is just greetings or small talk with zero factual content, return an empty triples list.
+
+Text:
+{text}
+
+Return the extracted facts ONLY as a valid JSON block in this exact format:
+{{
+  "triples": [
+    {{"subject": "SUBJECT", "relation": "RELATION", "object": "OBJECT"}}
+  ]
+}}
+Do not write any other explanation or thoughts outside the JSON block. If there are no facts, return: {{"triples": []}}"""
+
+    response = llm.invoke([HumanMessage(content=prompt)])
+    content = response.content.strip()
+    
+    print("\n--- Raw LLM Response ---")
+    print(content)
+    
+    # Extract JSON block using regex
+    match = re.search(r'\{.*\}', content, re.DOTALL)
+    if match:
+        json_str = match.group(0)
+        try:
+            data = json.loads(json_str)
+            triples = []
+            for t in data.get("triples", []):
+                triples.append(RelationshipTriple(
+                    subject=t.get("subject", ""),
+                    relation=t.get("relation", ""),
+                    object=t.get("object", "")
+                ))
+            return triples
+        except Exception as e:
+            print("Failed to parse JSON:", e)
+            return []
+    else:
+        print("No JSON block found.")
+        return []
+
+def main():
+    print("Testing robust extraction method...")
+    test_text = "Dr. Aris is the creator of the Soma Project. The Soma Project is an advanced cognitive architecture."
+    triples = extract_knowledge_robust(test_text, "komal")
+    print("\n--- Extracted Triples ---")
+    for t in triples:
+        print(f"- {t.subject} --{t.relation}--> {t.object}")
+
+if __name__ == "__main__":
+    main()

scratch/test_semantic_alias.py

@@ -0,0 +1,53 @@
+"""Test meaning-based entity resolution: synonyms merge, related-but-different things don't."""
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from app.services.neocortex import extract_and_store_knowledge
+from app.db.neo4j_driver import neo4j_db
+
+TEST_USER = "test_semantic"
+
+
+def show_graph(label):
+    res = neo4j_db.query(
+        "MATCH (n:Entity)-[r]->(m:Entity) WHERE n.user_id = $u AND m.user_id = $u "
+        "RETURN n.name AS s, type(r) AS rel, m.name AS o",
+        {"u": TEST_USER},
+    )
+    print(f"{label}:")
+    for row in res or []:
+        print(f"  {row['s']} --{row['rel']}--> {row['o']}")
+    print()
+
+
+def main():
+    if not neo4j_db.driver:
+        from app.core.config import settings
+        neo4j_db.connect(settings.NEO4J_URI, settings.NEO4J_USER, settings.NEO4J_PASSWORD)
+
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+
+    print("=== Step 1: 'I love soccer and I live in Delhi' ===")
+    extract_and_store_knowledge("I love soccer and I live in Delhi", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 2: 'I hate football' (synonym! should hit the SOCCER node) ===")
+    extract_and_store_knowledge("I hate football", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 3: 'I like pasta' (related to nothing — must be a NEW node) ===")
+    extract_and_store_knowledge("I like pasta", TEST_USER)
+    show_graph("Graph")
+
+    print("=== Step 4: 'I moved to Mumbai' (must NOT merge with Delhi, must replace LIVES_IN) ===")
+    extract_and_store_knowledge("I moved to Mumbai", TEST_USER)
+    show_graph("Graph")
+
+    neo4j_db.query("MATCH (n:Entity) WHERE n.user_id = $u DETACH DELETE n", {"u": TEST_USER})
+    print("Test nodes cleaned up.")
+
+
+if __name__ == "__main__":
+    main()