"""C12: one principal's durable conversation, in bounded store documents. The legacy Query document is an immutable migration source. Sequence numbers locate turns without an ever-growing message-id index; old ids remain in payload for ratings. """ import copy import datetime import hashlib import inspect from core import freeze_notice, grid_events CHUNK_TURNS = 200 ROLES = {"user", "assistant", "system"} DOORS = {"app", "whatsapp", "wake"} KINDS = {"text", "ping", "confirm", "divider", "notice"} def _get(rt, key): """Bypass a process cache when the bound runtime supports fresh reads.""" try: if "fresh" in inspect.signature(rt.get).parameters: return rt.get(key, fresh=True) return rt.get(key) except Exception as exc: raise grid_events.StoreUnavailable("Conversation store read failed.") from exc def _update(rt, key, fn): try: return rt.update(key, fn, flush="sync") except ValueError: raise except Exception as exc: raise grid_events.StoreUnavailable("Conversation store write failed.") from exc def _tenant_of(rt): # TenantRuntime.tenant is a Tenant object; its key is the session's tenant slug. tenant = getattr(rt, "key", None) or getattr(rt, "tenant", None) if not isinstance(tenant, str) or not tenant: raise ValueError("The conversation needs a tenant-bound runtime.") return tenant def _hash(rt, username): if not isinstance(username, str) or not username: raise ValueError("The conversation needs a username.") return hashlib.sha256(f"{_tenant_of(rt)}:{username}".encode("utf-8")).hexdigest()[:24] def _index(raw): raw = raw or {} return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)), "chunks": copy.deepcopy(raw.get("chunks") or {}), "pending": copy.deepcopy(raw.get("pending") or {}), "once": copy.deepcopy(raw.get("once") or {}), "onceVersion": int(raw.get("onceVersion", 0)), "folded": bool(raw.get("folded")), "foldedFrom": copy.deepcopy(raw.get("foldedFrom"))} def _chunk_key(h, n): return f"assistant_log_{h}_{n}" def _turn(h, seq, value): return {**copy.deepcopy(value), "id": f"{h}.{seq}", "seq": seq} def _put_rows(rt, h, n, rows): def update(raw): existing = {row["seq"]: row for row in (raw or {}).get("turns", [])} for row in rows: if row["seq"] // CHUNK_TURNS != n: raise ValueError("A turn belongs to a different conversation chunk.") # A retry or simultaneous fold never duplicates a turn or erases a later rating. existing.setdefault(row["seq"], copy.deepcopy(row)) if len(existing) > CHUNK_TURNS: raise ValueError("A conversation chunk is full.") return {"version": 1, "turns": [existing[seq] for seq in sorted(existing)]} _update(rt, _chunk_key(h, n), update) def _ack(rt, h, rows): sequences = {row["seq"] for row in rows} reservations = {key: seq for key, seq in _index(_get(rt, f"assistant_log_{h}_index"))["once"].items() if seq in sequences} # Publish the receipt after the chunk and before forgetting its allocation. for key, seq in reservations.items(): _update(rt, f"assistant_log_{h}_once_{key}", lambda raw, seq=seq: raw or {"seq": seq, "id": f"{h}.{seq}"}) committed = [] def finish(raw): current = _index(raw) for row in rows: current["pending"].pop(str(row["seq"]), None) for key, seq in reservations.items(): if current["once"].get(key) == seq: current["once"].pop(key, None) current["onceVersion"] += 1 committed[:] = [copy.deepcopy(current)] return current _update(rt, f"assistant_log_{h}_index", finish) return committed[0] def _recover(rt, h, index): """Publish journalled rows before removing their durable recovery copies.""" rows = list(index["pending"].values()) if not rows: return index chunks = {} for row in rows: chunks.setdefault(row["seq"] // CHUNK_TURNS, []).append(row) for n, values in chunks.items(): _put_rows(rt, h, n, values) _ack(rt, h, rows) # Keep this read's sequence boundary when other appends allocate during recovery. return {**index, "pending": {}} def _ensure_fold(rt, h): available = getattr(rt, "available", None) if callable(available) and not available(): raise grid_events.StoreUnavailable("Conversation store is unavailable.") key = f"assistant_log_{h}_index" index = _index(_get(rt, key)) if index["folded"]: return _recover(rt, h, index) old = _get(rt, "query_user_" + h) or {} threads, messages = old.get("threads") or {}, old.get("messages") or {} rows = sorted(messages.values(), key=lambda row: ( str(row.get("createdAt") or ""), 0 if row.get("role") == "user" else 1, str(row.get("threadId") or ""))) turns, seen = [], set() for row in rows: tid = row.get("threadId") if tid not in seen: seen.add(tid) turns.append(_turn(h, len(turns), { "role": "system", "door": "app", "kind": "divider", "text": (threads.get(tid) or {}).get("title") or "Previous chat", "payload": {"threadId": tid}, "at": row.get("createdAt")})) turns.append(_turn(h, len(turns), { "role": row.get("role"), "door": "app", "kind": "text", "text": row.get("content", ""), "at": row.get("createdAt"), "payload": {k: copy.deepcopy(v) for k, v in row.items() if k not in ("role", "content", "createdAt")}})) # Deterministic sequences make an interrupted fold resumable and simultaneous folds # harmless. Chunks merge by sequence, so a stale folder cannot erase fresh appends. for start in range(0, len(turns), CHUNK_TURNS): _put_rows(rt, h, start // CHUNK_TURNS, turns[start:start + CHUNK_TURNS]) committed = [] def finish(raw): current = _index(raw) if not current["folded"]: current.update(nextSeq=len(turns), folded=True, foldedFrom={"threads": len(threads), "messages": len(messages)}) current["chunks"] = {str(n): min(CHUNK_TURNS, len(turns) - n * CHUNK_TURNS) for n in range((len(turns) + CHUNK_TURNS - 1) // CHUNK_TURNS)} committed[:] = [copy.deepcopy(current)] return current _update(rt, key, finish) return _recover(rt, h, committed[0]) class _RetryAppend(ValueError): pass def append(rt, username, turn): """Append one principal-bound conversation turn.""" return _append(rt, username, turn) def append_once(rt, username, turn, *, idempotency_key): """Append one turn with a durable retry receipt.""" return _append(rt, username, turn, idempotency_key=idempotency_key) def _append(rt, username, turn, *, idempotency_key=None): """Append one C12 turn and return its opaque, principal-bound turn id.""" if not isinstance(turn, dict): raise ValueError("A conversation turn must be an object.") if turn.get("door") == "slack": raise ValueError("Slack conversations are never stored.") for key, allowed in (("role", ROLES), ("door", DOORS), ("kind", KINDS)): if not isinstance(turn.get(key), str) or turn[key] not in allowed: raise ValueError(f"That conversation {key} is not available.") if not isinstance(turn.get("text"), str): raise ValueError("A conversation turn needs text.") if "payload" in turn and not isinstance(turn["payload"], dict): raise ValueError("A conversation payload must be an object.") if turn.get("at") is not None and not isinstance(turn["at"], str): raise ValueError("A conversation timestamp must be text.") value = {key: copy.deepcopy(turn[key]) for key in ("role", "door", "kind", "text", "payload") if key in turn} value["at"] = turn.get("at") or datetime.datetime.now(datetime.timezone.utc).isoformat() h = _hash(rt, username) once = None if idempotency_key is not None: if not isinstance(idempotency_key, str) or not idempotency_key: raise ValueError("A conversation retry needs a result ID.") once = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest() _ensure_fold(rt, h) allocated = [] boundary = None def allocate(raw): index = _index(raw) if once is not None and (index["nextSeq"], index["onceVersion"]) != boundary: raise _RetryAppend("The conversation allocation changed.") if once is not None and once in index["once"]: seq = index["once"][once] allocated[:] = [copy.deepcopy(index["pending"][str(seq)])] return index seq = index["nextSeq"] n = seq // CHUNK_TURNS count = index["chunks"].get(str(n), 0) if count >= CHUNK_TURNS: raise ValueError("A conversation chunk is full.") if len(index["pending"]) >= CHUNK_TURNS: raise grid_events.StoreUnavailable("Conversation publication needs recovery.") index["nextSeq"] = seq + 1 index["chunks"][str(n)] = count + 1 index["pending"][str(seq)] = _turn(h, seq, value) if once is not None: index["once"][once] = seq allocated[:] = [copy.deepcopy(index["pending"][str(seq)])] return index # Allocation retains the row until its chunk is durable. while True: if once is not None: index = _index(_get(rt, f"assistant_log_{h}_index")) boundary = (index["nextSeq"], index["onceVersion"]) receipt = _get(rt, f"assistant_log_{h}_once_{once}") if receipt: return receipt["id"] try: _update(rt, f"assistant_log_{h}_index", allocate) break except _RetryAppend: continue row = allocated[0] _put_rows(rt, h, row["seq"] // CHUNK_TURNS, [row]) _ack(rt, h, [row]) return row["id"] def _seq(h, turn_id): if isinstance(turn_id, int) and not isinstance(turn_id, bool): return turn_id value = str(turn_id) if value.startswith(h + "."): value = value[len(h) + 1:] if value.isdecimal(): return int(value) return None def _rows(rt, h, index, n): rows = (_get(rt, _chunk_key(h, n)) or {}).get("turns", []) expected = set(range(n * CHUNK_TURNS, min((n + 1) * CHUNK_TURNS, index["nextSeq"]))) if not expected.issubset({row["seq"] for row in rows}): raise grid_events.StoreUnavailable("Conversation chunk is incomplete.") return rows def _newest(rt, h, index, before=None): end = min(index["nextSeq"], before) if before is not None else index["nextSeq"] for n in range((end - 1) // CHUNK_TURNS, -1, -1): if not index["chunks"].get(str(n)): continue for row in sorted(_rows(rt, h, index, n), key=lambda row: row["seq"], reverse=True): if row["seq"] < end: yield copy.deepcopy(row) class _Projection: """Fold and recover read results in memory while maintenance forbids durable writes.""" def __init__(self, source): self.source = source self.key = _tenant_of(source) self.documents = {} def get(self, key): return copy.deepcopy(self.documents[key] if key in self.documents else _get(self.source, key)) def available(self): return getattr(self.source, "available", lambda: True)() def update(self, key, fn, **kwargs): current = self.get(key) or {} changed = fn(current) self.documents[key] = copy.deepcopy(current if changed is None else changed) return self.get(key) def _read_runtime(rt): return _Projection(rt) if freeze_notice.enabled() and not isinstance(rt, _Projection) else rt def read(rt, username, *, before=None, limit=50): """Newest page by default; each page is oldest first and its first id pages backward.""" if freeze_notice.enabled(): rt = _read_runtime(rt) h = _hash(rt, username) index = _ensure_fold(rt, h) limit = max(1, min(CHUNK_TURNS, int(limit))) boundary = None if before is not None: boundary = _seq(h, before) if boundary is None: found = find(rt, username, before) if found is None: raise ValueError("That conversation cursor does not exist.") boundary = found["seq"] rows = [] for row in _newest(rt, h, index, boundary): rows.append(row) if len(rows) >= limit: break # Counts in the index answer whether earlier sequences exist without fetching a # preceding chunk solely for lookahead at an exact chunk boundary. first = rows[-1]["seq"] if rows else 0 has_more = bool(rows) and (first % CHUNK_TURNS > 0 or any( count for n, count in index["chunks"].items() if int(n) < first // CHUNK_TURNS)) return {"turns": list(reversed(rows)), "hasMore": has_more} def find(rt, username, turn_id): """Find a turn by C12 id, sequence, or the legacy message id in payload.""" rt = _read_runtime(rt) h = _hash(rt, username) index = _ensure_fold(rt, h) seq = _seq(h, turn_id) if seq is not None: if seq < 0 or seq >= index["nextSeq"]: return None return next((copy.deepcopy(row) for row in _rows(rt, h, index, seq // CHUNK_TURNS) if row["seq"] == seq), None) return next((row for row in _newest(rt, h, index) if (row.get("payload") or {}).get("id") == turn_id), None) def update_turn(rt, username, turn_id, fn): row = find(rt, username, turn_id) if row is None: return None h = _hash(rt, username) changed = [] def update(raw): current = copy.deepcopy(raw) for i, value in enumerate(current.get("turns", [])): if value["id"] == row["id"]: new = fn(copy.deepcopy(value)) if not isinstance(new, dict): raise ValueError("Updating a conversation turn needs an object.") new.update(id=value["id"], seq=value["seq"]) current["turns"][i] = new changed[:] = [copy.deepcopy(new)] return current _update(rt, _chunk_key(h, row["seq"] // CHUNK_TURNS), update) return changed[0] if changed else None def search(rt, username, query, limit=10): words = str(query or "").casefold()[:1000].split() if not words: return [] rt = _read_runtime(rt) h = _hash(rt, username) index = _ensure_fold(rt, h) limit = max(1, min(CHUNK_TURNS, int(limit))) found = [] for row in _newest(rt, h, index): if all(word in row.get("text", "").casefold() for word in words): found.append(row) if len(found) >= limit: break return found def _recall(session, args): from routes_query import _no_dashes result = [] for row in search(session.runtime, session.uname, args.get("query"), args.get("limit", 10)): text = _no_dashes(row["text"]) if len(text) > 400: text = text[:386] + " [text cut]" result.append({"id": row["id"], "role": row["role"], "at": row["at"], "text": text}) return result try: import agent_tools except ImportError: pass else: agent_tools.register("recall", { "type": "object", "description": "Search your own earlier conversation turns.", "properties": {"query": {"type": "string"}, "limit": {"type": "integer"}}, "required": ["query"], "additionalProperties": False}, _recall, effect="read")