loopable / api /assistant_log.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
c72f0a1 verified
Raw History Blame Contribute Delete
15.9 kB
"""C12: one principal's durable conversation, in bounded store documents.
The legacy Query document is an immutable migration source. Sequence numbers locate
turns without an ever-growing message-id index; old ids remain in payload for ratings.
"""
import copy
import datetime
import hashlib
import inspect
from core import freeze_notice, grid_events
CHUNK_TURNS = 200
ROLES = {"user", "assistant", "system"}
DOORS = {"app", "whatsapp", "wake"}
KINDS = {"text", "ping", "confirm", "divider", "notice"}
def _get(rt, key):
"""Bypass a process cache when the bound runtime supports fresh reads."""
try:
if "fresh" in inspect.signature(rt.get).parameters:
return rt.get(key, fresh=True)
return rt.get(key)
except Exception as exc:
raise grid_events.StoreUnavailable("Conversation store read failed.") from exc
def _update(rt, key, fn):
try:
return rt.update(key, fn, flush="sync")
except ValueError:
raise
except Exception as exc:
raise grid_events.StoreUnavailable("Conversation store write failed.") from exc
def _tenant_of(rt):
# TenantRuntime.tenant is a Tenant object; its key is the session's tenant slug.
tenant = getattr(rt, "key", None) or getattr(rt, "tenant", None)
if not isinstance(tenant, str) or not tenant:
raise ValueError("The conversation needs a tenant-bound runtime.")
return tenant
def _hash(rt, username):
if not isinstance(username, str) or not username:
raise ValueError("The conversation needs a username.")
return hashlib.sha256(f"{_tenant_of(rt)}:{username}".encode("utf-8")).hexdigest()[:24]
def _index(raw):
raw = raw or {}
return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)),
"chunks": copy.deepcopy(raw.get("chunks") or {}),
"pending": copy.deepcopy(raw.get("pending") or {}),
"once": copy.deepcopy(raw.get("once") or {}),
"onceVersion": int(raw.get("onceVersion", 0)),
"folded": bool(raw.get("folded")), "foldedFrom": copy.deepcopy(raw.get("foldedFrom"))}
def _chunk_key(h, n):
return f"assistant_log_{h}_{n}"
def _turn(h, seq, value):
return {**copy.deepcopy(value), "id": f"{h}.{seq}", "seq": seq}
def _put_rows(rt, h, n, rows):
def update(raw):
existing = {row["seq"]: row for row in (raw or {}).get("turns", [])}
for row in rows:
if row["seq"] // CHUNK_TURNS != n:
raise ValueError("A turn belongs to a different conversation chunk.")
# A retry or simultaneous fold never duplicates a turn or erases a later rating.
existing.setdefault(row["seq"], copy.deepcopy(row))
if len(existing) > CHUNK_TURNS:
raise ValueError("A conversation chunk is full.")
return {"version": 1, "turns": [existing[seq] for seq in sorted(existing)]}
_update(rt, _chunk_key(h, n), update)
def _ack(rt, h, rows):
sequences = {row["seq"] for row in rows}
reservations = {key: seq for key, seq in _index(_get(rt, f"assistant_log_{h}_index"))["once"].items()
if seq in sequences}
# Publish the receipt after the chunk and before forgetting its allocation.
for key, seq in reservations.items():
_update(rt, f"assistant_log_{h}_once_{key}",
lambda raw, seq=seq: raw or {"seq": seq, "id": f"{h}.{seq}"})
committed = []
def finish(raw):
current = _index(raw)
for row in rows:
current["pending"].pop(str(row["seq"]), None)
for key, seq in reservations.items():
if current["once"].get(key) == seq:
current["once"].pop(key, None)
current["onceVersion"] += 1
committed[:] = [copy.deepcopy(current)]
return current
_update(rt, f"assistant_log_{h}_index", finish)
return committed[0]
def _recover(rt, h, index):
"""Publish journalled rows before removing their durable recovery copies."""
rows = list(index["pending"].values())
if not rows:
return index
chunks = {}
for row in rows:
chunks.setdefault(row["seq"] // CHUNK_TURNS, []).append(row)
for n, values in chunks.items():
_put_rows(rt, h, n, values)
_ack(rt, h, rows)
# Keep this read's sequence boundary when other appends allocate during recovery.
return {**index, "pending": {}}
def _ensure_fold(rt, h):
available = getattr(rt, "available", None)
if callable(available) and not available():
raise grid_events.StoreUnavailable("Conversation store is unavailable.")
key = f"assistant_log_{h}_index"
index = _index(_get(rt, key))
if index["folded"]:
return _recover(rt, h, index)
old = _get(rt, "query_user_" + h) or {}
threads, messages = old.get("threads") or {}, old.get("messages") or {}
rows = sorted(messages.values(), key=lambda row: (
str(row.get("createdAt") or ""), 0 if row.get("role") == "user" else 1,
str(row.get("threadId") or "")))
turns, seen = [], set()
for row in rows:
tid = row.get("threadId")
if tid not in seen:
seen.add(tid)
turns.append(_turn(h, len(turns), {
"role": "system", "door": "app", "kind": "divider",
"text": (threads.get(tid) or {}).get("title") or "Previous chat",
"payload": {"threadId": tid}, "at": row.get("createdAt")}))
turns.append(_turn(h, len(turns), {
"role": row.get("role"), "door": "app", "kind": "text",
"text": row.get("content", ""), "at": row.get("createdAt"),
"payload": {k: copy.deepcopy(v) for k, v in row.items()
if k not in ("role", "content", "createdAt")}}))
# Deterministic sequences make an interrupted fold resumable and simultaneous folds
# harmless. Chunks merge by sequence, so a stale folder cannot erase fresh appends.
for start in range(0, len(turns), CHUNK_TURNS):
_put_rows(rt, h, start // CHUNK_TURNS, turns[start:start + CHUNK_TURNS])
committed = []
def finish(raw):
current = _index(raw)
if not current["folded"]:
current.update(nextSeq=len(turns), folded=True,
foldedFrom={"threads": len(threads), "messages": len(messages)})
current["chunks"] = {str(n): min(CHUNK_TURNS, len(turns) - n * CHUNK_TURNS)
for n in range((len(turns) + CHUNK_TURNS - 1) // CHUNK_TURNS)}
committed[:] = [copy.deepcopy(current)]
return current
_update(rt, key, finish)
return _recover(rt, h, committed[0])
class _RetryAppend(ValueError):
pass
def append(rt, username, turn):
"""Append one principal-bound conversation turn."""
return _append(rt, username, turn)
def append_once(rt, username, turn, *, idempotency_key):
"""Append one turn with a durable retry receipt."""
return _append(rt, username, turn, idempotency_key=idempotency_key)
def _append(rt, username, turn, *, idempotency_key=None):
"""Append one C12 turn and return its opaque, principal-bound turn id."""
if not isinstance(turn, dict):
raise ValueError("A conversation turn must be an object.")
if turn.get("door") == "slack":
raise ValueError("Slack conversations are never stored.")
for key, allowed in (("role", ROLES), ("door", DOORS), ("kind", KINDS)):
if not isinstance(turn.get(key), str) or turn[key] not in allowed:
raise ValueError(f"That conversation {key} is not available.")
if not isinstance(turn.get("text"), str):
raise ValueError("A conversation turn needs text.")
if "payload" in turn and not isinstance(turn["payload"], dict):
raise ValueError("A conversation payload must be an object.")
if turn.get("at") is not None and not isinstance(turn["at"], str):
raise ValueError("A conversation timestamp must be text.")
value = {key: copy.deepcopy(turn[key]) for key in ("role", "door", "kind", "text", "payload")
if key in turn}
value["at"] = turn.get("at") or datetime.datetime.now(datetime.timezone.utc).isoformat()
h = _hash(rt, username)
once = None
if idempotency_key is not None:
if not isinstance(idempotency_key, str) or not idempotency_key:
raise ValueError("A conversation retry needs a result ID.")
once = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
_ensure_fold(rt, h)
allocated = []
boundary = None
def allocate(raw):
index = _index(raw)
if once is not None and (index["nextSeq"], index["onceVersion"]) != boundary:
raise _RetryAppend("The conversation allocation changed.")
if once is not None and once in index["once"]:
seq = index["once"][once]
allocated[:] = [copy.deepcopy(index["pending"][str(seq)])]
return index
seq = index["nextSeq"]
n = seq // CHUNK_TURNS
count = index["chunks"].get(str(n), 0)
if count >= CHUNK_TURNS:
raise ValueError("A conversation chunk is full.")
if len(index["pending"]) >= CHUNK_TURNS:
raise grid_events.StoreUnavailable("Conversation publication needs recovery.")
index["nextSeq"] = seq + 1
index["chunks"][str(n)] = count + 1
index["pending"][str(seq)] = _turn(h, seq, value)
if once is not None:
index["once"][once] = seq
allocated[:] = [copy.deepcopy(index["pending"][str(seq)])]
return index
# Allocation retains the row until its chunk is durable.
while True:
if once is not None:
index = _index(_get(rt, f"assistant_log_{h}_index"))
boundary = (index["nextSeq"], index["onceVersion"])
receipt = _get(rt, f"assistant_log_{h}_once_{once}")
if receipt:
return receipt["id"]
try:
_update(rt, f"assistant_log_{h}_index", allocate)
break
except _RetryAppend:
continue
row = allocated[0]
_put_rows(rt, h, row["seq"] // CHUNK_TURNS, [row])
_ack(rt, h, [row])
return row["id"]
def _seq(h, turn_id):
if isinstance(turn_id, int) and not isinstance(turn_id, bool):
return turn_id
value = str(turn_id)
if value.startswith(h + "."):
value = value[len(h) + 1:]
if value.isdecimal():
return int(value)
return None
def _rows(rt, h, index, n):
rows = (_get(rt, _chunk_key(h, n)) or {}).get("turns", [])
expected = set(range(n * CHUNK_TURNS, min((n + 1) * CHUNK_TURNS, index["nextSeq"])))
if not expected.issubset({row["seq"] for row in rows}):
raise grid_events.StoreUnavailable("Conversation chunk is incomplete.")
return rows
def _newest(rt, h, index, before=None):
end = min(index["nextSeq"], before) if before is not None else index["nextSeq"]
for n in range((end - 1) // CHUNK_TURNS, -1, -1):
if not index["chunks"].get(str(n)):
continue
for row in sorted(_rows(rt, h, index, n), key=lambda row: row["seq"], reverse=True):
if row["seq"] < end:
yield copy.deepcopy(row)
class _Projection:
"""Fold and recover read results in memory while maintenance forbids durable writes."""
def __init__(self, source):
self.source = source
self.key = _tenant_of(source)
self.documents = {}
def get(self, key):
return copy.deepcopy(self.documents[key] if key in self.documents else _get(self.source, key))
def available(self):
return getattr(self.source, "available", lambda: True)()
def update(self, key, fn, **kwargs):
current = self.get(key) or {}
changed = fn(current)
self.documents[key] = copy.deepcopy(current if changed is None else changed)
return self.get(key)
def _read_runtime(rt):
return _Projection(rt) if freeze_notice.enabled() and not isinstance(rt, _Projection) else rt
def read(rt, username, *, before=None, limit=50):
"""Newest page by default; each page is oldest first and its first id pages backward."""
if freeze_notice.enabled():
rt = _read_runtime(rt)
h = _hash(rt, username)
index = _ensure_fold(rt, h)
limit = max(1, min(CHUNK_TURNS, int(limit)))
boundary = None
if before is not None:
boundary = _seq(h, before)
if boundary is None:
found = find(rt, username, before)
if found is None:
raise ValueError("That conversation cursor does not exist.")
boundary = found["seq"]
rows = []
for row in _newest(rt, h, index, boundary):
rows.append(row)
if len(rows) >= limit:
break
# Counts in the index answer whether earlier sequences exist without fetching a
# preceding chunk solely for lookahead at an exact chunk boundary.
first = rows[-1]["seq"] if rows else 0
has_more = bool(rows) and (first % CHUNK_TURNS > 0 or any(
count for n, count in index["chunks"].items() if int(n) < first // CHUNK_TURNS))
return {"turns": list(reversed(rows)), "hasMore": has_more}
def find(rt, username, turn_id):
"""Find a turn by C12 id, sequence, or the legacy message id in payload."""
rt = _read_runtime(rt)
h = _hash(rt, username)
index = _ensure_fold(rt, h)
seq = _seq(h, turn_id)
if seq is not None:
if seq < 0 or seq >= index["nextSeq"]:
return None
return next((copy.deepcopy(row) for row in _rows(rt, h, index, seq // CHUNK_TURNS)
if row["seq"] == seq), None)
return next((row for row in _newest(rt, h, index)
if (row.get("payload") or {}).get("id") == turn_id), None)
def update_turn(rt, username, turn_id, fn):
row = find(rt, username, turn_id)
if row is None:
return None
h = _hash(rt, username)
changed = []
def update(raw):
current = copy.deepcopy(raw)
for i, value in enumerate(current.get("turns", [])):
if value["id"] == row["id"]:
new = fn(copy.deepcopy(value))
if not isinstance(new, dict):
raise ValueError("Updating a conversation turn needs an object.")
new.update(id=value["id"], seq=value["seq"])
current["turns"][i] = new
changed[:] = [copy.deepcopy(new)]
return current
_update(rt, _chunk_key(h, row["seq"] // CHUNK_TURNS), update)
return changed[0] if changed else None
def search(rt, username, query, limit=10):
words = str(query or "").casefold()[:1000].split()
if not words:
return []
rt = _read_runtime(rt)
h = _hash(rt, username)
index = _ensure_fold(rt, h)
limit = max(1, min(CHUNK_TURNS, int(limit)))
found = []
for row in _newest(rt, h, index):
if all(word in row.get("text", "").casefold() for word in words):
found.append(row)
if len(found) >= limit:
break
return found
def _recall(session, args):
from routes_query import _no_dashes
result = []
for row in search(session.runtime, session.uname, args.get("query"), args.get("limit", 10)):
text = _no_dashes(row["text"])
if len(text) > 400:
text = text[:386] + " [text cut]"
result.append({"id": row["id"], "role": row["role"], "at": row["at"], "text": text})
return result
try:
import agent_tools
except ImportError:
pass
else:
agent_tools.register("recall", {
"type": "object", "description": "Search your own earlier conversation turns.",
"properties": {"query": {"type": "string"}, "limit": {"type": "integer"}},
"required": ["query"], "additionalProperties": False}, _recall, effect="read")