Download api/assistant_log.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 15.9 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/assistant_log.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable/api/assistant_log.py
-
curl -L -o assistant_log.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/api/assistant_log.py
15.9 kB
| """C12: one principal's durable conversation, in bounded store documents. | |
| The legacy Query document is an immutable migration source. Sequence numbers locate | |
| turns without an ever-growing message-id index; old ids remain in payload for ratings. | |
| """ | |
| import copy | |
| import datetime | |
| import hashlib | |
| import inspect | |
| from core import freeze_notice, grid_events | |
| CHUNK_TURNS = 200 | |
| ROLES = {"user", "assistant", "system"} | |
| DOORS = {"app", "whatsapp", "wake"} | |
| KINDS = {"text", "ping", "confirm", "divider", "notice"} | |
| def _get(rt, key): | |
| """Bypass a process cache when the bound runtime supports fresh reads.""" | |
| try: | |
| if "fresh" in inspect.signature(rt.get).parameters: | |
| return rt.get(key, fresh=True) | |
| return rt.get(key) | |
| except Exception as exc: | |
| raise grid_events.StoreUnavailable("Conversation store read failed.") from exc | |
| def _update(rt, key, fn): | |
| try: | |
| return rt.update(key, fn, flush="sync") | |
| except ValueError: | |
| raise | |
| except Exception as exc: | |
| raise grid_events.StoreUnavailable("Conversation store write failed.") from exc | |
| def _tenant_of(rt): | |
| # TenantRuntime.tenant is a Tenant object; its key is the session's tenant slug. | |
| tenant = getattr(rt, "key", None) or getattr(rt, "tenant", None) | |
| if not isinstance(tenant, str) or not tenant: | |
| raise ValueError("The conversation needs a tenant-bound runtime.") | |
| return tenant | |
| def _hash(rt, username): | |
| if not isinstance(username, str) or not username: | |
| raise ValueError("The conversation needs a username.") | |
| return hashlib.sha256(f"{_tenant_of(rt)}:{username}".encode("utf-8")).hexdigest()[:24] | |
| def _index(raw): | |
| raw = raw or {} | |
| return {"version": 1, "nextSeq": int(raw.get("nextSeq", 0)), | |
| "chunks": copy.deepcopy(raw.get("chunks") or {}), | |
| "pending": copy.deepcopy(raw.get("pending") or {}), | |
| "once": copy.deepcopy(raw.get("once") or {}), | |
| "onceVersion": int(raw.get("onceVersion", 0)), | |
| "folded": bool(raw.get("folded")), "foldedFrom": copy.deepcopy(raw.get("foldedFrom"))} | |
| def _chunk_key(h, n): | |
| return f"assistant_log_{h}_{n}" | |
| def _turn(h, seq, value): | |
| return {**copy.deepcopy(value), "id": f"{h}.{seq}", "seq": seq} | |
| def _put_rows(rt, h, n, rows): | |
| def update(raw): | |
| existing = {row["seq"]: row for row in (raw or {}).get("turns", [])} | |
| for row in rows: | |
| if row["seq"] // CHUNK_TURNS != n: | |
| raise ValueError("A turn belongs to a different conversation chunk.") | |
| # A retry or simultaneous fold never duplicates a turn or erases a later rating. | |
| existing.setdefault(row["seq"], copy.deepcopy(row)) | |
| if len(existing) > CHUNK_TURNS: | |
| raise ValueError("A conversation chunk is full.") | |
| return {"version": 1, "turns": [existing[seq] for seq in sorted(existing)]} | |
| _update(rt, _chunk_key(h, n), update) | |
| def _ack(rt, h, rows): | |
| sequences = {row["seq"] for row in rows} | |
| reservations = {key: seq for key, seq in _index(_get(rt, f"assistant_log_{h}_index"))["once"].items() | |
| if seq in sequences} | |
| # Publish the receipt after the chunk and before forgetting its allocation. | |
| for key, seq in reservations.items(): | |
| _update(rt, f"assistant_log_{h}_once_{key}", | |
| lambda raw, seq=seq: raw or {"seq": seq, "id": f"{h}.{seq}"}) | |
| committed = [] | |
| def finish(raw): | |
| current = _index(raw) | |
| for row in rows: | |
| current["pending"].pop(str(row["seq"]), None) | |
| for key, seq in reservations.items(): | |
| if current["once"].get(key) == seq: | |
| current["once"].pop(key, None) | |
| current["onceVersion"] += 1 | |
| committed[:] = [copy.deepcopy(current)] | |
| return current | |
| _update(rt, f"assistant_log_{h}_index", finish) | |
| return committed[0] | |
| def _recover(rt, h, index): | |
| """Publish journalled rows before removing their durable recovery copies.""" | |
| rows = list(index["pending"].values()) | |
| if not rows: | |
| return index | |
| chunks = {} | |
| for row in rows: | |
| chunks.setdefault(row["seq"] // CHUNK_TURNS, []).append(row) | |
| for n, values in chunks.items(): | |
| _put_rows(rt, h, n, values) | |
| _ack(rt, h, rows) | |
| # Keep this read's sequence boundary when other appends allocate during recovery. | |
| return {**index, "pending": {}} | |
| def _ensure_fold(rt, h): | |
| available = getattr(rt, "available", None) | |
| if callable(available) and not available(): | |
| raise grid_events.StoreUnavailable("Conversation store is unavailable.") | |
| key = f"assistant_log_{h}_index" | |
| index = _index(_get(rt, key)) | |
| if index["folded"]: | |
| return _recover(rt, h, index) | |
| old = _get(rt, "query_user_" + h) or {} | |
| threads, messages = old.get("threads") or {}, old.get("messages") or {} | |
| rows = sorted(messages.values(), key=lambda row: ( | |
| str(row.get("createdAt") or ""), 0 if row.get("role") == "user" else 1, | |
| str(row.get("threadId") or ""))) | |
| turns, seen = [], set() | |
| for row in rows: | |
| tid = row.get("threadId") | |
| if tid not in seen: | |
| seen.add(tid) | |
| turns.append(_turn(h, len(turns), { | |
| "role": "system", "door": "app", "kind": "divider", | |
| "text": (threads.get(tid) or {}).get("title") or "Previous chat", | |
| "payload": {"threadId": tid}, "at": row.get("createdAt")})) | |
| turns.append(_turn(h, len(turns), { | |
| "role": row.get("role"), "door": "app", "kind": "text", | |
| "text": row.get("content", ""), "at": row.get("createdAt"), | |
| "payload": {k: copy.deepcopy(v) for k, v in row.items() | |
| if k not in ("role", "content", "createdAt")}})) | |
| # Deterministic sequences make an interrupted fold resumable and simultaneous folds | |
| # harmless. Chunks merge by sequence, so a stale folder cannot erase fresh appends. | |
| for start in range(0, len(turns), CHUNK_TURNS): | |
| _put_rows(rt, h, start // CHUNK_TURNS, turns[start:start + CHUNK_TURNS]) | |
| committed = [] | |
| def finish(raw): | |
| current = _index(raw) | |
| if not current["folded"]: | |
| current.update(nextSeq=len(turns), folded=True, | |
| foldedFrom={"threads": len(threads), "messages": len(messages)}) | |
| current["chunks"] = {str(n): min(CHUNK_TURNS, len(turns) - n * CHUNK_TURNS) | |
| for n in range((len(turns) + CHUNK_TURNS - 1) // CHUNK_TURNS)} | |
| committed[:] = [copy.deepcopy(current)] | |
| return current | |
| _update(rt, key, finish) | |
| return _recover(rt, h, committed[0]) | |
| class _RetryAppend(ValueError): | |
| pass | |
| def append(rt, username, turn): | |
| """Append one principal-bound conversation turn.""" | |
| return _append(rt, username, turn) | |
| def append_once(rt, username, turn, *, idempotency_key): | |
| """Append one turn with a durable retry receipt.""" | |
| return _append(rt, username, turn, idempotency_key=idempotency_key) | |
| def _append(rt, username, turn, *, idempotency_key=None): | |
| """Append one C12 turn and return its opaque, principal-bound turn id.""" | |
| if not isinstance(turn, dict): | |
| raise ValueError("A conversation turn must be an object.") | |
| if turn.get("door") == "slack": | |
| raise ValueError("Slack conversations are never stored.") | |
| for key, allowed in (("role", ROLES), ("door", DOORS), ("kind", KINDS)): | |
| if not isinstance(turn.get(key), str) or turn[key] not in allowed: | |
| raise ValueError(f"That conversation {key} is not available.") | |
| if not isinstance(turn.get("text"), str): | |
| raise ValueError("A conversation turn needs text.") | |
| if "payload" in turn and not isinstance(turn["payload"], dict): | |
| raise ValueError("A conversation payload must be an object.") | |
| if turn.get("at") is not None and not isinstance(turn["at"], str): | |
| raise ValueError("A conversation timestamp must be text.") | |
| value = {key: copy.deepcopy(turn[key]) for key in ("role", "door", "kind", "text", "payload") | |
| if key in turn} | |
| value["at"] = turn.get("at") or datetime.datetime.now(datetime.timezone.utc).isoformat() | |
| h = _hash(rt, username) | |
| once = None | |
| if idempotency_key is not None: | |
| if not isinstance(idempotency_key, str) or not idempotency_key: | |
| raise ValueError("A conversation retry needs a result ID.") | |
| once = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest() | |
| _ensure_fold(rt, h) | |
| allocated = [] | |
| boundary = None | |
| def allocate(raw): | |
| index = _index(raw) | |
| if once is not None and (index["nextSeq"], index["onceVersion"]) != boundary: | |
| raise _RetryAppend("The conversation allocation changed.") | |
| if once is not None and once in index["once"]: | |
| seq = index["once"][once] | |
| allocated[:] = [copy.deepcopy(index["pending"][str(seq)])] | |
| return index | |
| seq = index["nextSeq"] | |
| n = seq // CHUNK_TURNS | |
| count = index["chunks"].get(str(n), 0) | |
| if count >= CHUNK_TURNS: | |
| raise ValueError("A conversation chunk is full.") | |
| if len(index["pending"]) >= CHUNK_TURNS: | |
| raise grid_events.StoreUnavailable("Conversation publication needs recovery.") | |
| index["nextSeq"] = seq + 1 | |
| index["chunks"][str(n)] = count + 1 | |
| index["pending"][str(seq)] = _turn(h, seq, value) | |
| if once is not None: | |
| index["once"][once] = seq | |
| allocated[:] = [copy.deepcopy(index["pending"][str(seq)])] | |
| return index | |
| # Allocation retains the row until its chunk is durable. | |
| while True: | |
| if once is not None: | |
| index = _index(_get(rt, f"assistant_log_{h}_index")) | |
| boundary = (index["nextSeq"], index["onceVersion"]) | |
| receipt = _get(rt, f"assistant_log_{h}_once_{once}") | |
| if receipt: | |
| return receipt["id"] | |
| try: | |
| _update(rt, f"assistant_log_{h}_index", allocate) | |
| break | |
| except _RetryAppend: | |
| continue | |
| row = allocated[0] | |
| _put_rows(rt, h, row["seq"] // CHUNK_TURNS, [row]) | |
| _ack(rt, h, [row]) | |
| return row["id"] | |
| def _seq(h, turn_id): | |
| if isinstance(turn_id, int) and not isinstance(turn_id, bool): | |
| return turn_id | |
| value = str(turn_id) | |
| if value.startswith(h + "."): | |
| value = value[len(h) + 1:] | |
| if value.isdecimal(): | |
| return int(value) | |
| return None | |
| def _rows(rt, h, index, n): | |
| rows = (_get(rt, _chunk_key(h, n)) or {}).get("turns", []) | |
| expected = set(range(n * CHUNK_TURNS, min((n + 1) * CHUNK_TURNS, index["nextSeq"]))) | |
| if not expected.issubset({row["seq"] for row in rows}): | |
| raise grid_events.StoreUnavailable("Conversation chunk is incomplete.") | |
| return rows | |
| def _newest(rt, h, index, before=None): | |
| end = min(index["nextSeq"], before) if before is not None else index["nextSeq"] | |
| for n in range((end - 1) // CHUNK_TURNS, -1, -1): | |
| if not index["chunks"].get(str(n)): | |
| continue | |
| for row in sorted(_rows(rt, h, index, n), key=lambda row: row["seq"], reverse=True): | |
| if row["seq"] < end: | |
| yield copy.deepcopy(row) | |
| class _Projection: | |
| """Fold and recover read results in memory while maintenance forbids durable writes.""" | |
| def __init__(self, source): | |
| self.source = source | |
| self.key = _tenant_of(source) | |
| self.documents = {} | |
| def get(self, key): | |
| return copy.deepcopy(self.documents[key] if key in self.documents else _get(self.source, key)) | |
| def available(self): | |
| return getattr(self.source, "available", lambda: True)() | |
| def update(self, key, fn, **kwargs): | |
| current = self.get(key) or {} | |
| changed = fn(current) | |
| self.documents[key] = copy.deepcopy(current if changed is None else changed) | |
| return self.get(key) | |
| def _read_runtime(rt): | |
| return _Projection(rt) if freeze_notice.enabled() and not isinstance(rt, _Projection) else rt | |
| def read(rt, username, *, before=None, limit=50): | |
| """Newest page by default; each page is oldest first and its first id pages backward.""" | |
| if freeze_notice.enabled(): | |
| rt = _read_runtime(rt) | |
| h = _hash(rt, username) | |
| index = _ensure_fold(rt, h) | |
| limit = max(1, min(CHUNK_TURNS, int(limit))) | |
| boundary = None | |
| if before is not None: | |
| boundary = _seq(h, before) | |
| if boundary is None: | |
| found = find(rt, username, before) | |
| if found is None: | |
| raise ValueError("That conversation cursor does not exist.") | |
| boundary = found["seq"] | |
| rows = [] | |
| for row in _newest(rt, h, index, boundary): | |
| rows.append(row) | |
| if len(rows) >= limit: | |
| break | |
| # Counts in the index answer whether earlier sequences exist without fetching a | |
| # preceding chunk solely for lookahead at an exact chunk boundary. | |
| first = rows[-1]["seq"] if rows else 0 | |
| has_more = bool(rows) and (first % CHUNK_TURNS > 0 or any( | |
| count for n, count in index["chunks"].items() if int(n) < first // CHUNK_TURNS)) | |
| return {"turns": list(reversed(rows)), "hasMore": has_more} | |
| def find(rt, username, turn_id): | |
| """Find a turn by C12 id, sequence, or the legacy message id in payload.""" | |
| rt = _read_runtime(rt) | |
| h = _hash(rt, username) | |
| index = _ensure_fold(rt, h) | |
| seq = _seq(h, turn_id) | |
| if seq is not None: | |
| if seq < 0 or seq >= index["nextSeq"]: | |
| return None | |
| return next((copy.deepcopy(row) for row in _rows(rt, h, index, seq // CHUNK_TURNS) | |
| if row["seq"] == seq), None) | |
| return next((row for row in _newest(rt, h, index) | |
| if (row.get("payload") or {}).get("id") == turn_id), None) | |
| def update_turn(rt, username, turn_id, fn): | |
| row = find(rt, username, turn_id) | |
| if row is None: | |
| return None | |
| h = _hash(rt, username) | |
| changed = [] | |
| def update(raw): | |
| current = copy.deepcopy(raw) | |
| for i, value in enumerate(current.get("turns", [])): | |
| if value["id"] == row["id"]: | |
| new = fn(copy.deepcopy(value)) | |
| if not isinstance(new, dict): | |
| raise ValueError("Updating a conversation turn needs an object.") | |
| new.update(id=value["id"], seq=value["seq"]) | |
| current["turns"][i] = new | |
| changed[:] = [copy.deepcopy(new)] | |
| return current | |
| _update(rt, _chunk_key(h, row["seq"] // CHUNK_TURNS), update) | |
| return changed[0] if changed else None | |
| def search(rt, username, query, limit=10): | |
| words = str(query or "").casefold()[:1000].split() | |
| if not words: | |
| return [] | |
| rt = _read_runtime(rt) | |
| h = _hash(rt, username) | |
| index = _ensure_fold(rt, h) | |
| limit = max(1, min(CHUNK_TURNS, int(limit))) | |
| found = [] | |
| for row in _newest(rt, h, index): | |
| if all(word in row.get("text", "").casefold() for word in words): | |
| found.append(row) | |
| if len(found) >= limit: | |
| break | |
| return found | |
| def _recall(session, args): | |
| from routes_query import _no_dashes | |
| result = [] | |
| for row in search(session.runtime, session.uname, args.get("query"), args.get("limit", 10)): | |
| text = _no_dashes(row["text"]) | |
| if len(text) > 400: | |
| text = text[:386] + " [text cut]" | |
| result.append({"id": row["id"], "role": row["role"], "at": row["at"], "text": text}) | |
| return result | |
| try: | |
| import agent_tools | |
| except ImportError: | |
| pass | |
| else: | |
| agent_tools.register("recall", { | |
| "type": "object", "description": "Search your own earlier conversation turns.", | |
| "properties": {"query": {"type": "string"}, "limit": {"type": "integer"}}, | |
| "required": ["query"], "additionalProperties": False}, _recall, effect="read") | |