sibyllabs's picture
Snapshot mirror of Sibyl-Labs/Sibyl-Memory@08ce4c0ec1c1d93053da2191963e8c711156a9f6
729bfb8
Raw History Blame Contribute Delete
24.3 kB
-- sibyl-memory-client SQLite schema v1
--
-- Port of the canonical sibyl_memory.* Postgres schema (scripts/sibyl-memory-schema.sql,
-- applied to Neon 2026-05-01) to SQLite for the local-first plugin v1.
--
-- Dialect translations:
-- UUID → TEXT (Python uuid.uuid4() at write-time)
-- JSONB → TEXT with CHECK(json_valid(col)) using SQLite json1
-- TIMESTAMPTZ + now() → TEXT ISO 8601 UTC via strftime('%Y-%m-%dT%H:%M:%fZ','now')
-- gin jsonb_path_ops → SQLite json_extract expression indexes where useful
-- NUMERIC → REAL (sufficient precision for plugin v1 use)
-- tsvector → FTS5 virtual tables for text search
--
-- Multi-tenant: every table carries tenant_id. Local-first means typically
-- one tenant per machine, but the schema accepts N tenants (paid Team-tier
-- federation forward-compatible).
--
-- Idempotent. Apply via CREATE TABLE IF NOT EXISTS. Schema version recorded
-- in sibyl_memory_schema_version for future migrations.
PRAGMA foreign_keys = ON;
PRAGMA journal_mode = WAL;
-- ============================================================================
-- WARM tier: entities (single source of truth per rule 43)
-- ============================================================================
CREATE TABLE IF NOT EXISTS entities (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
category TEXT NOT NULL,
name TEXT NOT NULL,
status TEXT,
body TEXT NOT NULL CHECK (json_valid(body)),
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
UNIQUE (tenant_id, category, name)
);
CREATE INDEX IF NOT EXISTS entities_tenant_cat_status
ON entities (tenant_id, category, status);
CREATE INDEX IF NOT EXISTS entities_updated_at
ON entities (tenant_id, updated_at DESC);
-- ============================================================================
-- Cross-references: typed relations between entities
-- ============================================================================
CREATE TABLE IF NOT EXISTS entity_relations (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
from_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
to_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
relation_type TEXT NOT NULL,
metadata TEXT CHECK (metadata IS NULL OR json_valid(metadata)),
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
);
CREATE INDEX IF NOT EXISTS entity_relations_from
ON entity_relations (tenant_id, from_id, relation_type);
CREATE INDEX IF NOT EXISTS entity_relations_to
ON entity_relations (tenant_id, to_id, relation_type);
-- ============================================================================
-- HOT tier: state documents (treasury, priorities, session, index analogs)
-- ============================================================================
CREATE TABLE IF NOT EXISTS state_documents (
tenant_id TEXT NOT NULL,
document_key TEXT NOT NULL,
body TEXT NOT NULL CHECK (json_valid(body)),
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
PRIMARY KEY (tenant_id, document_key)
);
-- ============================================================================
-- COLD tier: append-only journal of events
-- ============================================================================
CREATE TABLE IF NOT EXISTS journal_events (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
ts TEXT NOT NULL,
evaluated TEXT CHECK (evaluated IS NULL OR json_valid(evaluated)),
acted TEXT CHECK (acted IS NULL OR json_valid(acted)),
forward TEXT CHECK (forward IS NULL OR json_valid(forward)),
extra TEXT CHECK (extra IS NULL OR json_valid(extra))
);
CREATE INDEX IF NOT EXISTS journal_events_tenant_ts
ON journal_events (tenant_id, ts DESC);
-- ============================================================================
-- COLD tier: revenue events with optional entity ref
-- ============================================================================
CREATE TABLE IF NOT EXISTS revenue_events (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
ts TEXT NOT NULL,
event_type TEXT,
gross_usd REAL,
operator_share_usd REAL,
source TEXT,
tx TEXT,
entity_id TEXT REFERENCES entities(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS revenue_events_tenant_ts
ON revenue_events (tenant_id, ts DESC);
-- ============================================================================
-- COLD tier: error events
-- ============================================================================
CREATE TABLE IF NOT EXISTS error_events (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
ts TEXT NOT NULL,
code TEXT,
message TEXT,
context TEXT CHECK (context IS NULL OR json_valid(context))
);
CREATE INDEX IF NOT EXISTS error_events_tenant_ts
ON error_events (tenant_id, ts DESC);
-- ============================================================================
-- REFERENCE tier: static documents (markdown bodies, lookup-only)
-- ============================================================================
CREATE TABLE IF NOT EXISTS reference_documents (
tenant_id TEXT NOT NULL,
doc_key TEXT NOT NULL,
body TEXT,
metadata TEXT CHECK (metadata IS NULL OR json_valid(metadata)),
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
PRIMARY KEY (tenant_id, doc_key)
);
-- ============================================================================
-- ARCHIVE tier: frozen entities (out of working set, retrievable)
-- ============================================================================
CREATE TABLE IF NOT EXISTS archived_entities (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
original_entity_id TEXT,
category TEXT,
name TEXT,
body TEXT CHECK (body IS NULL OR json_valid(body)),
archived_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
archive_reason TEXT
);
CREATE INDEX IF NOT EXISTS archived_entities_tenant_cat
ON archived_entities (tenant_id, category, name);
-- ============================================================================
-- FLAGGED tier: actors flagged for social-engineering / fraud (rule 13/14/15)
-- ============================================================================
CREATE TABLE IF NOT EXISTS flagged_actors (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
actor_handle TEXT,
actor_address TEXT,
flagged_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
reason TEXT,
evidence TEXT CHECK (evidence IS NULL OR json_valid(evidence))
);
CREATE INDEX IF NOT EXISTS flagged_actors_tenant
ON flagged_actors (tenant_id);
-- ============================================================================
-- FTS5 virtual tables for full-text retrieval (the tsvector port)
-- ============================================================================
-- v3 (2026-05-18): all FTS5 tables now use external-content (or contentless
-- for journal). Body lives in the base table, FTS5 stores only the index.
-- Triggers fire transparently. Disk footprint stays flat (vs v2's 2x dup).
-- Cross-tier search lands here: entities + state + reference + journal.
-- v2 → v3 migration is handled in storage.py:_migrate_to_v3.
-- ENTITIES: external-content FTS5 over entities table
CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(
name, category, body, tenant_id UNINDEXED,
content='entities', content_rowid='rowid',
tokenize = 'porter unicode61'
);
CREATE TRIGGER IF NOT EXISTS entities_ai_fts
AFTER INSERT ON entities BEGIN
INSERT INTO entities_fts(rowid, name, category, body, tenant_id)
VALUES (new.rowid, new.name, new.category, new.body, new.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS entities_ad_fts
AFTER DELETE ON entities BEGIN
INSERT INTO entities_fts(entities_fts, rowid, name, category, body, tenant_id)
VALUES ('delete', old.rowid, old.name, old.category, old.body, old.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS entities_au_fts
AFTER UPDATE ON entities BEGIN
INSERT INTO entities_fts(entities_fts, rowid, name, category, body, tenant_id)
VALUES ('delete', old.rowid, old.name, old.category, old.body, old.tenant_id);
INSERT INTO entities_fts(rowid, name, category, body, tenant_id)
VALUES (new.rowid, new.name, new.category, new.body, new.tenant_id);
END;
-- STATE: external-content FTS5 over state_documents
CREATE VIRTUAL TABLE IF NOT EXISTS state_documents_fts USING fts5(
document_key, body, tenant_id UNINDEXED,
content='state_documents', content_rowid='rowid',
tokenize = 'porter unicode61'
);
CREATE TRIGGER IF NOT EXISTS state_documents_ai_fts
AFTER INSERT ON state_documents BEGIN
INSERT INTO state_documents_fts(rowid, document_key, body, tenant_id)
VALUES (new.rowid, new.document_key, new.body, new.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS state_documents_ad_fts
AFTER DELETE ON state_documents BEGIN
INSERT INTO state_documents_fts(state_documents_fts, rowid, document_key, body, tenant_id)
VALUES ('delete', old.rowid, old.document_key, old.body, old.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS state_documents_au_fts
AFTER UPDATE ON state_documents BEGIN
INSERT INTO state_documents_fts(state_documents_fts, rowid, document_key, body, tenant_id)
VALUES ('delete', old.rowid, old.document_key, old.body, old.tenant_id);
INSERT INTO state_documents_fts(rowid, document_key, body, tenant_id)
VALUES (new.rowid, new.document_key, new.body, new.tenant_id);
END;
-- REFERENCE: external-content FTS5 over reference_documents
CREATE VIRTUAL TABLE IF NOT EXISTS reference_documents_fts USING fts5(
doc_key, body, tenant_id UNINDEXED,
content='reference_documents', content_rowid='rowid',
tokenize = 'porter unicode61'
);
CREATE TRIGGER IF NOT EXISTS reference_ai_fts
AFTER INSERT ON reference_documents BEGIN
INSERT INTO reference_documents_fts(rowid, doc_key, body, tenant_id)
VALUES (new.rowid, new.doc_key, new.body, new.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS reference_ad_fts
AFTER DELETE ON reference_documents BEGIN
INSERT INTO reference_documents_fts(reference_documents_fts, rowid, doc_key, body, tenant_id)
VALUES ('delete', old.rowid, old.doc_key, old.body, old.tenant_id);
END;
CREATE TRIGGER IF NOT EXISTS reference_au_fts
AFTER UPDATE ON reference_documents BEGIN
INSERT INTO reference_documents_fts(reference_documents_fts, rowid, doc_key, body, tenant_id)
VALUES ('delete', old.rowid, old.doc_key, old.body, old.tenant_id);
INSERT INTO reference_documents_fts(rowid, doc_key, body, tenant_id)
VALUES (new.rowid, new.doc_key, new.body, new.tenant_id);
END;
-- JOURNAL: standalone FTS5 over journal_events (concatenated payload).
-- Standalone (not external-content) because journal_events has 4 separate
-- JSON payload columns we want searchable as one concatenated field, and
-- there's no single base-table column we could external-content against.
-- Acceptable cost: journal is append-only (no updates), so the body
-- duplication doesn't compound on edits like it would on warm entities.
CREATE VIRTUAL TABLE IF NOT EXISTS journal_events_fts USING fts5(
ts UNINDEXED, payload, tenant_id UNINDEXED, event_id UNINDEXED,
tokenize = 'porter unicode61'
);
CREATE TRIGGER IF NOT EXISTS journal_events_ai_fts
AFTER INSERT ON journal_events BEGIN
INSERT INTO journal_events_fts(rowid, ts, payload, tenant_id, event_id)
VALUES (
new.rowid, new.ts,
COALESCE(new.evaluated, '') || ' ' || COALESCE(new.acted, '') || ' ' ||
COALESCE(new.forward, '') || ' ' || COALESCE(new.extra, ''),
new.tenant_id, new.id
);
END;
-- ============================================================================
-- Schema version tracking
-- ============================================================================
CREATE TABLE IF NOT EXISTS sibyl_memory_schema_version (
version INTEGER PRIMARY KEY,
applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
description TEXT
);
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (1, 'sibyl-memory-client v1. SQLite port of sibyl_memory.* Postgres schema. 10 tables (entities, entity_relations, state_documents, journal_events, revenue_events, error_events, reference_documents, archived_entities, flagged_actors, schema_version) + 2 FTS5 virtual tables. Local-first plugin foundation.');
-- ============================================================================
-- Schema v2 — self-learning skill proposals (review queue)
-- ============================================================================
-- The Learner module scans journal_events for repeating patterns and writes
-- proposed skill documents here. The user reviews via `sibyl learn review`
-- and either accepts (which writes to reference_documents under skill/<slug>)
-- or rejects. Idempotent; safe to re-apply against a v1 database.
CREATE TABLE IF NOT EXISTS skill_proposals (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-- detector output
pattern_kind TEXT NOT NULL, -- 'repeated_action' / 'structural_similarity' / 'temporal_routine' / 'co_occurrence'
proposed_slug TEXT NOT NULL, -- the reference_documents.doc_key it would land under (skill/<slug>)
proposed_title TEXT, -- one-line human-readable title
proposed_body TEXT NOT NULL, -- the actual skill body (markdown text)
-- evidence + provenance
evidence TEXT NOT NULL CHECK (json_valid(evidence)), -- list of source journal_event ids + snippets
confidence REAL NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
summarizer TEXT NOT NULL, -- 'local-deterministic' / 'byok-anthropic' / 'byok-openai' / 'venice-x402' / etc.
-- review state
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'accepted', 'rejected', 'superseded')),
reviewed_at TEXT,
review_note TEXT,
-- when accepted, points at the reference_documents row that was created
accepted_doc_key TEXT
);
CREATE INDEX IF NOT EXISTS skill_proposals_tenant_status
ON skill_proposals (tenant_id, status, created_at DESC);
CREATE INDEX IF NOT EXISTS skill_proposals_slug
ON skill_proposals (tenant_id, proposed_slug);
-- ============================================================================
-- Schema v2 — learning run log (so detectors don't re-scan ground they covered)
-- ============================================================================
CREATE TABLE IF NOT EXISTS learning_runs (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
started_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
completed_at TEXT,
summarizer TEXT NOT NULL,
events_scanned INTEGER NOT NULL DEFAULT 0,
proposals_made INTEGER NOT NULL DEFAULT 0,
cursor_after_ts TEXT, -- watermark — newest journal ts processed
notes TEXT
);
CREATE INDEX IF NOT EXISTS learning_runs_tenant
ON learning_runs (tenant_id, started_at DESC);
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (2, 'sibyl-memory-client v2. Adds skill_proposals (self-learning review queue) and learning_runs (detector watermark log). Idempotent migration; v1 databases auto-upgrade on first open. Free tier uses local-deterministic summarizer; paid tier can opt into BYOK or Venice/x402-routed summarization.');
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (3, 'sibyl-memory-client v3. External-content FTS5 across entities + state_documents + reference_documents + contentless FTS5 over journal_events. Fixes the v0.3.0 "search covers warm entities only" bug. Eliminates body duplication (v2 stored body twice — base table + FTS5). v2 to v3 migration handled in storage.py:_migrate_to_v3: drops the standalone FTS5 tables and rebuilds in external-content shape from existing base-table data. No data loss.');
-- ============================================================================
-- Schema v4 — folded-trigram search shadow (v0.5.0 multi-language search)
-- ============================================================================
-- The standalone ``search_shadow`` FTS5 trigram table + its 10 maintenance
-- triggers are NOT declared here: their DDL is GENERATED at runtime by
-- sibyl_memory_client/shadow.py, because two pieces cannot live in a static file
-- — (a) the fold map (the ł/ß/ø/... non-decomposable → ASCII rendering baked into
-- the trigger + backfill SQL, single source of truth in shadow.FOLD_MAP), and
-- (b) the tokenizer clause, which is selected at runtime across the SQLite 3.45
-- boundary ('trigram remove_diacritics 1' on >= 3.45, bare 'trigram' below).
-- The shadow holds a FOLDED copy of the searchable text across all four tiers and
-- is consulted ONLY as a zero-hit fallback (client.py search()), giving substring
-- + non-decomposable-fold matching for scripts the porter-unicode61 primary index
-- cannot reach. v3 → v4 migration + crash-atomic marker + heal: storage.py
-- (_migrate_if_needed, _SHADOW_MARKER). Rollback to exact v3 behaviour: DROP the
-- 10 *_shadow triggers + DROP TABLE search_shadow + PRAGMA user_version = 3.
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (4, 'sibyl-memory-client v4. Adds the standalone folded-trigram search_shadow table + maintenance triggers (DDL generated by shadow.py — fold map + runtime tokenizer clause cannot live in a static file). A FOLDED copy of searchable text across all four tiers, maintained by DB-side triggers, consulted only as a zero-hit fallback in MemoryClient.search() to give substring + non-decomposable-diacritic (ł ß ø æ đ ı œ þ ð) matching for the scripts the porter-unicode61 primary index cannot reach. Strictly additive: primary results are byte-identical to v3. v3 → v4 migration + crash-atomic PRAGMA user_version marker + portability/heal handled in storage.py:_migrate_if_needed. Derived state, always rebuildable from the base tables; base data untouched. Rollback: DROP the *_shadow triggers + search_shadow, PRAGMA user_version = 3.');
-- ============================================================================
-- Schema v5 (0.9.0) - pre-sign-in adoption audit (not tenant-scoped)
-- ============================================================================
-- One row per adoption of DEFAULT_TENANT rows into an account tenant
-- (_adopt.py), written in the SAME transaction as the move, plus a 'seen' row
-- the first time a verified account uses this store (so automatic adoption runs
-- for an account's FIRST sign-in only, and never merges into a different
-- account once another one has used this store). Counts and renamed business
-- keys only; never bodies. Local only; nothing reads it over the network.
CREATE TABLE IF NOT EXISTS tenant_adoptions (
id TEXT PRIMARY KEY,
from_tenant TEXT NOT NULL,
to_tenant TEXT NOT NULL,
account_id TEXT NOT NULL,
kind TEXT NOT NULL CHECK (kind IN ('auto', 'command', 'seen')),
moved_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
counts TEXT NOT NULL CHECK (json_valid(counts)),
renamed TEXT CHECK (renamed IS NULL OR json_valid(renamed))
);
CREATE INDEX IF NOT EXISTS tenant_adoptions_account
ON tenant_adoptions (account_id, to_tenant);
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (5, 'sibyl-memory-client v5 (0.9.0). Adds tenant_adoptions: the local audit of moving pre-sign-in DEFAULT_TENANT rows into the signed-in account tenant (_adopt.py), written in the same transaction as the move. Additive; no existing table changes.');
-- ============================================================================
-- Schema v6 (spec tiered-retrieval 2.B) - memory_links: typed edges across tiers
-- ============================================================================
-- Edges between nodes addressed by BUSINESS KEYS (entity: category + name;
-- reference: doc_key; state: document_key), so they survive archive_entity and
-- re-creation, which id foreign keys with ON DELETE CASCADE would not. The
-- category columns are NOT NULL: entity endpoints always carry a non-empty
-- category and reference/state endpoints store '' (SQLite UNIQUE treats NULLs
-- as distinct, so a NULL would defeat the uniqueness of reference/state edges);
-- the CHECKs enforce (tier = 'entity') = (category <> '').
-- These statements are ALSO the Python constant _links.V6_DDL (a test asserts
-- the two are identical). A fresh store gets them here; a pre-v6 store gets
-- them from Storage._ensure_schema in ONE transaction with the one-time copy of
-- entity_relations rows, so the table's existence is the crash-atomic "copied"
-- marker. from_entity_id / to_entity_id stamp entity endpoints with the row id
-- the edge was written against (NULL for reference/state, or unknown); a key now
-- held by another id reads stale (_links.STALE_SIDE_SQL). Stores created before
-- the stamp get these columns by ALTER on open (_links.apply_identity_columns).
-- entity_relations stays in place, unwritten (frozen), until R4. Nothing gates on
-- the version-6 row below (another release may claim the same number).
CREATE TABLE IF NOT EXISTS memory_links (
id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
from_tier TEXT NOT NULL CHECK (from_tier IN ('entity','reference','state')),
from_category TEXT NOT NULL,
from_key TEXT NOT NULL,
to_tier TEXT NOT NULL CHECK (to_tier IN ('entity','reference','state')),
to_category TEXT NOT NULL,
to_key TEXT NOT NULL,
relation_type TEXT NOT NULL,
metadata TEXT CHECK (metadata IS NULL OR json_valid(metadata)),
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
from_entity_id TEXT,
to_entity_id TEXT,
updated_at TEXT,
CHECK ((from_tier = 'entity') = (from_category <> '')),
CHECK ((to_tier = 'entity') = (to_category <> '')),
UNIQUE (tenant_id, from_tier, from_category, from_key, to_tier, to_category, to_key, relation_type)
);
CREATE INDEX IF NOT EXISTS memory_links_from
ON memory_links (tenant_id, from_tier, from_category, from_key, relation_type);
CREATE INDEX IF NOT EXISTS memory_links_to
ON memory_links (tenant_id, to_tier, to_category, to_key, relation_type);
CREATE INDEX IF NOT EXISTS memory_links_created
ON memory_links (tenant_id, created_at, id);
INSERT OR IGNORE INTO sibyl_memory_schema_version (version, description)
VALUES (6, 'sibyl-memory-client v6. Adds memory_links: typed, directed edges between entity / reference / state nodes keyed by business keys (link, unlink, neighbors, links_of, links= on set_entity and set_reference). Pre-v6 entity_relations rows are copied once, in the same transaction that creates the table; entity_relations stays, unwritten, until it is dropped in a later major release. Additive.');
-- ============================================================================
-- Stage-0 exact-name index (spec tiered-retrieval 2.B DDL / 2.C)
-- ============================================================================
-- multi_record_search's stage 0 resolves a query that IS an entity name with a
-- parameterised `lower(name) = ?` equality (no MATCH syntax). This expression
-- index makes that one probe, not a tenant scan. Idempotent and applied on
-- every open like the rest of this file, so existing stores get it on their
-- next open (one index build, no data change). Not part of _links.V6_DDL: it
-- belongs to search, not to the memory_links migration transaction.
CREATE INDEX IF NOT EXISTS entities_tenant_lname
ON entities (tenant_id, lower(name));