runloopable / platform /harness /live_read.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
8133df9 verified
Raw History Blame Contribute Delete
34.7 kB
"""Read-only live delta reader: source rows written since an entity's last sync cursor, overlaid on
the mirror rows by id in memory.
Contract (S82-T104, ideation R6 and O2):
* The cursor is `_sync_state.cursor_wd`, read through a read cursor. Never `datastore.connect()`:
one sync worker holds the writer lease.
* The source read is `write_date >= cursor`, the same domain `datastore._sync_entity` builds.
During an id-walk `cursor_wd` reads `cursor|id`, and the plain cursor before the pipe covers it.
* An edited row replaces its mirror row by source id. A new row is added. Nothing is stored.
* Odoo unlinks since the last sync stay invisible: a deleted record keeps its mirror row.
* Every source read admits through `core.live_limiter` first. A refusal raises
`LiveLimitRefused` before any source call, and the caller answers from the mirror.
S82-T106: `live_answer` dispatches on the table's declared live mode (`connector_spec.LIVE_MODES`):
cursor -> `live_read`, slice -> `slice_read`, live API -> `api_read`, none -> `none_answer`.
"""
import json
import re
from core import connector_spec, live_limiter
from harness import datastore
#: Rows per source page. Paging only: the whole delta is read (rule 1).
PAGE = 2000
_IDENT = re.compile(r"^[a-z][a-z0-9_]*$")
#: Columns the mirror stamps on a row. A replaced row keeps the mirror's values for these.
_MIRROR_IDENTITY = ("id", "Store", "_connection_id", "_source_id")
class LiveReadError(RuntimeError):
"""A live read that cannot run. Carries `cause`, `message` and `fix`."""
def __init__(self, cause, message, fix):
super().__init__(message)
self.cause = cause
self.message = message
self.fix = fix
def as_dict(self):
"""The refusal as a client payload."""
return {'code': 'live_read_refused', 'cause': self.cause, 'message': self.message,
'fix': self.fix}
def parse_cursor(raw):
"""The `write_date` cursor in a stored `cursor_wd`: plain or `cursor|id`. None when unset."""
text = str(raw or '').strip()
if not text:
return None
cursor = text.partition('|')[0].strip()
return cursor or None
def _open_cursor(tenant_key=None):
"""A read cursor on the mirror, through the datastore's public read paths."""
if tenant_key is not None:
return datastore.tenant_handle(tenant_key).ro_cursor()
return datastore.ro_cursor()
def _entity_spec(entity):
mode = connector_spec.live_mode_for(entity, 'odoo')
if mode is None or not _IDENT.match(entity) or entity not in datastore.ENTITIES:
raise LiveReadError('unknown_entity', f'{entity} is not a mirrored source table.',
'Ask about a table the Odoo connector syncs.')
if mode.mode != connector_spec.LIVE_CURSOR:
raise LiveReadError('no_cursor_mode', f'{entity} has no change cursor to read past.',
'Read this table from the mirror.')
return datastore.ENTITIES[entity], mode
def _rows(cur, sql, params=()):
res = cur.execute(sql, list(params))
names = [d[0] for d in res.description]
return [dict(zip(names, row)) for row in res.fetchall()]
def read_cursor(cur, entity, state_entity=None):
"""`{raw, cursor, phase, updatedAt}` for one entity's sync state. Refuses with no live cursor."""
found = cur.execute(
"SELECT phase, cursor_wd, updated_at FROM _sync_state WHERE entity = ?",
[state_entity or entity]).fetchone()
phase, raw, updated = found if found else (None, None, None)
cursor = parse_cursor(raw)
if not found or phase == 'backfill' or cursor is None:
raise LiveReadError('no_cursor', f'{entity} has not finished its first sync.',
'Answer from the mirror until the first sync completes.')
return {'raw': raw, 'cursor': cursor, 'phase': phase, 'updatedAt': updated}
def _source_fields(spec):
"""The fields a live read asks for. Named tags and analytic names need a second source read,
so those columns keep their mirror value."""
skip = set(spec.get('named_m2m') or {}) | {'analytic_distribution'}
return [f for f in spec['fields'] if f not in skip] + list(spec.get('m2m') or [])
def _flatten(spec, recs):
fields = [f for f in _source_fields(spec) if f not in (spec.get('m2m') or [])]
rows = [datastore._flatten(rec, fields, spec.get('m2m') or ()) for rec in recs]
for rec, row in zip(recs, rows):
for output, source in (spec.get('raw_m2m') or {}).items():
if source in rec:
row[output] = json.dumps(list(rec[source] or []))
return rows
def delta_rows(client, entity, cursor, admission=None):
"""Every source row of `entity` written at or after `cursor`, flattened to mirror columns."""
spec, mode = _entity_spec(entity)
base = [('active', 'in', [True, False])] if spec.get('archivable') else []
domain = base + [(mode.cursor_field, '>=', cursor)]
fields = ['id'] + _source_fields(spec)
out, last = [], 0
while True:
page = client.search_read(spec['model'], domain + [('id', '>', last)], fields,
limit=PAGE, order='id asc')
out.extend(page or [])
if admission is not None:
admission.check_rows(len(out))
# Stop on an EMPTY page: a server-side page cap returns short pages before the end.
if not page:
break
last = page[-1]['id']
return _flatten(spec, out)
def _source_id(row):
sid = row.get('_source_id')
return row.get('id') if sid is None else sid
def overlay(mirror_rows, delta):
"""`(rows, replaced, added)`: each delta row replaces its mirror row by source id, else is added."""
fresh = {}
for row in delta:
fresh[row['id']] = row
rows, replaced = [], 0
for row in mirror_rows:
new = fresh.pop(_source_id(row), None)
if new is None:
rows.append(dict(row))
continue
merged = {**row, **new}
merged.update({k: row[k] for k in _MIRROR_IDENTITY if k in row})
rows.append(merged)
replaced += 1
added = [{**row, '_source_id': row['id']} for row in fresh.values()]
rows.extend(added)
return rows, replaced, len(added)
def live_read(entity, tenant, *, client=None, tenant_key=None, where=None, params=(),
keep=None, state_entity=None, caller='live_read'):
"""The mirror rows of `entity` with every row changed since the last sync overlaid by id.
`tenant` is the tenant-bound store handle the limiter admits through. `where` and `params`
slice the mirror (SQL we author, values bound). `keep` filters the overlaid rows, so an edited
row that left or entered the slice is judged on its new values. Returns
`{entity, rows, cursor, cursorRaw, phase, syncedAt, mirrorRows, delta, replaced, added}`.
"""
_entity_spec(entity)
if client is None:
from core import odoo as client
cur = _open_cursor(tenant_key)
try:
state = read_cursor(cur, entity, state_entity)
mirror = _rows(cur, f"SELECT * FROM {entity} WHERE {where or 'TRUE'}", params)
finally:
try:
cur.close()
except Exception: # noqa: BLE001
pass
with live_limiter.admit(tenant, caller=caller) as admission:
delta = delta_rows(client, entity, state['cursor'], admission)
rows, replaced, added = overlay(mirror, delta)
if keep is not None:
rows = [row for row in rows if keep(row)]
return {'entity': entity, 'rows': rows, 'cursor': state['cursor'], 'cursorRaw': state['raw'],
'phase': state['phase'], 'syncedAt': state['updatedAt'], 'mirrorRows': len(mirror),
'delta': len(delta), 'replaced': replaced, 'added': added}
# ── S82-T106: the slice, live API and none dispatch ──────────────────────────────────────────
#: The answer a table with no live read gives: the mirror answers.
NONE_FIX = 'Answer from the mirror. The next sync refreshes it.'
#: Odoo read operators a slice term may use.
_SLICE_OPS = frozenset(('=', '!=', '>', '>=', '<', '<=', 'in', 'not in', 'like', 'ilike',
'=like', '=ilike'))
_FIELD_PATH = re.compile(r'^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$')
_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$')
def resolve_table(table, connector=None):
"""`(connector, mirror table, LiveMode)` for a mirror table or a `ut_*` key. Refuses an unknown one."""
name = str(table or '').strip()
spec = connector_spec.spec_for(name)
if spec is not None:
connector = connector or connector_spec.connector_of(name)
name = spec.source
if connector is None:
connector = next((key for key in connector_spec.LIVE_MODES
if name in connector_spec.live_modes(key)), None)
mode = connector_spec.live_mode_for(name, connector) if connector else None
if mode is None or not _IDENT.match(name):
raise LiveReadError('unknown_table', 'This table has no declared live mode.',
'Ask about a table a connector syncs.')
return connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector), name, mode
def none_answer(table, connector=None):
"""The answer for a table with no live read: its stated reason, and the mirror answers."""
connector, name, mode = resolve_table(table, connector)
if mode.mode != connector_spec.LIVE_NONE:
raise LiveReadError('has_live_mode', f'{name} has a live read.', 'Call live_answer.')
return {'table': name, 'connector': connector, 'mode': mode.mode, 'live': False,
'cause': 'no_live_read', 'reason': mode.reason, 'fix': NONE_FIX}
def _slice_spec(entity):
_connector, name, mode = resolve_table(entity, 'odoo')
if (mode.mode != connector_spec.LIVE_SLICE or name not in datastore.ENTITIES
or name not in datastore.REPLACE_EACH_PASS):
raise LiveReadError('no_slice_mode', f'{name} has no bounded slice read.',
'Call live_answer for this table.')
return name, datastore.ENTITIES[name], datastore.REPLACE_EACH_PASS[name]
def slice_domain(spec, domain):
"""The caller's Odoo domain, checked term by term: a column of the table, a read operator."""
allowed = {'id', *spec['fields'], *(spec.get('m2m') or ())}
out = []
for term in domain or ():
if not isinstance(term, (list, tuple)) or len(term) != 3:
raise LiveReadError('bad_domain', 'A slice term is a field, an operator and a value.',
'Send each term as field, operator, value.')
field, op, value = str(term[0]), term[1], term[2]
if not _FIELD_PATH.match(field) or field.split('.', 1)[0] not in allowed \
or op not in _SLICE_OPS:
raise LiveReadError('bad_domain', f'{field} {op} is not a filter this table allows.',
'Filter on a column of this table with a comparison.')
out.append((field, op, value))
return out
def slice_rows(client, entity, plan, domain, admission=None):
"""Source rows in the plan's scope and `domain`, flattened. Reads at most max rows plus one."""
spec = datastore.ENTITIES[entity]
full = list(plan.get('domain') or []) + list(domain)
cap = admission.limits.get('maxRowsPerCall') if admission is not None else None
fields = ['id'] + _source_fields(spec)
found, last = [], 0
while True:
size = PAGE if cap is None else max(1, min(PAGE, int(cap) + 1 - len(found)))
page = client.search_read(spec['model'], full + [('id', '>', last)], fields,
limit=size, order='id asc')
found.extend(page or [])
if admission is not None:
admission.check_rows(len(found))
# Stop on an EMPTY page: a server-side page cap returns short pages before the end.
if not page:
break
last = page[-1]['id']
return _flatten(spec, found)
def slice_read(entity, tenant, *, domain=(), client=None, caller='live_slice'):
"""A bounded live read of an Odoo table the sync replaces whole each pass (no change cursor).
`domain` is Odoo terms on the table's columns, ANDed with the sync plan's own scope. Over the
tenant's max rows per call it raises `LiveLimitRefused` with cause `rows` and its fix.
Returns `{table, connector, mode, rows, rowCount, domain, maxRows, readAt}`.
"""
name, spec, plan = _slice_spec(entity)
terms = slice_domain(spec, domain)
if client is None:
from core import odoo as client
with live_limiter.admit(tenant, caller=caller) as slice_admission:
rows = slice_rows(client, name, plan, terms, slice_admission)
return {'table': name, 'connector': 'odoo', 'mode': connector_spec.LIVE_SLICE, 'live': True,
'rows': rows, 'rowCount': len(rows), 'domain': [list(t) for t in terms],
'maxRows': slice_admission.limits.get('maxRowsPerCall'),
'readAt': live_limiter.iso_utc(slice_admission.admitted_at)}
def _meta_connections(tenant_key):
"""`[(connection id, label, token, legacy)]` for the tenant's Meta Ads credentials."""
from harness import meta_store
rt = meta_store._runtime_for(tenant_key)
if rt is None:
tok = meta_store.token(tenant_key=tenant_key)
return [('', 'Meta Ads', tok, True)] if tok else []
return meta_store._connection_rows(rt, tenant_key, {'problems': []})
def _graph_pages(get, path, tok, sink, admission=None, **params):
"""Every page of one Graph edge into `sink`. A heavy page shrinks; a rate limit refuses at once."""
from harness import meta_store
limit = int(params.pop('limit', None) or meta_store.PAGE)
after, cursors = None, set()
while True:
call = dict(params, limit=limit, **({'after': after} if after else {}))
try:
body = get(path, tok, **call)
except meta_store.MetaError as exc:
text = str(exc)
if meta_store._TOO_MUCH in text and limit > 5:
limit = max(5, limit // 4)
continue
if meta_store._RATE_LIMITED in text.lower():
raise LiveReadError('source_rate', 'Meta is limiting calls to this ad account.',
'Try again in a few minutes, or answer from the mirror.') from None
raise LiveReadError('source_error', 'Meta refused the live read.',
'Try again, or answer from the mirror.') from None
sink.extend(body.get('data') or [])
if admission is not None:
admission.check_rows(len(sink))
paging = body.get('paging') or {}
after = (paging.get('cursors') or {}).get('after')
if not (paging.get('next') and after):
return sink
if after in cursors:
raise LiveReadError('source_error', 'Meta repeated a page of the live read.',
'Try again, or answer from the mirror.')
cursors.add(after)
def _day_windows(since, until, size):
from datetime import date, timedelta
start, end = date.fromisoformat(since), date.fromisoformat(until)
out = []
while start <= end:
stop = min(start + timedelta(days=max(1, size) - 1), end)
out.append((start.isoformat(), stop.isoformat()))
start = stop + timedelta(days=1)
return out
def api_read(table, tenant, *, tenant_key=None, since=None, until=None, connections=None,
get=None, caller='live_api', windows=None):
"""A live Meta Graph read of one Meta mirror table, every page (rule 1).
Insights read `windows` or `since..until` (default tenant today) at the sync's grain.
`connections` and `get` are the credential list and the Graph call, injectable for fixtures.
Returns `{table, connector, mode, rows, rowCount, accounts, since, until, maxRows, readAt}`.
"""
from core import periods
from harness import meta_store
connector, name, mode = resolve_table(table)
if mode.mode != connector_spec.LIVE_API or connector != 'meta':
raise LiveReadError('no_api_mode', f'{name} has no live API read.',
'Call live_answer for this table.')
insights = name == meta_store.INSIGHTS_TABLE
since = str(since or until or periods.today().isoformat())
until = str(until or since)
asked = sorted(windows if windows is not None else [(since, until)])
spans = []
for lo, hi in asked:
if not (_ISO_DAY.match(str(lo or '')) and _ISO_DAY.match(str(hi or ''))) or lo > hi:
raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.',
'Send since and until as YYYY-MM-DD.')
if spans and lo <= spans[-1][1]:
spans[-1] = (spans[-1][0], max(hi, spans[-1][1]))
else:
spans.append((lo, hi))
if connections is None:
connections = _meta_connections(tenant_key)
if not connections:
raise LiveReadError('no_credential', 'No Meta Ads credential is connected.',
'Connect Meta Ads under Connectors.')
get = get or meta_store._get
rows, accounts = [], []
with live_limiter.admit(tenant, caller=caller) as api_admission:
for cid, label, tok, _legacy in connections:
accts = _graph_pages(get, 'me/adaccounts', tok, [], fields='id,name')
for acct in accts:
aid = acct['id']
accounts.append(aid)
start = len(rows)
if insights:
for first, last in spans:
for lo, hi in _day_windows(first, last, meta_store.INSIGHTS_SLICE):
_graph_pages(get, f'{aid}/insights', tok, rows, api_admission,
fields=','.join(meta_store.INSIGHT_FIELDS),
level=meta_store.INSIGHTS_LEVEL, time_increment='1',
limit=meta_store.INSIGHTS_PAGE,
time_range=json.dumps({'since': lo, 'until': hi}))
else:
edge, fields, _parent = meta_store.SPECS[name]
if edge is None:
rows.append(get(aid, tok, fields=','.join(fields)))
api_admission.check_rows(len(rows))
else:
_graph_pages(get, f'{aid}/{edge}', tok, rows, api_admission,
fields=','.join(fields), limit=meta_store.PAGE)
for row in rows[start:]:
row['_connection_id'] = cid
if insights:
# The sync's stable Insights id, so a row matches its mirror row.
row['id'] = ':'.join(str(row.get(k) or '') for k in
(f'{meta_store.INSIGHTS_LEVEL}_id', 'date_start',
'date_stop'))
for index in range(start, len(rows)):
raw = rows[index]
_cols, normalized = meta_store._origin_rows(list(raw), [raw], cid, label)
rows[index] = {key: meta_store._cell(value) for key, value in normalized[0].items()}
return {'table': name, 'connector': connector, 'mode': connector_spec.LIVE_API, 'live': True,
'rows': rows, 'rowCount': len(rows), 'accounts': accounts,
'since': spans[0][0] if insights and spans else None,
'until': spans[-1][1] if insights and spans else None,
'windows': [{'since': lo, 'until': hi} for lo, hi in spans] if insights else None,
'maxRows': api_admission.limits.get('maxRowsPerCall'),
'readAt': live_limiter.iso_utc(api_admission.admitted_at)}
def live_answer(table, tenant, *, connector=None, tenant_key=None, client=None, where=None,
params=(), keep=None, state_entity=None, domain=(), since=None, until=None,
connections=None, get=None, caller='live', windows=None):
"""One live question about one table, routed by its declared live mode.
`table` is a mirror table or a `ut_*` key. Cursor tables take `where`, `params` and `keep`.
Slice tables take `domain`. Meta tables take `since` and `until`. A table with no live read
returns its reason and reads nothing. Every source read admits through the live limiter.
Each result carries `mode` and `connector`.
"""
connector, name, mode = resolve_table(table, connector)
if mode.mode == connector_spec.LIVE_CURSOR:
out = live_read(name, tenant, client=client, tenant_key=tenant_key, where=where,
params=params, keep=keep, state_entity=state_entity, caller=caller)
return {**out, 'table': name, 'connector': connector, 'mode': mode.mode, 'live': True}
if mode.mode == connector_spec.LIVE_SLICE:
return slice_read(name, tenant, domain=domain, client=client, caller=caller)
if mode.mode == connector_spec.LIVE_API:
return api_read(name, tenant, tenant_key=tenant_key, since=since, until=until,
connections=connections, get=get, caller=caller, windows=windows)
return none_answer(name, connector)
# ── S82-T105: the Live tool's windowed answer ────────────────────────────────────────────────
#: The ISO form of a business-day window bound.
_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$')
_QUALIFIED = re.compile(r'\b([a-z_][a-z0-9_]*)\.([a-z_][a-z0-9_]*)\b')
_WINDOW_KEY = '_live_window_at'
def _instant(value):
"""A stored instant as a naive UTC `datetime`, or None."""
from datetime import datetime, timezone
if value is None or value == '':
return None
if isinstance(value, datetime):
moment = value
else:
try:
moment = datetime.fromisoformat(str(value).strip().replace('Z', '+00:00'))
except ValueError:
return None
if moment.tzinfo is not None:
moment = moment.astimezone(timezone.utc).replace(tzinfo=None)
return moment
def _window_bounds(date_from, date_to, instants):
"""`(lo, hi)` for one window: naive UTC `[lo, hi)` for an instant column, ISO days otherwise."""
from core import periods
if not (_DAY.match(str(date_from or '')) and _DAY.match(str(date_to or ''))) \
or date_from > date_to:
raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.',
'Send date_from and date_to as YYYY-MM-DD.')
return periods.business_day_bounds(date_from, date_to) if instants else (date_from, date_to)
def _in_window(value, bounds, instants):
lo, hi = bounds
if instants:
moment = _instant(value)
return moment is not None and lo <= moment < hi
day = str(value or '')[:10]
return bool(day) and lo <= day <= hi
def mirror_slice(entity, alias, where, params=(), tenant_key=None):
"""The mirror rows of `entity` the SQL `where` keeps, read through a read cursor."""
import duckdb
cur = _open_cursor(tenant_key)
try:
return _rows(cur, f"SELECT * FROM {entity} AS {alias} WHERE {where or 'TRUE'}", params)
except duckdb.Error as exc:
raise LiveReadError('unsupported_definition', 'This database slice cannot answer at source row grain.',
'Read this database from the mirror or ask about its source table.') from exc
finally:
try:
cur.close()
except Exception: # noqa: BLE001
pass
def _definition_columns(entity, alias, where):
"""The mirror columns the definition SQL reads. Refuses a definition naming another table."""
known = {c.split()[0]: c for c in datastore._cols(datastore.ENTITIES[entity])}
text = re.sub(r"'(?:[^']|'')*'", "''", where or '')
if re.search(r'\b(select|join|from)\b', text, re.IGNORECASE):
raise LiveReadError('unsupported_definition',
'This database definition needs another table.',
'Read this database from the mirror or ask about its source table.')
used = []
for table, column in _QUALIFIED.findall(text):
if table != alias or column not in known:
raise LiveReadError('unsupported_definition',
'This database is defined over more than one table.',
'Ask about a database defined over one synced table.')
if column not in used:
used.append(column)
if '{' in text:
raise LiveReadError('unsupported_definition', 'This database definition is not static.',
'Ask about a database defined over one synced table.')
return [known[column] for column in used]
def defined_rows(entity, alias, where, rows):
"""The rows the database's own definition SQL keeps, each judged on its current values.
The rows load into an in-memory DuckDB table, never the mirror."""
rows = list(rows)
if not where or not where.strip() or where.strip().upper() == 'TRUE':
return rows
import duckdb
cols = _definition_columns(entity, alias, where)
names = [c.split()[0] for c in cols]
con = duckdb.connect(':memory:')
try:
con.execute(f"CREATE TABLE {alias} (_k BIGINT, {', '.join(cols) or '_none INTEGER'})")
if rows:
values = [[i] + [row.get(n) for n in names] for i, row in enumerate(rows)]
marks = ', '.join('?' for _ in range(len(names) + 1))
con.executemany(f"INSERT INTO {alias} (_k{''.join(', ' + n for n in names)}) "
f"VALUES ({marks})", values)
kept = {r[0] for r in con.execute(f"SELECT _k FROM {alias} WHERE {where}").fetchall()}
except duckdb.Error as exc:
raise LiveReadError('unsupported_definition', 'This database definition needs another binding.',
'Read this database from the mirror or ask about its source table.') from exc
finally:
con.close()
return [row for i, row in enumerate(rows) if i in kept]
def walled_rows(rows, wall):
"""The projected rows the caller may see, through the row wall every door applies."""
from core import perm_scope
if not isinstance(wall, dict) or 'user' not in wall or not wall.get('database'):
raise LiveReadError('no_wall', 'A live answer needs the caller and the database.',
'Call the Live tool as a signed-in member.')
return perm_scope.apply_row_scope(rows, wall['user'], wall['database'], wall['fields'],
st=wall.get('st'), defn=wall.get('defn'))
def _total(rows, measure):
if measure is None:
return len(rows)
total = 0.0
for row in rows:
value = row.get(measure)
if value is None or value == '':
continue
try:
total += float(value)
except (TypeError, ValueError):
raise LiveReadError('invalid_measure_value', 'The measure contains a non-numeric value.',
'Choose a numeric source field or correct its source value.') from None
return round(total, 6)
def project_definition(entity, rows, spec, keys, tenant_key=None):
"""Evaluate the database SQL over source rows and mirrored dimensions in memory."""
import duckdb
cur = _open_cursor(tenant_key)
con = duckdb.connect(':memory:')
quote = lambda key: '"' + key.replace('"', '""') + '"'
tables = set((spec.get('tables') or {}).values()) | {entity}
try:
for table in tables:
if not _IDENT.fullmatch(table):
raise ValueError('Invalid mirror table')
cur.execute(f'SELECT * FROM {table} LIMIT 0')
columns = [(d[0], str(d[1])) for d in cur.description]
source = list(rows) if table == entity else _rows(cur, f'SELECT * FROM {table}')
con.execute(f'CREATE TABLE {table} (' + ', '.join(
f'{quote(name)} {kind}' for name, kind in columns) + ')')
if source:
con.executemany(f'INSERT INTO {table} VALUES (' + ','.join('?' for _ in columns) + ')',
[[row.get(name) for name, _ in columns] for row in source])
bindings = {key: binding for key, binding in spec['cols'].items() if key in keys}
expressions = [f'{expr} AS {quote(key)}' for key, (expr, _convert) in bindings.items()]
frm = spec.get('from_sql') or (entity if spec.get('table') else
entity + ' AS ' + spec['id'].split('.')[0])
values = _rows(con, f"SELECT {spec['id']} AS pid" +
(', ' + ', '.join(expressions) if expressions else '') +
f" FROM {frm} WHERE {spec.get('where') or 'TRUE'}")
ids = [r['pid'] for r in values]
if len(set(ids)) != len(ids):
raise ValueError('The join repeats a source row')
for row in values:
for key, (_expr, convert) in bindings.items():
try:
row[key] = convert(row[key])
except (TypeError, ValueError):
row[key] = None
return values
except (duckdb.Error, ValueError) as exc:
raise LiveReadError('unsupported_definition',
'This database join or expression cannot answer at source row grain.',
'Read this database from the mirror or ask about its source table.') from exc
finally:
cur.close()
con.close()
def window_answer(entity, tenant, *, alias, where, date_column, instants, windows, project,
wall, measure=None, tenant_key=None, client=None, caller='live.answer',
definition=None, domain=(), filters=None):
"""The screen (mirror) and live totals of one database per window, live rows overlaid by id.
`windows` is `{name: (date_from, date_to)}` in tenant business days. `where` is the database's
own definition SQL over `alias`. `project(row)` maps a mirror row to the database's field keys,
and every projected row passes `walled_rows`. `measure` is a field key to sum, else rows count.
Over the limit, or with no live read, the mirror answers with the cause, next allowed time and
fix. Returns `{entity, served, syncedAt, delta, replaced, added, limit_refusal,
live_unavailable, windows: {name: {date_from, date_to, screen, live, difference}}, rows}`.
"""
bounds = {name: _window_bounds(lo, hi, instants) for name, (lo, hi) in windows.items()
if date_column is not None}
cast = f"CAST({date_column} AS TIMESTAMP)" if instants else f"CAST({date_column} AS DATE)"
terms, params = [], []
for lo, hi in bounds.values():
terms.append(f"({cast} >= ? AND {cast} < ?)" if instants
else f"({cast} >= CAST(? AS DATE) AND {cast} <= CAST(? AS DATE))")
params += [lo, hi]
sliced = (' OR '.join(terms) or 'FALSE') if date_column else 'TRUE'
def in_any(row):
return date_column is None or any(
_in_window(row.get(date_column), b, instants) for b in bounds.values())
defined = f"({sliced}) AND ({where})" if where and where.strip() else sliced
if not definition:
_definition_columns(entity, alias, where)
screen_rows = mirror_slice(entity, alias, 'TRUE' if definition else defined,
() if definition else params, tenant_key)
live, limit, unavailable = None, None, None
try:
live = live_answer(entity, tenant, client=client, tenant_key=tenant_key, where=sliced,
params=params, keep=in_any, domain=domain, caller=caller,
windows=list(windows.values()) if date_column else None)
if not live.get('live'):
unavailable = live
live = None
except live_limiter.LiveLimitRefused as refusal:
live, limit = None, refusal.as_dict()
except LiveReadError as exc:
live, unavailable = None, exc.as_dict()
def shown(rows):
if definition:
rows = project_definition(entity, rows, definition, project, tenant_key)
shown_rows = walled_rows(rows, wall)
if filters:
from harness import filter_eval
shown_rows = [r for r in shown_rows if filter_eval.permits(filters, r, wall['fields'])]
return shown_rows
# The window key rides each projected row through the wall and is dropped after.
return walled_rows([{**project(row), _WINDOW_KEY: row.get(date_column)} for row in rows],
wall)
def split(rows, b):
return [{k: v for k, v in r.items() if k != _WINDOW_KEY} for r in rows
if date_column is None or _in_window(r.get(_WINDOW_KEY), b, instants)]
screen = shown(screen_rows)
fresh = shown(live['rows'] if definition else defined_rows(
entity, alias, where, live['rows'])) if live is not None else None
out_windows, out_rows = {}, {}
for name, (lo, hi) in windows.items():
screen_in = split(screen, bounds.get(name))
entry = {'date_from': lo, 'date_to': hi,
'screen': {'value': _total(screen_in, measure), 'rows': len(screen_in)},
'live': None, 'difference': None}
out_rows[name] = screen_in
if fresh is not None:
live_in = split(fresh, bounds.get(name))
entry['live'] = {'value': _total(live_in, measure), 'rows': len(live_in)}
entry['difference'] = round(entry['live']['value'] - entry['screen']['value'], 6)
out_rows[name] = live_in
out_windows[name] = entry
return {'entity': entity, 'served': 'live' if live is not None else 'mirror',
'syncedAt': (live or {}).get('syncedAt'),
# Source-wide counters disclose rows outside the database and reader's walls.
'delta': None, 'replaced': None,
'added': None, 'limit_refusal': limit,
'live_unavailable': unavailable, 'windows': out_windows, 'rows': out_rows}