Download platform/harness/live_read.py from fsanyoto/loopable: direct link, hf CLI and curl.
- Browser
- Download file 34.7 kB
-
https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/platform/harness/live_read.py
- Command line
-
hf download hf://spaces/fsanyoto/loopable/platform/harness/live_read.py
-
curl -L -o live_read.py https://huggingface.co/spaces/fsanyoto/loopable/resolve/main/platform/harness/live_read.py
34.7 kB
| """Read-only live delta reader: source rows written since an entity's last sync cursor, overlaid on | |
| the mirror rows by id in memory. | |
| Contract (S82-T104, ideation R6 and O2): | |
| * The cursor is `_sync_state.cursor_wd`, read through a read cursor. Never `datastore.connect()`: | |
| one sync worker holds the writer lease. | |
| * The source read is `write_date >= cursor`, the same domain `datastore._sync_entity` builds. | |
| During an id-walk `cursor_wd` reads `cursor|id`, and the plain cursor before the pipe covers it. | |
| * An edited row replaces its mirror row by source id. A new row is added. Nothing is stored. | |
| * Odoo unlinks since the last sync stay invisible: a deleted record keeps its mirror row. | |
| * Every source read admits through `core.live_limiter` first. A refusal raises | |
| `LiveLimitRefused` before any source call, and the caller answers from the mirror. | |
| S82-T106: `live_answer` dispatches on the table's declared live mode (`connector_spec.LIVE_MODES`): | |
| cursor -> `live_read`, slice -> `slice_read`, live API -> `api_read`, none -> `none_answer`. | |
| """ | |
| import json | |
| import re | |
| from core import connector_spec, live_limiter | |
| from harness import datastore | |
| #: Rows per source page. Paging only: the whole delta is read (rule 1). | |
| PAGE = 2000 | |
| _IDENT = re.compile(r"^[a-z][a-z0-9_]*$") | |
| #: Columns the mirror stamps on a row. A replaced row keeps the mirror's values for these. | |
| _MIRROR_IDENTITY = ("id", "Store", "_connection_id", "_source_id") | |
| class LiveReadError(RuntimeError): | |
| """A live read that cannot run. Carries `cause`, `message` and `fix`.""" | |
| def __init__(self, cause, message, fix): | |
| super().__init__(message) | |
| self.cause = cause | |
| self.message = message | |
| self.fix = fix | |
| def as_dict(self): | |
| """The refusal as a client payload.""" | |
| return {'code': 'live_read_refused', 'cause': self.cause, 'message': self.message, | |
| 'fix': self.fix} | |
| def parse_cursor(raw): | |
| """The `write_date` cursor in a stored `cursor_wd`: plain or `cursor|id`. None when unset.""" | |
| text = str(raw or '').strip() | |
| if not text: | |
| return None | |
| cursor = text.partition('|')[0].strip() | |
| return cursor or None | |
| def _open_cursor(tenant_key=None): | |
| """A read cursor on the mirror, through the datastore's public read paths.""" | |
| if tenant_key is not None: | |
| return datastore.tenant_handle(tenant_key).ro_cursor() | |
| return datastore.ro_cursor() | |
| def _entity_spec(entity): | |
| mode = connector_spec.live_mode_for(entity, 'odoo') | |
| if mode is None or not _IDENT.match(entity) or entity not in datastore.ENTITIES: | |
| raise LiveReadError('unknown_entity', f'{entity} is not a mirrored source table.', | |
| 'Ask about a table the Odoo connector syncs.') | |
| if mode.mode != connector_spec.LIVE_CURSOR: | |
| raise LiveReadError('no_cursor_mode', f'{entity} has no change cursor to read past.', | |
| 'Read this table from the mirror.') | |
| return datastore.ENTITIES[entity], mode | |
| def _rows(cur, sql, params=()): | |
| res = cur.execute(sql, list(params)) | |
| names = [d[0] for d in res.description] | |
| return [dict(zip(names, row)) for row in res.fetchall()] | |
| def read_cursor(cur, entity, state_entity=None): | |
| """`{raw, cursor, phase, updatedAt}` for one entity's sync state. Refuses with no live cursor.""" | |
| found = cur.execute( | |
| "SELECT phase, cursor_wd, updated_at FROM _sync_state WHERE entity = ?", | |
| [state_entity or entity]).fetchone() | |
| phase, raw, updated = found if found else (None, None, None) | |
| cursor = parse_cursor(raw) | |
| if not found or phase == 'backfill' or cursor is None: | |
| raise LiveReadError('no_cursor', f'{entity} has not finished its first sync.', | |
| 'Answer from the mirror until the first sync completes.') | |
| return {'raw': raw, 'cursor': cursor, 'phase': phase, 'updatedAt': updated} | |
| def _source_fields(spec): | |
| """The fields a live read asks for. Named tags and analytic names need a second source read, | |
| so those columns keep their mirror value.""" | |
| skip = set(spec.get('named_m2m') or {}) | {'analytic_distribution'} | |
| return [f for f in spec['fields'] if f not in skip] + list(spec.get('m2m') or []) | |
| def _flatten(spec, recs): | |
| fields = [f for f in _source_fields(spec) if f not in (spec.get('m2m') or [])] | |
| rows = [datastore._flatten(rec, fields, spec.get('m2m') or ()) for rec in recs] | |
| for rec, row in zip(recs, rows): | |
| for output, source in (spec.get('raw_m2m') or {}).items(): | |
| if source in rec: | |
| row[output] = json.dumps(list(rec[source] or [])) | |
| return rows | |
| def delta_rows(client, entity, cursor, admission=None): | |
| """Every source row of `entity` written at or after `cursor`, flattened to mirror columns.""" | |
| spec, mode = _entity_spec(entity) | |
| base = [('active', 'in', [True, False])] if spec.get('archivable') else [] | |
| domain = base + [(mode.cursor_field, '>=', cursor)] | |
| fields = ['id'] + _source_fields(spec) | |
| out, last = [], 0 | |
| while True: | |
| page = client.search_read(spec['model'], domain + [('id', '>', last)], fields, | |
| limit=PAGE, order='id asc') | |
| out.extend(page or []) | |
| if admission is not None: | |
| admission.check_rows(len(out)) | |
| # Stop on an EMPTY page: a server-side page cap returns short pages before the end. | |
| if not page: | |
| break | |
| last = page[-1]['id'] | |
| return _flatten(spec, out) | |
| def _source_id(row): | |
| sid = row.get('_source_id') | |
| return row.get('id') if sid is None else sid | |
| def overlay(mirror_rows, delta): | |
| """`(rows, replaced, added)`: each delta row replaces its mirror row by source id, else is added.""" | |
| fresh = {} | |
| for row in delta: | |
| fresh[row['id']] = row | |
| rows, replaced = [], 0 | |
| for row in mirror_rows: | |
| new = fresh.pop(_source_id(row), None) | |
| if new is None: | |
| rows.append(dict(row)) | |
| continue | |
| merged = {**row, **new} | |
| merged.update({k: row[k] for k in _MIRROR_IDENTITY if k in row}) | |
| rows.append(merged) | |
| replaced += 1 | |
| added = [{**row, '_source_id': row['id']} for row in fresh.values()] | |
| rows.extend(added) | |
| return rows, replaced, len(added) | |
| def live_read(entity, tenant, *, client=None, tenant_key=None, where=None, params=(), | |
| keep=None, state_entity=None, caller='live_read'): | |
| """The mirror rows of `entity` with every row changed since the last sync overlaid by id. | |
| `tenant` is the tenant-bound store handle the limiter admits through. `where` and `params` | |
| slice the mirror (SQL we author, values bound). `keep` filters the overlaid rows, so an edited | |
| row that left or entered the slice is judged on its new values. Returns | |
| `{entity, rows, cursor, cursorRaw, phase, syncedAt, mirrorRows, delta, replaced, added}`. | |
| """ | |
| _entity_spec(entity) | |
| if client is None: | |
| from core import odoo as client | |
| cur = _open_cursor(tenant_key) | |
| try: | |
| state = read_cursor(cur, entity, state_entity) | |
| mirror = _rows(cur, f"SELECT * FROM {entity} WHERE {where or 'TRUE'}", params) | |
| finally: | |
| try: | |
| cur.close() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| with live_limiter.admit(tenant, caller=caller) as admission: | |
| delta = delta_rows(client, entity, state['cursor'], admission) | |
| rows, replaced, added = overlay(mirror, delta) | |
| if keep is not None: | |
| rows = [row for row in rows if keep(row)] | |
| return {'entity': entity, 'rows': rows, 'cursor': state['cursor'], 'cursorRaw': state['raw'], | |
| 'phase': state['phase'], 'syncedAt': state['updatedAt'], 'mirrorRows': len(mirror), | |
| 'delta': len(delta), 'replaced': replaced, 'added': added} | |
| # ββ S82-T106: the slice, live API and none dispatch ββββββββββββββββββββββββββββββββββββββββββ | |
| #: The answer a table with no live read gives: the mirror answers. | |
| NONE_FIX = 'Answer from the mirror. The next sync refreshes it.' | |
| #: Odoo read operators a slice term may use. | |
| _SLICE_OPS = frozenset(('=', '!=', '>', '>=', '<', '<=', 'in', 'not in', 'like', 'ilike', | |
| '=like', '=ilike')) | |
| _FIELD_PATH = re.compile(r'^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$') | |
| _ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') | |
| def resolve_table(table, connector=None): | |
| """`(connector, mirror table, LiveMode)` for a mirror table or a `ut_*` key. Refuses an unknown one.""" | |
| name = str(table or '').strip() | |
| spec = connector_spec.spec_for(name) | |
| if spec is not None: | |
| connector = connector or connector_spec.connector_of(name) | |
| name = spec.source | |
| if connector is None: | |
| connector = next((key for key in connector_spec.LIVE_MODES | |
| if name in connector_spec.live_modes(key)), None) | |
| mode = connector_spec.live_mode_for(name, connector) if connector else None | |
| if mode is None or not _IDENT.match(name): | |
| raise LiveReadError('unknown_table', 'This table has no declared live mode.', | |
| 'Ask about a table a connector syncs.') | |
| return connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector), name, mode | |
| def none_answer(table, connector=None): | |
| """The answer for a table with no live read: its stated reason, and the mirror answers.""" | |
| connector, name, mode = resolve_table(table, connector) | |
| if mode.mode != connector_spec.LIVE_NONE: | |
| raise LiveReadError('has_live_mode', f'{name} has a live read.', 'Call live_answer.') | |
| return {'table': name, 'connector': connector, 'mode': mode.mode, 'live': False, | |
| 'cause': 'no_live_read', 'reason': mode.reason, 'fix': NONE_FIX} | |
| def _slice_spec(entity): | |
| _connector, name, mode = resolve_table(entity, 'odoo') | |
| if (mode.mode != connector_spec.LIVE_SLICE or name not in datastore.ENTITIES | |
| or name not in datastore.REPLACE_EACH_PASS): | |
| raise LiveReadError('no_slice_mode', f'{name} has no bounded slice read.', | |
| 'Call live_answer for this table.') | |
| return name, datastore.ENTITIES[name], datastore.REPLACE_EACH_PASS[name] | |
| def slice_domain(spec, domain): | |
| """The caller's Odoo domain, checked term by term: a column of the table, a read operator.""" | |
| allowed = {'id', *spec['fields'], *(spec.get('m2m') or ())} | |
| out = [] | |
| for term in domain or (): | |
| if not isinstance(term, (list, tuple)) or len(term) != 3: | |
| raise LiveReadError('bad_domain', 'A slice term is a field, an operator and a value.', | |
| 'Send each term as field, operator, value.') | |
| field, op, value = str(term[0]), term[1], term[2] | |
| if not _FIELD_PATH.match(field) or field.split('.', 1)[0] not in allowed \ | |
| or op not in _SLICE_OPS: | |
| raise LiveReadError('bad_domain', f'{field} {op} is not a filter this table allows.', | |
| 'Filter on a column of this table with a comparison.') | |
| out.append((field, op, value)) | |
| return out | |
| def slice_rows(client, entity, plan, domain, admission=None): | |
| """Source rows in the plan's scope and `domain`, flattened. Reads at most max rows plus one.""" | |
| spec = datastore.ENTITIES[entity] | |
| full = list(plan.get('domain') or []) + list(domain) | |
| cap = admission.limits.get('maxRowsPerCall') if admission is not None else None | |
| fields = ['id'] + _source_fields(spec) | |
| found, last = [], 0 | |
| while True: | |
| size = PAGE if cap is None else max(1, min(PAGE, int(cap) + 1 - len(found))) | |
| page = client.search_read(spec['model'], full + [('id', '>', last)], fields, | |
| limit=size, order='id asc') | |
| found.extend(page or []) | |
| if admission is not None: | |
| admission.check_rows(len(found)) | |
| # Stop on an EMPTY page: a server-side page cap returns short pages before the end. | |
| if not page: | |
| break | |
| last = page[-1]['id'] | |
| return _flatten(spec, found) | |
| def slice_read(entity, tenant, *, domain=(), client=None, caller='live_slice'): | |
| """A bounded live read of an Odoo table the sync replaces whole each pass (no change cursor). | |
| `domain` is Odoo terms on the table's columns, ANDed with the sync plan's own scope. Over the | |
| tenant's max rows per call it raises `LiveLimitRefused` with cause `rows` and its fix. | |
| Returns `{table, connector, mode, rows, rowCount, domain, maxRows, readAt}`. | |
| """ | |
| name, spec, plan = _slice_spec(entity) | |
| terms = slice_domain(spec, domain) | |
| if client is None: | |
| from core import odoo as client | |
| with live_limiter.admit(tenant, caller=caller) as slice_admission: | |
| rows = slice_rows(client, name, plan, terms, slice_admission) | |
| return {'table': name, 'connector': 'odoo', 'mode': connector_spec.LIVE_SLICE, 'live': True, | |
| 'rows': rows, 'rowCount': len(rows), 'domain': [list(t) for t in terms], | |
| 'maxRows': slice_admission.limits.get('maxRowsPerCall'), | |
| 'readAt': live_limiter.iso_utc(slice_admission.admitted_at)} | |
| def _meta_connections(tenant_key): | |
| """`[(connection id, label, token, legacy)]` for the tenant's Meta Ads credentials.""" | |
| from harness import meta_store | |
| rt = meta_store._runtime_for(tenant_key) | |
| if rt is None: | |
| tok = meta_store.token(tenant_key=tenant_key) | |
| return [('', 'Meta Ads', tok, True)] if tok else [] | |
| return meta_store._connection_rows(rt, tenant_key, {'problems': []}) | |
| def _graph_pages(get, path, tok, sink, admission=None, **params): | |
| """Every page of one Graph edge into `sink`. A heavy page shrinks; a rate limit refuses at once.""" | |
| from harness import meta_store | |
| limit = int(params.pop('limit', None) or meta_store.PAGE) | |
| after, cursors = None, set() | |
| while True: | |
| call = dict(params, limit=limit, **({'after': after} if after else {})) | |
| try: | |
| body = get(path, tok, **call) | |
| except meta_store.MetaError as exc: | |
| text = str(exc) | |
| if meta_store._TOO_MUCH in text and limit > 5: | |
| limit = max(5, limit // 4) | |
| continue | |
| if meta_store._RATE_LIMITED in text.lower(): | |
| raise LiveReadError('source_rate', 'Meta is limiting calls to this ad account.', | |
| 'Try again in a few minutes, or answer from the mirror.') from None | |
| raise LiveReadError('source_error', 'Meta refused the live read.', | |
| 'Try again, or answer from the mirror.') from None | |
| sink.extend(body.get('data') or []) | |
| if admission is not None: | |
| admission.check_rows(len(sink)) | |
| paging = body.get('paging') or {} | |
| after = (paging.get('cursors') or {}).get('after') | |
| if not (paging.get('next') and after): | |
| return sink | |
| if after in cursors: | |
| raise LiveReadError('source_error', 'Meta repeated a page of the live read.', | |
| 'Try again, or answer from the mirror.') | |
| cursors.add(after) | |
| def _day_windows(since, until, size): | |
| from datetime import date, timedelta | |
| start, end = date.fromisoformat(since), date.fromisoformat(until) | |
| out = [] | |
| while start <= end: | |
| stop = min(start + timedelta(days=max(1, size) - 1), end) | |
| out.append((start.isoformat(), stop.isoformat())) | |
| start = stop + timedelta(days=1) | |
| return out | |
| def api_read(table, tenant, *, tenant_key=None, since=None, until=None, connections=None, | |
| get=None, caller='live_api', windows=None): | |
| """A live Meta Graph read of one Meta mirror table, every page (rule 1). | |
| Insights read `windows` or `since..until` (default tenant today) at the sync's grain. | |
| `connections` and `get` are the credential list and the Graph call, injectable for fixtures. | |
| Returns `{table, connector, mode, rows, rowCount, accounts, since, until, maxRows, readAt}`. | |
| """ | |
| from core import periods | |
| from harness import meta_store | |
| connector, name, mode = resolve_table(table) | |
| if mode.mode != connector_spec.LIVE_API or connector != 'meta': | |
| raise LiveReadError('no_api_mode', f'{name} has no live API read.', | |
| 'Call live_answer for this table.') | |
| insights = name == meta_store.INSIGHTS_TABLE | |
| since = str(since or until or periods.today().isoformat()) | |
| until = str(until or since) | |
| asked = sorted(windows if windows is not None else [(since, until)]) | |
| spans = [] | |
| for lo, hi in asked: | |
| if not (_ISO_DAY.match(str(lo or '')) and _ISO_DAY.match(str(hi or ''))) or lo > hi: | |
| raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.', | |
| 'Send since and until as YYYY-MM-DD.') | |
| if spans and lo <= spans[-1][1]: | |
| spans[-1] = (spans[-1][0], max(hi, spans[-1][1])) | |
| else: | |
| spans.append((lo, hi)) | |
| if connections is None: | |
| connections = _meta_connections(tenant_key) | |
| if not connections: | |
| raise LiveReadError('no_credential', 'No Meta Ads credential is connected.', | |
| 'Connect Meta Ads under Connectors.') | |
| get = get or meta_store._get | |
| rows, accounts = [], [] | |
| with live_limiter.admit(tenant, caller=caller) as api_admission: | |
| for cid, label, tok, _legacy in connections: | |
| accts = _graph_pages(get, 'me/adaccounts', tok, [], fields='id,name') | |
| for acct in accts: | |
| aid = acct['id'] | |
| accounts.append(aid) | |
| start = len(rows) | |
| if insights: | |
| for first, last in spans: | |
| for lo, hi in _day_windows(first, last, meta_store.INSIGHTS_SLICE): | |
| _graph_pages(get, f'{aid}/insights', tok, rows, api_admission, | |
| fields=','.join(meta_store.INSIGHT_FIELDS), | |
| level=meta_store.INSIGHTS_LEVEL, time_increment='1', | |
| limit=meta_store.INSIGHTS_PAGE, | |
| time_range=json.dumps({'since': lo, 'until': hi})) | |
| else: | |
| edge, fields, _parent = meta_store.SPECS[name] | |
| if edge is None: | |
| rows.append(get(aid, tok, fields=','.join(fields))) | |
| api_admission.check_rows(len(rows)) | |
| else: | |
| _graph_pages(get, f'{aid}/{edge}', tok, rows, api_admission, | |
| fields=','.join(fields), limit=meta_store.PAGE) | |
| for row in rows[start:]: | |
| row['_connection_id'] = cid | |
| if insights: | |
| # The sync's stable Insights id, so a row matches its mirror row. | |
| row['id'] = ':'.join(str(row.get(k) or '') for k in | |
| (f'{meta_store.INSIGHTS_LEVEL}_id', 'date_start', | |
| 'date_stop')) | |
| for index in range(start, len(rows)): | |
| raw = rows[index] | |
| _cols, normalized = meta_store._origin_rows(list(raw), [raw], cid, label) | |
| rows[index] = {key: meta_store._cell(value) for key, value in normalized[0].items()} | |
| return {'table': name, 'connector': connector, 'mode': connector_spec.LIVE_API, 'live': True, | |
| 'rows': rows, 'rowCount': len(rows), 'accounts': accounts, | |
| 'since': spans[0][0] if insights and spans else None, | |
| 'until': spans[-1][1] if insights and spans else None, | |
| 'windows': [{'since': lo, 'until': hi} for lo, hi in spans] if insights else None, | |
| 'maxRows': api_admission.limits.get('maxRowsPerCall'), | |
| 'readAt': live_limiter.iso_utc(api_admission.admitted_at)} | |
| def live_answer(table, tenant, *, connector=None, tenant_key=None, client=None, where=None, | |
| params=(), keep=None, state_entity=None, domain=(), since=None, until=None, | |
| connections=None, get=None, caller='live', windows=None): | |
| """One live question about one table, routed by its declared live mode. | |
| `table` is a mirror table or a `ut_*` key. Cursor tables take `where`, `params` and `keep`. | |
| Slice tables take `domain`. Meta tables take `since` and `until`. A table with no live read | |
| returns its reason and reads nothing. Every source read admits through the live limiter. | |
| Each result carries `mode` and `connector`. | |
| """ | |
| connector, name, mode = resolve_table(table, connector) | |
| if mode.mode == connector_spec.LIVE_CURSOR: | |
| out = live_read(name, tenant, client=client, tenant_key=tenant_key, where=where, | |
| params=params, keep=keep, state_entity=state_entity, caller=caller) | |
| return {**out, 'table': name, 'connector': connector, 'mode': mode.mode, 'live': True} | |
| if mode.mode == connector_spec.LIVE_SLICE: | |
| return slice_read(name, tenant, domain=domain, client=client, caller=caller) | |
| if mode.mode == connector_spec.LIVE_API: | |
| return api_read(name, tenant, tenant_key=tenant_key, since=since, until=until, | |
| connections=connections, get=get, caller=caller, windows=windows) | |
| return none_answer(name, connector) | |
| # ββ S82-T105: the Live tool's windowed answer ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| #: The ISO form of a business-day window bound. | |
| _DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') | |
| _QUALIFIED = re.compile(r'\b([a-z_][a-z0-9_]*)\.([a-z_][a-z0-9_]*)\b') | |
| _WINDOW_KEY = '_live_window_at' | |
| def _instant(value): | |
| """A stored instant as a naive UTC `datetime`, or None.""" | |
| from datetime import datetime, timezone | |
| if value is None or value == '': | |
| return None | |
| if isinstance(value, datetime): | |
| moment = value | |
| else: | |
| try: | |
| moment = datetime.fromisoformat(str(value).strip().replace('Z', '+00:00')) | |
| except ValueError: | |
| return None | |
| if moment.tzinfo is not None: | |
| moment = moment.astimezone(timezone.utc).replace(tzinfo=None) | |
| return moment | |
| def _window_bounds(date_from, date_to, instants): | |
| """`(lo, hi)` for one window: naive UTC `[lo, hi)` for an instant column, ISO days otherwise.""" | |
| from core import periods | |
| if not (_DAY.match(str(date_from or '')) and _DAY.match(str(date_to or ''))) \ | |
| or date_from > date_to: | |
| raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.', | |
| 'Send date_from and date_to as YYYY-MM-DD.') | |
| return periods.business_day_bounds(date_from, date_to) if instants else (date_from, date_to) | |
| def _in_window(value, bounds, instants): | |
| lo, hi = bounds | |
| if instants: | |
| moment = _instant(value) | |
| return moment is not None and lo <= moment < hi | |
| day = str(value or '')[:10] | |
| return bool(day) and lo <= day <= hi | |
| def mirror_slice(entity, alias, where, params=(), tenant_key=None): | |
| """The mirror rows of `entity` the SQL `where` keeps, read through a read cursor.""" | |
| import duckdb | |
| cur = _open_cursor(tenant_key) | |
| try: | |
| return _rows(cur, f"SELECT * FROM {entity} AS {alias} WHERE {where or 'TRUE'}", params) | |
| except duckdb.Error as exc: | |
| raise LiveReadError('unsupported_definition', 'This database slice cannot answer at source row grain.', | |
| 'Read this database from the mirror or ask about its source table.') from exc | |
| finally: | |
| try: | |
| cur.close() | |
| except Exception: # noqa: BLE001 | |
| pass | |
| def _definition_columns(entity, alias, where): | |
| """The mirror columns the definition SQL reads. Refuses a definition naming another table.""" | |
| known = {c.split()[0]: c for c in datastore._cols(datastore.ENTITIES[entity])} | |
| text = re.sub(r"'(?:[^']|'')*'", "''", where or '') | |
| if re.search(r'\b(select|join|from)\b', text, re.IGNORECASE): | |
| raise LiveReadError('unsupported_definition', | |
| 'This database definition needs another table.', | |
| 'Read this database from the mirror or ask about its source table.') | |
| used = [] | |
| for table, column in _QUALIFIED.findall(text): | |
| if table != alias or column not in known: | |
| raise LiveReadError('unsupported_definition', | |
| 'This database is defined over more than one table.', | |
| 'Ask about a database defined over one synced table.') | |
| if column not in used: | |
| used.append(column) | |
| if '{' in text: | |
| raise LiveReadError('unsupported_definition', 'This database definition is not static.', | |
| 'Ask about a database defined over one synced table.') | |
| return [known[column] for column in used] | |
| def defined_rows(entity, alias, where, rows): | |
| """The rows the database's own definition SQL keeps, each judged on its current values. | |
| The rows load into an in-memory DuckDB table, never the mirror.""" | |
| rows = list(rows) | |
| if not where or not where.strip() or where.strip().upper() == 'TRUE': | |
| return rows | |
| import duckdb | |
| cols = _definition_columns(entity, alias, where) | |
| names = [c.split()[0] for c in cols] | |
| con = duckdb.connect(':memory:') | |
| try: | |
| con.execute(f"CREATE TABLE {alias} (_k BIGINT, {', '.join(cols) or '_none INTEGER'})") | |
| if rows: | |
| values = [[i] + [row.get(n) for n in names] for i, row in enumerate(rows)] | |
| marks = ', '.join('?' for _ in range(len(names) + 1)) | |
| con.executemany(f"INSERT INTO {alias} (_k{''.join(', ' + n for n in names)}) " | |
| f"VALUES ({marks})", values) | |
| kept = {r[0] for r in con.execute(f"SELECT _k FROM {alias} WHERE {where}").fetchall()} | |
| except duckdb.Error as exc: | |
| raise LiveReadError('unsupported_definition', 'This database definition needs another binding.', | |
| 'Read this database from the mirror or ask about its source table.') from exc | |
| finally: | |
| con.close() | |
| return [row for i, row in enumerate(rows) if i in kept] | |
| def walled_rows(rows, wall): | |
| """The projected rows the caller may see, through the row wall every door applies.""" | |
| from core import perm_scope | |
| if not isinstance(wall, dict) or 'user' not in wall or not wall.get('database'): | |
| raise LiveReadError('no_wall', 'A live answer needs the caller and the database.', | |
| 'Call the Live tool as a signed-in member.') | |
| return perm_scope.apply_row_scope(rows, wall['user'], wall['database'], wall['fields'], | |
| st=wall.get('st'), defn=wall.get('defn')) | |
| def _total(rows, measure): | |
| if measure is None: | |
| return len(rows) | |
| total = 0.0 | |
| for row in rows: | |
| value = row.get(measure) | |
| if value is None or value == '': | |
| continue | |
| try: | |
| total += float(value) | |
| except (TypeError, ValueError): | |
| raise LiveReadError('invalid_measure_value', 'The measure contains a non-numeric value.', | |
| 'Choose a numeric source field or correct its source value.') from None | |
| return round(total, 6) | |
| def project_definition(entity, rows, spec, keys, tenant_key=None): | |
| """Evaluate the database SQL over source rows and mirrored dimensions in memory.""" | |
| import duckdb | |
| cur = _open_cursor(tenant_key) | |
| con = duckdb.connect(':memory:') | |
| quote = lambda key: '"' + key.replace('"', '""') + '"' | |
| tables = set((spec.get('tables') or {}).values()) | {entity} | |
| try: | |
| for table in tables: | |
| if not _IDENT.fullmatch(table): | |
| raise ValueError('Invalid mirror table') | |
| cur.execute(f'SELECT * FROM {table} LIMIT 0') | |
| columns = [(d[0], str(d[1])) for d in cur.description] | |
| source = list(rows) if table == entity else _rows(cur, f'SELECT * FROM {table}') | |
| con.execute(f'CREATE TABLE {table} (' + ', '.join( | |
| f'{quote(name)} {kind}' for name, kind in columns) + ')') | |
| if source: | |
| con.executemany(f'INSERT INTO {table} VALUES (' + ','.join('?' for _ in columns) + ')', | |
| [[row.get(name) for name, _ in columns] for row in source]) | |
| bindings = {key: binding for key, binding in spec['cols'].items() if key in keys} | |
| expressions = [f'{expr} AS {quote(key)}' for key, (expr, _convert) in bindings.items()] | |
| frm = spec.get('from_sql') or (entity if spec.get('table') else | |
| entity + ' AS ' + spec['id'].split('.')[0]) | |
| values = _rows(con, f"SELECT {spec['id']} AS pid" + | |
| (', ' + ', '.join(expressions) if expressions else '') + | |
| f" FROM {frm} WHERE {spec.get('where') or 'TRUE'}") | |
| ids = [r['pid'] for r in values] | |
| if len(set(ids)) != len(ids): | |
| raise ValueError('The join repeats a source row') | |
| for row in values: | |
| for key, (_expr, convert) in bindings.items(): | |
| try: | |
| row[key] = convert(row[key]) | |
| except (TypeError, ValueError): | |
| row[key] = None | |
| return values | |
| except (duckdb.Error, ValueError) as exc: | |
| raise LiveReadError('unsupported_definition', | |
| 'This database join or expression cannot answer at source row grain.', | |
| 'Read this database from the mirror or ask about its source table.') from exc | |
| finally: | |
| cur.close() | |
| con.close() | |
| def window_answer(entity, tenant, *, alias, where, date_column, instants, windows, project, | |
| wall, measure=None, tenant_key=None, client=None, caller='live.answer', | |
| definition=None, domain=(), filters=None): | |
| """The screen (mirror) and live totals of one database per window, live rows overlaid by id. | |
| `windows` is `{name: (date_from, date_to)}` in tenant business days. `where` is the database's | |
| own definition SQL over `alias`. `project(row)` maps a mirror row to the database's field keys, | |
| and every projected row passes `walled_rows`. `measure` is a field key to sum, else rows count. | |
| Over the limit, or with no live read, the mirror answers with the cause, next allowed time and | |
| fix. Returns `{entity, served, syncedAt, delta, replaced, added, limit_refusal, | |
| live_unavailable, windows: {name: {date_from, date_to, screen, live, difference}}, rows}`. | |
| """ | |
| bounds = {name: _window_bounds(lo, hi, instants) for name, (lo, hi) in windows.items() | |
| if date_column is not None} | |
| cast = f"CAST({date_column} AS TIMESTAMP)" if instants else f"CAST({date_column} AS DATE)" | |
| terms, params = [], [] | |
| for lo, hi in bounds.values(): | |
| terms.append(f"({cast} >= ? AND {cast} < ?)" if instants | |
| else f"({cast} >= CAST(? AS DATE) AND {cast} <= CAST(? AS DATE))") | |
| params += [lo, hi] | |
| sliced = (' OR '.join(terms) or 'FALSE') if date_column else 'TRUE' | |
| def in_any(row): | |
| return date_column is None or any( | |
| _in_window(row.get(date_column), b, instants) for b in bounds.values()) | |
| defined = f"({sliced}) AND ({where})" if where and where.strip() else sliced | |
| if not definition: | |
| _definition_columns(entity, alias, where) | |
| screen_rows = mirror_slice(entity, alias, 'TRUE' if definition else defined, | |
| () if definition else params, tenant_key) | |
| live, limit, unavailable = None, None, None | |
| try: | |
| live = live_answer(entity, tenant, client=client, tenant_key=tenant_key, where=sliced, | |
| params=params, keep=in_any, domain=domain, caller=caller, | |
| windows=list(windows.values()) if date_column else None) | |
| if not live.get('live'): | |
| unavailable = live | |
| live = None | |
| except live_limiter.LiveLimitRefused as refusal: | |
| live, limit = None, refusal.as_dict() | |
| except LiveReadError as exc: | |
| live, unavailable = None, exc.as_dict() | |
| def shown(rows): | |
| if definition: | |
| rows = project_definition(entity, rows, definition, project, tenant_key) | |
| shown_rows = walled_rows(rows, wall) | |
| if filters: | |
| from harness import filter_eval | |
| shown_rows = [r for r in shown_rows if filter_eval.permits(filters, r, wall['fields'])] | |
| return shown_rows | |
| # The window key rides each projected row through the wall and is dropped after. | |
| return walled_rows([{**project(row), _WINDOW_KEY: row.get(date_column)} for row in rows], | |
| wall) | |
| def split(rows, b): | |
| return [{k: v for k, v in r.items() if k != _WINDOW_KEY} for r in rows | |
| if date_column is None or _in_window(r.get(_WINDOW_KEY), b, instants)] | |
| screen = shown(screen_rows) | |
| fresh = shown(live['rows'] if definition else defined_rows( | |
| entity, alias, where, live['rows'])) if live is not None else None | |
| out_windows, out_rows = {}, {} | |
| for name, (lo, hi) in windows.items(): | |
| screen_in = split(screen, bounds.get(name)) | |
| entry = {'date_from': lo, 'date_to': hi, | |
| 'screen': {'value': _total(screen_in, measure), 'rows': len(screen_in)}, | |
| 'live': None, 'difference': None} | |
| out_rows[name] = screen_in | |
| if fresh is not None: | |
| live_in = split(fresh, bounds.get(name)) | |
| entry['live'] = {'value': _total(live_in, measure), 'rows': len(live_in)} | |
| entry['difference'] = round(entry['live']['value'] - entry['screen']['value'], 6) | |
| out_rows[name] = live_in | |
| out_windows[name] = entry | |
| return {'entity': entity, 'served': 'live' if live is not None else 'mirror', | |
| 'syncedAt': (live or {}).get('syncedAt'), | |
| # Source-wide counters disclose rows outside the database and reader's walls. | |
| 'delta': None, 'replaced': None, | |
| 'added': None, 'limit_refusal': limit, | |
| 'live_unavailable': unavailable, 'windows': out_windows, 'rows': out_rows} | |