"""Read-only live delta reader: source rows written since an entity's last sync cursor, overlaid on the mirror rows by id in memory. Contract (S82-T104, ideation R6 and O2): * The cursor is `_sync_state.cursor_wd`, read through a read cursor. Never `datastore.connect()`: one sync worker holds the writer lease. * The source read is `write_date >= cursor`, the same domain `datastore._sync_entity` builds. During an id-walk `cursor_wd` reads `cursor|id`, and the plain cursor before the pipe covers it. * An edited row replaces its mirror row by source id. A new row is added. Nothing is stored. * Odoo unlinks since the last sync stay invisible: a deleted record keeps its mirror row. * Every source read admits through `core.live_limiter` first. A refusal raises `LiveLimitRefused` before any source call, and the caller answers from the mirror. S82-T106: `live_answer` dispatches on the table's declared live mode (`connector_spec.LIVE_MODES`): cursor -> `live_read`, slice -> `slice_read`, live API -> `api_read`, none -> `none_answer`. """ import json import re from core import connector_spec, live_limiter from harness import datastore #: Rows per source page. Paging only: the whole delta is read (rule 1). PAGE = 2000 _IDENT = re.compile(r"^[a-z][a-z0-9_]*$") #: Columns the mirror stamps on a row. A replaced row keeps the mirror's values for these. _MIRROR_IDENTITY = ("id", "Store", "_connection_id", "_source_id") class LiveReadError(RuntimeError): """A live read that cannot run. Carries `cause`, `message` and `fix`.""" def __init__(self, cause, message, fix): super().__init__(message) self.cause = cause self.message = message self.fix = fix def as_dict(self): """The refusal as a client payload.""" return {'code': 'live_read_refused', 'cause': self.cause, 'message': self.message, 'fix': self.fix} def parse_cursor(raw): """The `write_date` cursor in a stored `cursor_wd`: plain or `cursor|id`. None when unset.""" text = str(raw or '').strip() if not text: return None cursor = text.partition('|')[0].strip() return cursor or None def _open_cursor(tenant_key=None): """A read cursor on the mirror, through the datastore's public read paths.""" if tenant_key is not None: return datastore.tenant_handle(tenant_key).ro_cursor() return datastore.ro_cursor() def _entity_spec(entity): mode = connector_spec.live_mode_for(entity, 'odoo') if mode is None or not _IDENT.match(entity) or entity not in datastore.ENTITIES: raise LiveReadError('unknown_entity', f'{entity} is not a mirrored source table.', 'Ask about a table the Odoo connector syncs.') if mode.mode != connector_spec.LIVE_CURSOR: raise LiveReadError('no_cursor_mode', f'{entity} has no change cursor to read past.', 'Read this table from the mirror.') return datastore.ENTITIES[entity], mode def _rows(cur, sql, params=()): res = cur.execute(sql, list(params)) names = [d[0] for d in res.description] return [dict(zip(names, row)) for row in res.fetchall()] def read_cursor(cur, entity, state_entity=None): """`{raw, cursor, phase, updatedAt}` for one entity's sync state. Refuses with no live cursor.""" found = cur.execute( "SELECT phase, cursor_wd, updated_at FROM _sync_state WHERE entity = ?", [state_entity or entity]).fetchone() phase, raw, updated = found if found else (None, None, None) cursor = parse_cursor(raw) if not found or phase == 'backfill' or cursor is None: raise LiveReadError('no_cursor', f'{entity} has not finished its first sync.', 'Answer from the mirror until the first sync completes.') return {'raw': raw, 'cursor': cursor, 'phase': phase, 'updatedAt': updated} def _source_fields(spec): """The fields a live read asks for. Named tags and analytic names need a second source read, so those columns keep their mirror value.""" skip = set(spec.get('named_m2m') or {}) | {'analytic_distribution'} return [f for f in spec['fields'] if f not in skip] + list(spec.get('m2m') or []) def _flatten(spec, recs): fields = [f for f in _source_fields(spec) if f not in (spec.get('m2m') or [])] rows = [datastore._flatten(rec, fields, spec.get('m2m') or ()) for rec in recs] for rec, row in zip(recs, rows): for output, source in (spec.get('raw_m2m') or {}).items(): if source in rec: row[output] = json.dumps(list(rec[source] or [])) return rows def delta_rows(client, entity, cursor, admission=None): """Every source row of `entity` written at or after `cursor`, flattened to mirror columns.""" spec, mode = _entity_spec(entity) base = [('active', 'in', [True, False])] if spec.get('archivable') else [] domain = base + [(mode.cursor_field, '>=', cursor)] fields = ['id'] + _source_fields(spec) out, last = [], 0 while True: page = client.search_read(spec['model'], domain + [('id', '>', last)], fields, limit=PAGE, order='id asc') out.extend(page or []) if admission is not None: admission.check_rows(len(out)) # Stop on an EMPTY page: a server-side page cap returns short pages before the end. if not page: break last = page[-1]['id'] return _flatten(spec, out) def _source_id(row): sid = row.get('_source_id') return row.get('id') if sid is None else sid def overlay(mirror_rows, delta): """`(rows, replaced, added)`: each delta row replaces its mirror row by source id, else is added.""" fresh = {} for row in delta: fresh[row['id']] = row rows, replaced = [], 0 for row in mirror_rows: new = fresh.pop(_source_id(row), None) if new is None: rows.append(dict(row)) continue merged = {**row, **new} merged.update({k: row[k] for k in _MIRROR_IDENTITY if k in row}) rows.append(merged) replaced += 1 added = [{**row, '_source_id': row['id']} for row in fresh.values()] rows.extend(added) return rows, replaced, len(added) def live_read(entity, tenant, *, client=None, tenant_key=None, where=None, params=(), keep=None, state_entity=None, caller='live_read'): """The mirror rows of `entity` with every row changed since the last sync overlaid by id. `tenant` is the tenant-bound store handle the limiter admits through. `where` and `params` slice the mirror (SQL we author, values bound). `keep` filters the overlaid rows, so an edited row that left or entered the slice is judged on its new values. Returns `{entity, rows, cursor, cursorRaw, phase, syncedAt, mirrorRows, delta, replaced, added}`. """ _entity_spec(entity) if client is None: from core import odoo as client cur = _open_cursor(tenant_key) try: state = read_cursor(cur, entity, state_entity) mirror = _rows(cur, f"SELECT * FROM {entity} WHERE {where or 'TRUE'}", params) finally: try: cur.close() except Exception: # noqa: BLE001 pass with live_limiter.admit(tenant, caller=caller) as admission: delta = delta_rows(client, entity, state['cursor'], admission) rows, replaced, added = overlay(mirror, delta) if keep is not None: rows = [row for row in rows if keep(row)] return {'entity': entity, 'rows': rows, 'cursor': state['cursor'], 'cursorRaw': state['raw'], 'phase': state['phase'], 'syncedAt': state['updatedAt'], 'mirrorRows': len(mirror), 'delta': len(delta), 'replaced': replaced, 'added': added} # ── S82-T106: the slice, live API and none dispatch ────────────────────────────────────────── #: The answer a table with no live read gives: the mirror answers. NONE_FIX = 'Answer from the mirror. The next sync refreshes it.' #: Odoo read operators a slice term may use. _SLICE_OPS = frozenset(('=', '!=', '>', '>=', '<', '<=', 'in', 'not in', 'like', 'ilike', '=like', '=ilike')) _FIELD_PATH = re.compile(r'^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$') _ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') def resolve_table(table, connector=None): """`(connector, mirror table, LiveMode)` for a mirror table or a `ut_*` key. Refuses an unknown one.""" name = str(table or '').strip() spec = connector_spec.spec_for(name) if spec is not None: connector = connector or connector_spec.connector_of(name) name = spec.source if connector is None: connector = next((key for key in connector_spec.LIVE_MODES if name in connector_spec.live_modes(key)), None) mode = connector_spec.live_mode_for(name, connector) if connector else None if mode is None or not _IDENT.match(name): raise LiveReadError('unknown_table', 'This table has no declared live mode.', 'Ask about a table a connector syncs.') return connector_spec.CONNECTOR_SPEC_KEYS.get(connector, connector), name, mode def none_answer(table, connector=None): """The answer for a table with no live read: its stated reason, and the mirror answers.""" connector, name, mode = resolve_table(table, connector) if mode.mode != connector_spec.LIVE_NONE: raise LiveReadError('has_live_mode', f'{name} has a live read.', 'Call live_answer.') return {'table': name, 'connector': connector, 'mode': mode.mode, 'live': False, 'cause': 'no_live_read', 'reason': mode.reason, 'fix': NONE_FIX} def _slice_spec(entity): _connector, name, mode = resolve_table(entity, 'odoo') if (mode.mode != connector_spec.LIVE_SLICE or name not in datastore.ENTITIES or name not in datastore.REPLACE_EACH_PASS): raise LiveReadError('no_slice_mode', f'{name} has no bounded slice read.', 'Call live_answer for this table.') return name, datastore.ENTITIES[name], datastore.REPLACE_EACH_PASS[name] def slice_domain(spec, domain): """The caller's Odoo domain, checked term by term: a column of the table, a read operator.""" allowed = {'id', *spec['fields'], *(spec.get('m2m') or ())} out = [] for term in domain or (): if not isinstance(term, (list, tuple)) or len(term) != 3: raise LiveReadError('bad_domain', 'A slice term is a field, an operator and a value.', 'Send each term as field, operator, value.') field, op, value = str(term[0]), term[1], term[2] if not _FIELD_PATH.match(field) or field.split('.', 1)[0] not in allowed \ or op not in _SLICE_OPS: raise LiveReadError('bad_domain', f'{field} {op} is not a filter this table allows.', 'Filter on a column of this table with a comparison.') out.append((field, op, value)) return out def slice_rows(client, entity, plan, domain, admission=None): """Source rows in the plan's scope and `domain`, flattened. Reads at most max rows plus one.""" spec = datastore.ENTITIES[entity] full = list(plan.get('domain') or []) + list(domain) cap = admission.limits.get('maxRowsPerCall') if admission is not None else None fields = ['id'] + _source_fields(spec) found, last = [], 0 while True: size = PAGE if cap is None else max(1, min(PAGE, int(cap) + 1 - len(found))) page = client.search_read(spec['model'], full + [('id', '>', last)], fields, limit=size, order='id asc') found.extend(page or []) if admission is not None: admission.check_rows(len(found)) # Stop on an EMPTY page: a server-side page cap returns short pages before the end. if not page: break last = page[-1]['id'] return _flatten(spec, found) def slice_read(entity, tenant, *, domain=(), client=None, caller='live_slice'): """A bounded live read of an Odoo table the sync replaces whole each pass (no change cursor). `domain` is Odoo terms on the table's columns, ANDed with the sync plan's own scope. Over the tenant's max rows per call it raises `LiveLimitRefused` with cause `rows` and its fix. Returns `{table, connector, mode, rows, rowCount, domain, maxRows, readAt}`. """ name, spec, plan = _slice_spec(entity) terms = slice_domain(spec, domain) if client is None: from core import odoo as client with live_limiter.admit(tenant, caller=caller) as slice_admission: rows = slice_rows(client, name, plan, terms, slice_admission) return {'table': name, 'connector': 'odoo', 'mode': connector_spec.LIVE_SLICE, 'live': True, 'rows': rows, 'rowCount': len(rows), 'domain': [list(t) for t in terms], 'maxRows': slice_admission.limits.get('maxRowsPerCall'), 'readAt': live_limiter.iso_utc(slice_admission.admitted_at)} def _meta_connections(tenant_key): """`[(connection id, label, token, legacy)]` for the tenant's Meta Ads credentials.""" from harness import meta_store rt = meta_store._runtime_for(tenant_key) if rt is None: tok = meta_store.token(tenant_key=tenant_key) return [('', 'Meta Ads', tok, True)] if tok else [] return meta_store._connection_rows(rt, tenant_key, {'problems': []}) def _graph_pages(get, path, tok, sink, admission=None, **params): """Every page of one Graph edge into `sink`. A heavy page shrinks; a rate limit refuses at once.""" from harness import meta_store limit = int(params.pop('limit', None) or meta_store.PAGE) after, cursors = None, set() while True: call = dict(params, limit=limit, **({'after': after} if after else {})) try: body = get(path, tok, **call) except meta_store.MetaError as exc: text = str(exc) if meta_store._TOO_MUCH in text and limit > 5: limit = max(5, limit // 4) continue if meta_store._RATE_LIMITED in text.lower(): raise LiveReadError('source_rate', 'Meta is limiting calls to this ad account.', 'Try again in a few minutes, or answer from the mirror.') from None raise LiveReadError('source_error', 'Meta refused the live read.', 'Try again, or answer from the mirror.') from None sink.extend(body.get('data') or []) if admission is not None: admission.check_rows(len(sink)) paging = body.get('paging') or {} after = (paging.get('cursors') or {}).get('after') if not (paging.get('next') and after): return sink if after in cursors: raise LiveReadError('source_error', 'Meta repeated a page of the live read.', 'Try again, or answer from the mirror.') cursors.add(after) def _day_windows(since, until, size): from datetime import date, timedelta start, end = date.fromisoformat(since), date.fromisoformat(until) out = [] while start <= end: stop = min(start + timedelta(days=max(1, size) - 1), end) out.append((start.isoformat(), stop.isoformat())) start = stop + timedelta(days=1) return out def api_read(table, tenant, *, tenant_key=None, since=None, until=None, connections=None, get=None, caller='live_api', windows=None): """A live Meta Graph read of one Meta mirror table, every page (rule 1). Insights read `windows` or `since..until` (default tenant today) at the sync's grain. `connections` and `get` are the credential list and the Graph call, injectable for fixtures. Returns `{table, connector, mode, rows, rowCount, accounts, since, until, maxRows, readAt}`. """ from core import periods from harness import meta_store connector, name, mode = resolve_table(table) if mode.mode != connector_spec.LIVE_API or connector != 'meta': raise LiveReadError('no_api_mode', f'{name} has no live API read.', 'Call live_answer for this table.') insights = name == meta_store.INSIGHTS_TABLE since = str(since or until or periods.today().isoformat()) until = str(until or since) asked = sorted(windows if windows is not None else [(since, until)]) spans = [] for lo, hi in asked: if not (_ISO_DAY.match(str(lo or '')) and _ISO_DAY.match(str(hi or ''))) or lo > hi: raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.', 'Send since and until as YYYY-MM-DD.') if spans and lo <= spans[-1][1]: spans[-1] = (spans[-1][0], max(hi, spans[-1][1])) else: spans.append((lo, hi)) if connections is None: connections = _meta_connections(tenant_key) if not connections: raise LiveReadError('no_credential', 'No Meta Ads credential is connected.', 'Connect Meta Ads under Connectors.') get = get or meta_store._get rows, accounts = [], [] with live_limiter.admit(tenant, caller=caller) as api_admission: for cid, label, tok, _legacy in connections: accts = _graph_pages(get, 'me/adaccounts', tok, [], fields='id,name') for acct in accts: aid = acct['id'] accounts.append(aid) start = len(rows) if insights: for first, last in spans: for lo, hi in _day_windows(first, last, meta_store.INSIGHTS_SLICE): _graph_pages(get, f'{aid}/insights', tok, rows, api_admission, fields=','.join(meta_store.INSIGHT_FIELDS), level=meta_store.INSIGHTS_LEVEL, time_increment='1', limit=meta_store.INSIGHTS_PAGE, time_range=json.dumps({'since': lo, 'until': hi})) else: edge, fields, _parent = meta_store.SPECS[name] if edge is None: rows.append(get(aid, tok, fields=','.join(fields))) api_admission.check_rows(len(rows)) else: _graph_pages(get, f'{aid}/{edge}', tok, rows, api_admission, fields=','.join(fields), limit=meta_store.PAGE) for row in rows[start:]: row['_connection_id'] = cid if insights: # The sync's stable Insights id, so a row matches its mirror row. row['id'] = ':'.join(str(row.get(k) or '') for k in (f'{meta_store.INSIGHTS_LEVEL}_id', 'date_start', 'date_stop')) for index in range(start, len(rows)): raw = rows[index] _cols, normalized = meta_store._origin_rows(list(raw), [raw], cid, label) rows[index] = {key: meta_store._cell(value) for key, value in normalized[0].items()} return {'table': name, 'connector': connector, 'mode': connector_spec.LIVE_API, 'live': True, 'rows': rows, 'rowCount': len(rows), 'accounts': accounts, 'since': spans[0][0] if insights and spans else None, 'until': spans[-1][1] if insights and spans else None, 'windows': [{'since': lo, 'until': hi} for lo, hi in spans] if insights else None, 'maxRows': api_admission.limits.get('maxRowsPerCall'), 'readAt': live_limiter.iso_utc(api_admission.admitted_at)} def live_answer(table, tenant, *, connector=None, tenant_key=None, client=None, where=None, params=(), keep=None, state_entity=None, domain=(), since=None, until=None, connections=None, get=None, caller='live', windows=None): """One live question about one table, routed by its declared live mode. `table` is a mirror table or a `ut_*` key. Cursor tables take `where`, `params` and `keep`. Slice tables take `domain`. Meta tables take `since` and `until`. A table with no live read returns its reason and reads nothing. Every source read admits through the live limiter. Each result carries `mode` and `connector`. """ connector, name, mode = resolve_table(table, connector) if mode.mode == connector_spec.LIVE_CURSOR: out = live_read(name, tenant, client=client, tenant_key=tenant_key, where=where, params=params, keep=keep, state_entity=state_entity, caller=caller) return {**out, 'table': name, 'connector': connector, 'mode': mode.mode, 'live': True} if mode.mode == connector_spec.LIVE_SLICE: return slice_read(name, tenant, domain=domain, client=client, caller=caller) if mode.mode == connector_spec.LIVE_API: return api_read(name, tenant, tenant_key=tenant_key, since=since, until=until, connections=connections, get=get, caller=caller, windows=windows) return none_answer(name, connector) # ── S82-T105: the Live tool's windowed answer ──────────────────────────────────────────────── #: The ISO form of a business-day window bound. _DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') _QUALIFIED = re.compile(r'\b([a-z_][a-z0-9_]*)\.([a-z_][a-z0-9_]*)\b') _WINDOW_KEY = '_live_window_at' def _instant(value): """A stored instant as a naive UTC `datetime`, or None.""" from datetime import datetime, timezone if value is None or value == '': return None if isinstance(value, datetime): moment = value else: try: moment = datetime.fromisoformat(str(value).strip().replace('Z', '+00:00')) except ValueError: return None if moment.tzinfo is not None: moment = moment.astimezone(timezone.utc).replace(tzinfo=None) return moment def _window_bounds(date_from, date_to, instants): """`(lo, hi)` for one window: naive UTC `[lo, hi)` for an instant column, ISO days otherwise.""" from core import periods if not (_DAY.match(str(date_from or '')) and _DAY.match(str(date_to or ''))) \ or date_from > date_to: raise LiveReadError('bad_window', 'The window needs a start day on or before its end day.', 'Send date_from and date_to as YYYY-MM-DD.') return periods.business_day_bounds(date_from, date_to) if instants else (date_from, date_to) def _in_window(value, bounds, instants): lo, hi = bounds if instants: moment = _instant(value) return moment is not None and lo <= moment < hi day = str(value or '')[:10] return bool(day) and lo <= day <= hi def mirror_slice(entity, alias, where, params=(), tenant_key=None): """The mirror rows of `entity` the SQL `where` keeps, read through a read cursor.""" import duckdb cur = _open_cursor(tenant_key) try: return _rows(cur, f"SELECT * FROM {entity} AS {alias} WHERE {where or 'TRUE'}", params) except duckdb.Error as exc: raise LiveReadError('unsupported_definition', 'This database slice cannot answer at source row grain.', 'Read this database from the mirror or ask about its source table.') from exc finally: try: cur.close() except Exception: # noqa: BLE001 pass def _definition_columns(entity, alias, where): """The mirror columns the definition SQL reads. Refuses a definition naming another table.""" known = {c.split()[0]: c for c in datastore._cols(datastore.ENTITIES[entity])} text = re.sub(r"'(?:[^']|'')*'", "''", where or '') if re.search(r'\b(select|join|from)\b', text, re.IGNORECASE): raise LiveReadError('unsupported_definition', 'This database definition needs another table.', 'Read this database from the mirror or ask about its source table.') used = [] for table, column in _QUALIFIED.findall(text): if table != alias or column not in known: raise LiveReadError('unsupported_definition', 'This database is defined over more than one table.', 'Ask about a database defined over one synced table.') if column not in used: used.append(column) if '{' in text: raise LiveReadError('unsupported_definition', 'This database definition is not static.', 'Ask about a database defined over one synced table.') return [known[column] for column in used] def defined_rows(entity, alias, where, rows): """The rows the database's own definition SQL keeps, each judged on its current values. The rows load into an in-memory DuckDB table, never the mirror.""" rows = list(rows) if not where or not where.strip() or where.strip().upper() == 'TRUE': return rows import duckdb cols = _definition_columns(entity, alias, where) names = [c.split()[0] for c in cols] con = duckdb.connect(':memory:') try: con.execute(f"CREATE TABLE {alias} (_k BIGINT, {', '.join(cols) or '_none INTEGER'})") if rows: values = [[i] + [row.get(n) for n in names] for i, row in enumerate(rows)] marks = ', '.join('?' for _ in range(len(names) + 1)) con.executemany(f"INSERT INTO {alias} (_k{''.join(', ' + n for n in names)}) " f"VALUES ({marks})", values) kept = {r[0] for r in con.execute(f"SELECT _k FROM {alias} WHERE {where}").fetchall()} except duckdb.Error as exc: raise LiveReadError('unsupported_definition', 'This database definition needs another binding.', 'Read this database from the mirror or ask about its source table.') from exc finally: con.close() return [row for i, row in enumerate(rows) if i in kept] def walled_rows(rows, wall): """The projected rows the caller may see, through the row wall every door applies.""" from core import perm_scope if not isinstance(wall, dict) or 'user' not in wall or not wall.get('database'): raise LiveReadError('no_wall', 'A live answer needs the caller and the database.', 'Call the Live tool as a signed-in member.') return perm_scope.apply_row_scope(rows, wall['user'], wall['database'], wall['fields'], st=wall.get('st'), defn=wall.get('defn')) def _total(rows, measure): if measure is None: return len(rows) total = 0.0 for row in rows: value = row.get(measure) if value is None or value == '': continue try: total += float(value) except (TypeError, ValueError): raise LiveReadError('invalid_measure_value', 'The measure contains a non-numeric value.', 'Choose a numeric source field or correct its source value.') from None return round(total, 6) def project_definition(entity, rows, spec, keys, tenant_key=None): """Evaluate the database SQL over source rows and mirrored dimensions in memory.""" import duckdb cur = _open_cursor(tenant_key) con = duckdb.connect(':memory:') quote = lambda key: '"' + key.replace('"', '""') + '"' tables = set((spec.get('tables') or {}).values()) | {entity} try: for table in tables: if not _IDENT.fullmatch(table): raise ValueError('Invalid mirror table') cur.execute(f'SELECT * FROM {table} LIMIT 0') columns = [(d[0], str(d[1])) for d in cur.description] source = list(rows) if table == entity else _rows(cur, f'SELECT * FROM {table}') con.execute(f'CREATE TABLE {table} (' + ', '.join( f'{quote(name)} {kind}' for name, kind in columns) + ')') if source: con.executemany(f'INSERT INTO {table} VALUES (' + ','.join('?' for _ in columns) + ')', [[row.get(name) for name, _ in columns] for row in source]) bindings = {key: binding for key, binding in spec['cols'].items() if key in keys} expressions = [f'{expr} AS {quote(key)}' for key, (expr, _convert) in bindings.items()] frm = spec.get('from_sql') or (entity if spec.get('table') else entity + ' AS ' + spec['id'].split('.')[0]) values = _rows(con, f"SELECT {spec['id']} AS pid" + (', ' + ', '.join(expressions) if expressions else '') + f" FROM {frm} WHERE {spec.get('where') or 'TRUE'}") ids = [r['pid'] for r in values] if len(set(ids)) != len(ids): raise ValueError('The join repeats a source row') for row in values: for key, (_expr, convert) in bindings.items(): try: row[key] = convert(row[key]) except (TypeError, ValueError): row[key] = None return values except (duckdb.Error, ValueError) as exc: raise LiveReadError('unsupported_definition', 'This database join or expression cannot answer at source row grain.', 'Read this database from the mirror or ask about its source table.') from exc finally: cur.close() con.close() def window_answer(entity, tenant, *, alias, where, date_column, instants, windows, project, wall, measure=None, tenant_key=None, client=None, caller='live.answer', definition=None, domain=(), filters=None): """The screen (mirror) and live totals of one database per window, live rows overlaid by id. `windows` is `{name: (date_from, date_to)}` in tenant business days. `where` is the database's own definition SQL over `alias`. `project(row)` maps a mirror row to the database's field keys, and every projected row passes `walled_rows`. `measure` is a field key to sum, else rows count. Over the limit, or with no live read, the mirror answers with the cause, next allowed time and fix. Returns `{entity, served, syncedAt, delta, replaced, added, limit_refusal, live_unavailable, windows: {name: {date_from, date_to, screen, live, difference}}, rows}`. """ bounds = {name: _window_bounds(lo, hi, instants) for name, (lo, hi) in windows.items() if date_column is not None} cast = f"CAST({date_column} AS TIMESTAMP)" if instants else f"CAST({date_column} AS DATE)" terms, params = [], [] for lo, hi in bounds.values(): terms.append(f"({cast} >= ? AND {cast} < ?)" if instants else f"({cast} >= CAST(? AS DATE) AND {cast} <= CAST(? AS DATE))") params += [lo, hi] sliced = (' OR '.join(terms) or 'FALSE') if date_column else 'TRUE' def in_any(row): return date_column is None or any( _in_window(row.get(date_column), b, instants) for b in bounds.values()) defined = f"({sliced}) AND ({where})" if where and where.strip() else sliced if not definition: _definition_columns(entity, alias, where) screen_rows = mirror_slice(entity, alias, 'TRUE' if definition else defined, () if definition else params, tenant_key) live, limit, unavailable = None, None, None try: live = live_answer(entity, tenant, client=client, tenant_key=tenant_key, where=sliced, params=params, keep=in_any, domain=domain, caller=caller, windows=list(windows.values()) if date_column else None) if not live.get('live'): unavailable = live live = None except live_limiter.LiveLimitRefused as refusal: live, limit = None, refusal.as_dict() except LiveReadError as exc: live, unavailable = None, exc.as_dict() def shown(rows): if definition: rows = project_definition(entity, rows, definition, project, tenant_key) shown_rows = walled_rows(rows, wall) if filters: from harness import filter_eval shown_rows = [r for r in shown_rows if filter_eval.permits(filters, r, wall['fields'])] return shown_rows # The window key rides each projected row through the wall and is dropped after. return walled_rows([{**project(row), _WINDOW_KEY: row.get(date_column)} for row in rows], wall) def split(rows, b): return [{k: v for k, v in r.items() if k != _WINDOW_KEY} for r in rows if date_column is None or _in_window(r.get(_WINDOW_KEY), b, instants)] screen = shown(screen_rows) fresh = shown(live['rows'] if definition else defined_rows( entity, alias, where, live['rows'])) if live is not None else None out_windows, out_rows = {}, {} for name, (lo, hi) in windows.items(): screen_in = split(screen, bounds.get(name)) entry = {'date_from': lo, 'date_to': hi, 'screen': {'value': _total(screen_in, measure), 'rows': len(screen_in)}, 'live': None, 'difference': None} out_rows[name] = screen_in if fresh is not None: live_in = split(fresh, bounds.get(name)) entry['live'] = {'value': _total(live_in, measure), 'rows': len(live_in)} entry['difference'] = round(entry['live']['value'] - entry['screen']['value'], 6) out_rows[name] = live_in out_windows[name] = entry return {'entity': entity, 'served': 'live' if live is not None else 'mirror', 'syncedAt': (live or {}).get('syncedAt'), # Source-wide counters disclose rows outside the database and reader's walls. 'delta': None, 'replaced': None, 'added': None, 'limit_refusal': limit, 'live_unavailable': unavailable, 'windows': out_windows, 'rows': out_rows}