fsanyoto commited on
Commit
a97d6f1
·
verified ·
1 Parent(s): 2291176

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "51321e4939f",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "24b7ac5bbe",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 51321e4939f
 
1
+ 24b7ac5bbe
api/amazon_relational.py CHANGED
@@ -847,26 +847,38 @@ def _integer(value):
847
  return None if value in (None, "") else int(value)
848
 
849
 
850
- def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED):
851
- """One binding from the current mirror schema, or an explicit unloaded-mirror reason.
852
-
853
- `status` is a `source_status` answer the caller already holds.
854
- """
855
- columns = mirror_columns(cur, SOURCE[key]) if columns is None else columns
856
- columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
857
- if not columns:
858
- return missing_source(key, cur=cur)
859
- names = {name for name, _dtype in columns}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
860
  table = key.removeprefix(PREFIX)
861
- prepared = {row[0] for row in cur.execute(
862
- "SELECT column_name FROM information_schema.columns WHERE table_catalog = current_database() "
863
- "AND table_schema = current_schema() AND table_name = ? AND column_name IN (?, ?, ?)",
864
- [SOURCE[key], amazon_schema.IDENTITY_OCCURRENCE, amazon_schema.IDENTITY_PID,
865
- amazon_schema.CONNECTION_ID]).fetchall()}
866
  identity_names = names | prepared
867
  identity = _identity_columns(table, identity_names)
868
- fallback = sorted(set((*identity, *_fallback_columns(table))))
869
  source = _quoted(SOURCE[key])
 
 
870
 
871
  def value(name, owner=source):
872
  return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
@@ -875,55 +887,62 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
875
  def digest(values):
876
  return f"sha256(to_json(list_value({', '.join(values)})))"
877
 
878
- complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in identity)
879
- peer = '"_amazon_identity_peer"'
880
- peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in identity)
881
- partition = ", ".join(value(name, peer) for name in fallback)
882
- # Unprepared mirrors rank once in a materialized relation, outside the row lookup.
883
- occurrence = (
884
- '(WITH "_amazon_missing_identity" AS MATERIALIZED '
885
- f'(SELECT {peer}.rowid AS "_identity_rowid", '
886
- f'row_number() OVER (PARTITION BY {partition} ORDER BY {peer}.rowid) - 1 '
887
- f'AS "_identity_occurrence" FROM {source} AS {peer} WHERE NOT ({peer_complete})) '
888
- 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
889
- f'WHERE "_identity_rowid" = {source}.rowid)')
890
- if amazon_schema.IDENTITY_OCCURRENCE in prepared:
891
- occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
892
- fallback_digest = digest(["'__amazon_missing_identity__'",
893
- *(value(name) for name in fallback), occurrence])
894
- digest_sql = (f"CASE WHEN {complete} THEN {digest([value(name) for name in identity])} "
895
- f"ELSE {fallback_digest} END")
896
- if table in amazon_store.DERIVED_TABLES:
897
- # Derived views construct complete keys and have no physical rowid for fallback order.
898
- digest_sql = digest([value(name) for name in identity])
899
- # The shared grid, relation and selection contracts all use numeric pids. Thirteen
900
- # hex digits retain a stable 52-bit identity that JavaScript represents exactly.
901
- ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
 
 
902
  if amazon_schema.IDENTITY_PID in prepared:
903
  ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
904
- elif amazon_schema.CONNECTION_ID in prepared:
905
- legacy_owner = amazon_store.legacy_identity_owner(cur, table)
 
906
  if legacy_owner:
907
  literal = "'" + str(legacy_owner).replace("'", "''") + "'"
908
  old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
909
- old_fallback = sorted(set((*old_identity, *_fallback_columns(table))))
910
- old_complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in old_identity)
911
- old_peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in old_identity)
912
- old_partition = ", ".join(value(name, peer) for name in old_fallback)
913
- old_occurrence = (
914
- '(WITH "_amazon_legacy_identity" AS MATERIALIZED '
915
- f'(SELECT {peer}.rowid AS "_identity_rowid", '
916
- f'row_number() OVER (PARTITION BY {old_partition} ORDER BY {peer}.rowid) - 1 '
917
- f'AS "_identity_occurrence" FROM {source} AS {peer} '
918
- f'WHERE {peer}."_connection_id"={literal} AND NOT ({old_peer_complete})) '
919
- 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_legacy_identity" '
920
- f'WHERE "_identity_rowid" = {source}.rowid)')
921
- old_missing = digest(["'__amazon_missing_identity__'", *(value(name) for name in old_fallback),
922
- old_occurrence])
923
- old_digest = (f"CASE WHEN {old_complete} THEN {digest([value(name) for name in old_identity])} "
924
- f"ELSE {old_missing} END")
925
- old_ident = f"greatest(1, CAST(('0x' || substr(({old_digest}), 1, 13)) AS BIGINT))"
926
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
927
  cols = {}
928
  personal = set(amazon_store.personal_columns(table))
929
  for name, dtype in columns:
@@ -942,10 +961,8 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
942
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
943
  for lookup in lookups(cur, key, names, allowed=allow_lookups):
944
  cols[lookup["key"]] = (lookup["expr"], _text)
945
- spec = {"table": SOURCE[key], "where": "", "id": ident,
946
- "needs_excluded": False, "cols": cols}
947
- if table not in amazon_store.DERIVED_TABLES:
948
- spec["page_key"] = f"{source}.rowid"
949
  # D-2643: an empty table carries its code and reason to every door that reads this binding.
950
  status = source_status(cur, key) if status is _UNRESOLVED else status
951
  if status is not None:
@@ -953,6 +970,22 @@ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVE
953
  return spec
954
 
955
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
956
  def page_source(cur, key, definition=None, *, allow_lookups=True):
957
  """Resolve one target's storage policy on the admitted cursor without provider discovery."""
958
  columns = mirror_columns(cur, SOURCE[key], strict=True)
 
847
  return None if value in (None, "") else int(value)
848
 
849
 
850
+ def _mirror_schemas(cur, tables):
851
+ rows = cur.execute(
852
+ 'SELECT c.table_name, c.column_name, c.data_type, t.table_type '
853
+ 'FROM information_schema.columns c JOIN information_schema.tables t '
854
+ 'USING (table_catalog, table_schema, table_name) '
855
+ 'WHERE c.table_catalog = current_database() AND c.table_schema = current_schema() '
856
+ 'AND c.table_name IN (SELECT unnest(?::VARCHAR[])) ORDER BY c.table_name, c.ordinal_position',
857
+ [list(tables)]).fetchall()
858
+ schemas = {}
859
+ for table, name, kind, table_type in rows:
860
+ columns, _view = schemas.setdefault(table, ([], table_type == 'VIEW'))
861
+ columns.append((name, kind))
862
+ return schemas
863
+
864
+
865
+ def _missing_occurrence(source, partition, complete, predicate=''):
866
+ return ('(WITH "_amazon_missing_identity" AS MATERIALIZED '
867
+ f'(SELECT "_amazon_identity_peer".rowid AS "_identity_rowid", '
868
+ f'row_number() OVER (PARTITION BY {partition} ORDER BY "_amazon_identity_peer".rowid) - 1 '
869
+ f'AS "_identity_occurrence" FROM {source} AS "_amazon_identity_peer" '
870
+ f'WHERE {predicate}NOT ({complete})) '
871
+ 'SELECT CAST("_identity_occurrence" AS VARCHAR) FROM "_amazon_missing_identity" '
872
+ f'WHERE "_identity_rowid" = {source}.rowid)')
873
+
874
+
875
+ def _grid_identity(cur, key, names, prepared, is_view):
876
  table = key.removeprefix(PREFIX)
 
 
 
 
 
877
  identity_names = names | prepared
878
  identity = _identity_columns(table, identity_names)
 
879
  source = _quoted(SOURCE[key])
880
+ peer = '"_amazon_identity_peer"'
881
+ ranks = []
882
 
883
  def value(name, owner=source):
884
  return (f"NULLIF(CAST({owner}.{_quoted(name)} AS VARCHAR), '')"
 
887
  def digest(values):
888
  return f"sha256(to_json(list_value({', '.join(values)})))"
889
 
890
+ def identity_sql(keys, label, legacy=None):
891
+ if table in amazon_store.DERIVED_TABLES:
892
+ hashed = digest([value(name) for name in keys])
893
+ return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
894
+ complete = " AND ".join(f"{value(name)} IS NOT NULL" for name in keys)
895
+ fallback = sorted(set((*keys, *_fallback_columns(table))))
896
+ partition = ", ".join(value(name, peer) for name in fallback)
897
+ if is_view:
898
+ if legacy:
899
+ partition += f', ({peer}."_connection_id" = {legacy})'
900
+ ranks.append(f'row_number() OVER (PARTITION BY {partition} '
901
+ f'ORDER BY {peer}."_amazon_publication_order") - 1 AS {_quoted(label)}')
902
+ occurrence = f'CAST({source}.{_quoted(label)} AS VARCHAR)'
903
+ else:
904
+ peer_complete = " AND ".join(f"{value(name, peer)} IS NOT NULL" for name in keys)
905
+ predicate = f'{peer}."_connection_id" = {legacy} AND ' if legacy else ''
906
+ occurrence = _missing_occurrence(source, partition, peer_complete, predicate)
907
+ if not legacy and amazon_schema.IDENTITY_OCCURRENCE in prepared:
908
+ occurrence = f'CAST({source}.{_quoted(amazon_schema.IDENTITY_OCCURRENCE)} AS VARCHAR)'
909
+ if is_view:
910
+ ranks.pop()
911
+ fallback_digest = digest(["'__amazon_missing_identity__'", *(value(name) for name in fallback), occurrence])
912
+ hashed = digest([value(name) for name in keys])
913
+ hashed = f'CASE WHEN {complete} THEN {hashed} ELSE {fallback_digest} END'
914
+ return f"greatest(1, CAST(('0x' || substr(({hashed}), 1, 13)) AS BIGINT))"
915
+
916
  if amazon_schema.IDENTITY_PID in prepared:
917
  ident = f'{source}.{_quoted(amazon_schema.IDENTITY_PID)}'
918
+ else:
919
+ ident = identity_sql(identity, '_amazon_publication_occurrence')
920
+ legacy_owner = amazon_store.legacy_identity_owner(cur, table) if amazon_schema.CONNECTION_ID in prepared else None
921
  if legacy_owner:
922
  literal = "'" + str(legacy_owner).replace("'", "''") + "'"
923
  old_identity = _identity_columns(table, identity_names - {amazon_schema.CONNECTION_ID})
924
+ old_ident = identity_sql(old_identity, '_amazon_publication_legacy_occurrence', literal)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
925
  ident = f'CASE WHEN {source}."_connection_id"={literal} THEN {old_ident} ELSE {ident} END'
926
+ spec = {'id': ident}
927
+ if ranks:
928
+ spec['from_sql'] = (f'(SELECT *, {", ".join(ranks)} FROM '
929
+ f'(SELECT *, row_number() OVER () - 1 AS "_amazon_publication_order" '
930
+ f'FROM {source}) AS {peer}) AS {source}')
931
+ if not is_view and table not in amazon_store.DERIVED_TABLES:
932
+ spec['page_key'] = f'{source}.rowid'
933
+ return spec
934
+
935
+
936
+ def grid_source(cur, key, *, columns=None, allow_lookups=True, status=_UNRESOLVED, schema=None):
937
+ """Bind the canonical grid identity and fields to the admitted table or publication view."""
938
+ raw, is_view = schema if schema is not None else _mirror_schemas(cur, [SOURCE[key]]).get(SOURCE[key], ([], False))
939
+ columns = raw if columns is None else columns
940
+ columns = [(name, kind) for name, kind in columns if name not in amazon_schema.MIRROR_INTERNAL_COLUMNS]
941
+ if not columns:
942
+ return missing_source(key, cur=cur)
943
+ names = {name for name, _dtype in columns}
944
+ prepared = {name for name, _kind in raw} & amazon_schema.MIRROR_INTERNAL_COLUMNS
945
+ table = key.removeprefix(PREFIX)
946
  cols = {}
947
  personal = set(amazon_store.personal_columns(table))
948
  for name, dtype in columns:
 
961
  # group (count by parent) and a sort compile against them in SQL with no per-row Python.
962
  for lookup in lookups(cur, key, names, allowed=allow_lookups):
963
  cols[lookup["key"]] = (lookup["expr"], _text)
964
+ spec = {"table": SOURCE[key], "where": "", "needs_excluded": False, "cols": cols,
965
+ **_grid_identity(cur, key, names, prepared, is_view)}
 
 
966
  # D-2643: an empty table carries its code and reason to every door that reads this binding.
967
  status = source_status(cur, key) if status is _UNRESOLVED else status
968
  if status is not None:
 
970
  return spec
971
 
972
 
973
+ def publication_sources(cur, keys):
974
+ """Resolve completed raw publications once without request readiness or store discovery."""
975
+ keys = [key for key in keys if key in SOURCE]
976
+ schemas = _mirror_schemas(cur, [SOURCE[key] for key in keys])
977
+ result = {}
978
+ for key in keys:
979
+ schema = schemas.get(SOURCE[key])
980
+ if schema is None:
981
+ continue
982
+ spec = grid_source(cur, key, allow_lookups=False, status=None, schema=schema)
983
+ if amazon_schema.CONNECTION_ID in dict(schema[0]):
984
+ spec['cols'][amazon_schema.CONNECTION_ID] = ('"_connection_id"', _text)
985
+ result[key] = spec
986
+ return result
987
+
988
+
989
  def page_source(cur, key, definition=None, *, allow_lookups=True):
990
  """Resolve one target's storage policy on the admitted cursor without provider discovery."""
991
  columns = mirror_columns(cur, SOURCE[key], strict=True)
api/registry_source.py CHANGED
@@ -526,6 +526,19 @@ class CanonicalStore(BatchStore):
526
  key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
527
  self.pool_cache[key] = (time.time(), rows)
528
 
 
 
 
 
 
 
 
 
 
 
 
 
 
529
  def get_projection(self, name, drop=()):
530
  if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
531
  return self.get(name)
@@ -846,6 +859,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
846
  # A caller's request snapshot carries the generation taken before its first read.
847
  metadata = {'_scan': ids is None} if metadata is None else metadata
848
  before = metadata.get('_generation') or generation(session, table_key)
 
849
  # A whole-field read needs no schema pass: its one row assembly serves every visible field.
850
  definition = metadata.get('_definition') or (None if needed is None else
851
  canonical_metadata(session, table_key, metadata=metadata, cur=cur))
@@ -890,7 +904,7 @@ def _canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric
890
  cached = session.runtime.pool_cache.get(pool_key)
891
  fresh = bool(cached) and time.time() - cached[0] < 900
892
  # A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
893
- paused = not fresh and routes_keychain.odoo_paused(session.runtime)
894
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
895
  size = None if table_key == 'product_data' else BATCH_ROWS
896
  if cached and (paused or fresh):
 
526
  key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
527
  self.pool_cache[key] = (time.time(), rows)
528
 
529
+ def revision(self, name):
530
+ before = self.metadata.get('_generation')
531
+ mapped = getattr(self._runtime, 'store_key', str)(name)
532
+ if before is not None and mapped in before[1]:
533
+ return copy.deepcopy(before[1][mapped])
534
+ return self._runtime.revision(name)
535
+
536
+ def odoo_paused(self):
537
+ slot = ('odoo_paused',)
538
+ if slot not in self.metadata:
539
+ self.metadata[slot] = self._runtime.odoo_paused()
540
+ return self.metadata[slot]
541
+
542
  def get_projection(self, name, drop=()):
543
  if name in {'loopable_route_system', 'table_zones'} or name.endswith('_cohorts'):
544
  return self.get(name)
 
859
  # A caller's request snapshot carries the generation taken before its first read.
860
  metadata = {'_scan': ids is None} if metadata is None else metadata
861
  before = metadata.get('_generation') or generation(session, table_key)
862
+ metadata['_generation'] = before
863
  # A whole-field read needs no schema pass: its one row assembly serves every visible field.
864
  definition = metadata.get('_definition') or (None if needed is None else
865
  canonical_metadata(session, table_key, metadata=metadata, cur=cur))
 
904
  cached = session.runtime.pool_cache.get(pool_key)
905
  fresh = bool(cached) and time.time() - cached[0] < 900
906
  # A fresh pool serves paused or not, so only a stale or absent one reads the connector state.
907
+ paused = not fresh and routes_keychain.odoo_paused(runtime)
908
  # One product assembly per drill: its store reads cost per call, not per row (sdlc85:23).
909
  size = None if table_key == 'product_data' else BATCH_ROWS
910
  if cached and (paused or fresh):
api/routes_automation.py CHANGED
@@ -2873,14 +2873,13 @@ def _received_projection(connector):
2873
  from fastapi import HTTPException
2874
  from core import user_tables
2875
  import routes_odoo_tables
2876
- result = {}
2877
  if connector == 'amazon':
2878
- # Amazon specs raise BinderException on a publication's raw views and cost about 7 s
2879
- # a table in the worker hand-off (sdlc85:88), so the projection waits for that fix.
2880
- logging.getLogger(__name__).debug('received-change projection skips amazon until its specs bind')
2881
- return result
2882
- prefix = 'ut_amzn_' if connector == 'amazon' else 'ut_odoo_'
2883
- for key in user_tables.all_defs(st=rt, schema_only=True):
2884
  if not str(key).startswith(prefix):
2885
  continue
2886
  try:
 
2873
  from fastapi import HTTPException
2874
  from core import user_tables
2875
  import routes_odoo_tables
2876
+ definitions = user_tables.all_defs(st=rt, schema_only=True)
2877
  if connector == 'amazon':
2878
+ import amazon_relational
2879
+ return amazon_relational.publication_sources(cursor, definitions)
2880
+ result = {}
2881
+ prefix = 'ut_odoo_'
2882
+ for key in definitions:
 
2883
  if not str(key).startswith(prefix):
2884
  continue
2885
  try:
api/routes_customers.py CHANGED
@@ -138,7 +138,10 @@ def _assembly_revisions(runtime, names, uses_roster=False, roster_tenant=None, r
138
  """Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
139
  from core import store, store_pg
140
  names = tuple(sorted(set(names)))
141
- handle = _assembly_pg_store(runtime)
 
 
 
142
  users_stamp = None
143
 
144
  def roster_stamp(revision):
 
138
  """Validate all tracked PostgreSQL documents in one round trip, with identical tokens."""
139
  from core import store, store_pg
140
  names = tuple(sorted(set(names)))
141
+ # The outer canonical generation check validates these borrowed pre-read revisions after hydration.
142
+ canonical = (getattr(runtime, '_canonical_read_only', False)
143
+ and getattr(runtime, 'metadata', {}).get('_generation') is not None)
144
+ handle = None if canonical else _assembly_pg_store(runtime)
145
  users_stamp = None
146
 
147
  def roster_stamp(revision):
api/routes_sheet.py CHANGED
@@ -7084,23 +7084,20 @@ def _drill_tokens(session, databases):
7084
 
7085
  @contextlib.contextmanager
7086
  def _drill_mirror(session, database, window, needed, databases, sources=None):
7087
- """A small reader-scoped LRU of query mirrors: one lease per key, short registry guard.
7088
-
7089
- `sources` is the caller's opening token pass. The lease key reuses it, so one request reads
7090
- the tokens once before the build. The build-miss and end-of-request rechecks still read anew.
7091
- """
7092
  from core import sheet_binding, perm_scope
7093
  from core.request_timing import raise_if_resource_error
7094
  import datetime
7095
 
7096
  for key in sorted(databases):
7097
  _require_caught_up(session, key)
 
7098
  if _native_table_definition(session, database) is not None:
7099
  # Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
7100
  # across requests; the request's ExitStack closes it after the page is produced.
7101
  yield _mirror(session)(database, metric_window=window, needed=needed), None
 
7102
  return
7103
- sources = _drill_tokens(session, databases) if sources is None else dict(sources)
7104
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7105
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7106
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
@@ -7118,23 +7115,11 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
7118
  getattr(session.runtime.tenant, "config", None)])
7119
  key = (session.tenant, session.uname, database, token, reader,
7120
  sheet_binding.fingerprint(window), datetime.date.today().isoformat())
7121
- attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
7122
- "_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor",
7123
- "_sheet_drill_entries")
7124
- absent = object()
7125
- previous = {name: getattr(session, name, absent) for name in attrs}
7126
  with _drill_lease(key if token else None) as lease:
7127
  with _drill_mirror_lock():
7128
  hit = _DRILL_MIRRORS.get(key) if token else None
7129
  lease["entry"] = hit
7130
- session._sheet_drill_isolated = True
7131
- session._sheet_drill_cursor = hit["owned"] if hit else None
7132
- session._sheet_drill_entries = hit["mirrors"] if hit else {}
7133
- session._sheet_mirrors = {}
7134
- session._sheet_assemblies = {}
7135
- session._sheet_stored_mirrors = {}
7136
- session._sheet_amzn_cursor = session._sheet_drill_cursor
7137
- try:
7138
  want = set(needed) | (hit["needed"] if hit else set())
7139
  if hit and want <= hit["needed"]:
7140
  made = hit["made"]
@@ -7150,42 +7135,70 @@ def _drill_mirror(session, database, window, needed, databases, sources=None):
7150
  owned = getattr(session, "_sheet_drill_cursor", None)
7151
  hit = {"made": made, "needed": want, "owned": owned,
7152
  "mirrors": session._sheet_drill_entries}
7153
- # A build racing a source change is usable for this request, never reusable.
7154
- if token and _drill_tokens(session, databases) == sources:
7155
- with _DRILL_MIRROR_LOCK:
7156
- _DRILL_MIRRORS[key] = hit
7157
- _DRILL_MIRRORS.move_to_end(key)
7158
- lease["entry"] = hit
7159
- while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
7160
- gone, evicted = _DRILL_MIRRORS.popitem(last=False)
7161
- # A leased cursor closes when its holder leaves, never under it.
7162
- held = _DRILL_LEASES.get(gone)
7163
- if evicted["owned"] and not (held and held["entry"] is evicted):
7164
- evicted["owned"][0].close()
7165
- else:
7166
- token = None
7167
  # Related leaves may revisit the primary registry. Keep its table intact and widen
7168
  # only within this cursor. Assemblies remain request-local and leave with the call.
7169
  session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
7170
  want, made)
7171
- yield made, token
7172
- finally:
7173
- owned = getattr(session, "_sheet_drill_cursor", None)
7174
- entry = lease["entry"]
7175
- with _DRILL_MIRROR_LOCK:
7176
- lease["entry"] = None
7177
- kept = [item["owned"] for item in _DRILL_MIRRORS.values()]
7178
- # The last lease on an uncached or evicted cursor closes it.
7179
- spent = [owned, entry["owned"] if entry else None]
7180
- for index, cursor in enumerate(spent):
7181
- if (cursor and not any(seen is cursor for seen in spent[:index])
7182
- and not any(item is cursor for item in kept)):
7183
- cursor[0].close()
7184
- for name, value in previous.items():
7185
- if value is absent:
7186
- session.__dict__.pop(name, None)
7187
- else:
7188
- setattr(session, name, value)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7189
 
7190
 
7191
  def _drill_page(session, database, record_ids, offset, limit, total):
@@ -7533,21 +7546,19 @@ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
7533
  binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
7534
  if all(sources.values()) and wall_stamp else None
7535
 
7536
- limits = list(limits) + list(spec.get("limits") or [])
7537
- out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
7538
- "record_ids_scope": "page", "id_limit": None, "limits": limits,
7539
- "population": population, "query": query}
7540
- if scope.get("view_id"):
7541
- out["view_id"] = scope["view_id"]
7542
- else:
7543
- out["filter"] = scope.get("filter")
7544
- if summary is not None:
7545
- out["summary"] = {"ok": True, **summary}
7546
- else:
7547
- out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
7548
- out["page"]["totalUnfiltered"] = population_total
7549
- if _drill_tokens(session, databases) != sources:
7550
- raise err(409, "drill_population_changed", "These records changed while loading. Try again.")
7551
  out["token"] = token
7552
  return out
7553
 
 
7084
 
7085
  @contextlib.contextmanager
7086
  def _drill_mirror(session, database, window, needed, databases, sources=None):
7087
+ """Lease a reader-scoped mirror and validate its source tokens after page hydration."""
 
 
 
 
7088
  from core import sheet_binding, perm_scope
7089
  from core.request_timing import raise_if_resource_error
7090
  import datetime
7091
 
7092
  for key in sorted(databases):
7093
  _require_caught_up(session, key)
7094
+ sources = _drill_tokens(session, databases) if sources is None else dict(sources)
7095
  if _native_table_definition(session, database) is not None:
7096
  # Native rows are a fresh scoped Data payload. Never cache their cursor/permissions
7097
  # across requests; the request's ExitStack closes it after the page is produced.
7098
  yield _mirror(session)(database, metric_window=window, needed=needed), None
7099
+ _require_drill_tokens(session, databases, sources)
7100
  return
 
7101
  token = sheet_binding.fingerprint(sources) if all(sources.values()) else None
7102
  # C1 also fingerprints derived wall sources, which can change without the primary table.
7103
  # An unavailable stamp must cost a rebuild, never reuse a mirror with unknown permissions.
 
7115
  getattr(session.runtime.tenant, "config", None)])
7116
  key = (session.tenant, session.uname, database, token, reader,
7117
  sheet_binding.fingerprint(window), datetime.date.today().isoformat())
 
 
 
 
 
7118
  with _drill_lease(key if token else None) as lease:
7119
  with _drill_mirror_lock():
7120
  hit = _DRILL_MIRRORS.get(key) if token else None
7121
  lease["entry"] = hit
7122
+ with _drill_reader(session, hit, lease):
 
 
 
 
 
 
 
7123
  want = set(needed) | (hit["needed"] if hit else set())
7124
  if hit and want <= hit["needed"]:
7125
  made = hit["made"]
 
7135
  owned = getattr(session, "_sheet_drill_cursor", None)
7136
  hit = {"made": made, "needed": want, "owned": owned,
7137
  "mirrors": session._sheet_drill_entries}
7138
+ # Same-key readers wait for this lease until final validation succeeds.
7139
+ if token:
7140
+ _retain_drill_mirror(key, hit, lease)
 
 
 
 
 
 
 
 
 
 
 
7141
  # Related leaves may revisit the primary registry. Keep its table intact and widen
7142
  # only within this cursor. Assemblies remain request-local and leave with the call.
7143
  session._sheet_mirrors[(database, json.dumps(window, sort_keys=True, default=str))] = (
7144
  want, made)
7145
+ try:
7146
+ yield made, token
7147
+ _require_drill_tokens(session, databases, sources)
7148
+ except BaseException:
7149
+ with _DRILL_MIRROR_LOCK:
7150
+ if _DRILL_MIRRORS.get(key) is hit:
7151
+ _DRILL_MIRRORS.pop(key)
7152
+ raise
7153
+
7154
+
7155
+ @contextlib.contextmanager
7156
+ def _drill_reader(session, hit, lease):
7157
+ """Restore request attributes and close cursors whose leases no longer retain them."""
7158
+ attrs = ("_sheet_drill_cursor", "_sheet_drill_isolated", "_sheet_mirrors",
7159
+ "_sheet_assemblies", "_sheet_stored_mirrors", "_sheet_amzn_cursor", "_sheet_drill_entries")
7160
+ absent = object()
7161
+ previous = {name: getattr(session, name, absent) for name in attrs}
7162
+ session._sheet_drill_isolated = True
7163
+ session._sheet_drill_cursor = hit['owned'] if hit else None
7164
+ session._sheet_drill_entries = hit['mirrors'] if hit else {}
7165
+ session._sheet_mirrors, session._sheet_assemblies, session._sheet_stored_mirrors = {}, {}, {}
7166
+ session._sheet_amzn_cursor = session._sheet_drill_cursor
7167
+ try:
7168
+ yield
7169
+ finally:
7170
+ owned = getattr(session, '_sheet_drill_cursor', None)
7171
+ entry = lease['entry']
7172
+ with _DRILL_MIRROR_LOCK:
7173
+ lease['entry'] = None
7174
+ kept = [item['owned'] for item in _DRILL_MIRRORS.values()]
7175
+ spent = [owned, entry['owned'] if entry else None]
7176
+ for index, cursor in enumerate(spent):
7177
+ if (cursor and not any(seen is cursor for seen in spent[:index])
7178
+ and not any(item is cursor for item in kept)):
7179
+ cursor[0].close()
7180
+ for name, value in previous.items():
7181
+ if value is absent:
7182
+ session.__dict__.pop(name, None)
7183
+ else:
7184
+ setattr(session, name, value)
7185
+
7186
+
7187
+ def _require_drill_tokens(session, databases, sources):
7188
+ if _drill_tokens(session, databases) != sources:
7189
+ raise err(409, 'drill_population_changed', 'These records changed while loading. Try again.')
7190
+
7191
+
7192
+ def _retain_drill_mirror(key, hit, lease):
7193
+ with _DRILL_MIRROR_LOCK:
7194
+ _DRILL_MIRRORS[key] = hit
7195
+ _DRILL_MIRRORS.move_to_end(key)
7196
+ lease['entry'] = hit
7197
+ while len(_DRILL_MIRRORS) > _DRILL_MIRROR_MAX:
7198
+ gone, evicted = _DRILL_MIRRORS.popitem(last=False)
7199
+ held = _DRILL_LEASES.get(gone)
7200
+ if evicted['owned'] and not (held and held['entry'] is evicted):
7201
+ evicted['owned'][0].close()
7202
 
7203
 
7204
  def _drill_page(session, database, record_ids, offset, limit, total):
 
7546
  binding, scope_nodes, scope_conj, window, sources, token, wall_stamp, population_query]) \
7547
  if all(sources.values()) and wall_stamp else None
7548
 
7549
+ limits = list(limits) + list(spec.get("limits") or [])
7550
+ out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
7551
+ "record_ids_scope": "page", "id_limit": None, "limits": limits,
7552
+ "population": population, "query": query}
7553
+ if scope.get("view_id"):
7554
+ out["view_id"] = scope["view_id"]
7555
+ else:
7556
+ out["filter"] = scope.get("filter")
7557
+ if summary is not None:
7558
+ out["summary"] = {"ok": True, **summary}
7559
+ else:
7560
+ out["page"] = _drill_page(session, database, record_ids, offset, limit, total)
7561
+ out["page"]["totalUnfiltered"] = population_total
 
 
7562
  out["token"] = token
7563
  return out
7564
 
platform/core/amazon_rivals.py CHANGED
@@ -121,11 +121,11 @@ def _backend(rt):
121
  # --------------------------------------------------------------------------- mirror reads
122
 
123
  def weekly_period_sql():
124
- """Only finite dates or timestamps with an inclusive seven-day span define a report week."""
125
  start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
126
  end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
127
  return (f"isfinite({start}) AND isfinite({end}) AND "
128
- f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6")
129
 
130
 
131
  def _week_sql():
 
121
  # --------------------------------------------------------------------------- mirror reads
122
 
123
  def weekly_period_sql():
124
+ """Admit connector weeks with exclusive ends and legacy weeks with inclusive ends."""
125
  start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
126
  end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
127
  return (f"isfinite({start}) AND isfinite({end}) AND "
128
+ f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) IN (6, 7)")
129
 
130
 
131
  def _week_sql():
platform/core/received_changes.py CHANGED
@@ -406,6 +406,21 @@ def _move_baseline(rt, identity, expected, updated):
406
  raise BaselineUnavailable("The observation reference was not saved.")
407
 
408
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
410
  diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
411
  "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
@@ -417,28 +432,29 @@ def _compare_baseline(rt, source, database, spec, con, baseline, current, snapsh
417
  return
418
  cursor, done = con.cursor(), 0
419
  try:
420
- cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
421
- + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
422
- while batch := cursor.fetchmany(256):
423
- for row_id, before_json, after_json in batch:
424
- # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
425
- if deadline is not None and time.monotonic() >= deadline[0]:
426
- print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
427
- f"after {done} of {count} rows; the rest are not journaled", flush=True)
428
- return
429
- before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
430
- changes = {}
431
- for key, (_sql, formatter) in spec["cols"].items():
432
- left, right = before.get(key), after.get(key)
433
- if formatter:
434
- left = formatter(left) if left is not None else None
435
- right = formatter(right) if right is not None else None
436
- if left != right:
437
- changes[key] = {"before": left, "after": right}
438
- operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
439
- publish(rt, source=source, database=database, row_id=row_id, changes=changes,
440
- receipt_id=snapshot, operation=operation)
441
- done += 1
 
442
  finally:
443
  cursor.close()
444
 
@@ -517,18 +533,31 @@ def mirror_hook(connector, tenant_key, manifest, _state_rows, paths):
517
  deadline = (time.monotonic() + seconds, seconds)
518
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
519
  try:
520
- for name, files in paths.items():
521
- con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
522
- for database, spec in resolver(rt, con).items():
523
- # A skipped database keeps its stored baseline, so the next publication compares again.
524
- if time.monotonic() >= deadline[0]:
525
- print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
526
- continue
527
- try:
528
- _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
529
- except Exception as exc:
530
- _LOG.warning("Received-change projection deferred for %s: %s",
531
- database, type(exc).__name__)
 
 
 
 
 
 
 
 
 
 
 
 
 
532
  finally:
533
  con.close()
534
 
 
406
  raise BaselineUnavailable("The observation reference was not saved.")
407
 
408
 
409
+ @contextmanager
410
+ def _query_watchdog(con, deadline):
411
+ """Use the runtime's registered statement deadlines for publication queries."""
412
+ if deadline is None:
413
+ yield
414
+ return
415
+ from core import request_timing
416
+ parent = request_timing.CURRENT.get()
417
+ until = min(deadline[0], parent.deadline) if parent is not None and parent.deadline else deadline[0]
418
+ with request_timing.request_scope() as acc:
419
+ acc.deadline = until
420
+ with request_timing.duckdb_operation(con):
421
+ yield
422
+
423
+
424
  def _compare_baseline(rt, source, database, spec, con, baseline, current, snapshot, deadline=None):
425
  diff = ("FROM read_parquet(?) b FULL OUTER JOIN read_parquet(?) a "
426
  "USING (row_id) WHERE b.image IS DISTINCT FROM a.image")
 
432
  return
433
  cursor, done = con.cursor(), 0
434
  try:
435
+ with _query_watchdog(cursor, deadline):
436
+ cursor.execute("SELECT coalesce(b.row_id, a.row_id), b.image, a.image " + diff
437
+ + " ORDER BY coalesce(b.row_id, a.row_id)", [str(baseline), str(current)])
438
+ while batch := cursor.fetchmany(256):
439
+ for row_id, before_json, after_json in batch:
440
+ # Stop and commit: a rerun cannot resume, since each journaled receipt still costs a store write.
441
+ if deadline is not None and time.monotonic() >= deadline[0]:
442
+ print(f"[received-changes] {database}: journal deadline {deadline[1]:g} s passed "
443
+ f"after {done} of {count} rows; the rest are not journaled", flush=True)
444
+ return
445
+ before, after = json.loads(before_json or "{}"), json.loads(after_json or "{}")
446
+ changes = {}
447
+ for key, (_sql, formatter) in spec["cols"].items():
448
+ left, right = before.get(key), after.get(key)
449
+ if formatter:
450
+ left = formatter(left) if left is not None else None
451
+ right = formatter(right) if right is not None else None
452
+ if left != right:
453
+ changes[key] = {"before": left, "after": right}
454
+ operation = "created" if before_json is None else "deleted" if after_json is None else "updated"
455
+ publish(rt, source=source, database=database, row_id=row_id, changes=changes,
456
+ receipt_id=snapshot, operation=operation)
457
+ done += 1
458
  finally:
459
  cursor.close()
460
 
 
533
  deadline = (time.monotonic() + seconds, seconds)
534
  con = datastore.apply_limits(datastore.duckdb.connect(":memory:"))
535
  try:
536
+ with _query_watchdog(con, deadline):
537
+ for name, files in paths.items():
538
+ con.read_parquet([str(path) for path in files], union_by_name=True).create_view(str(name))
539
+ started = time.monotonic()
540
+ specs = resolver(rt, con)
541
+ resolution_seconds = time.monotonic() - started
542
+ snapshots_started = time.monotonic()
543
+ completed = deferred = skipped = 0
544
+ for database, spec in specs.items():
545
+ # A skipped database keeps its stored baseline for the next publication.
546
+ if time.monotonic() >= deadline[0]:
547
+ skipped += 1
548
+ print(f"[received-changes] {database}: skipped, journal deadline passed", flush=True)
549
+ continue
550
+ try:
551
+ _snapshot(rt, connector, str(database), spec, con, root, manifest, deadline)
552
+ completed += 1
553
+ except Exception as exc:
554
+ deferred += 1
555
+ _LOG.warning("Received-change projection deferred for %s: %s",
556
+ database, type(exc).__name__)
557
+ print(f'[received-changes] publication connector={connector} tables={len(specs)} '
558
+ f'completed={completed} deferred={deferred} skipped={skipped} '
559
+ f'resolution={resolution_seconds:.3f}s snapshots={time.monotonic() - snapshots_started:.3f}s',
560
+ flush=True)
561
  finally:
562
  con.close()
563
 
platform/core/sheet_binding.py CHANGED
@@ -58,7 +58,7 @@ def database_change_token(st, database):
58
  key(connector_sync.JOBS_KEY), database,
59
  database, key(connector_sync.STATE_KEY))).fetchone()
60
  token, definition = row[0], row[1] or {}
61
- if store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
62
  from core import user_table_rows
63
  from psycopg import sql
64
  definition = user_table_rows.definition_view(user_table_rows.read_definition(
 
58
  key(connector_sync.JOBS_KEY), database,
59
  database, key(connector_sync.STATE_KEY))).fetchone()
60
  token, definition = row[0], row[1] or {}
61
+ if database.startswith('ut_') and store_pg._ut_rows(key(user_tables.STORE_KEY), handle.tenant_slug):
62
  from core import user_table_rows
63
  from psycopg import sql
64
  definition = user_table_rows.definition_view(user_table_rows.read_definition(
platform/harness/canonical_source.py CHANGED
@@ -6,6 +6,7 @@ import logging
6
  import re
7
  import uuid
8
  from contextlib import contextmanager
 
9
 
10
  from core import odoo as _odoo
11
 
@@ -53,6 +54,17 @@ class Rows:
53
  return bool(len(self))
54
 
55
 
 
 
 
 
 
 
 
 
 
 
 
56
  class Source:
57
  """Own bounded source operands and release every temporary snapshot on exit."""
58
 
@@ -119,6 +131,30 @@ class Source:
119
  ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
120
  return self.cache[key]
121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  def sku_revenue(self, groups):
123
  products = self.products()
124
  code = code_sql('p')
@@ -171,7 +207,7 @@ class Source:
171
 
172
  def categories(self, keep_blank_code=False):
173
  rows = self.products()
174
- cats = self.search_read('product.category', [], ['id', 'complete_name'])
175
  blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
176
  return {code: category for code, category in self.cur.execute(f"""SELECT
177
  {code_sql('p')} code,
@@ -206,7 +242,7 @@ class Source:
206
  ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
207
  products = self.search_read('product.product', [('default_code', '!=', False)],
208
  ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
209
- cats = self.search_read('product.category', [], ['id', 'complete_name'])
210
  query = f"""WITH raw AS (SELECT p.ordinal,
211
  trim(json_extract_string(p.body, '/default_code')) sku,
212
  json_extract_string(p.body, '/name') AS product_name,
@@ -231,17 +267,18 @@ class Source:
231
  def tiers(self, prods):
232
  from core import periods
233
  from modules import products
234
- lists = self.search_read('product.pricelist', [], ['id', 'name'])
235
  today = periods.today().isoformat()
236
- rules = self.search_read('product.pricelist.item', [('compute_price', '=', 'fixed'),
237
- ('applied_on', 'in', ['0_product_variant', '1_product']),
238
- '|', ('date_start', '=', False), ('date_start', '<=', today),
239
- '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)],
240
  ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
241
- 'fixed_price', 'packaging_id', 'packaging_price'])
 
 
 
242
  normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
243
  canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
244
- prods = products.active_products() if prods is None else prods
245
  declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
246
  # Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
247
  # (234 s for the whole catalogue on the staging mirror).
@@ -250,7 +287,7 @@ class Source:
250
  json_extract_string(p.body, '/product_tmpl_id/0') tmpl
251
  FROM "{prods.table}" p),
252
  rr AS (
253
- SELECT r.ordinal rule_order, json_extract_string(r.body, '/applied_on') applied_on,
254
  json_extract_string(r.body, '/product_id/0') rpid,
255
  json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
256
  json_extract_string(r.body, '/pricelist_id/0') plid,
@@ -289,10 +326,11 @@ class Source:
289
  entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
290
  if name in declared and n > products.PRICELIST_TIER_DEPTH:
291
  entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
292
- return out, {'bounded': True,
293
- 'rules_total': self.search_count('product.pricelist.item', []),
294
- 'rules_not_fixed': self.search_count('product.pricelist.item',
295
- [('compute_price', '!=', 'fixed')])}
 
296
 
297
  def search_read(self, model, domain=None, fields=None, **kw):
298
  domain = self.domain(model, domain)
@@ -615,6 +653,9 @@ class _Query:
615
  expr = expr[0]
616
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
617
  raise _Unmirrored(f'name search on {path}')
 
 
 
618
  if kind == 'bool':
619
  expr = f'coalesce({expr}, false)'
620
  value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
@@ -721,6 +762,11 @@ class MirrorSource(Source):
721
  """The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
722
  if model not in self._relations:
723
  target = _MIRROR_MODELS[model]
 
 
 
 
 
724
  if not target.startswith('(') and not self.cur.execute(
725
  'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
726
  [target]).fetchone():
@@ -750,6 +796,24 @@ class MirrorSource(Source):
750
  # Live `_code_category` keys every `default_code != False` product, a blank '' code included.
751
  return Source.categories(self, keep_blank_code)
752
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753
  def servable(self, model, names):
754
  """The `names` the mirror declares for `model`; the rest are logged once and dropped."""
755
  kept, dropped = [], []
@@ -922,9 +986,10 @@ class MirrorSource(Source):
922
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
923
  FROM {prods.from_sql()} WHERE {pw}),
924
  quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
925
- FROM {quants.from_sql()} WHERE {qw} GROUP BY 1),
926
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
927
- sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw} GROUP BY 1)
 
928
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
929
  bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
930
  sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
@@ -962,6 +1027,7 @@ class MirrorSource(Source):
962
  "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
963
  "JOIN stock_location l ON l.id = q.location_id "
964
  "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
 
965
  "GROUP BY 1) f ON f.product_id = p.id "
966
  "WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
967
  [view[1], ids])
 
6
  import re
7
  import uuid
8
  from contextlib import contextmanager
9
+ from dataclasses import dataclass
10
 
11
  from core import odoo as _odoo
12
 
 
54
  return bool(len(self))
55
 
56
 
57
+ @dataclass(frozen=True)
58
+ class RowIds:
59
+ """A SQL-backed selection of related identities from one source snapshot."""
60
+ rows: Rows
61
+ path: str
62
+
63
+ def sql(self):
64
+ return (f'SELECT DISTINCT try_cast(json_extract_string(body, \'{self.path}\') AS BIGINT) id '
65
+ f'FROM "{self.rows.table}" WHERE id IS NOT NULL')
66
+
67
+
68
  class Source:
69
  """Own bounded source operands and release every temporary snapshot on exit."""
70
 
 
131
  ['id', 'default_code', 'categ_id', 'type', 'standard_price', 'name', 'active'])
132
  return self.cache[key]
133
 
134
+ def related_read(self, model, selections, fields, domain=()):
135
+ """Page related source reads by selected identities without retaining the population in Python."""
136
+ def records():
137
+ for field, ids, extra in selections:
138
+ last = -1
139
+ while True:
140
+ page = [row[0] for row in self.cur.execute(
141
+ f'SELECT id FROM ({ids.sql()}) selected WHERE id > ? ORDER BY id LIMIT ?',
142
+ [last, BATCH]).fetchall()]
143
+ if not page:
144
+ break
145
+ selected = self.search_read(model, [*domain, *extra, (field, 'in', page)], fields)
146
+ try:
147
+ yield from selected
148
+ finally:
149
+ self.cur.execute(f'DROP TABLE "{selected.table}"')
150
+ self.tables.remove(selected.table)
151
+ last = page[-1]
152
+ return self.snapshot(records())
153
+
154
+ def category_rows(self, products):
155
+ return self.related_read('product.category', [('id', RowIds(products, '/categ_id/0'), [])],
156
+ ['id', 'complete_name'])
157
+
158
  def sku_revenue(self, groups):
159
  products = self.products()
160
  code = code_sql('p')
 
207
 
208
  def categories(self, keep_blank_code=False):
209
  rows = self.products()
210
+ cats = self.category_rows(rows)
211
  blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
212
  return {code: category for code, category in self.cur.execute(f"""SELECT
213
  {code_sql('p')} code,
 
242
  ['product_id', 'price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
243
  products = self.search_read('product.product', [('default_code', '!=', False)],
244
  ['id', 'default_code', 'name', 'categ_id', 'standard_price', 'sale_ok', 'type'])
245
+ cats = self.category_rows(products)
246
  query = f"""WITH raw AS (SELECT p.ordinal,
247
  trim(json_extract_string(p.body, '/default_code')) sku,
248
  json_extract_string(p.body, '/name') AS product_name,
 
267
  def tiers(self, prods):
268
  from core import periods
269
  from modules import products
270
+ prods = products.active_products() if prods is None else prods
271
  today = periods.today().isoformat()
272
+ selections = [('product_id', RowIds(prods, '/id'), [('applied_on', '=', '0_product_variant')]),
273
+ ('product_tmpl_id', RowIds(prods, '/product_tmpl_id/0'), [('applied_on', '=', '1_product')])]
274
+ rules = self.related_read('product.pricelist.item', selections,
 
275
  ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on',
276
+ 'fixed_price', 'packaging_id', 'packaging_price'], [('compute_price', '=', 'fixed'),
277
+ '|', ('date_start', '=', False), ('date_start', '<=', today),
278
+ '|', ('date_end', '=', False), ('date_end', '>=', today), ('fixed_price', '>', 0)])
279
+ lists = self.related_read('product.pricelist', [('id', RowIds(rules, '/pricelist_id/0'), [])], ['id', 'name'])
280
  normalized = self.snapshot(dict(r, canonical_price=round(r.get('packaging_price') or r.get('fixed_price') or 0.0, 2),
281
  canonical_unit=str(_odoo.m2o_name(r.get('packaging_id')) or '').strip()) for r in rules)
 
282
  declared = sorted({name for _p, _u, name, _n in products.PRICELIST_TIER_COLUMNS})
283
  # Two equality joins and an anti-join: an OR join with a correlated NOT EXISTS is a nested loop
284
  # (234 s for the whole catalogue on the staging mirror).
 
287
  json_extract_string(p.body, '/product_tmpl_id/0') tmpl
288
  FROM "{prods.table}" p),
289
  rr AS (
290
+ SELECT try_cast(json_extract_string(r.body, '/id') AS BIGINT) rule_order, json_extract_string(r.body, '/applied_on') applied_on,
291
  json_extract_string(r.body, '/product_id/0') rpid,
292
  json_extract_string(r.body, '/product_tmpl_id/0') rtmpl,
293
  json_extract_string(r.body, '/pricelist_id/0') plid,
 
326
  entry.append({'pricelist': name, 'unit_price': price, 'unit': unit})
327
  if name in declared and n > products.PRICELIST_TIER_DEPTH:
328
  entry.overflow[name] = n - products.PRICELIST_TIER_DEPTH
329
+ return out, self.tier_report()
330
+
331
+ def tier_report(self):
332
+ return {'bounded': True, 'rules_total': self.search_count('product.pricelist.item', []),
333
+ 'rules_not_fixed': self.search_count('product.pricelist.item', [('compute_price', '!=', 'fixed')])}
334
 
335
  def search_read(self, model, domain=None, fields=None, **kw):
336
  domain = self.domain(model, domain)
 
653
  expr = expr[0]
654
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
655
  raise _Unmirrored(f'name search on {path}')
656
+ if op == 'in' and isinstance(value, RowIds):
657
+ cond = f'{expr} IN ({value.sql()})'
658
+ return f'coalesce(({guard} AND {cond}), false)' if guard else f'coalesce(({cond}), false)'
659
  if kind == 'bool':
660
  expr = f'coalesce({expr}, false)'
661
  value = [_plain(v) for v in value] if isinstance(value, (list, tuple, set)) else _plain(value)
 
762
  """The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
763
  if model not in self._relations:
764
  target = _MIRROR_MODELS[model]
765
+ if model == 'product.category' and self.kind == 'product':
766
+ products = self.products()
767
+ target = ("(SELECT DISTINCT try_cast(json_extract_string(body, '/categ_id/0') AS BIGINT) AS id, "
768
+ "json_extract_string(body, '/categ_id/1') AS complete_name "
769
+ f'FROM "{products.table}" WHERE id IS NOT NULL)')
770
  if not target.startswith('(') and not self.cur.execute(
771
  'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
772
  [target]).fetchone():
 
796
  # Live `_code_category` keys every `default_code != False` product, a blank '' code included.
797
  return Source.categories(self, keep_blank_code)
798
 
799
+ def related_read(self, model, selections, fields, domain=()):
800
+ branches = [['&'] * len(extra) + [*extra, (field, 'in', ids)]
801
+ for field, ids, extra in selections]
802
+ scoped = ['|'] * (len(branches) - 1) + [term for branch in branches for term in branch]
803
+ return self.search_read(model, [*domain, *scoped], fields)
804
+
805
+ def tier_report(self):
806
+ owner = self.parent or self
807
+ if 'tier_report' not in owner.cache:
808
+ query = _Query(self, 'product.pricelist.item')
809
+ price = query.field('compute_price')[0]
810
+ where = query.where([])
811
+ total, other = self.cur.execute(
812
+ f"SELECT count(*), count(*) FILTER (WHERE {price} IS DISTINCT FROM 'fixed') "
813
+ f'FROM {query.from_sql()} WHERE {where}', query.params).fetchone()
814
+ owner.cache['tier_report'] = {'bounded': True, 'rules_total': total, 'rules_not_fixed': other}
815
+ return dict(owner.cache['tier_report'])
816
+
817
  def servable(self, model, names):
818
  """The `names` the mirror declares for `model`; the rest are logged once and dropped."""
819
  kept, dropped = [], []
 
986
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
987
  FROM {prods.from_sql()} WHERE {pw}),
988
  quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
989
+ FROM {quants.from_sql()} WHERE {qw} AND b.product_id IN (SELECT id FROM prods) GROUP BY 1),
990
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
991
+ sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw}
992
+ AND b.product_id IN (SELECT id FROM prods) GROUP BY 1)
993
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
994
  bool_or(p.storable) storable, sum(coalesce(q.qty, 0)) on_hand,
995
  sum(coalesce(q.qty, 0) * p.cost) inv_value, sum(coalesce(s.qty, 0) * p.cost) cogs_ltm,
 
1027
  "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
1028
  "JOIN stock_location l ON l.id = q.location_id "
1029
  "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
1030
+ "AND q.product_id IN (SELECT unnest($2::BIGINT[])) "
1031
  "GROUP BY 1) f ON f.product_id = p.id "
1032
  "WHERE p.id IN (SELECT unnest($2::BIGINT[]))",
1033
  [view[1], ids])
platform/modules/competitor_watch.py CHANGED
@@ -26,6 +26,8 @@ import re
26
  from pathlib import Path
27
  from urllib.parse import unquote_plus
28
 
 
 
29
  TRIGGERS = {
30
  "new_asin": "New ASIN in top 3",
31
  "lost_spot": "Lost spot",
@@ -438,14 +440,6 @@ def _columns(cur, table):
438
  raise _ReadFailed(type(exc).__name__) from exc
439
 
440
 
441
- def _weekly_period_sql():
442
- """Only finite dates or timestamps with an inclusive seven-day span define a report week."""
443
- start = 'TRY_CAST("_period_start" AS TIMESTAMP)'
444
- end = 'TRY_CAST("_period_end" AS TIMESTAMP)'
445
- return (f"isfinite({start}) AND isfinite({end}) AND "
446
- f"date_diff('day', CAST({start} AS DATE), CAST({end} AS DATE)) = 6")
447
-
448
-
449
  def _ba_columns(cur):
450
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
451
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
 
26
  from pathlib import Path
27
  from urllib.parse import unquote_plus
28
 
29
+ from core.amazon_rivals import weekly_period_sql as _weekly_period_sql
30
+
31
  TRIGGERS = {
32
  "new_asin": "New ASIN in top 3",
33
  "lost_spot": "Lost spot",
 
440
  raise _ReadFailed(type(exc).__name__) from exc
441
 
442
 
 
 
 
 
 
 
 
 
443
  def _ba_columns(cur):
444
  """Check weekly readiness with metadata and scalar existence checks, without loading source rows."""
445
  cols = _columns(cur, BA_TABLE) if cur is not None else set()
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 69849e1f35135e55d83001d1325a5c4984a2203bdb5f6c3ec4ab08fc430482aa
 
1
+ 7b3fa972918f4775d1a39386caef6dd64b506f516c56895555ce39a2bdf027c1