fsanyoto commited on
Commit
c2aa70d
Β·
verified Β·
1 Parent(s): 366442c

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "1b5017e01b",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "5d0a33974e",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 1b5017e01b
 
1
+ 5d0a33974e
api/connector_onboarding.py CHANGED
@@ -87,6 +87,12 @@ def ensure_salespeople_selection(rt, *, namespace=None):
87
  rt.update(rel.CONFIG_KEY, admit, flush="sync")
88
 
89
 
 
 
 
 
 
 
90
  def selected_mirror_entities(rt):
91
  """None preserves old sync policy; an explicit setup limits source pulls to selected dependencies."""
92
  import odoo_relational as rel
@@ -106,8 +112,12 @@ def selected_mirror_entities(rt):
106
  for key, enabled in grids.items():
107
  if not enabled:
108
  continue
109
- if key in {"customer_data", "product_data"}:
110
- entities.add({"customer_data": "res_partner", "product_data": "product_product"}[key])
 
 
 
 
111
  continue
112
  if key not in sources:
113
  continue
 
87
  rt.update(rel.CONFIG_KEY, admit, flush="sync")
88
 
89
 
90
+ #: Every mirror table the product pool reads. A selected tenant with product data syncs all of them.
91
+ PRODUCT_DATA_ENTITIES = ("product_product", "product_pricelist", "product_pricelist_item",
92
+ "product_tag", "stock_quant", "stock_location", "sale_order",
93
+ "sale_order_line", "res_partner")
94
+
95
+
96
  def selected_mirror_entities(rt):
97
  """None preserves old sync policy; an explicit setup limits source pulls to selected dependencies."""
98
  import odoo_relational as rel
 
112
  for key, enabled in grids.items():
113
  if not enabled:
114
  continue
115
+ if key == "customer_data":
116
+ entities.add("res_partner")
117
+ continue
118
+ if key == "product_data":
119
+ # The product pool reads only the mirror, so its data pulls every table the pool reads.
120
+ entities.update(PRODUCT_DATA_ENTITIES)
121
  continue
122
  if key not in sources:
123
  continue
platform/harness/canonical_source.py CHANGED
@@ -2,6 +2,8 @@
2
  import contextvars
3
  import itertools
4
  import json
 
 
5
  import uuid
6
  from contextlib import contextmanager
7
 
@@ -9,6 +11,7 @@ from core import odoo as _odoo
9
 
10
  BATCH = 1024
11
  _ACTIVE = contextvars.ContextVar('canonical_source', default=None)
 
12
 
13
 
14
  def code_sql(alias):
@@ -164,9 +167,10 @@ class Source:
164
  return {pid: {'top_category': cat, 'top_category_pct': pct, 'sku_count': n, 'top_sku': name}
165
  for pid, cat, pct, n, name in self.cur.execute(query).fetchall()}
166
 
167
- def categories(self):
168
  rows = self.products()
169
  cats = self.search_read('product.category', [], ['id', 'complete_name'])
 
170
  return {code: category for code, category in self.cur.execute(f"""SELECT
171
  {code_sql('p')} code,
172
  first(coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''),
@@ -174,7 +178,7 @@ class Source:
174
  FROM "{rows.table}" p LEFT JOIN "{cats.table}" c
175
  ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id')
176
  WHERE json_extract_string(p.body, '/active') = 'true'
177
- AND json_type(p.body, '/default_code') = 'VARCHAR' AND json_extract_string(p.body, '/default_code') != ''
178
  GROUP BY code""").fetchall()}
179
 
180
  def backlog(self, groups):
@@ -356,9 +360,6 @@ class Source:
356
  #: Tables a product pool needs before it may read the mirror instead of Odoo.
357
  MIRROR_TABLES = ('product_product', 'stock_quant', 'stock_location', 'sale_order',
358
  'sale_order_line', 'res_partner')
359
- #: Models the mirror does not carry that a product pool may still read live, each read recorded.
360
- LIVE_RESIDUAL = frozenset({'product.product', 'product.pricelist', 'product.pricelist.item',
361
- 'product.tag'})
362
  _MIRROR_MODELS = {
363
  'product.product': 'product_product',
364
  'sale.order.line': 'sale_order_line',
@@ -366,6 +367,9 @@ _MIRROR_MODELS = {
366
  'stock.quant': 'stock_quant',
367
  'stock.location': 'stock_location',
368
  'res.partner': 'res_partner',
 
 
 
369
  # product.category is not synced: its id and complete_name ride on product_product.
370
  'product.category': ('(SELECT DISTINCT categ_id AS id, categ_name AS complete_name '
371
  'FROM product_product WHERE categ_id IS NOT NULL)'),
@@ -384,6 +388,45 @@ _MIRROR_ALIASES = {('sale.order.line', 'state'): 'order_id.state'}
384
  _NUMERIC = ('DOUBLE', 'FLOAT', 'REAL', 'INTEGER', 'BIGINT', 'SMALLINT', 'TINYINT', 'HUGEINT',
385
  'UBIGINT', 'UINTEGER', 'USMALLINT', 'UTINYINT', 'DECIMAL')
386
  _CODE_FIELD = '__code__'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
387
 
388
 
389
  class _Unmirrored(Exception):
@@ -441,11 +484,20 @@ class _Query:
441
  cols = self.source.columns(model)
442
  kind = cols.get(name)
443
  if kind is None:
444
- raise _Unmirrored(f'{model}.{name}')
 
 
 
 
 
 
 
445
  if name == 'id':
446
  return f'{alias}.id', 'int'
447
  if name.endswith('_id') and kind == 'num':
448
  label = name[:-3] + '_name'
 
 
449
  return (f'{alias}.{name}', f'{alias}.{label}' if label in cols else 'NULL'), 'm2o'
450
  return f'{alias}.{name}', kind
451
 
@@ -473,6 +525,11 @@ class _Query:
473
  path, op, value = term
474
  op = {'==': '=', '<>': '!='}.get(str(op).lower(), str(op).lower())
475
  expr, kind, guard = self.field(path)
 
 
 
 
 
476
  if kind == 'm2o':
477
  expr = expr[0]
478
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
@@ -530,12 +587,16 @@ class _Query:
530
  return ' AND '.join(reversed(stack)) or 'true'
531
 
532
  def from_sql(self):
533
- joins = ''.join(f' LEFT JOIN {_MIRROR_MODELS[comodel]} {alias} ON {alias}.id = b.{head}'
534
  for head, (comodel, alias) in self.joins.items())
535
- return f'{_MIRROR_MODELS[self.model]} b{joins}'
536
 
537
 
538
  def _json_value(expr, kind, grouped=False):
 
 
 
 
539
  if kind == 'm2o':
540
  ident, label = expr
541
  label = f'min({label})' if grouped else label
@@ -549,26 +610,80 @@ def _json_value(expr, kind, grouped=False):
549
  return f"CASE WHEN {expr} IS NULL THEN 'false'::JSON ELSE to_json({expr}) END"
550
 
551
 
552
- def _domain_key(model, domain):
553
- return model, repr(list(domain or []))
554
-
555
-
556
- #: The one live read a mirrored build makes: the mirror's stock_quant drifts from Odoo.
557
- LIVE_QUANT_READ = ('stock.quant', [('location_id.usage', '=', 'internal')], ['quantity:sum'],
558
- ['product_id', 'location_id'])
559
 
560
 
561
  class MirrorSource(Source):
562
- """A `Source` that answers sales and product reads from the tenant's Odoo mirror."""
 
 
 
 
563
 
564
  def __init__(self, cur, kind, identities, store=None):
565
  super().__init__(cur, kind, identities, store)
566
  self.live_reads = []
 
 
567
  self.parent = None
568
- self._origin = {}
569
  self._columns = {}
570
- self._quants = None
571
- self._quant_table = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
572
 
573
  @property
574
  def cur(self):
@@ -593,13 +708,8 @@ class MirrorSource(Source):
593
 
594
  def columns(self, model):
595
  if model not in self._columns:
596
- rows = self.cur.execute(f'DESCRIBE SELECT * FROM {_MIRROR_MODELS[model]} AS t').fetchall()
597
- kinds = {}
598
- for name, typ, *_ in rows:
599
- typ = str(typ).upper()
600
- kinds[name] = ('bool' if typ == 'BOOLEAN' else
601
- 'num' if typ.startswith(_NUMERIC) else 'str')
602
- self._columns[model] = kinds
603
  return self._columns[model]
604
 
605
  def _scoped(self, model, domain):
@@ -612,14 +722,6 @@ class MirrorSource(Source):
612
  domain = Source.domain(self, model, domain)
613
  return domain
614
 
615
- def _live(self, method, model, domain, fields, cause, call):
616
- if model not in LIVE_RESIDUAL:
617
- raise RuntimeError(f'canonical_source_unmirrored: {model}.{method} needs {cause}, '
618
- 'which the Odoo mirror does not carry')
619
- self.live_reads.append({'model': model, 'method': method, 'fields': list(fields or [])})
620
- self._origin[_domain_key(model, domain)] = 'live'
621
- return call()
622
-
623
  def _materialize(self, query, select, where, order, group=None):
624
  name = 'canonical_' + uuid.uuid4().hex
625
  self.cur.execute(f'CREATE TEMP TABLE "{name}" (ordinal BIGINT, body JSON)')
@@ -645,9 +747,7 @@ class MirrorSource(Source):
645
  select = [(f, _json_value(*query.field(f)[:2])) for f in wanted]
646
  where = query.where(self._scoped(model, domain))
647
  except _Unmirrored as exc:
648
- return self._live('search_read', model, domain, fields, exc,
649
- lambda: Source.search_read(self, model, domain, fields, **kw))
650
- self._origin[_domain_key(model, domain)] = 'mirror'
651
  return self._materialize(query, select, where, 'b.id')
652
 
653
  def read_group(self, model, domain, fields, groupby, **kw):
@@ -659,6 +759,8 @@ class MirrorSource(Source):
659
  if ':' in g:
660
  raise _Unmirrored('date grouping ' + g)
661
  expr, kind, _ = query.field(g)
 
 
662
  keys.append(expr[0] if kind == 'm2o' else expr)
663
  select.append((g, _json_value(expr, kind, grouped=True)))
664
  for spec in fields or []:
@@ -666,6 +768,8 @@ class MirrorSource(Source):
666
  if name in groupby or name in {'id', '__count'}:
667
  continue
668
  expr, kind, _ = query.field(name)
 
 
669
  ident = expr[0] if kind == 'm2o' else expr
670
  agg = agg or ('sum' if kind == 'num' else '')
671
  if agg in {'sum', 'min', 'max', 'avg'}:
@@ -679,27 +783,18 @@ class MirrorSource(Source):
679
  select.append(('__count', 'count(*)'))
680
  where = query.where(self._scoped(model, domain))
681
  except _Unmirrored as exc:
682
- return self._live('read_group', model, domain, fields, exc,
683
- lambda: Source.read_group(self, model, domain, fields, groupby, **kw))
684
  order = ', '.join(f'{k} NULLS LAST' for k in keys) or '1'
685
  return self._materialize(query, select, where, order, group=', '.join(keys) or None)
686
 
687
  def search_count(self, model, domain=None, **kw):
688
- if self._origin.get(_domain_key(model, domain)) != 'live':
689
- try:
690
- query = _Query(self, model)
691
- where = query.where(self._scoped(model, domain))
692
- return self.cur.execute(f'SELECT count(*) FROM {query.from_sql()} WHERE {where}',
693
- query.params).fetchone()[0]
694
- except _Unmirrored as exc:
695
- cause = exc
696
- else:
697
- cause = 'the source its matching search_read used'
698
- def live():
699
- if self.kind == 'product' and model == 'product.product':
700
- return len(Source.search_read(self, model, domain, ['id', 'default_code']))
701
- return _odoo.get_odoo().search_count(model, Source.domain(self, model, domain), **kw)
702
- return self._live('search_count', model, domain, [], cause, live)
703
 
704
  def excluded_partner_ids(self):
705
  """`core.odoo.excluded_partner_ids` resolved from res_partner, shared through its cache."""
@@ -721,63 +816,23 @@ class MirrorSource(Source):
721
  self.excluded_partner_ids()
722
  return _odoo.sale_line_domain(*args, **kw)
723
 
724
- def live_quants(self):
725
- """`[(product_id, location_id, location_name, quantity)]`, read live once per root build."""
726
- if self.parent is not None:
727
- return self.parent.live_quants()
728
- if isinstance(self._quants, Exception):
729
- raise self._quants
730
- if self._quants is None:
731
- model, domain, fields, groupby = LIVE_QUANT_READ
732
- self.live_reads.append({'model': model, 'method': 'read_group', 'fields': list(fields)})
733
- try:
734
- # One RPC, unpaged: Odoo's read_group has no default limit, so this is complete.
735
- rows = _odoo.get_odoo().execute_kw(model, 'read_group',
736
- [list(domain), list(fields), list(groupby)], {'lazy': False})
737
- out = []
738
- for r in rows or []:
739
- pid, loc = _odoo.m2o_id(r.get('product_id')), r.get('location_id')
740
- if pid is not None:
741
- out.append((int(pid), _odoo.m2o_id(loc), _odoo.m2o_name(loc) or None,
742
- float(r.get('quantity') or 0.0)))
743
- except Exception as exc:
744
- # A failed stock read blanks the stock columns, as `_inventory_by_code` documents.
745
- self.failure, self._quants = None, exc
746
- raise
747
- self._quants = out
748
- return self._quants
749
-
750
- def _quants_sql(self):
751
- """A temp table on this cursor holding `live_quants()`."""
752
- if self._quant_table is None:
753
- rows = self.live_quants()
754
- name = 'canonical_' + uuid.uuid4().hex
755
- self.cur.execute(f'CREATE TEMP TABLE "{name}" (product_id BIGINT, location_id BIGINT, '
756
- 'location_name VARCHAR, quantity DOUBLE)')
757
- self.tables.append(name)
758
- if rows:
759
- cols = list(zip(*rows))
760
- self.cur.execute(f'INSERT INTO "{name}" SELECT unnest($1::BIGINT[]), unnest($2::BIGINT[]), '
761
- 'unnest($3::VARCHAR[]), unnest($4::DOUBLE[])', [list(c) for c in cols])
762
- self._quant_table = name
763
- return self._quant_table
764
-
765
  def inventory_frame(self, t):
766
  from core import periods
767
  import pandas as pd
768
  lf, lt = periods.ltm(t)
769
- quants = self._quants_sql()
770
  params = []
771
  prods = _Query(self, 'product.product', params)
 
772
  sales = _Query(self, 'sale.order.line', params)
773
  pw = prods.where(self._scoped('product.product', [('default_code', '!=', False)]))
 
774
  sw = sales.where(_odoo.sale_line_domain(lf, lt, all_channels=True))
775
  result = self.cur.execute(f"""WITH prods AS (SELECT b.id, trim(b.default_code) AS sku, b.name,
776
  b.type = 'product' AS storable, coalesce(b.standard_price, 0) AS cost,
777
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
778
  FROM {prods.from_sql()} WHERE {pw}),
779
- quant AS (SELECT product_id AS id, sum(coalesce(quantity, 0)) AS qty
780
- FROM "{quants}" GROUP BY 1),
781
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
782
  sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw} GROUP BY 1)
783
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
@@ -796,11 +851,8 @@ class MirrorSource(Source):
796
  return pd.DataFrame(rows, columns=names), (lf, lt)
797
 
798
  def fba_qty(self, product_ids=None):
799
- """`warehouse.fba_qty_by_product_id`'s `(by_pid | None, report)` from the live quant read.
800
-
801
- A location sits in FBA by the mirror's parent_path of the FBA view; a location the mirror
802
- does not know counts by its live name, 'FBA' or 'FBA/...'.
803
- """
804
  from modules import warehouse
805
  view = self.cur.execute(
806
  "SELECT id, parent_path, warehouse_id, warehouse_name FROM stock_location "
@@ -813,18 +865,16 @@ class MirrorSource(Source):
813
  'SELECT id FROM product_product WHERE coalesce(active, false) ORDER BY id').fetchall()]
814
  else:
815
  ids = [int(i) for i in product_ids if i is not None]
816
- quants = self._quants_sql()
817
  out = {}
818
  # Odoo's search_read on product.product skips archived ids, so they stay absent here too.
819
  result = self.cur.execute(
820
  "SELECT p.id, coalesce(f.qty, 0) FROM product_product p LEFT JOIN ("
821
- f'SELECT g.product_id, sum(coalesce(g.quantity, 0)) qty FROM "{quants}" g '
822
- "LEFT JOIN stock_location l ON l.id = g.location_id WHERE CASE WHEN l.id IS NOT NULL "
823
- "THEN starts_with(l.parent_path, $1) ELSE (g.location_name = $3 "
824
- "OR starts_with(g.location_name, $3 || '/')) END "
825
  "GROUP BY 1) f ON f.product_id = p.id "
826
  "WHERE p.id IN (SELECT unnest($2::BIGINT[])) AND coalesce(p.active, false)",
827
- [view[1], ids, warehouse.FBA_WAREHOUSE_CODE])
828
  while True:
829
  batch = result.fetchmany(BATCH)
830
  if not batch:
@@ -832,13 +882,15 @@ class MirrorSource(Source):
832
  out.update({pid: float(qty) for pid, qty in batch})
833
  wh = {'id': view[2], 'name': view[3], 'code': warehouse.FBA_WAREHOUSE_CODE}
834
  return out, warehouse.fba_report(wh, len(ids), len(out), 1 if ids else 0,
835
- 'the FBA view location in the Odoo mirror, never a pinned id; '
836
- 'quantities read live')
837
 
838
  def close(self):
839
  super().close()
840
- self._origin.clear()
841
- self._quant_table = None
 
 
 
842
  if self._factory is not None and self._cur is not None:
843
  self._cur.close()
844
  self._cur = None
@@ -865,7 +917,8 @@ def bounded(cur, kind, identities, store=None):
865
  parent = active()
866
  if isinstance(parent, MirrorSource):
867
  source = MirrorSource(cur, kind, identities, store)
868
- source.live_reads, source.parent = parent.live_reads, parent
 
869
  else:
870
  source = Source(cur, kind, identities, store)
871
  with _activate(source) as entered:
@@ -874,7 +927,7 @@ def bounded(cur, kind, identities, store=None):
874
 
875
  @contextmanager
876
  def mirrored(cur, store=None):
877
- """Serve every proxied read in this block from the Odoo mirror; never falls back live.
878
 
879
  `cur` is a mirror cursor, or a zero-argument factory opened on the first read and closed on exit.
880
  """
 
2
  import contextvars
3
  import itertools
4
  import json
5
+ import logging
6
+ import re
7
  import uuid
8
  from contextlib import contextmanager
9
 
 
11
 
12
  BATCH = 1024
13
  _ACTIVE = contextvars.ContextVar('canonical_source', default=None)
14
+ _LOG = logging.getLogger('canonical_source')
15
 
16
 
17
  def code_sql(alias):
 
167
  return {pid: {'top_category': cat, 'top_category_pct': pct, 'sku_count': n, 'top_sku': name}
168
  for pid, cat, pct, n, name in self.cur.execute(query).fetchall()}
169
 
170
+ def categories(self, keep_blank_code=False):
171
  rows = self.products()
172
  cats = self.search_read('product.category', [], ['id', 'complete_name'])
173
+ blank = '' if keep_blank_code else "AND json_extract_string(p.body, '/default_code') != ''"
174
  return {code: category for code, category in self.cur.execute(f"""SELECT
175
  {code_sql('p')} code,
176
  first(coalesce(nullif(trim(list_extract(string_split(json_extract_string(c.body, '/complete_name'), '/'), 2)), ''),
 
178
  FROM "{rows.table}" p LEFT JOIN "{cats.table}" c
179
  ON json_extract_string(p.body, '/categ_id/0')=json_extract_string(c.body, '/id')
180
  WHERE json_extract_string(p.body, '/active') = 'true'
181
+ AND json_type(p.body, '/default_code') = 'VARCHAR' {blank}
182
  GROUP BY code""").fetchall()}
183
 
184
  def backlog(self, groups):
 
360
  #: Tables a product pool needs before it may read the mirror instead of Odoo.
361
  MIRROR_TABLES = ('product_product', 'stock_quant', 'stock_location', 'sale_order',
362
  'sale_order_line', 'res_partner')
 
 
 
363
  _MIRROR_MODELS = {
364
  'product.product': 'product_product',
365
  'sale.order.line': 'sale_order_line',
 
367
  'stock.quant': 'stock_quant',
368
  'stock.location': 'stock_location',
369
  'res.partner': 'res_partner',
370
+ 'product.pricelist': 'product_pricelist',
371
+ 'product.pricelist.item': 'product_pricelist_item',
372
+ 'product.tag': 'product_tag',
373
  # product.category is not synced: its id and complete_name ride on product_product.
374
  'product.category': ('(SELECT DISTINCT categ_id AS id, categ_name AS complete_name '
375
  'FROM product_product WHERE categ_id IS NOT NULL)'),
 
388
  _NUMERIC = ('DOUBLE', 'FLOAT', 'REAL', 'INTEGER', 'BIGINT', 'SMALLINT', 'TINYINT', 'HUGEINT',
389
  'UBIGINT', 'UINTEGER', 'USMALLINT', 'UTINYINT', 'DECIMAL')
390
  _CODE_FIELD = '__code__'
391
+ #: The product pool's mirror contract, read as declared even before `datastore.ENTITIES` names it.
392
+ _DECLARED = {
393
+ 'product_product': {
394
+ 'create_date': 'VARCHAR', 'product_tmpl_id': 'BIGINT', 'product_tmpl_name': 'VARCHAR',
395
+ 'volume': 'DOUBLE', 'display_name': 'VARCHAR', 'product_tag_ids': 'VARCHAR',
396
+ 'incoming_qty': 'DOUBLE', 'free_qty': 'DOUBLE', 'virtual_available': 'DOUBLE'},
397
+ 'product_pricelist': {'id': 'BIGINT', 'name': 'VARCHAR'},
398
+ 'product_pricelist_item': {
399
+ 'id': 'BIGINT', 'pricelist_id': 'BIGINT', 'pricelist_name': 'VARCHAR',
400
+ 'product_id': 'BIGINT', 'product_name': 'VARCHAR', 'product_tmpl_id': 'BIGINT',
401
+ 'product_tmpl_name': 'VARCHAR', 'applied_on': 'VARCHAR', 'compute_price': 'VARCHAR',
402
+ 'fixed_price': 'DOUBLE', 'min_quantity': 'DOUBLE', 'date_start': 'VARCHAR',
403
+ 'date_end': 'VARCHAR', 'packaging_id': 'BIGINT', 'packaging_name': 'VARCHAR',
404
+ 'packaging_price': 'DOUBLE'},
405
+ 'product_tag': {'id': 'BIGINT', 'name': 'VARCHAR'},
406
+ }
407
+ _DATE_ONLY = re.compile(r'\d{4}-\d{2}-\d{2}')
408
+
409
+
410
+ def declared_columns(table):
411
+ """`{column: SQL type}` the mirror sync declares for `table`, as `datastore._cols` names them."""
412
+ from harness import datastore
413
+ out = {}
414
+ spec = datastore.ENTITIES.get(table)
415
+ for col in datastore._cols(spec) if spec else ():
416
+ name, typ = col.split()[:2]
417
+ out[name] = typ
418
+ out.update(_DECLARED.get(table) or {})
419
+ return out
420
+
421
+
422
+ def _kind(name, typ):
423
+ typ = str(typ).upper()
424
+ if typ == 'BOOLEAN':
425
+ return 'bool'
426
+ if typ.startswith(_NUMERIC):
427
+ return 'num'
428
+ # A many2many lands as a VARCHAR JSON list of ids.
429
+ return 'm2m' if name.endswith('_ids') else 'str'
430
 
431
 
432
  class _Unmirrored(Exception):
 
484
  cols = self.source.columns(model)
485
  kind = cols.get(name)
486
  if kind is None:
487
+ # A declared column the sync has not filled yet reads blank; an undeclared one refuses.
488
+ typ = self.source.pending_column(model, name)
489
+ if typ is None:
490
+ raise _Unmirrored(f'{model}.{name}')
491
+ kind, expr = _kind(name, typ), f'CAST(NULL AS {typ})'
492
+ if name.endswith('_id') and kind == 'num':
493
+ return (expr, 'CAST(NULL AS VARCHAR)'), 'm2o'
494
+ return expr, kind
495
  if name == 'id':
496
  return f'{alias}.id', 'int'
497
  if name.endswith('_id') and kind == 'num':
498
  label = name[:-3] + '_name'
499
+ if label not in cols:
500
+ self.source.pending_column(model, label)
501
  return (f'{alias}.{name}', f'{alias}.{label}' if label in cols else 'NULL'), 'm2o'
502
  return f'{alias}.{name}', kind
503
 
 
525
  path, op, value = term
526
  op = {'==': '=', '<>': '!='}.get(str(op).lower(), str(op).lower())
527
  expr, kind, guard = self.field(path)
528
+ if kind == 'm2m':
529
+ raise _Unmirrored(f'many2many filter on {self.model}.{path}')
530
+ if kind == 'str' and op in {'<=', '>'} and isinstance(value, str) and _DATE_ONLY.fullmatch(value):
531
+ # Odoo reads a bare date as its midnight, so a value of exactly midnight is <= that day.
532
+ value += ' 00:00:00'
533
  if kind == 'm2o':
534
  expr = expr[0]
535
  if isinstance(value, str) or op in {'like', 'ilike', '=like', '=ilike'}:
 
587
  return ' AND '.join(reversed(stack)) or 'true'
588
 
589
  def from_sql(self):
590
+ joins = ''.join(f' LEFT JOIN {self.source.relation(comodel)} {alias} ON {alias}.id = b.{head}'
591
  for head, (comodel, alias) in self.joins.items())
592
+ return f'{self.source.relation(self.model)} b{joins}'
593
 
594
 
595
  def _json_value(expr, kind, grouped=False):
596
+ if isinstance(expr, str) and expr.startswith('CAST(NULL AS'):
597
+ return "'[]'::JSON" if kind == 'm2m' else "'false'::JSON"
598
+ if kind == 'm2m':
599
+ return f"coalesce(TRY_CAST({expr} AS JSON), '[]'::JSON)"
600
  if kind == 'm2o':
601
  ident, label = expr
602
  label = f'min({label})' if grouped else label
 
610
  return f"CASE WHEN {expr} IS NULL THEN 'false'::JSON ELSE to_json({expr}) END"
611
 
612
 
613
+ def _refuse(method, model, cause):
614
+ return RuntimeError(f'canonical_source_unmirrored: {model}.{method} needs {cause}, '
615
+ 'which the Odoo mirror does not carry')
 
 
 
 
616
 
617
 
618
  class MirrorSource(Source):
619
+ """A `Source` that answers every product pool read from the tenant's Odoo mirror, never live.
620
+
621
+ `live_reads` stays `[]`. `pending` names each declared `table.column` or `table` the sync has
622
+ not filled yet: those read blank or empty, logged once per root build.
623
+ """
624
 
625
  def __init__(self, cur, kind, identities, store=None):
626
  super().__init__(cur, kind, identities, store)
627
  self.live_reads = []
628
+ self.pending = []
629
+ self.dropped = []
630
  self.parent = None
 
631
  self._columns = {}
632
+ self._relations = {}
633
+
634
+ def _note(self, bucket, name):
635
+ if name not in bucket:
636
+ bucket.append(name)
637
+
638
+ def relation(self, model):
639
+ """The FROM operand for `model`: its mirror table, or an empty typed one while unsynced."""
640
+ if model not in self._relations:
641
+ target = _MIRROR_MODELS[model]
642
+ if not target.startswith('(') and not self.cur.execute(
643
+ 'SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1',
644
+ [target]).fetchone():
645
+ declared = declared_columns(target)
646
+ if target in MIRROR_TABLES:
647
+ raise RuntimeError(_unready([target]))
648
+ if not declared:
649
+ raise _Unmirrored('table ' + target)
650
+ self._note(self.pending, target)
651
+ target = '(SELECT ' + ', '.join(f'CAST(NULL AS {t}) AS {c}' for c, t in declared.items()) \
652
+ + ' WHERE false)'
653
+ self._relations[model] = target
654
+ return self._relations[model]
655
+
656
+ def pending_column(self, model, name):
657
+ """The SQL type of a declared column the mirror table lacks, recorded as pending, or None."""
658
+ target = _MIRROR_MODELS[model]
659
+ if target.startswith('('):
660
+ return None
661
+ typ = declared_columns(target).get(name)
662
+ if typ is not None:
663
+ self._note(self.pending, f'{target}.{name}')
664
+ return typ
665
+
666
+ def categories(self, keep_blank_code=True):
667
+ # Live `_code_category` keys every `default_code != False` product, a blank '' code included.
668
+ return Source.categories(self, keep_blank_code)
669
+
670
+ def servable(self, model, names):
671
+ """The `names` the mirror declares for `model`; the rest are logged once and dropped."""
672
+ kept, dropped = [], []
673
+ for name in names:
674
+ try:
675
+ _Query(self, model).field(name)
676
+ kept.append(name)
677
+ except _Unmirrored:
678
+ dropped.append(name)
679
+ fresh = [n for n in dropped if f'{model}.{n}' not in self.dropped]
680
+ for name in fresh:
681
+ self._note(self.dropped, f'{model}.{name}')
682
+ if fresh:
683
+ _LOG.warning('canonical_source_dropped: the Odoo mirror does not declare %s, so the product '
684
+ 'pool leaves those fields blank. Declare them in harness.datastore.ENTITIES to '
685
+ 'sync them.', ', '.join(f'{model}.{n}' for n in fresh))
686
+ return kept
687
 
688
  @property
689
  def cur(self):
 
708
 
709
  def columns(self, model):
710
  if model not in self._columns:
711
+ rows = self.cur.execute(f'DESCRIBE SELECT * FROM {self.relation(model)} AS t').fetchall()
712
+ self._columns[model] = {name: _kind(name, typ) for name, typ, *_ in rows}
 
 
 
 
 
713
  return self._columns[model]
714
 
715
  def _scoped(self, model, domain):
 
722
  domain = Source.domain(self, model, domain)
723
  return domain
724
 
 
 
 
 
 
 
 
 
725
  def _materialize(self, query, select, where, order, group=None):
726
  name = 'canonical_' + uuid.uuid4().hex
727
  self.cur.execute(f'CREATE TEMP TABLE "{name}" (ordinal BIGINT, body JSON)')
 
747
  select = [(f, _json_value(*query.field(f)[:2])) for f in wanted]
748
  where = query.where(self._scoped(model, domain))
749
  except _Unmirrored as exc:
750
+ raise _refuse('search_read', model, exc) from None
 
 
751
  return self._materialize(query, select, where, 'b.id')
752
 
753
  def read_group(self, model, domain, fields, groupby, **kw):
 
759
  if ':' in g:
760
  raise _Unmirrored('date grouping ' + g)
761
  expr, kind, _ = query.field(g)
762
+ if kind == 'm2m':
763
+ raise _Unmirrored('many2many grouping ' + g)
764
  keys.append(expr[0] if kind == 'm2o' else expr)
765
  select.append((g, _json_value(expr, kind, grouped=True)))
766
  for spec in fields or []:
 
768
  if name in groupby or name in {'id', '__count'}:
769
  continue
770
  expr, kind, _ = query.field(name)
771
+ if kind == 'm2m':
772
+ raise _Unmirrored('many2many aggregate ' + spec)
773
  ident = expr[0] if kind == 'm2o' else expr
774
  agg = agg or ('sum' if kind == 'num' else '')
775
  if agg in {'sum', 'min', 'max', 'avg'}:
 
783
  select.append(('__count', 'count(*)'))
784
  where = query.where(self._scoped(model, domain))
785
  except _Unmirrored as exc:
786
+ raise _refuse('read_group', model, exc) from None
 
787
  order = ', '.join(f'{k} NULLS LAST' for k in keys) or '1'
788
  return self._materialize(query, select, where, order, group=', '.join(keys) or None)
789
 
790
  def search_count(self, model, domain=None, **kw):
791
+ try:
792
+ query = _Query(self, model)
793
+ where = query.where(self._scoped(model, domain))
794
+ except _Unmirrored as exc:
795
+ raise _refuse('search_count', model, exc) from None
796
+ return self.cur.execute(f'SELECT count(*) FROM {query.from_sql()} WHERE {where}',
797
+ query.params).fetchone()[0]
 
 
 
 
 
 
 
 
798
 
799
  def excluded_partner_ids(self):
800
  """`core.odoo.excluded_partner_ids` resolved from res_partner, shared through its cache."""
 
816
  self.excluded_partner_ids()
817
  return _odoo.sale_line_domain(*args, **kw)
818
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
819
  def inventory_frame(self, t):
820
  from core import periods
821
  import pandas as pd
822
  lf, lt = periods.ltm(t)
 
823
  params = []
824
  prods = _Query(self, 'product.product', params)
825
+ quants = _Query(self, 'stock.quant', params)
826
  sales = _Query(self, 'sale.order.line', params)
827
  pw = prods.where(self._scoped('product.product', [('default_code', '!=', False)]))
828
+ qw = quants.where([('location_id.usage', '=', 'internal')])
829
  sw = sales.where(_odoo.sale_line_domain(lf, lt, all_channels=True))
830
  result = self.cur.execute(f"""WITH prods AS (SELECT b.id, trim(b.default_code) AS sku, b.name,
831
  b.type = 'product' AS storable, coalesce(b.standard_price, 0) AS cost,
832
  coalesce(nullif(trim(list_extract(string_split(b.categ_name, '/'), 2)), ''), '(uncategorized)') AS category
833
  FROM {prods.from_sql()} WHERE {pw}),
834
+ quant AS (SELECT b.product_id id, sum(coalesce(b.quantity, 0)) qty
835
+ FROM {quants.from_sql()} WHERE {qw} GROUP BY 1),
836
  sold AS (SELECT b.product_id id, sum(coalesce(b.product_uom_qty, 0)) qty,
837
  sum(coalesce(b.price_subtotal, 0)) rev FROM {sales.from_sql()} WHERE {sw} GROUP BY 1)
838
  SELECT p.sku, first(p.name ORDER BY p.id) AS name, first(p.category ORDER BY p.id) category,
 
851
  return pd.DataFrame(rows, columns=names), (lf, lt)
852
 
853
  def fba_qty(self, product_ids=None):
854
+ """`warehouse.fba_qty_by_product_id`'s `(by_pid | None, report)` from the mirror's internal
855
+ quants under the FBA view location's parent_path."""
 
 
 
856
  from modules import warehouse
857
  view = self.cur.execute(
858
  "SELECT id, parent_path, warehouse_id, warehouse_name FROM stock_location "
 
865
  'SELECT id FROM product_product WHERE coalesce(active, false) ORDER BY id').fetchall()]
866
  else:
867
  ids = [int(i) for i in product_ids if i is not None]
 
868
  out = {}
869
  # Odoo's search_read on product.product skips archived ids, so they stay absent here too.
870
  result = self.cur.execute(
871
  "SELECT p.id, coalesce(f.qty, 0) FROM product_product p LEFT JOIN ("
872
+ "SELECT q.product_id, sum(coalesce(q.quantity, 0)) qty FROM stock_quant q "
873
+ "JOIN stock_location l ON l.id = q.location_id "
874
+ "WHERE l.usage = 'internal' AND starts_with(l.parent_path, $1) "
 
875
  "GROUP BY 1) f ON f.product_id = p.id "
876
  "WHERE p.id IN (SELECT unnest($2::BIGINT[])) AND coalesce(p.active, false)",
877
+ [view[1], ids])
878
  while True:
879
  batch = result.fetchmany(BATCH)
880
  if not batch:
 
882
  out.update({pid: float(qty) for pid, qty in batch})
883
  wh = {'id': view[2], 'name': view[3], 'code': warehouse.FBA_WAREHOUSE_CODE}
884
  return out, warehouse.fba_report(wh, len(ids), len(out), 1 if ids else 0,
885
+ 'the FBA view location in the Odoo mirror, never a pinned id')
 
886
 
887
  def close(self):
888
  super().close()
889
+ self._relations.clear()
890
+ if self.parent is None and self.pending:
891
+ _LOG.warning('canonical_source_pending: the Odoo mirror has not synced %s yet, so the '
892
+ 'product pool reads those cells blank until the next sync pass',
893
+ ', '.join(self.pending))
894
  if self._factory is not None and self._cur is not None:
895
  self._cur.close()
896
  self._cur = None
 
917
  parent = active()
918
  if isinstance(parent, MirrorSource):
919
  source = MirrorSource(cur, kind, identities, store)
920
+ source.live_reads, source.pending, source.dropped = parent.live_reads, parent.pending, parent.dropped
921
+ source.parent = parent
922
  else:
923
  source = Source(cur, kind, identities, store)
924
  with _activate(source) as entered:
 
927
 
928
  @contextmanager
929
  def mirrored(cur, store=None):
930
+ """Serve every proxied read in this block from the Odoo mirror; never reads Odoo live.
931
 
932
  `cur` is a mirror cursor, or a zero-argument factory opened on the first read and closed on exit.
933
  """
platform/harness/datastore.py CHANGED
@@ -311,7 +311,12 @@ ENTITIES = {
311
  # carry a real barcode and 1,997 carry none. It is a `store=True` char column on
312
  # `product.product`, so the mirror can hold it.
313
  "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
314
- "barcode", "uom_id", "write_date"],
 
 
 
 
 
315
  # β›” WITHOUT THIS THE COLUMN ALTERS IN AND STAYS NULL FOREVER β€” the same trap
316
  # `account_move.invoice_origin` and the `res_partner` address block are both commented
317
  # for. WE moved the schema; no product's `write_date` moved, so an incremental
@@ -327,7 +332,7 @@ ENTITIES = {
327
  # DOES NOT COME FROM HERE: it comes live from `modules/products.py::catalogue()`. This
328
  # column exists so the mirror is not permanently blind to a field the source has, which is
329
  # D-165's shape; widening `read_products` is a separate ticket with its own fence.
330
- "backfill_fields": ["barcode", "uom_id"],
331
  },
332
  "account_move": {
333
  "model": "account.move", "archivable": False,
@@ -620,8 +625,10 @@ def check_odoo_snapshot(rt, candidate, published, row_counts):
620
  return {row[0] for row in con.execute(
621
  "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()}
622
 
 
623
  previous_counts = {name: published.execute(f'SELECT count(*) FROM "{name}"').fetchone()[0]
624
- for name in ENTITIES if columns(published, name)}
 
625
  record("row_count_drop", [
626
  {"table": name, "previous": count, "candidate": row_counts.get(name, 0),
627
  "removed": count - row_counts.get(name, 0),
@@ -1930,6 +1937,8 @@ def _flatten(rec, fields, m2m=(), *, named_m2m=None, analytic_names=None, named_
1930
  for account_id in key.split(",")]}
1931
  for key, share in v.items()
1932
  ], ensure_ascii=False)
 
 
1933
  elif isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
1934
  out[f"{f.removesuffix('_id')}_id"] = v[0]
1935
  out[f"{f.removesuffix('_id')}_name"] = str(v[1] or "")
@@ -1994,6 +2003,8 @@ NUMERIC_FIELDS = ("amount_untaxed", "amount_untaxed_signed", "amount_residual_si
1994
 
1995
  # Odoo also names this many2one without the usual `_id` suffix.
1996
  MANY2ONE_FIELDS = frozenset({"product_uom", "create_uid"})
 
 
1997
 
1998
  # S72-T43 / R107: the audited Odoo fields the Data grids show. Each one is declared AND
1999
  # backfilled, because existing rows never re-sync when the schema grows. The list must equal
@@ -2117,6 +2128,38 @@ ENTITIES.update({
2117
  },
2118
  })
2119
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2120
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
2121
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
2122
 
@@ -2347,7 +2390,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
2347
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2348
  candidate = getattr(_SYNC_TLS, "candidate", None)
2349
  if candidate is not None:
2350
- phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2351
  # A recurring refresh may report changed pages to a bespoke cursor, but a
2352
  # tenant pass cannot claim completion until those private pages finish too.
2353
  pending = candidate["con"].execute(
@@ -2358,7 +2401,7 @@ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
2358
  return phase, pulled
2359
  rt = _resident_odoo_runtime()
2360
  if rt is None:
2361
- return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2362
  from harness import mirror_store
2363
  report = {}
2364
  with mirror_store.sync_mirror("odoo", rt.key, rt, report=report, log=log, expected_tables=[key]):
@@ -2502,6 +2545,98 @@ def _sync_entity(key, max_batches=200, batch=BATCH, log=print):
2502
  con.close()
2503
 
2504
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2505
  def _already(con, key, rec):
2506
  return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
2507
 
 
311
  # carry a real barcode and 1,997 carry none. It is a `store=True` char column on
312
  # `product.product`, so the mirror can hold it.
313
  "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
314
+ "barcode", "uom_id", "create_date", "product_tmpl_id", "volume",
315
+ "display_name", "product_tag_ids", "incoming_qty", "free_qty",
316
+ "virtual_available", "write_date"],
317
+ # Computed or template-related in Odoo 16: they change with no product.product write_date.
318
+ "refresh_fields": ["display_name", "product_tag_ids", "incoming_qty", "free_qty",
319
+ "virtual_available"],
320
  # β›” WITHOUT THIS THE COLUMN ALTERS IN AND STAYS NULL FOREVER β€” the same trap
321
  # `account_move.invoice_origin` and the `res_partner` address block are both commented
322
  # for. WE moved the schema; no product's `write_date` moved, so an incremental
 
332
  # DOES NOT COME FROM HERE: it comes live from `modules/products.py::catalogue()`. This
333
  # column exists so the mirror is not permanently blind to a field the source has, which is
334
  # D-165's shape; widening `read_products` is a separate ticket with its own fence.
335
+ "backfill_fields": ["barcode", "uom_id", "create_date", "product_tmpl_id", "volume"],
336
  },
337
  "account_move": {
338
  "model": "account.move", "archivable": False,
 
625
  return {row[0] for row in con.execute(
626
  "SELECT column_name FROM information_schema.columns WHERE table_name=?", [table]).fetchall()}
627
 
628
+ # A whole-table replace is a complete count-checked read of Odoo, so its drop is real, not a lost copy.
629
  previous_counts = {name: published.execute(f'SELECT count(*) FROM "{name}"').fetchone()[0]
630
+ for name in ENTITIES if columns(published, name)
631
+ and (name not in REPLACE_EACH_PASS or REPLACE_EACH_PASS[name].get("scope") == "internal")}
632
  record("row_count_drop", [
633
  {"table": name, "previous": count, "candidate": row_counts.get(name, 0),
634
  "removed": count - row_counts.get(name, 0),
 
1937
  for account_id in key.split(",")]}
1938
  for key, share in v.items()
1939
  ], ensure_ascii=False)
1940
+ elif f in ID_LIST_FIELDS: # a 2-id list would otherwise read as an m2o pair
1941
+ out[f] = json.dumps(list(v or []))
1942
  elif isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
1943
  out[f"{f.removesuffix('_id')}_id"] = v[0]
1944
  out[f"{f.removesuffix('_id')}_name"] = str(v[1] or "")
 
2003
 
2004
  # Odoo also names this many2one without the usual `_id` suffix.
2005
  MANY2ONE_FIELDS = frozenset({"product_uom", "create_uid"})
2006
+ # Plain x2many fields stored whole as a JSON id list, `[]` for none.
2007
+ ID_LIST_FIELDS = frozenset({"product_tag_ids"})
2008
 
2009
  # S72-T43 / R107: the audited Odoo fields the Data grids show. Each one is declared AND
2010
  # backfilled, because existing rows never re-sync when the schema grows. The list must equal
 
2128
  },
2129
  })
2130
 
2131
+ # The product pool's pricing and tags. Each table holds exactly Odoo's default (active_test) read,
2132
+ # so it carries no `active` column and a reader needs no active filter.
2133
+ ENTITIES.update({
2134
+ "product_pricelist": {
2135
+ "model": "product.pricelist", "archivable": False, "optional": True,
2136
+ "fields": ["name"],
2137
+ },
2138
+ "product_pricelist_item": {
2139
+ "model": "product.pricelist.item", "archivable": False, "optional": True,
2140
+ "fields": ["pricelist_id", "product_id", "product_tmpl_id", "applied_on", "compute_price",
2141
+ "fixed_price", "min_quantity", "date_start", "date_end", "packaging_id",
2142
+ "packaging_price"],
2143
+ },
2144
+ "product_tag": {
2145
+ "model": "product.tag", "archivable": False, "optional": True,
2146
+ "fields": ["name"],
2147
+ },
2148
+ })
2149
+ NUMERIC_FIELDS += ("volume", "incoming_qty", "free_qty", "virtual_available", "fixed_price",
2150
+ "min_quantity")
2151
+
2152
+ #: Entities read in full every pass and swapped into the mirror in one transaction. A cursor sync
2153
+ #: cannot see an Odoo delete, or a quant quantity that moved without a write_date change.
2154
+ #: Scope `internal` replaces the internal-location quants only. Any other scope is the whole table.
2155
+ REPLACE_EACH_PASS = {
2156
+ "stock_quant": {"domain": [("location_id.usage", "=", "internal")], "scope": "internal"},
2157
+ "product_pricelist": {"domain": []},
2158
+ "product_pricelist_item": {"domain": []},
2159
+ "product_tag": {"domain": []},
2160
+ }
2161
+ REPLACE_PAGE = 5000
2162
+
2163
  _connector_spec.register("odoo", _connector_spec.tables_for("odoo"),
2164
  mirror=_connector_spec.ConnectorMirrorSpec(tuple(ENTITIES)))
2165
 
 
2390
  return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2391
  candidate = getattr(_SYNC_TLS, "candidate", None)
2392
  if candidate is not None:
2393
+ phase, pulled = _sync_one(key, max_batches=max_batches, batch=batch, log=log)
2394
  # A recurring refresh may report changed pages to a bespoke cursor, but a
2395
  # tenant pass cannot claim completion until those private pages finish too.
2396
  pending = candidate["con"].execute(
 
2401
  return phase, pulled
2402
  rt = _resident_odoo_runtime()
2403
  if rt is None:
2404
+ return _sync_one(key, max_batches=max_batches, batch=batch, log=log)
2405
  from harness import mirror_store
2406
  report = {}
2407
  with mirror_store.sync_mirror("odoo", rt.key, rt, report=report, log=log, expected_tables=[key]):
 
2545
  con.close()
2546
 
2547
 
2548
+ def _sync_one(key, max_batches=200, batch=BATCH, log=print):
2549
+ """One entity's pass: the cursor sync, the full replace, or the cursor sync then the replace."""
2550
+ plan = REPLACE_EACH_PASS.get(key)
2551
+ if plan is None or source_paused():
2552
+ return _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2553
+ if plan.get("scope") != "internal":
2554
+ return _replace_entity(key, log=log)
2555
+ phase, pulled = _sync_entity(key, max_batches=max_batches, batch=batch, log=log)
2556
+ return phase, pulled + _replace_entity(key, log=log)[1]
2557
+
2558
+
2559
+ def _replace_scope(con, key, plan):
2560
+ """SQL predicate for the mirror rows a replace owns. `_replace_stage` holds the fresh read."""
2561
+ if plan.get("scope") != "internal":
2562
+ return "TRUE"
2563
+ terms = ["id IN (SELECT id FROM _replace_stage)",
2564
+ "location_id IN (SELECT location_id FROM _replace_stage WHERE location_id IS NOT NULL)"]
2565
+ if {"id", "usage"} <= set(columns_of("stock_location", cur=con) or ()):
2566
+ terms.append("location_id IN (SELECT id FROM stock_location WHERE usage = 'internal')")
2567
+ return "(" + " OR ".join(terms) + ")"
2568
+
2569
+
2570
+ def _replace_entity(key, log=print):
2571
+ """Read every row the plan's domain matches, then swap the mirror's scope for it in one transaction.
2572
+
2573
+ Returns (phase, changed_rows). A failed or short read raises before any delete.
2574
+ """
2575
+ if source_paused():
2576
+ _paused_notice(f"full read of {key}", log)
2577
+ return PAUSED_PHASE, 0
2578
+ spec, plan = ENTITIES[key], REPLACE_EACH_PASS[key]
2579
+ domain = list(plan["domain"])
2580
+ fields = spec["fields"] + list(spec.get("m2m") or [])
2581
+ expected = int(O.get_odoo().search_count(spec["model"], domain))
2582
+ rows, last = [], 0
2583
+ while True:
2584
+ recs = O.search_read(spec["model"], domain + [("id", ">", last)], ["id"] + fields,
2585
+ limit=REPLACE_PAGE, order="id asc")
2586
+ if any(field not in rec for rec in recs for field in fields):
2587
+ raise RuntimeError(f"{key}: full read omitted a requested field. The mirror keeps its previous rows.")
2588
+ rows += _flatten_records(recs, spec["fields"], spec.get("m2m") or (),
2589
+ named_m2m=spec.get("named_m2m"))
2590
+ # Stop on an EMPTY page: a server-side page cap returns short pages before the end.
2591
+ if not recs:
2592
+ break
2593
+ last = recs[-1]["id"]
2594
+ # Rows moved during the read when the read matches a second count. Refuse only a read
2595
+ # that disagrees with both counts, before any delete.
2596
+ if len(rows) != expected:
2597
+ after = int(O.get_odoo().search_count(spec["model"], domain))
2598
+ if len(rows) != after:
2599
+ raise RuntimeError(f"{key}: full read returned {len(rows)} rows, counts were "
2600
+ f"{expected} and {after}. The mirror keeps its previous rows.")
2601
+ con = connect()
2602
+ try:
2603
+ _ensure_table(con, key, spec)
2604
+ _ensure_columns(con, key, spec)
2605
+ colnames = [c.split()[0] for c in _cols(spec)]
2606
+ cols = ", ".join(colnames)
2607
+ con.execute(f"CREATE OR REPLACE TEMP TABLE _replace_stage AS SELECT {cols} FROM {key} LIMIT 0")
2608
+ if rows:
2609
+ con.executemany(f"INSERT INTO _replace_stage ({cols}) VALUES ({', '.join('?' for _ in colnames)})",
2610
+ [[r.get(c) for c in colnames] for r in rows])
2611
+ scope = _replace_scope(con, key, plan)
2612
+ changed = con.execute(
2613
+ f"SELECT count(DISTINCT id) FROM ((SELECT {cols} FROM _replace_stage "
2614
+ f"EXCEPT SELECT {cols} FROM {key} WHERE {scope}) UNION ALL "
2615
+ f"(SELECT {cols} FROM {key} WHERE {scope} EXCEPT SELECT {cols} FROM _replace_stage))"
2616
+ ).fetchone()[0]
2617
+ if changed:
2618
+ con.execute("BEGIN TRANSACTION")
2619
+ try:
2620
+ con.execute(f"DELETE FROM {key} WHERE {scope}")
2621
+ con.execute(f"INSERT INTO {key} ({cols}) SELECT {cols} FROM _replace_stage")
2622
+ except BaseException:
2623
+ con.execute("ROLLBACK")
2624
+ raise
2625
+ con.execute("COMMIT")
2626
+ if plan.get("scope") == "internal":
2627
+ n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
2628
+ con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key])
2629
+ else:
2630
+ _save_state(con, key, "live", max((r["id"] for r in rows), default=0), None)
2631
+ log(f" {key}: full read {len(rows):,} rows, {changed:,} changed")
2632
+ return "live", changed
2633
+ finally:
2634
+ try:
2635
+ con.execute("DROP TABLE IF EXISTS _replace_stage")
2636
+ finally:
2637
+ con.close()
2638
+
2639
+
2640
  def _already(con, key, rec):
2641
  return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
2642
 
platform/modules/inventory.py CHANGED
@@ -312,7 +312,7 @@ def validate_mirror(cur, t=None, codes=None, sample=40):
312
  """Reconcile the mirror-built `sku_inventory` to a live bounded build; this oracle reads live Odoo.
313
 
314
  Returns `{'checked', 'mismatches': [{sku, column, live, mirror}], 'columns'}` at tolerance 0.01.
315
- Both sides read on_hand live, so a mismatch names drift in cost, category, names or sales.
316
  """
317
  from harness.canonical_source import MirrorSource, active, bounded, mirrored
318
  if isinstance(active(), MirrorSource):
 
312
  """Reconcile the mirror-built `sku_inventory` to a live bounded build; this oracle reads live Odoo.
313
 
314
  Returns `{'checked', 'mismatches': [{sku, column, live, mirror}], 'columns'}` at tolerance 0.01.
315
+ The mirror side reads on_hand from the mirror's internal quants, so a mismatch also names quant drift.
316
  """
317
  from harness.canonical_source import MirrorSource, active, bounded, mirrored
318
  if isinstance(active(), MirrorSource):
platform/modules/products.py CHANGED
@@ -333,6 +333,12 @@ def inventory_value_per_quantity(inv_value, quantity):
333
  return inventory_value_per_quantity_result(inv_value, quantity)[0]
334
 
335
 
 
 
 
 
 
 
336
  def catalogue():
337
  """`{sku_code: {'product', 'category', 'id', 'created_at', 'upc', 'discontinued',
338
  'incoming', 'available', 'forecasted', 'product_tmpl_id', 'volume'}}` β€” EVERY ACTIVE product, sold or not.
@@ -405,7 +411,9 @@ def catalogue():
405
  f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
406
  f"nothing reporting it; raise the limit before shipping this.")
407
  code_cat = _code_category()
408
- disc_tag = discontinued_tag_id()
 
 
409
  out = {}
410
  for p in prods:
411
  code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
@@ -437,19 +445,19 @@ def catalogue():
437
  'volume': (float(p['volume']) if isinstance(p.get('volume'),
438
  (int, float)) and not isinstance(p['volume'], bool)
439
  else None),
440
- 'discontinued': 'No', 'incoming': 0.0,
441
- 'available': 0.0, 'forecasted': 0.0})
442
  # β›” ACROSS EVERY RECORD SHARING THE CODE, not just the first one that won above. A SKU
443
  # whose variants are separate records carries its inbound on whichever record the PO named,
444
  # and 'first record wins' would drop the rest β€” a buy list that under-counts what is
445
  # already on the water tells you to re-order stock you have already bought.
446
- row['incoming'] += float(p.get('incoming_qty') or 0.0)
447
  # S72-T40 (R104): Available is Odoo's `free_qty` (on hand minus reserved) and Forecasted is
448
  # `virtual_available` (on hand plus incoming minus outgoing). Both are unstored computed
449
  # fields Odoo refuses to aggregate, so they ride this read and accumulate per code like
450
  # `incoming`. `validate_available_forecasted` ties the totals to stock.quant and stock.move.
451
- row['available'] += float(p.get('free_qty') or 0.0)
452
- row['forecasted'] += float(p.get('virtual_available') or 0.0)
453
  # ⭐ W40-T08 β€” the EARLIEST create_date across the records sharing this code, not the
454
  # first record's. A row here is a SKU, and the honest answer to "when was this record
455
  # created" for a merged SKU is when the earliest of its records appeared. Only `2112-12`
@@ -503,7 +511,9 @@ def discontinued_tag_id():
503
  RED on the difference instead of shipping a column that quietly says nobody is discontinued.
504
  """
505
  rows = O.search_read('product.tag', [('name', '=ilike', DISCONTINUED_TAG)], ['id'], limit=2)
506
- return rows[0]['id'] if rows else None
 
 
507
 
508
 
509
  def incoming_from_po(prods=None):
@@ -657,7 +667,12 @@ def active_products(limit=50000, extra_fields=()):
657
  on this connection, on a path that is somebody's first page load.
658
  """
659
  dom = [('active', '=', True)]
660
- fields = ['id', 'default_code', 'product_tmpl_id'] + [str(f) for f in extra_fields if str(f) not in {'id', 'default_code', 'product_tmpl_id'}]
 
 
 
 
 
661
  prods = O.search_read('product.product', dom, fields,
662
  limit=limit)
663
  n = O.get_odoo().search_count('product.product', dom)
 
333
  return inventory_value_per_quantity_result(inv_value, quantity)[0]
334
 
335
 
336
+ def _add_qty(row, key, value):
337
+ """Add a record's quantity to its SKU row. None (a mirror column not yet synced) stays blank, never 0."""
338
+ if value is not None and value is not False:
339
+ row[key] = (row[key] or 0.0) + float(value)
340
+
341
+
342
  def catalogue():
343
  """`{sku_code: {'product', 'category', 'id', 'created_at', 'upc', 'discontinued',
344
  'incoming', 'available', 'forecasted', 'product_tmpl_id', 'volume'}}` β€” EVERY ACTIVE product, sold or not.
 
411
  f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
412
  f"nothing reporting it; raise the limit before shipping this.")
413
  code_cat = _code_category()
414
+ from harness.canonical_source import active
415
+ # A source-bound read must not fill the process cache: an unsynced tag table would pin "No".
416
+ disc_tag = discontinued_tag_id.__wrapped__() if active() is not None else discontinued_tag_id()
417
  out = {}
418
  for p in prods:
419
  code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
 
445
  'volume': (float(p['volume']) if isinstance(p.get('volume'),
446
  (int, float)) and not isinstance(p['volume'], bool)
447
  else None),
448
+ 'discontinued': 'No', 'incoming': None,
449
+ 'available': None, 'forecasted': None})
450
  # β›” ACROSS EVERY RECORD SHARING THE CODE, not just the first one that won above. A SKU
451
  # whose variants are separate records carries its inbound on whichever record the PO named,
452
  # and 'first record wins' would drop the rest β€” a buy list that under-counts what is
453
  # already on the water tells you to re-order stock you have already bought.
454
+ _add_qty(row, 'incoming', p.get('incoming_qty'))
455
  # S72-T40 (R104): Available is Odoo's `free_qty` (on hand minus reserved) and Forecasted is
456
  # `virtual_available` (on hand plus incoming minus outgoing). Both are unstored computed
457
  # fields Odoo refuses to aggregate, so they ride this read and accumulate per code like
458
  # `incoming`. `validate_available_forecasted` ties the totals to stock.quant and stock.move.
459
+ _add_qty(row, 'available', p.get('free_qty'))
460
+ _add_qty(row, 'forecasted', p.get('virtual_available'))
461
  # ⭐ W40-T08 β€” the EARLIEST create_date across the records sharing this code, not the
462
  # first record's. A row here is a SKU, and the honest answer to "when was this record
463
  # created" for a merged SKU is when the earliest of its records appeared. Only `2112-12`
 
511
  RED on the difference instead of shipping a column that quietly says nobody is discontinued.
512
  """
513
  rows = O.search_read('product.tag', [('name', '=ilike', DISCONTINUED_TAG)], ['id'], limit=2)
514
+ # A source-bound read returns iterable `Rows`, not a list.
515
+ first = next(iter(rows), None)
516
+ return first['id'] if first else None
517
 
518
 
519
  def incoming_from_po(prods=None):
 
667
  on this connection, on a path that is somebody's first page load.
668
  """
669
  dom = [('active', '=', True)]
670
+ extra = [str(f) for f in extra_fields if str(f) not in {'id', 'default_code', 'product_tmpl_id'}]
671
+ from harness.canonical_source import active, MirrorSource
672
+ if extra and isinstance(active(), MirrorSource):
673
+ # Standing rule 1: a field the mirror cannot serve is dropped and logged, never read live.
674
+ extra = active().servable('product.product', extra)
675
+ fields = ['id', 'default_code', 'product_tmpl_id'] + extra
676
  prods = O.search_read('product.product', dom, fields,
677
  limit=limit)
678
  n = O.get_odoo().search_count('product.product', dom)
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- 6d121476c2dd8aa30ffd7e889e1e65347ef34406966e9f75fe655ea23f070f7d
 
1
+ b2b8f4407a2c83b27f7a01a8589a5d7ec4737a35f984ce2f9bc81b8812cc64fe