fsanyoto commited on
Commit
b2aa446
·
verified ·
1 Parent(s): 64006c1

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/agent_directory.py +20 -3
  4. api/amazon_quotes_loop.py +128 -0
  5. api/app_templates.py +10 -5
  6. api/application_registry.py +274 -17
  7. api/assistant_memory.py +69 -0
  8. api/assistant_mirror.py +19 -12
  9. api/automation_engine.py +383 -14
  10. api/connector_onboarding.py +6 -5
  11. api/deps.py +228 -58
  12. api/diag_aging.py +8 -13
  13. api/external_freshness.py +22 -9
  14. api/loopy_file_tools.py +3 -1
  15. api/loopy_tools.py +1 -1
  16. api/main.py +610 -62
  17. api/odoo_relational.py +144 -35
  18. api/pages.py +6 -4
  19. api/process_role.py +366 -161
  20. api/registry_source.py +193 -1
  21. api/rollup_sql.py +19 -7
  22. api/routes_account_health.py +141 -10
  23. api/routes_admin.py +107 -8
  24. api/routes_aged_stock.py +8 -5
  25. api/routes_amazon_dashboard.py +18 -43
  26. api/routes_amazon_operator.py +103 -15
  27. api/routes_amazon_profit.py +163 -19
  28. api/routes_amazon_projects.py +389 -0
  29. api/routes_app_dashboard.py +72 -26
  30. api/routes_applications.py +1 -1
  31. api/routes_assets.py +22 -4
  32. api/routes_auth.py +5 -1
  33. api/routes_competitor_watch.py +80 -16
  34. api/routes_competitors.py +203 -0
  35. api/routes_connectors.py +127 -28
  36. api/routes_customers.py +86 -16
  37. api/routes_external.py +517 -54
  38. api/routes_feedback.py +45 -4
  39. api/routes_grid.py +44 -2
  40. api/routes_keychain.py +282 -3
  41. api/routes_nav.py +12 -8
  42. api/routes_odoo_tables.py +177 -19
  43. api/routes_opportunity_finder.py +47 -24
  44. api/routes_payouts.py +38 -15
  45. api/routes_products.py +24 -2
  46. api/routes_query.py +10 -4
  47. api/routes_ranking_diagnostic.py +58 -34
  48. api/routes_reimbursements.py +60 -18
  49. api/routes_repeat_buyers.py +16 -11
  50. api/routes_restock_planner.py +96 -6
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "eaf53db9bc88",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "53755382a",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- eaf53db9bc88
 
1
+ 53755382a
api/agent_directory.py CHANGED
@@ -166,11 +166,28 @@ def _filter_sentences(tree, seen, label):
166
  return [LIMITED if p is None else prefix + p for p in parts]
167
 
168
 
169
- def _legacy_limits(rec):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170
  out = []
171
  labels = users.allowed_bus_labels(rec)
172
  if "All" not in labels and labels:
173
- out.append("Business unit is " + " or ".join(labels))
174
  agent = perms.scope_agent(rec)
175
  if agent:
176
  out.append(f"Sales agent is {agent}")
@@ -210,7 +227,7 @@ def _entry(session, uname, rec, ctx):
210
  if unnamed:
211
  named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
212
  if not admin and not perm_scope.is_migrated(principal):
213
- limits = _legacy_limits(rec) + limits
214
  return {"username": str(rec.get("username_original") or uname),
215
  "name": str(rec.get("name") or uname),
216
  "role": "Administrator" if admin else "Member",
 
166
  return [LIMITED if p is None else prefix + p for p in parts]
167
 
168
 
169
+ #: The Odoo field a legacy team scope narrows, and Odoo's own name for it (R66).
170
+ _TEAM_TABLE, _TEAM_FIELD, _TEAM_LABEL = "ut_odoo_orders", "team", "Sales team"
171
+
172
+
173
+ def _team_label(st, caller):
174
+ """The caller-visible team label, with Odoo's name for an absent field."""
175
+ try:
176
+ from core import user_tables
177
+ fields = (user_tables.get_one(_TEAM_TABLE, st) or {}).get("fields") or ()
178
+ field = next((f for f in fields if isinstance(f, dict) and f.get("key") == _TEAM_FIELD), {})
179
+ if field:
180
+ return _name_field(_fields(caller, _TEAM_TABLE, st), _TEAM_FIELD)
181
+ return _TEAM_LABEL
182
+ except Exception: # noqa: BLE001
183
+ return _TEAM_LABEL
184
+
185
+
186
+ def _legacy_limits(rec, st=None, caller=None):
187
  out = []
188
  labels = users.allowed_bus_labels(rec)
189
  if "All" not in labels and labels:
190
+ out.append(f"{_team_label(st, caller)} is " + " or ".join(labels))
191
  agent = perms.scope_agent(rec)
192
  if agent:
193
  out.append(f"Sales agent is {agent}")
 
227
  if unnamed:
228
  named.append(f"{unnamed} database{'' if unnamed == 1 else 's'} you cannot open")
229
  if not admin and not perm_scope.is_migrated(principal):
230
+ limits = _legacy_limits(rec, st, caller) + limits
231
  return {"username": str(rec.get("username_original") or uname),
232
  "name": str(rec.get("name") or uname),
233
  "role": "Administrator" if admin else "Member",
api/amazon_quotes_loop.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Price and rank every rival once a day per tenant in the sync worker, outside the sync pass (C7).
2
+
3
+ The loop holds the SP-API client and passes its reads into `core.amazon_rivals.run_quotes`, because
4
+ core imports no harness module. It never joins `amazon_store.AFTER_SYNC` and never runs as an HF job.
5
+ """
6
+ from contextlib import contextmanager
7
+ import importlib
8
+ import logging
9
+ from pathlib import Path
10
+ import threading
11
+ import time
12
+
13
+ JOB = "amazon-rival-quotes"
14
+ #: The loop asks each tenant whether its daily run is due this often.
15
+ POLL_SECONDS = 600
16
+
17
+ #: Called as `fn(tenant, changed_asins)` after a run finishes, where `changed_asins` is a tuple of the
18
+ #: ASINs whose newest quote changed. Each import is lazy and a failure is one log line, never a failed run.
19
+ #: Command Center patches the competitor cells of the rows those ASINs move (S82-T89).
20
+ AFTER_QUOTES = ("modules.amazon_profit:on_quotes", "modules.competitor_watch:on_quotes")
21
+
22
+ _log = logging.getLogger(__name__)
23
+ _missing = False
24
+ _unavailable = set()
25
+ _thread = None
26
+ _stop = None
27
+
28
+
29
+ def _say(message):
30
+ print(f"[{JOB}] {message}", flush=True)
31
+
32
+
33
+ def readers(fields):
34
+ """The three SP-API reads of one tenant's credential. Each takes a list of at most 20 ASINs."""
35
+ from harness import amazon_client as cx
36
+ return {"offers": lambda asins: cx.item_offers(fields, asins),
37
+ # The pricing sync draws this same `competitive_summary` pacer in this worker process.
38
+ "summaries": lambda asins: cx.competitive_summaries(fields, asins),
39
+ "ranks": lambda asins: cx.catalog_sales_ranks(fields, asins)}
40
+
41
+
42
+ @contextmanager
43
+ def _mirror(runtime):
44
+ """A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
45
+ from harness import runtime as R
46
+ if not Path(runtime.datastore_path).is_file():
47
+ yield None
48
+ return
49
+ cur = R.mirror_cursor(runtime, require_ready=False)
50
+ try:
51
+ yield cur
52
+ finally:
53
+ cur.close()
54
+
55
+
56
+ def _after_quotes(tenant, changed):
57
+ for target in AFTER_QUOTES:
58
+ try:
59
+ module_name, _, name = target.partition(":")
60
+ getattr(importlib.import_module(module_name), name)(tenant, changed)
61
+ _say(f"tenant={tenant} after-quotes callback {target} completed")
62
+ except Exception as exc: # noqa: BLE001
63
+ _say(f"tenant={tenant} after-quotes callback {target} failed ({type(exc).__name__}); "
64
+ "the run result is unchanged.")
65
+
66
+
67
+ def run_tenant(tenant, runtime, *, now=None, fields=None, sleep=time.sleep):
68
+ """Run today's quotes for one tenant when due. None when it is not due or has no Amazon store."""
69
+ from core import amazon_rivals as rivals
70
+ try:
71
+ if not rivals.due(runtime, now=now):
72
+ return None
73
+ except rivals.BackendUnavailable:
74
+ if tenant not in _unavailable:
75
+ _unavailable.add(tenant)
76
+ _say(f"tenant={tenant} has no PostgreSQL store, so no rival quotes run")
77
+ return None
78
+ if fields is None:
79
+ from harness import amazon_store
80
+ fields = amazon_store.credential(tenant, runtime)
81
+ if not fields:
82
+ return None
83
+ result = rivals.run_quotes(runtime, lambda: _mirror(runtime), **readers(fields),
84
+ seller_id=str(fields.get("seller_id") or "").strip() or None, now=now,
85
+ log=lambda message: _say(f"tenant={tenant} {message}"), sleep=sleep)
86
+ if result["cause"] is None:
87
+ _after_quotes(tenant, result["changed"])
88
+ return result
89
+
90
+
91
+ def run_once(*, for_each_tenant=None, now=None):
92
+ """Admit one independent per-tenant job without waiting for its body."""
93
+ global _missing
94
+ if for_each_tenant is None:
95
+ try:
96
+ from tenant_loops import for_each_tenant
97
+ except ImportError:
98
+ if not _missing:
99
+ _log.warning("Rival quotes unavailable: tenant_loops is not installed.")
100
+ _missing = True
101
+ return {}
102
+ _missing = False
103
+ return for_each_tenant(lambda tenant, runtime: run_tenant(tenant, runtime, now=now),
104
+ name=JOB, logger=_log, kind="heavy")
105
+
106
+
107
+ def start():
108
+ """Start the daily loop only when process_role admits its heavy coordinator (the sync worker)."""
109
+ global _thread, _stop
110
+ import process_role
111
+ if _thread is not None and _thread.is_alive():
112
+ return _thread
113
+ stop_event = threading.Event()
114
+ _stop = stop_event
115
+
116
+ def run():
117
+ while not stop_event.is_set():
118
+ run_once()
119
+ stop_event.wait(POLL_SECONDS)
120
+
121
+ # A literal name: verify_background_loops classifies a job by it.
122
+ _thread = process_role.start("amazon-rival-quotes", "heavy", run)
123
+ return _thread
124
+
125
+
126
+ def stop():
127
+ if _stop is not None:
128
+ _stop.set()
api/app_templates.py CHANGED
@@ -78,11 +78,12 @@ TEMPLATES = (
78
  "module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
79
  "proposes": False},
80
  _amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
81
- "Acquire", ("ut_amzn_brand_analytics",), proposes=False),
82
  _amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
83
  "Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
84
  "ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
85
- "ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False),
 
86
  _amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
87
  "Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
88
  "ut_amzn_economics"), connector="amazon_ads", proposes=False),
@@ -100,11 +101,15 @@ TEMPLATES = (
100
  "Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
101
  optional_connectors=("odoo",)),
102
  _amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
103
- "Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "ut_amzn_fba_inventory",
104
- "ut_amzn_removal_orders", "ut_amzn_removal_shipments", "ut_amzn_storage_fees"),
 
 
105
  optional_connectors=("odoo",)),
106
  _amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
107
- "Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False),
 
 
108
  _amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
109
  "Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
110
  optional_connectors=("odoo",)),
 
78
  "module": None, "router": None, "view": None, "tools": ("catalog_build", "catalog_pdf"),
79
  "proposes": False},
80
  _amazon("competitor_watch", "Competitor watch", "See who ranks on your search terms each week.",
81
+ "Acquire", ("ut_amzn_brand_analytics", "ut_amzn_listings"), proposes=False),
82
  _amazon("ranking_diagnostic", "Ranking diagnostic", "Find why a listing lost its rank.",
83
  "Acquire", ("ut_amzn_pricing", "ut_amzn_catalog_items", "ut_amzn_brand_analytics",
84
  "ut_amzn_economics", "ut_amzn_fba_inventory", "ut_amzn_suppressed_listings",
85
+ "ut_amzn_stranded_inventory", "ut_amzn_inventory_planning"), proposes=False,
86
+ optional_sources=("ut_amzn_search_query_performance", "ut_amzn_listing_items")),
87
  _amazon("amazon_operator", "Operator inbox", "Approve ad changes in one batch.",
88
  "Acquire", ("ut_amzn_ads_sp_campaigns", "ut_amzn_ads_sb_campaigns", "ut_amzn_ads_sd_campaigns",
89
  "ut_amzn_economics"), connector="amazon_ads", proposes=False),
 
101
  "Create value", ("ut_amzn_brand_analytics", "ut_amzn_search_query_performance", "ut_amzn_listings"),
102
  optional_connectors=("odoo",)),
103
  _amazon("returns_refunds", "Returns and refunds", "See why buyers return and track each fix.",
104
+ "Retain", ("ut_amzn_fba_customer_returns", "ut_amzn_fba_shipped_sales", "ut_amzn_mfn_returns",
105
+ "ut_amzn_settlements", "ut_amzn_transactions", "ut_amzn_order_items"),
106
+ optional_sources=("ut_amzn_fba_inventory", "ut_amzn_removal_orders",
107
+ "ut_amzn_removal_shipments", "ut_amzn_storage_fees"),
108
  optional_connectors=("odoo",)),
109
  _amazon("repeat_buyers", "Repeat buyers", "See who buys again and what they buy together.",
110
+ "Retain", ("ut_amzn_repeat_purchase", "ut_amzn_market_basket"), proposes=False,
111
+ optional_sources=("ut_amzn_replenishment", "ut_amzn_subscribe_save_performance",
112
+ "ut_amzn_fba_shipped_sales", "ut_amzn_listings")),
113
  _amazon("payouts", "Payouts", "Tie each payout to its orders and fees.",
114
  "Finance and cash", ("ut_amzn_settlements", "ut_amzn_economics", "ut_amzn_transactions"),
115
  optional_connectors=("odoo",)),
api/application_registry.py CHANGED
@@ -185,6 +185,7 @@ def _connector_gate(connector):
185
 
186
  # Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
187
  _TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
 
188
  else _connector_gate(contract["connector"])
189
  for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
190
  #: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
@@ -552,7 +553,8 @@ def load(runtime, *, strict=False):
552
  _AVAILABILITY_MESSAGES = {
553
  "router_unavailable": "This App cannot open because its routes are unavailable.",
554
  "no_grant": "Your access limit prevents this App from reading its source.",
555
- "not_connected": "Connect the App's connector to open this App.",
 
556
  "no_rows": "This App has no source rows yet.",
557
  "source_unavailable": "This App cannot read its source right now.",
558
  "held": "This App is not available yet.",
@@ -565,7 +567,7 @@ def availability_reason(state, connector=None):
565
  if state == "not_connected":
566
  label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
567
  if label:
568
- message = f"Connect {label} to open this App."
569
  return {"state": state, "message": message,
570
  "lock": "Access limit" if state == "no_grant" else None}
571
 
@@ -631,8 +633,116 @@ def _readable_template_app(session, app_id):
631
  return app, catalogue
632
 
633
 
634
- def _availability_for_app(session, app, catalogue):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
635
  from core import perm_scope
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
636
  from routes_connectors import _entry_types
637
 
638
  kind = app["templateKind"]
@@ -643,28 +753,51 @@ def _availability_for_app(session, app, catalogue):
643
  return mount
644
  if not _app_templates.seams_ready(kind):
645
  return availability_reason("router_unavailable")
 
 
 
 
 
 
 
646
  try:
647
  sources = catalogue["sources"]
648
- for source in sources:
649
- if not perm_scope.may_read(session.user, source, st=session.runtime):
650
- return availability_reason("no_grant")
651
- # These App readers cannot apply a row or field wall, so refuse before probing.
652
- if perm_scope.wall_declared(session.user, source, st=session.runtime):
653
- return availability_reason("no_grant")
 
 
 
 
 
 
 
 
654
  types, _scopes, failure = _entry_types(session.runtime, session.uname)
655
  if failure is not None:
656
- return availability_reason("source_unavailable")
657
  connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
658
  if not connected and catalogue["connector"] == "odoo":
659
  from routes_keychain import env_odoo_available
660
  connected = env_odoo_available(session.runtime)
661
  if not connected:
662
- return availability_reason("not_connected", catalogue["connector"])
663
- if sources and not _source_rows_exist(session, sources):
664
- return availability_reason("no_rows")
665
- except Exception:
666
- return availability_reason("source_unavailable")
667
- return availability_reason("ready")
 
 
 
 
 
 
 
 
668
 
669
 
670
  def availability(session, app_id):
@@ -1179,7 +1312,8 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
1179
  mount = _mount_reason(row.get("templateKind"))
1180
  if mount:
1181
  row["availability"] = mount
1182
- elif not _database_free(app) and not compatible:
 
1183
  row["availability"] = availability_reason("no_grant")
1184
  if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
1185
  # A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
@@ -1644,6 +1778,129 @@ def migrate_returns_split(runtime, tenant):
1644
  return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
1645
 
1646
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1647
  def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
1648
  """R15: boot migrations, once per tenant, never from an Application read.
1649
 
 
185
 
186
  # Competitor watch's gate already requires the Amazon connection; it adds brand analytics (R21).
187
  _TEMPLATE_GATES = {kind: brand_analytics_offered if kind == "competitor_watch"
188
+ else _connector_gate("amazon") if kind == "amazon_operator"
189
  else _connector_gate(contract["connector"])
190
  for kind, contract in _GATED_TEMPLATE_CONTRACTS.items()}
191
  #: S79-T02 (instr 20): Routing and Catalog are offered only to a reader with Odoo.
 
553
  _AVAILABILITY_MESSAGES = {
554
  "router_unavailable": "This App cannot open because its routes are unavailable.",
555
  "no_grant": "Your access limit prevents this App from reading its source.",
556
+ "not_connected": "Connect the App's connector in Connectors to open this App.",
557
+ "refreshing": "This App's source is refreshing and opens when the refresh ends",
558
  "no_rows": "This App has no source rows yet.",
559
  "source_unavailable": "This App cannot read its source right now.",
560
  "held": "This App is not available yet.",
 
567
  if state == "not_connected":
568
  label = {"amazon": "Amazon", "amazon_ads": "Amazon Ads", "odoo": "Odoo"}.get(connector)
569
  if label:
570
+ message = f"Connect {label} in Connectors to open this App."
571
  return {"state": state, "message": message,
572
  "lock": "Access limit" if state == "no_grant" else None}
573
 
 
633
  return app, catalogue
634
 
635
 
636
+ #: R62: App kinds whose readers apply a source row and field wall to their own rows.
637
+ #: Any other kind refuses its App while such a wall applies, because it cannot narrow.
638
+ NARROWING_KINDS = frozenset({"account_health", "opportunity_finder", "ranking_diagnostic", "repeat_buyers"})
639
+
640
+
641
+ def app_access(user, runtime, app, sources):
642
+ """R46, R62: the one source-wall answer for an App, its Dashboard and its readers.
643
+
644
+ Takes a user record, not a session, so a digest can ask for any recipient. Returns
645
+ `app` and `dashboard` ("ready" or refused), the Dashboard refusal's `status`, `code`
646
+ and one-line `message`, and `narrow`: per source, the row wall and hidden field keys.
647
+ """
648
+ from core import perm_scope, perms, route_system, user_tables
649
+ from core.request_timing import remaining_budget
650
+
651
+ user = user if isinstance(user, dict) else {}
652
+ kind = app.get("templateKind") if isinstance(app, dict) else None
653
+ granted = may_read(app, user.get("username"), perms.is_admin(user))
654
+ refusal = []
655
+
656
+ def refuse(status, code, message):
657
+ refusal.append((status, code, message))
658
+
659
+ all_units = "All" in perms.allowed_bu_labels(user)
660
+ raw_units = user.get("bus", "all")
661
+ if raw_units != "all":
662
+ from core import users
663
+ by_label = {label: key for key, label in users.BU_LABELS.items()}
664
+ identities = []
665
+ valid_units = isinstance(raw_units, (tuple, list)) and bool(raw_units)
666
+ for unit in raw_units if valid_units else ():
667
+ if type(unit) is int and unit in users.BU_LABELS:
668
+ identities.append(unit)
669
+ elif isinstance(unit, str) and unit in by_label:
670
+ identities.append(by_label[unit])
671
+ elif unit != "All":
672
+ valid_units = False
673
+ all_units = bool(valid_units and identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
674
+ if not all_units:
675
+ refuse(403, "app_dashboard_scope", "The Dashboard needs access to every business unit.")
676
+ elif perms.scope_agent(user) is not None:
677
+ refuse(403, "app_dashboard_scope", "The Dashboard needs access to every salesperson's records.")
678
+ if not isinstance(sources, (tuple, list)) or not sources:
679
+ refuse(503, "app_dashboard_sources", "The Dashboard has no source to read.")
680
+ sources = ()
681
+ config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
682
+ enabled = config.get("modules", "all")
683
+ narrow = {}
684
+ for source in sources:
685
+ remaining_budget()
686
+ label = _source_label(source) or "a source database"
687
+ definition = user_tables.get_definition(source, st=runtime)
688
+ fields = definition.get("fields") if isinstance(definition, dict) else None
689
+ declared = perm_scope.entry(user, source) or {}
690
+ walled = bool(perm_scope.wall_declared(user, source, st=runtime, defn=definition))
691
+ rows = bool(declared.get("filter")) or bool(
692
+ fields and perm_scope.row_scope_applies(user, source, st=runtime, defn=definition))
693
+ hidden = set(declared.get("hiddenFields") or ()) | set(
694
+ perm_scope.hidden_keys(user, source, fields, st=runtime) if fields else ())
695
+ narrow[source] = {"rows": rows or (walled and not hidden), "hidden": tuple(sorted(map(str, hidden)))}
696
+ # The Dashboard totals the raw source, so it keeps the source grant and refuses any narrowing.
697
+ if (enabled != "all" and source not in (enabled or ())) or not perm_scope.may_read(user, source, st=runtime):
698
+ refuse(403, "app_dashboard_source", f"The Dashboard needs read access to {label}.")
699
+ elif not fields:
700
+ refuse(503, "app_dashboard_source", f"The Dashboard cannot read {label} right now.")
701
+ elif (declared.get("access") is False or walled or rows or hidden
702
+ or route_system.is_managed(definition) or route_system.is_managed(source)):
703
+ refuse(403, "app_dashboard_scope", f"The Dashboard needs {label} without a row or field limit.")
704
+ narrowed = any(wall["rows"] or wall["hidden"] for wall in narrow.values())
705
+ status, code, message = refusal[0] if refusal else (None, None, None)
706
+ return {"app": "ready" if granted and (kind in NARROWING_KINDS or not narrowed) else "no_grant",
707
+ "dashboard": "ready" if not refusal else "refused",
708
+ "status": status, "code": code, "message": message, "narrow": narrow}
709
+
710
+
711
+ def app_source_scope(user, runtime, source, *, app, records=None):
712
+ """Return source fields and row identities after live admission of the named App."""
713
  from core import perm_scope
714
+ # Importing the API adapter registers the core admission callback.
715
+ import routes_app_dashboard
716
+ fields, rows = perm_scope.scoped_app_source(
717
+ user, source, st=runtime, app_id=app.get("id") if isinstance(app, dict) else None, records=records)
718
+ identity = "__scope_key" if records is not None else "pid"
719
+ return {str(field["key"]) for field in fields if field.get("key")}, {
720
+ str(row[identity]) for row in rows if identity in row}
721
+
722
+
723
+ def readable_app(runtime, tenant, uname, is_admin, kind):
724
+ """The caller's live readable App of `kind` in `tenant`, their own copy first, or None."""
725
+ apps = [row for _key, row in sorted(load(runtime)["applications"].items())
726
+ if isinstance(row, dict) and row.get("templateKind") == kind and row.get("tenant") == tenant
727
+ and row.get("retired") is not True and may_read(row, uname, is_admin)]
728
+ return next((row for row in apps if row.get("createdBy") == uname), apps[0] if apps else None)
729
+
730
+
731
+ def admit_app(session, app, sources, *, code):
732
+ """R62: admit an App read on the App grant plus `app_access`, never the source open grant."""
733
+ from deps import err
734
+ if (session.tenant != getattr(session.runtime, "key", None)
735
+ or session.user.get("tenant", session.tenant) != session.tenant
736
+ or not isinstance(app, dict) or app.get("tenant") != session.tenant
737
+ or app.get("retired") is True or not may_read(app, session.uname, session.admin)):
738
+ raise err(404, f"{code}_app_not_found", "This App is unavailable or is not shared with you.")
739
+ access = app_access(session.user, session.runtime, app, sources)
740
+ if access["app"] != "ready":
741
+ raise err(403, f"{code}_limited", "This App cannot apply your access limit on its source.")
742
+ return access
743
+
744
+
745
+ def _availability_for_app(session, app, catalogue):
746
  from routes_connectors import _entry_types
747
 
748
  kind = app["templateKind"]
 
753
  return mount
754
  if not _app_templates.seams_ready(kind):
755
  return availability_reason("router_unavailable")
756
+ amazon = catalogue.get("connector") in ("amazon", "amazon_ads")
757
+ dashboard = {"state": "source_unavailable", "message": _AVAILABILITY_MESSAGES["source_unavailable"]}
758
+
759
+ def reason(state, connector=None):
760
+ result = availability_reason(state, connector)
761
+ return {**result, "dashboard": dashboard} if amazon else result
762
+
763
  try:
764
  sources = catalogue["sources"]
765
+ if amazon:
766
+ # Availability and the Dashboard use one source-wall answer.
767
+ access = app_access(session.user, session.runtime, app, sources)
768
+ dashboard = {"state": "ready" if access["dashboard"] == "ready" else
769
+ "no_grant" if access["status"] == 403 else "source_unavailable",
770
+ "message": access["message"]}
771
+ if access["app"] != "ready":
772
+ return reason("no_grant")
773
+ else:
774
+ from core import perm_scope
775
+ for source in sources:
776
+ if (not perm_scope.may_read(session.user, source, st=session.runtime)
777
+ or perm_scope.wall_declared(session.user, source, st=session.runtime)):
778
+ return reason("no_grant")
779
  types, _scopes, failure = _entry_types(session.runtime, session.uname)
780
  if failure is not None:
781
+ return reason("source_unavailable")
782
  connected = _app_templates.entry_type(catalogue["connector"]) in set(types or ())
783
  if not connected and catalogue["connector"] == "odoo":
784
  from routes_keychain import env_odoo_available
785
  connected = env_odoo_available(session.runtime)
786
  if not connected:
787
+ return reason("not_connected", catalogue["connector"])
788
+ row_sources = ((*sources, *catalogue.get("optional_sources", ()))
789
+ if kind == "repeat_buyers" else sources)
790
+ if amazon:
791
+ from harness.mirror_store import restore_in_progress
792
+ if restore_in_progress(session.tenant, catalogue["connector"]):
793
+ return reason("refreshing")
794
+ if row_sources and not _source_rows_exist(session, row_sources):
795
+ return reason("no_rows")
796
+ except Exception as exc: # noqa: BLE001
797
+ if amazon:
798
+ raise_if_resource_error(exc)
799
+ return reason("source_unavailable")
800
+ return reason("ready")
801
 
802
 
803
  def availability(session, app_id):
 
1312
  mount = _mount_reason(row.get("templateKind"))
1313
  if mount:
1314
  row["availability"] = mount
1315
+ elif (catalogue.get("connector") not in ("amazon", "amazon_ads")
1316
+ and not _database_free(app) and not compatible):
1317
  row["availability"] = availability_reason("no_grant")
1318
  if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
1319
  # A Finance App is a user's own copy: owner controls apply. Defaults bind a database.
 
1778
  return {"outcome": "migrated", "itemIds": copy.deepcopy(copied["itemIds"])}
1779
 
1780
 
1781
+ #: Marker of the one-copy step: each removed later copy and the kept copy that holds its slot now.
1782
+ TEMPLATE_ONE_COPY_MARKER = "templateOneCopy"
1783
+ #: The one-copy step deletes as the account Loopable seeds Apps under.
1784
+ _ONE_COPY_ACTOR = "loopable"
1785
+
1786
+
1787
+ def _template_double_plan(documents):
1788
+ """`[(identity, (key, kept), [(key, later), ...])]` per tenant-wide template with two live copies.
1789
+
1790
+ Copies sort as `_refuse_template_held` sorts them. Routing is one private copy per account by design.
1791
+ """
1792
+ groups = {}
1793
+ for key, doc in documents.items():
1794
+ for app_id, app in doc["applications"].items():
1795
+ if not isinstance(app, dict) or app.get("retired") is True:
1796
+ continue
1797
+ identity = template_identity(app)
1798
+ if identity and identity != "route":
1799
+ groups.setdefault(identity, {}).setdefault(app_id, (key, app))
1800
+ plan = []
1801
+ for identity, copies in sorted(groups.items()):
1802
+ rows = sorted(copies.values(), key=lambda row: (str(row[1].get("createdAt") or ""),
1803
+ str(row[1].get("id") or "")))
1804
+ if len(rows) > 1:
1805
+ plan.append((identity, rows[0], rows[1:]))
1806
+ return plan
1807
+
1808
+
1809
+ def _fold_template_copy(runtime, kept_key, kept_id, later):
1810
+ """Give the kept copy the later copy's readers and stars, and repoint its defaults."""
1811
+ def fold(raw):
1812
+ doc = _clean_doc(raw)
1813
+ app = doc["applications"].get(kept_id)
1814
+ if not isinstance(app, dict) or app.get("retired") is True:
1815
+ raise ApplicationError("The kept App copy is unavailable.")
1816
+ before = copy.deepcopy(app)
1817
+ share = app.get("share") if isinstance(app.get("share"), dict) else {}
1818
+ other = later.get("share") if isinstance(later.get("share"), dict) else {}
1819
+ if share.get("mode") != "tenant":
1820
+ if other.get("mode") == "tenant":
1821
+ app["share"] = {"mode": "tenant", "users": []}
1822
+ else:
1823
+ users = [user for user in _users([
1824
+ *(share.get("users") or () if share.get("mode") == "users" else ()),
1825
+ *(other.get("users") or () if other.get("mode") == "users" else ()),
1826
+ later.get("createdBy")]) if user != app.get("createdBy")]
1827
+ if users:
1828
+ app["share"] = {"mode": "users", "users": users}
1829
+ stars = {*(app.get("starredBy") or ()), *(later.get("starredBy") or ())}
1830
+ if stars != set(app.get("starredBy") or ()):
1831
+ app["starredBy"] = sorted(stars)
1832
+ if app != before:
1833
+ app["revision"] = int(app.get("revision") or 0) + 1
1834
+ app["updatedAt"] = write_execution.utcnow().isoformat()
1835
+ return doc
1836
+ return raw
1837
+ _update_document(runtime, fold, key=kept_key)
1838
+
1839
+ def repoint(raw):
1840
+ doc = _clean_doc(raw)
1841
+ names = [name for name, value in doc["defaults"].items() if value == later.get("id")]
1842
+ if not names:
1843
+ return raw
1844
+ for name in names:
1845
+ doc["defaults"][name] = kept_id
1846
+ return doc
1847
+ if later.get("id") in _clean_doc(_read_strict(runtime, _KEY))["defaults"].values():
1848
+ _update_document(runtime, repoint)
1849
+ index = _read_strict(runtime, _APP_INDEX_KEY)
1850
+ owners = index.get("users") if isinstance(index, dict) else None
1851
+ if isinstance(owners, dict):
1852
+ _move_placements(runtime, {name: {later["id"]: kept_id} for name in owners})
1853
+
1854
+
1855
+ def remove_template_doubles(runtime, tenant):
1856
+ """Keep the oldest live copy of each tenant-wide template and delete every later copy.
1857
+
1858
+ `delete` keeps the later copy's databases, Views and records. A second run writes nothing.
1859
+ Never raises: a failed copy is reported and retried at the next boot.
1860
+ """
1861
+ if (getattr(runtime, "key", None) != tenant
1862
+ or getattr(getattr(runtime, "tenant", None), "key", tenant) != tenant):
1863
+ return {"outcome": "skipped", "removed": {}, "reason": "the runtime belongs to another tenant"}
1864
+ try:
1865
+ documents = {_KEY: _clean_doc(_read_strict(runtime, _KEY))}
1866
+ documents.update(_record_documents(runtime))
1867
+ except Exception as exc:
1868
+ return {"outcome": "skipped", "removed": {},
1869
+ "reason": f"registry read failed: {type(exc).__name__}: {exc}"}
1870
+ removed, failed = {}, []
1871
+ for identity, (kept_key, kept), later in _template_double_plan(documents):
1872
+ for _key, app in later:
1873
+ try:
1874
+ _fold_template_copy(runtime, kept_key, kept["id"], app)
1875
+ delete(runtime, app["id"], _ONE_COPY_ACTOR, True)
1876
+ except Exception as exc:
1877
+ failed.append(f"{app['id']}: {type(exc).__name__}")
1878
+ print(f"[aios-api] one template copy {tenant}: {app['id']} not removed: "
1879
+ f"{type(exc).__name__}")
1880
+ continue
1881
+ removed[app["id"]] = kept["id"]
1882
+ print(f"[aios-api] one template copy {tenant}: removed {app['id']}, "
1883
+ f"kept {kept['id']} ({identity})")
1884
+ if removed:
1885
+ def mark(raw):
1886
+ doc = _clean_doc(raw)
1887
+ migrations = doc.setdefault("migrations", {})
1888
+ marker = migrations.get(TEMPLATE_ONE_COPY_MARKER)
1889
+ marker = copy.deepcopy(marker) if isinstance(marker, dict) else {}
1890
+ held = marker.get("removed") if isinstance(marker.get("removed"), dict) else {}
1891
+ marker.update(removed={**held, **removed}, at=write_execution.utcnow().isoformat())
1892
+ migrations[TEMPLATE_ONE_COPY_MARKER] = marker
1893
+ return doc
1894
+ try:
1895
+ _update_document(runtime, mark)
1896
+ except Exception as exc:
1897
+ failed.append(f"marker: {type(exc).__name__}")
1898
+ outcome = "removed" if removed else ("failed" if failed else "unchanged")
1899
+ reason = ", ".join(f"{later_id} into {kept_id}" for later_id, kept_id in removed.items())
1900
+ return {"outcome": outcome, "removed": removed,
1901
+ "reason": "; ".join(part for part in (reason, ", ".join(failed)) if part)}
1902
+
1903
+
1904
  def ensure_route_application(runtime, tenant, *, accounts=None, wall=None):
1905
  """R15: boot migrations, once per tenant, never from an Application read.
1906
 
api/assistant_memory.py CHANGED
@@ -1,4 +1,5 @@
1
  """C13: a user's sourced facts, separate from the C12 conversation transcript."""
 
2
  import copy
3
  import datetime
4
  import json
@@ -493,6 +494,74 @@ def recall(rt, principal, query=None, limit=20):
493
  return result
494
 
495
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
496
  #: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
497
  #: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
498
  #: neither the store read nor the model prompt grows with the conversation (D-803).
 
1
  """C13: a user's sourced facts, separate from the C12 conversation transcript."""
2
+ import base64
3
  import copy
4
  import datetime
5
  import json
 
494
  return result
495
 
496
 
497
+ INSPECT_ORDERS = ("newest", "oldest")
498
+
499
+
500
+ class InspectRequestError(ValueError):
501
+ """A page request the route answers with 400, raised before any store read."""
502
+
503
+
504
+ def _encode_cursor(key, id):
505
+ stamp, company, position = key
506
+ raw = json.dumps([None if stamp == float("-inf") else stamp, company, position, id], separators=(",", ":"))
507
+ return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=")
508
+
509
+
510
+ def _decode_cursor(cursor):
511
+ try:
512
+ stamp, company, position, id = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)))
513
+ if ((stamp is not None and type(stamp) not in (int, float)) or company not in (0, 1)
514
+ or type(position) is not int or not isinstance(id, str)):
515
+ raise TypeError
516
+ return (float("-inf") if stamp is None else float(stamp), company, position), id
517
+ except Exception:
518
+ raise InspectRequestError("That memory page is out of date. Reload the list.") from None
519
+
520
+
521
+ def inspect_page(rt, principal, *, cursor=None, limit=MAX_ENTRIES, order="newest"):
522
+ """Page the whole authorized memory for the inspect and erase panel. Turn context keeps recall."""
523
+ if order not in INSPECT_ORDERS:
524
+ raise InspectRequestError("Choose newest or oldest first.")
525
+ after = _decode_cursor(cursor) if cursor else None
526
+ limit = max(1, min(MAX_ENTRIES, int(limit)))
527
+ if principal["tenant"] != assistant_log._tenant_of(rt):
528
+ return {"entries": [], "total": 0, "next": None}
529
+ doc = document(rt, principal["username"])
530
+ company = company_document(rt)
531
+ entries = [(entry, "personal", doc["tombstones"], 0, n) for n, entry in enumerate(doc["entries"])]
532
+ entries.extend((entry, "company", company["tombstones"], 1, n) for n, entry in enumerate(company["entries"]))
533
+ # Same walls as recall: the prevailing note is chosen before the source filter.
534
+ superseded = _superseded(company["entries"], company["tombstones"])
535
+ readable, rows = {}, []
536
+ for entry, scope, tombstones, rank, position in entries:
537
+ if (not isinstance(entry, dict) or not all(key in entry for key in ("id", "text", "sources", "door", "at"))
538
+ or not isinstance(entry["text"], str) or not isinstance(entry["id"], str)):
539
+ continue
540
+ if scope == "company" and id(entry) in superseded:
541
+ continue
542
+ if _normalized(entry["text"]) in tombstones:
543
+ continue
544
+ # Permissions are fixed for one request, so one answer per source set is enough.
545
+ wall = (scope, json.dumps(entry["sources"], sort_keys=True, default=str))
546
+ if wall not in readable:
547
+ readable[wall] = _readable(rt, principal, entry["sources"], company=scope == "company")
548
+ if readable[wall]:
549
+ # Age, then document position: the order recall uses, so equal stamps never reorder.
550
+ rows.append(((_oldest(entry), rank, position), entry, scope))
551
+ newest = order == "newest"
552
+ rows.sort(key=lambda row: row[0], reverse=newest)
553
+ start = 0
554
+ if after is not None:
555
+ key, anchor = after
556
+ # Resume after the anchor id. If it was erased, resume after its sort key.
557
+ start = next((n + 1 for n, row in enumerate(rows) if row[1]["id"] == anchor and row[0][1] == key[1]),
558
+ next((n for n, row in enumerate(rows) if (row[0] < key if newest else row[0] > key)), len(rows)))
559
+ page = rows[start:start + limit]
560
+ more = start + limit < len(rows)
561
+ return {"entries": [{**copy.deepcopy(entry), "scope": scope} for _, entry, scope in page],
562
+ "total": len(rows), "next": _encode_cursor(page[-1][0], page[-1][1]["id"]) if more else None}
563
+
564
+
565
  #: One compaction reads and sends at most one log chunk. A tail that grew while compaction could
566
  #: not run (no model, no credit) is worked through in order, one bounded batch per attempt, so
567
  #: neither the store read nor the model prompt grows with the conversation (D-803).
api/assistant_mirror.py CHANGED
@@ -58,6 +58,12 @@ def _rot():
58
  return routes_odoo_tables
59
 
60
 
 
 
 
 
 
 
61
  def open_cursor(session, table_key=None):
62
  """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
63
 
@@ -77,17 +83,21 @@ def open_cursor(session, table_key=None):
77
  try:
78
  cur, mismatch = rot._mirror_cur(session, table_key)
79
  except RuntimeError as exc:
80
- return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
 
 
 
 
81
  except Exception as exc: # noqa: BLE001
82
  # ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
83
  # `deps`, which every fixture and gate in the repo imports and drives with runtimes that
84
  # are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
85
  # whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
86
  # which reads to a user as the product being broken rather than the mirror being shut.
87
- # Named and fail-closed beats a stack trace.
88
- return None, MirrorRefusal(409, "assistant_snapshot_unavailable",
89
- f"this database's connector mirror could not be opened "
90
- f"({exc})")
91
  if mismatch:
92
  return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
93
  return cur, None
@@ -167,17 +177,14 @@ def unavailable_reason(table_key, definition=None, *, snapshot=None):
167
  if key.startswith("ut_amzn_"):
168
  import amazon_relational as amazon
169
  if key in amazon.SOURCE:
170
- if snapshot is not None:
171
- return amazon.missing_source(key, pending=snapshot.amazon_pending())["missing"]
172
- return amazon.missing_source(key)["missing"]
173
  spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
174
  if spec and not spec.get("missing"):
175
- label = str((definition or {}).get("label") or key)
176
- return (f"{label} is still loading into this server after a restart, "
177
- "so ask again in an hour.")
178
  except Exception: # noqa: BLE001
179
  pass
180
- return NO_BINDING
181
 
182
 
183
  def still_loading(table_key, *, cur=None, snapshot=None):
 
58
  return routes_odoo_tables
59
 
60
 
61
+ def _cause(cause):
62
+ """The closed-set refusal sentence `deps` owns, imported lazily to break the `deps` cycle."""
63
+ import deps
64
+ return deps._assistant_source_message(cause)
65
+
66
+
67
  def open_cursor(session, table_key=None):
68
  """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
69
 
 
83
  try:
84
  cur, mismatch = rot._mirror_cur(session, table_key)
85
  except RuntimeError as exc:
86
+ if "could not be restored" in str(exc):
87
+ return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("restore_blocked"))
88
+ if "has not been attached" in str(exc):
89
+ return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("loading"))
90
+ return None, MirrorRefusal(503, "assistant_store_not_ready", _cause("syncing"))
91
  except Exception as exc: # noqa: BLE001
92
  # ⚠ A DELIBERATE THIRD CLAUSE, and it is not laziness. This module is reached from
93
  # `deps`, which every fixture and gate in the repo imports and drives with runtimes that
94
  # are not a real `TenantRuntime`; `mirror_cursor` calls `assert_datastore_matches` on
95
  # whatever it is handed. An unexpected failure there is a 500 on an assistant answer,
96
  # which reads to a user as the product being broken rather than the mirror being shut.
97
+ # Named and fail-closed beats a stack trace. The class goes to the server log, never the reply.
98
+ import deps
99
+ deps._assistant_log_cause("assistant mirror open", exc)
100
+ return None, MirrorRefusal(409, "assistant_snapshot_unavailable", _cause("mirror_open"))
101
  if mismatch:
102
  return None, MirrorRefusal(409, "assistant_cross_tenant_store", str(mismatch))
103
  return cur, None
 
177
  if key.startswith("ut_amzn_"):
178
  import amazon_relational as amazon
179
  if key in amazon.SOURCE:
180
+ pending = snapshot.amazon_pending() if snapshot is not None else None
181
+ return amazon.missing_source(key, pending=pending)["missing"]
 
182
  spec = snapshot.source(key) if snapshot is not None else _rot()._sources().get(key)
183
  if spec and not spec.get("missing"):
184
+ return _cause("loading")
 
 
185
  except Exception: # noqa: BLE001
186
  pass
187
+ return _cause("no_binding")
188
 
189
 
190
  def still_loading(table_key, *, cur=None, snapshot=None):
api/automation_engine.py CHANGED
@@ -6804,14 +6804,22 @@ def migrate_ig_tables(rt, log=print, only=""):
6804
  rows, merged = _merge_candidates(rows)
6805
  stats["merged"] += merged
6806
 
 
 
6807
  def _up(cur, _f=fields, _r=rows):
 
6808
  cur = cur or {}
6809
  t = dict(cur.get(table_key) or {})
 
6810
  t["fields"], t["rows"] = _f, _r
 
 
 
6811
  cur[table_key] = t
6812
  return cur
6813
 
6814
  rt.update(UT_STORE_KEY, _up)
 
6815
  # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
6816
  #
6817
  # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
@@ -14211,6 +14219,10 @@ def _module_table(rt, key, memo, want_rows=False):
14211
  "check this tenant's Odoo connector under Settings -> Connectors; the linked column "
14212
  "keeps its last resolved value until the source answers again")
14213
  table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
 
 
 
 
14214
  for rid, row in list(table['rows'].items()):
14215
  table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
14216
  table["_rows_built"] = True
@@ -14360,24 +14372,52 @@ def _resolve_table(rt, store, key, memo, want_rows=False):
14360
  return _module_table(rt, key, memo, want_rows=want_rows)
14361
 
14362
 
14363
- def _link_from_key(fields, bag):
14364
  """Which column on THIS table supplies the join value.
14365
 
14366
  Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
14367
  fallback chain is what makes an Instagram database link up with no configuration at all —
14368
  the "automatically" in the owner's instruction — and it is the same chain the grid already
14369
  uses to decide a table's identity column, rather than a second opinion about it.
 
 
14370
  """
14371
  declared = str((bag or {}).get("from") or "").strip()
14372
  if declared:
14373
  return declared
14374
- prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None)
 
14375
  if prof:
14376
  return str(prof.get("key") or "")
14377
- pin = next((f for f in fields if f.get("pinned") is True), None)
14378
  return str(pin.get("key") or "") if pin else ""
14379
 
14380
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14381
  def _join_norm(value):
14382
  """ONE normalisation of an `on` join value, applied to BOTH sides of the join.
14383
 
@@ -14564,6 +14604,18 @@ def relation_cells(rt, table_key, tables=None):
14564
  # --- resolve every link field ONCE per table, not once per row.
14565
  # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
14566
  resolved, linked_types, refused = {}, {}, set()
 
 
 
 
 
 
 
 
 
 
 
 
14567
  for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
14568
  lk_key = str((f or {}).get("key") or "")
14569
  if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
@@ -14630,7 +14682,7 @@ def relation_cells(rt, table_key, tables=None):
14630
  on_key = str(bag.get("on") or "")
14631
  hop_in = str(bag.get("throughOn") or "")
14632
  hop_out = str(bag.get("throughTo") or "")
14633
- from_key = _link_from_key(fields, bag)
14634
  if not (on_key and hop_in and hop_out and from_key):
14635
  # ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
14636
  # a missing leg has no second hop to make; resolving it to `[]` would render a
@@ -14668,7 +14720,7 @@ def relation_cells(rt, table_key, tables=None):
14668
  target_idx = _linked_rows_by_join(linked, on_key)
14669
  cap = int(_ut().LINK_THROUGH_MAX_IDS)
14670
 
14671
- def _walk(row, _bidx=bridge_idx, _tidx=target_idx, _cap=cap):
14672
  """One source row's DISTINCT target records, and whether the ceiling stopped it.
14673
 
14674
  ⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
@@ -14676,7 +14728,7 @@ def relation_cells(rt, table_key, tables=None):
14676
  label of a set of PRODUCTS.
14677
  """
14678
  seen, out = set(), []
14679
- for _bid, brow in _bidx.get(_join_norm((row or {}).get(from_key)), ()):
14680
  for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
14681
  if tid in seen:
14682
  continue
@@ -14688,7 +14740,7 @@ def relation_cells(rt, table_key, tables=None):
14688
 
14689
  two_hop, capped = {}, 0
14690
  for rid, row in rows.items():
14691
- hits_2, hit_cap = _walk(row)
14692
  two_hop[str(rid)] = hits_2
14693
  capped += 1 if hit_cap else 0
14694
  resolved[lk_key] = two_hop
@@ -14712,7 +14764,7 @@ def relation_cells(rt, table_key, tables=None):
14712
  effect="capped", limit=cap))
14713
  elif bag.get("on"):
14714
  idx = _linked_rows_by_join(linked, str(bag["on"]))
14715
- from_key = _link_from_key(fields, bag)
14716
  if not from_key:
14717
  # ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
14718
  # and then the PINNED one; a table with neither — which a registry module can be,
@@ -14728,7 +14780,7 @@ def relation_cells(rt, table_key, tables=None):
14728
  "database that is the id column, e.g. `partner_id` or `product_id`)"))
14729
  continue
14730
  resolved[lk_key] = {
14731
- str(rid): idx.get(_join_norm((row or {}).get(from_key)), [])
14732
  for rid, row in rows.items()}
14733
  else:
14734
  # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
@@ -15024,6 +15076,136 @@ def related_product_metric_values(rt, customer_id, products, fields, offer, *,
15024
  return out, list(dict.fromkeys(reasons))
15025
 
15026
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15027
  def compute_relation_cells(rt, table_key, tables=None):
15028
  """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
15029
 
@@ -15050,7 +15232,10 @@ def compute_relation_cells(rt, table_key, tables=None):
15050
  another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
15051
  caller that renders them reads them, and nothing about this door changes shape.
15052
  """
15053
- changes, _limits = relation_cells(rt, table_key, tables=tables)
 
 
 
15054
  if not changes:
15055
  return 0
15056
  if str(table_key) in LINKABLE_MODULES:
@@ -15093,24 +15278,36 @@ def _refresh_relations_inplace(tables, log=print, rt=None):
15093
  keyword this function briefly added (see `compute_relation_cells`).
15094
  """
15095
  touched = 0
 
15096
  for tk, table in list((tables or {}).items()):
15097
  fields = (table or {}).get("fields") or []
15098
  if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
 
 
15099
  continue
 
15100
  try:
15101
  touched += compute_relation_cells(rt, tk, tables=tables)
 
15102
  except Exception as e: # noqa: BLE001
15103
  log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
 
 
 
15104
  return touched
15105
 
15106
 
15107
  def refresh_relations(rt, log=print, tables=None, persist=True):
15108
  """Compute relations outside the write lock and merge only unchanged derived cells."""
15109
- snapshot = tables if tables is not None else ut_all(rt)
15110
  # Keep authored inputs before computation, including nested cell values.
15111
  before = copy.deepcopy(snapshot)
15112
  local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
15113
- if not local or not persist:
 
 
 
 
15114
  return local
15115
  patches = {}
15116
  for key, old in before.items():
@@ -15129,31 +15326,47 @@ def refresh_relations(rt, log=print, tables=None, persist=True):
15129
  if changed:
15130
  patches.setdefault(str(key), {})[str(rid)] = (row, changed)
15131
  if not patches:
 
15132
  return 0
15133
  actual = [0]
 
 
 
15134
 
15135
  def _up(cur):
15136
  actual[0] = 0
 
15137
  lock = getattr(cur, 'lock_records', None)
15138
  if callable(lock):
15139
  lock({key: list(rows) for key, rows in patches.items()})
15140
  for key, rows in patches.items():
15141
  table = (cur or {}).get(key)
15142
  if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
 
15143
  continue
 
15144
  current_rows = table.get('rows')
15145
  if current_rows is None:
 
15146
  continue
15147
  for rid, (old_row, changes) in rows.items():
15148
  current = current_rows.get(rid)
15149
  # A changed authored input or a deleted row waits for the next refresh.
15150
  if not isinstance(current, dict) or current != old_row:
 
15151
  continue
15152
  current.update(changes)
15153
  actual[0] += 1
 
 
 
15154
  return cur
15155
 
15156
- rt.update(UT_STORE_KEY, _up, flush="async")
 
 
 
 
15157
  return actual[0]
15158
 
15159
 
@@ -16256,13 +16469,153 @@ def _automation_slack_reply_text(result):
16256
  return "An authorized automation completed."
16257
 
16258
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16259
  def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
16260
  """Deliver one authorized event result, with readable linked records for an enters_view cohort."""
16261
  definition = defn if isinstance(defn, dict) else {}
16262
  aid = str(definition.get("id") or "").strip()
16263
  if not aid:
16264
  return {"sent": False, "reason": "unknown_automation"}
16265
- stored = all_definitions(rt).get(aid)
 
 
 
 
 
 
 
16266
  if not isinstance(stored, dict):
16267
  return {"sent": False, "reason": "unknown_automation"}
16268
  trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
@@ -16286,6 +16639,22 @@ def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
16286
  context = routes_slack.automation_answer_context(rt, aid)
16287
  if context is None:
16288
  return {"sent": False, "reason": "not_authorized"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16289
  provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
16290
  if trigger_key == "enters_view":
16291
  module = str(trigger.get("table") or _flow_table(stored) or "")
 
6804
  rows, merged = _merge_candidates(rows)
6805
  stats["merged"] += merged
6806
 
6807
+ field_changes = {}
6808
+
6809
  def _up(cur, _f=fields, _r=rows):
6810
+ field_changes.clear()
6811
  cur = cur or {}
6812
  t = dict(cur.get(table_key) or {})
6813
+ old_keys = _ut()._field_list_keys(t)
6814
  t["fields"], t["rows"] = _f, _r
6815
+ new_keys = _ut()._field_list_keys(t)
6816
+ field_changes.update(removed=[key for key in old_keys if key not in new_keys],
6817
+ added=[key for key in new_keys if key not in old_keys])
6818
  cur[table_key] = t
6819
  return cur
6820
 
6821
  rt.update(UT_STORE_KEY, _up)
6822
+ _ut()._log_field_list_change(rt, table_key, 'automation', 'migrate_ig_tables', field_changes)
6823
  # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION.
6824
  #
6825
  # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early
 
14219
  "check this tenant's Odoo connector under Settings -> Connectors; the linked column "
14220
  "keeps its last resolved value until the source answers again")
14221
  table["rows"] = {str(r.get("pid")): r for r in (pool or []) if r.get("pid") is not None}
14222
+ if key == "product_data":
14223
+ from modules import product_data
14224
+ table["rows"] = {str(row["pid"]): row for row in product_data.apply_lead_overlay(
14225
+ list(table["rows"].values()), table.get("_if_shared_cells"))}
14226
  for rid, row in list(table['rows'].items()):
14227
  table['rows'][rid] = {**row, **(table.get('_if_shared_cells', {}).get(rid) or {})}
14228
  table["_rows_built"] = True
 
14372
  return _module_table(rt, key, memo, want_rows=want_rows)
14373
 
14374
 
14375
+ def _link_from_key(fields, bag, overlay_fields=None):
14376
  """Which column on THIS table supplies the join value.
14377
 
14378
  Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
14379
  fallback chain is what makes an Instagram database link up with no configuration at all —
14380
  the "automatically" in the owner's instruction — and it is the same chain the grid already
14381
  uses to decide a table's identity column, rather than a second opinion about it.
14382
+ `overlay_fields` (`{key: Field}` from the shared overlay) joins the fallback after the
14383
+ definition's own fields: a typed overlay column can supply the join too (D-11171).
14384
  """
14385
  declared = str((bag or {}).get("from") or "").strip()
14386
  if declared:
14387
  return declared
14388
+ pool = list(fields) + [dict(f, key=k) for k, f in (overlay_fields or {}).items() if isinstance(f, dict)]
14389
+ prof = next((f for f in pool if isinstance(f.get("profile"), dict)), None)
14390
  if prof:
14391
  return str(prof.get("key") or "")
14392
+ pin = next((f for f in pool if f.get("pinned") is True), None)
14393
  return str(pin.get("key") or "") if pin else ""
14394
 
14395
 
14396
+ def _overlay_join_rows(rt, table_key, rows):
14397
+ """`({key: Field}, {row_id: {key: value}})`: this table's shared overlay columns and their
14398
+ typed cells for `rows`. A join reads a typed overlay value from here (D-11171).
14399
+
14400
+ Lenient like every display read: an unreadable overlay is `({}, {})`, so the join reads the
14401
+ stored row value only, exactly as it did before."""
14402
+ try:
14403
+ from core import shared_overlay
14404
+ declared = shared_overlay.fields(table_key, st=rt)
14405
+ if not declared:
14406
+ return {}, {}
14407
+ pids = [str(rid) for rid in rows if str(rid).isdigit()]
14408
+ return dict(declared), shared_overlay.cells(table_key, pids, st=rt)
14409
+ except Exception: # noqa: BLE001
14410
+ return {}, {}
14411
+
14412
+
14413
+ def _join_value(row, rid, from_key, overlay_cells):
14414
+ """The join value a source row supplies: the stored cell, else its typed overlay cell."""
14415
+ value = (row or {}).get(from_key)
14416
+ if value is None or value == "":
14417
+ value = ((overlay_cells or {}).get(str(rid)) or {}).get(from_key)
14418
+ return value
14419
+
14420
+
14421
  def _join_norm(value):
14422
  """ONE normalisation of an `on` join value, applied to BOTH sides of the join.
14423
 
 
14604
  # --- resolve every link field ONCE per table, not once per row.
14605
  # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
14606
  resolved, linked_types, refused = {}, {}, set()
14607
+ overlay = []
14608
+
14609
+ def _overlay(bag):
14610
+ """`(overlay fields, overlay cells, from key)` for one join. The shared overlay is read
14611
+ once, and only when the `from` column is not a field this table defines."""
14612
+ own = _link_from_key(fields, bag)
14613
+ if own and own in by_key:
14614
+ return {}, {}, own
14615
+ if not overlay:
14616
+ overlay.append(_overlay_join_rows(rt, table_key, rows))
14617
+ return overlay[0] + (_link_from_key(fields, bag, overlay[0][0]),)
14618
+
14619
  for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
14620
  lk_key = str((f or {}).get("key") or "")
14621
  if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
 
14682
  on_key = str(bag.get("on") or "")
14683
  hop_in = str(bag.get("throughOn") or "")
14684
  hop_out = str(bag.get("throughTo") or "")
14685
+ _of, overlay_cells, from_key = _overlay(bag)
14686
  if not (on_key and hop_in and hop_out and from_key):
14687
  # ⛔ REFUSED, NOT BLANK — the same law the one-hop arm below follows. A route with
14688
  # a missing leg has no second hop to make; resolving it to `[]` would render a
 
14720
  target_idx = _linked_rows_by_join(linked, on_key)
14721
  cap = int(_ut().LINK_THROUGH_MAX_IDS)
14722
 
14723
+ def _walk(row, rid, _bidx=bridge_idx, _tidx=target_idx, _cap=cap, _cells=overlay_cells):
14724
  """One source row's DISTINCT target records, and whether the ceiling stopped it.
14725
 
14726
  ⚠ DE-DUPED ON THE TARGET ROW ID. A customer orders the same SKU on twenty lines;
 
14728
  label of a set of PRODUCTS.
14729
  """
14730
  seen, out = set(), []
14731
+ for _bid, brow in _bidx.get(_join_norm(_join_value(row, rid, from_key, _cells)), ()):
14732
  for tid, trow in _tidx.get(_join_norm((brow or {}).get(hop_out)), ()):
14733
  if tid in seen:
14734
  continue
 
14740
 
14741
  two_hop, capped = {}, 0
14742
  for rid, row in rows.items():
14743
+ hits_2, hit_cap = _walk(row, rid)
14744
  two_hop[str(rid)] = hits_2
14745
  capped += 1 if hit_cap else 0
14746
  resolved[lk_key] = two_hop
 
14764
  effect="capped", limit=cap))
14765
  elif bag.get("on"):
14766
  idx = _linked_rows_by_join(linked, str(bag["on"]))
14767
+ _of, overlay_cells, from_key = _overlay(bag)
14768
  if not from_key:
14769
  # ⛔ REFUSED, NOT BLANK. `_link_from_key` falls back to the PROFILE-flagged column
14770
  # and then the PINNED one; a table with neither — which a registry module can be,
 
14780
  "database that is the id column, e.g. `partner_id` or `product_id`)"))
14781
  continue
14782
  resolved[lk_key] = {
14783
+ str(rid): idx.get(_join_norm(_join_value(row, rid, from_key, overlay_cells)), [])
14784
  for rid, row in rows.items()}
14785
  else:
14786
  # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
 
15076
  return out, list(dict.fromkeys(reasons))
15077
 
15078
 
15079
+ #: Compatibility memo for direct computation; GET reads the durable provenance document.
15080
+ _RELATION_LIMITS = {}
15081
+ _RELATION_LIMITS_LOCK = threading.Lock()
15082
+ #: The refusals of the table this thread just refreshed, for the refresh log.
15083
+ _RELATION_PASS = threading.local()
15084
+ RELATION_LIMITS_KEY = "relation_refusals"
15085
+ _RELATION_LIMIT_KEYS = ("subject", "effect", "cause", "recommendation", "code", "limit")
15086
+
15087
+
15088
+ def _relation_tenant(rt):
15089
+ rt = getattr(rt, "_runtime", None) or rt
15090
+ return str(getattr(rt, "key", "") or "")
15091
+
15092
+
15093
+ def remember_relation_limits(rt, table_key, limits):
15094
+ """Keep this table's newest link refusals for its tenant; an empty list clears them."""
15095
+ tenant = _relation_tenant(rt)
15096
+ if not tenant:
15097
+ return
15098
+ with _RELATION_LIMITS_LOCK:
15099
+ if limits:
15100
+ _RELATION_LIMITS[(tenant, str(table_key))] = [dict(limit) for limit in limits]
15101
+ else:
15102
+ _RELATION_LIMITS.pop((tenant, str(table_key)), None)
15103
+
15104
+
15105
+ def relation_limits(rt, table_key):
15106
+ """Read the last saved relation provenance without refreshing or writing cells."""
15107
+ rt = getattr(rt, "_runtime", None) or rt
15108
+ document = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
15109
+ return copy.deepcopy(document.get(str(table_key), []))
15110
+
15111
+
15112
+ def _relation_read(rt, name):
15113
+ """Use the bound backend's strict read and its existing bounded store waits."""
15114
+ if not _relation_tenant(rt) or not rt.available():
15115
+ raise RuntimeError("relation_provenance_unavailable")
15116
+ resolver = getattr(rt, "_document_store", None)
15117
+ if callable(resolver):
15118
+ backend = resolver() # Validates the store handle's tenant before reading.
15119
+ reader = getattr(backend, "_read_strict", None)
15120
+ if callable(reader):
15121
+ return reader(rt.store_key(name))
15122
+ return backend.get(rt.store_key(name), fresh=True)
15123
+ return rt.get(name, fresh=True)
15124
+
15125
+
15126
+ def _relation_document(rt, raw):
15127
+ """Reject foreign or malformed provenance instead of treating it as a successful pass."""
15128
+ if raw is None or raw == {}:
15129
+ return {}
15130
+ if (not isinstance(raw, dict) or raw.get("tenant") != _relation_tenant(rt)
15131
+ or type(raw.get("version")) is not int or raw.get("version") != 1
15132
+ or not isinstance(raw.get("tables"), dict)):
15133
+ raise RuntimeError("relation_provenance_unavailable")
15134
+ tables = raw["tables"]
15135
+ if any(not isinstance(limits, list) or any(not isinstance(limit, dict) for limit in limits)
15136
+ for limits in tables.values()):
15137
+ raise RuntimeError("relation_provenance_unavailable")
15138
+ for limits in tables.values():
15139
+ for limit in limits:
15140
+ if (any(not isinstance(limit.get(key), str) for key in
15141
+ ("subject", "effect", "cause", "recommendation", "fieldSignature"))
15142
+ or not re.fullmatch(r"[0-9a-f]{64}", limit["fieldSignature"])):
15143
+ raise RuntimeError("relation_provenance_unavailable")
15144
+ return tables
15145
+
15146
+
15147
+ def relation_field_signature(field):
15148
+ """Bind saved provenance to the field definition that produced it."""
15149
+ definition = {k: field[k] for k in ("key", "type", "link", "rollup", "if") if k in field}
15150
+ return hashlib.sha256(json.dumps(definition, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
15151
+
15152
+
15153
+ def _relation_provenance(table, limits):
15154
+ """Keep only field-bound diagnostics, excluding row identities, cells and exception payloads."""
15155
+ fields = [f for f in (table or {}).get("fields", []) if f.get("key")]
15156
+ out = []
15157
+ for limit in limits:
15158
+ subject = str(limit.get("subject") or "")
15159
+ matched = [f for f in fields if str(f["key"]) == subject]
15160
+ if not matched:
15161
+ matched = [f for f in fields if str((f.get("link") or {}).get("table") or "") == subject]
15162
+ if not matched and subject == str((table or {}).get("key") or ""):
15163
+ matched = [f for f in fields if f.get("type") in ("rollup", "if") or _ut().is_derived_link(f)]
15164
+ for field in matched:
15165
+ clean = {k: str(limit[k]) for k in _RELATION_LIMIT_KEYS if k in limit}
15166
+ if type(limit.get("limit")) is int:
15167
+ clean["limit"] = limit["limit"]
15168
+ clean["subject"] = str(field["key"])
15169
+ # Source exceptions can contain row values, SQL operands or provider response bodies.
15170
+ if any(marker in clean.get("cause", "") for marker in
15171
+ ("could not be read:", "could not answer for it:")):
15172
+ clean["cause"] = "The relation source could not be read. Its last resolved value is retained."
15173
+ clean["recommendation"] = "Check this tenant's source connector, then retry the relation refresh."
15174
+ clean["fieldSignature"] = relation_field_signature(field)
15175
+ out.append(clean)
15176
+ return out
15177
+
15178
+
15179
+ def _persist_relation_limits(rt, results):
15180
+ """Save one small tenant-bound document per pass, including recovery with no changed cells."""
15181
+ if not results:
15182
+ return
15183
+ old = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
15184
+ def merged(tables):
15185
+ tables = copy.deepcopy(tables)
15186
+ for key, limits in results.items():
15187
+ if limits:
15188
+ tables[str(key)] = copy.deepcopy(limits)
15189
+ else:
15190
+ tables.pop(str(key), None)
15191
+ return tables
15192
+ if merged(old) == old:
15193
+ return
15194
+ def up(raw):
15195
+ return {"version": 1, "tenant": _relation_tenant(rt),
15196
+ "tables": merged(_relation_document(rt, raw))}
15197
+ rt.update(RELATION_LIMITS_KEY, up, flush="sync")
15198
+
15199
+
15200
+ def relation_limit_line(table_key, limit):
15201
+ """One log line for one refused link: the field, the cause and the fix."""
15202
+ limit = limit or {}
15203
+ cause = str(limit.get("cause") or "").strip().rstrip(".")
15204
+ fix = str(limit.get("recommendation") or "").strip().rstrip(".")
15205
+ return (f"[aios-auto] relation refresh {table_key}: link {limit.get('subject')} refused: "
15206
+ f"{cause}. Fix: {fix}.")
15207
+
15208
+
15209
  def compute_relation_cells(rt, table_key, tables=None):
15210
  """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
15211
 
 
15232
  another section. ⇒ Refusals ride out on `relation_cells`' SECOND RETURN VALUE, where the
15233
  caller that renders them reads them, and nothing about this door changes shape.
15234
  """
15235
+ changes, limits = relation_cells(rt, table_key, tables=tables)
15236
+ # D-11172: the refusals leave through state, never through this signature (see above).
15237
+ _RELATION_PASS.limits = [dict(limit) for limit in limits]
15238
+ remember_relation_limits(rt, table_key, limits)
15239
  if not changes:
15240
  return 0
15241
  if str(table_key) in LINKABLE_MODULES:
 
15278
  keyword this function briefly added (see `compute_relation_cells`).
15279
  """
15280
  touched = 0
15281
+ _RELATION_PASS.results = {}
15282
  for tk, table in list((tables or {}).items()):
15283
  fields = (table or {}).get("fields") or []
15284
  if not any(f.get("type") in ('rollup', 'if') or _ut().is_derived_link(f) for f in fields):
15285
+ remember_relation_limits(rt, tk, [])
15286
+ _RELATION_PASS.results[str(tk)] = []
15287
  continue
15288
+ _RELATION_PASS.limits = []
15289
  try:
15290
  touched += compute_relation_cells(rt, tk, tables=tables)
15291
+ _RELATION_PASS.results[str(tk)] = _relation_provenance(table, _RELATION_PASS.limits)
15292
  except Exception as e: # noqa: BLE001
15293
  log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
15294
+ # D-11172: one line per refused link, its cause and its fix (standing rule 1).
15295
+ for limit in getattr(_RELATION_PASS, "limits", None) or ():
15296
+ log(relation_limit_line(tk, limit))
15297
  return touched
15298
 
15299
 
15300
  def refresh_relations(rt, log=print, tables=None, persist=True):
15301
  """Compute relations outside the write lock and merge only unchanged derived cells."""
15302
+ snapshot = tables if tables is not None else (_relation_read(rt, UT_STORE_KEY) or {})
15303
  # Keep authored inputs before computation, including nested cell values.
15304
  before = copy.deepcopy(snapshot)
15305
  local = _refresh_relations_inplace(snapshot, log=log, rt=rt)
15306
+ results = copy.deepcopy(_RELATION_PASS.results)
15307
+ if not persist:
15308
+ return local
15309
+ if not local:
15310
+ _persist_relation_limits(rt, results)
15311
  return local
15312
  patches = {}
15313
  for key, old in before.items():
 
15326
  if changed:
15327
  patches.setdefault(str(key), {})[str(rid)] = (row, changed)
15328
  if not patches:
15329
+ _persist_relation_limits(rt, results)
15330
  return 0
15331
  actual = [0]
15332
+ field_changes = {}
15333
+ previous_limits = _relation_document(rt, _relation_read(rt, RELATION_LIMITS_KEY))
15334
+ clearing_refusal = any(previous_limits.get(key) and not limits for key, limits in results.items())
15335
 
15336
  def _up(cur):
15337
  actual[0] = 0
15338
+ field_changes.clear()
15339
  lock = getattr(cur, 'lock_records', None)
15340
  if callable(lock):
15341
  lock({key: list(rows) for key, rows in patches.items()})
15342
  for key, rows in patches.items():
15343
  table = (cur or {}).get(key)
15344
  if not isinstance(table, dict) or table.get('fields') != before[key].get('fields'):
15345
+ results.pop(key, None)
15346
  continue
15347
+ old_keys = _ut()._field_list_keys(table)
15348
  current_rows = table.get('rows')
15349
  if current_rows is None:
15350
+ results.pop(key, None)
15351
  continue
15352
  for rid, (old_row, changes) in rows.items():
15353
  current = current_rows.get(rid)
15354
  # A changed authored input or a deleted row waits for the next refresh.
15355
  if not isinstance(current, dict) or current != old_row:
15356
+ results.pop(key, None)
15357
  continue
15358
  current.update(changes)
15359
  actual[0] += 1
15360
+ new_keys = _ut()._field_list_keys(table)
15361
+ field_changes[key] = {'removed': [name for name in old_keys if name not in new_keys],
15362
+ 'added': [name for name in new_keys if name not in old_keys]}
15363
  return cur
15364
 
15365
+ # A cold process must see recovered cells before their old refusal disappears.
15366
+ rt.update(UT_STORE_KEY, _up, flush="sync" if clearing_refusal else "async")
15367
+ _persist_relation_limits(rt, results)
15368
+ for key, change in field_changes.items():
15369
+ _ut()._log_field_list_change(rt, key, 'automation', 'refresh_relations', change)
15370
  return actual[0]
15371
 
15372
 
 
16469
  return "An authorized automation completed."
16470
 
16471
 
16472
+ NOTIFICATION_PAYLOAD_VERSION = 1
16473
+
16474
+
16475
+ def _authorized_notification_text(rt, stored, payload, context):
16476
+ """Authorize the current recipient, channel and every source before composing notification content."""
16477
+ from core import account_rows, notify_destinations, perm_scope, perms, user_tables, users
16478
+ if (not isinstance(payload, dict) or type(payload.get('version')) is not int
16479
+ or payload.get('version') != NOTIFICATION_PAYLOAD_VERSION):
16480
+ return None
16481
+ aid = stored.get('id')
16482
+ expected = {key: payload.get(key) for key in ('version', 'tenant', 'user', 'kind', 'channel')}
16483
+ if (expected != stored.get('notificationDestination') or payload.get('automationId') != aid
16484
+ or expected['tenant'] != getattr(rt, 'key', None) or expected['channel'] != context.get('channel')
16485
+ or not isinstance(expected['user'], str) or not expected['user']
16486
+ or stored.get('createdBy') != expected['user']
16487
+ or stored.get('trigger', {}).get('key') != 'notification'):
16488
+ return None
16489
+ if aid != notify_destinations.automation_id(rt.key, expected['user'], expected['kind'], expected['channel']):
16490
+ return None
16491
+ resolve = getattr(rt, '_document_store', None)
16492
+ record = account_rows.get(expected['user'], st=resolve() if callable(resolve) else rt, fresh=True)
16493
+ if not record or record.get('active', True) is False or record.get('tenant') != rt.key:
16494
+ return None
16495
+ recipient = users._public(expected['user'], record)
16496
+ principal = context.get('principal')
16497
+ if not isinstance(principal, dict) or not principal.get('username'):
16498
+ return None
16499
+ notify_destinations.kind_id(expected['kind'])
16500
+ raw_items = payload.get('items')
16501
+ if not isinstance(raw_items, list) or not raw_items:
16502
+ return None
16503
+ for item in raw_items:
16504
+ source = item.get('source') if isinstance(item, dict) else None
16505
+ if (not isinstance(source, dict) or set(source) != {'table', 'rowIds'}
16506
+ or not isinstance(source['table'], str) or not source['table']
16507
+ or not isinstance(source['rowIds'], list) or not source['rowIds']
16508
+ or any(not isinstance(rid, str) or not rid for rid in source['rowIds'])
16509
+ or not _notification_source_readable(rt, context, source)):
16510
+ return None
16511
+ items = notify_destinations.clean_items(raw_items)
16512
+ for item in items:
16513
+ source = item['source']
16514
+ table, wanted = source['table'], set(source['rowIds'])
16515
+ if table == 'applications':
16516
+ import application_registry
16517
+ import app_templates
16518
+ applications = application_registry.load(rt, strict=True)['applications']
16519
+ for app_id in wanted:
16520
+ app = applications.get(app_id)
16521
+ if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != rt.key
16522
+ or app.get('retired') is True):
16523
+ return None
16524
+ template = app_templates.BY_KIND.get(app.get('templateKind'))
16525
+ if not template or app_templates.held(app.get('templateKind')):
16526
+ return None
16527
+ sources = (*template['sources'], *template.get('optional_sources', ()))
16528
+ enabled = (getattr(getattr(rt, 'tenant', None), 'config', None) or {}).get('modules', 'all')
16529
+ if any((enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset))
16530
+ or source not in enabled))
16531
+ or not (user_tables.get_definition(source, st=rt) or {}).get('fields')
16532
+ for source in sources):
16533
+ return None
16534
+ for user in (recipient, principal):
16535
+ if not application_registry.may_read(app, user['username'], perms.is_admin(user)):
16536
+ return None
16537
+ access = application_registry.app_access(user, rt, app, sources)
16538
+ if access['app'] != 'ready' or any(wall['rows'] or wall['hidden']
16539
+ for wall in access['narrow'].values()):
16540
+ return None
16541
+ else:
16542
+ for user in (recipient, principal):
16543
+ if not perm_scope.may_read(user, table, st=rt) or perm_scope.wall_declared(user, table, st=rt):
16544
+ return None
16545
+ missing = set(wanted)
16546
+ for row in perm_scope.scoped_table(user, table, st=rt):
16547
+ missing.discard(str(row.get('pid')))
16548
+ if not missing:
16549
+ break
16550
+ if missing:
16551
+ return None
16552
+ return '\n\n'.join('\n'.join(str(item[key]) for key in ('title', 'body', 'url') if item.get(key))
16553
+ for item in items)
16554
+
16555
+
16556
+ def _notification_source_readable(rt, context, source):
16557
+ """Require destination read access to every source row and field before using supplied content."""
16558
+ import routes_slack
16559
+ from core import perm_scope
16560
+ agent = routes_slack._agents(rt).get(context["agentId"])
16561
+ module, ids = source["table"], set(source["rowIds"])
16562
+ if not agent or agent.get("active", True) is False:
16563
+ return False
16564
+ if module == "applications":
16565
+ import application_registry
16566
+ import app_templates
16567
+ apps = application_registry.load(rt, strict=True)["applications"]
16568
+ for rid in ids:
16569
+ app = apps.get(rid)
16570
+ if (not isinstance(app, dict) or app.get("tenant") != rt.key or app.get("retired")
16571
+ or not application_registry.may_read(app, context["principal"]["username"])):
16572
+ return False
16573
+ template = app_templates.BY_KIND.get(app.get("templateKind")) or {}
16574
+ sources = template.get("sources") or []
16575
+ if not sources or any(not _notification_source_readable(
16576
+ rt, context, {"table": key, "rowIds": []}) for key in sources):
16577
+ return False
16578
+ return bool(ids)
16579
+ if not routes_slack.agent_may_read(agent, module):
16580
+ return False
16581
+ definition = _ut().get_definition(module, st=rt)
16582
+ if not ids:
16583
+ table, refusal = definition, None
16584
+ elif definition is None:
16585
+ table, refusal = _module_table(rt, module, {}, want_rows=True)
16586
+ elif _ut().materialises(module, defn=definition):
16587
+ table, refusal = _ut().get_records(module, ids, st=rt), None
16588
+ else:
16589
+ table, refusal = _resolve_table(rt, {module: definition}, module, {}, want_rows=True)
16590
+ if refusal or not isinstance(table, dict):
16591
+ return False
16592
+ fields = table.get("fields") or []
16593
+ if not fields or perm_scope.hidden_keys(context["principal"], module, fields, st=rt):
16594
+ return False
16595
+ if not ids:
16596
+ return not perm_scope.row_scope_applies(context["principal"], module, st=rt, defn=definition)
16597
+ rows = table.get("rows") or {}
16598
+ if not ids <= set(rows):
16599
+ return False
16600
+ selected = [{**rows[rid], "pid": rid} for rid in ids]
16601
+ readable = routes_slack.agent_rows(agent, selected, module, fields, st=rt)
16602
+ return {str(row.get("pid")) for row in readable} == ids
16603
+
16604
+
16605
  def dispatch_authorized_slack_automation(rt, defn, result=None, *, post=None):
16606
  """Deliver one authorized event result, with readable linked records for an enters_view cohort."""
16607
  definition = defn if isinstance(defn, dict) else {}
16608
  aid = str(definition.get("id") or "").strip()
16609
  if not aid:
16610
  return {"sent": False, "reason": "unknown_automation"}
16611
+ if re.fullmatch(r"nd_[0-9a-f]{48}", aid):
16612
+ from core import notify_destinations
16613
+ try:
16614
+ stored = notify_destinations.slack_automation(rt, aid)
16615
+ except Exception: # noqa: BLE001
16616
+ return {"sent": False, "reason": "unknown_automation"}
16617
+ else:
16618
+ stored = all_definitions(rt).get(aid)
16619
  if not isinstance(stored, dict):
16620
  return {"sent": False, "reason": "unknown_automation"}
16621
  trigger = stored.get("trigger") if isinstance(stored.get("trigger"), dict) else {}
 
16639
  context = routes_slack.automation_answer_context(rt, aid)
16640
  if context is None:
16641
  return {"sent": False, "reason": "not_authorized"}
16642
+ if isinstance(result, dict) and "notification" in result:
16643
+ try:
16644
+ text = _authorized_notification_text(rt, stored, result['notification'], context)
16645
+ except Exception:
16646
+ text = None
16647
+ if text is None:
16648
+ return {'sent': False, 'reason': 'notification_not_authorized'}
16649
+ provenance = {'basis': 'authorized_app_data',
16650
+ 'citationIds': [item['id'] for item in result['notification']['items']],
16651
+ 'modelGuidance': False}
16652
+ delivered = routes_slack.deliver_channel_reply(
16653
+ rt, context['channel'], text, principal=context['principal'], automation_id=aid,
16654
+ provenance=provenance, post=post)
16655
+ if delivered.get('sent') is True and delivered.get('channel') == context['channel']:
16656
+ delivered['notificationVersion'] = NOTIFICATION_PAYLOAD_VERSION
16657
+ return delivered
16658
  provenance = {"basis": "model_guidance", "citationIds": [], "modelGuidance": False}
16659
  if trigger_key == "enters_view":
16660
  module = str(trigger.get("table") or _flow_table(stored) or "")
api/connector_onboarding.py CHANGED
@@ -136,14 +136,15 @@ def mirror_cursor(rt, table_key=None):
136
  if rt is None:
137
  return datastore.ro_con()
138
  rt.assert_datastore_matches()
 
139
  entities = selected_mirror_entities(rt)
140
  if entities is None:
141
  import odoo_relational as rel
142
  if (table_key is not None
143
  and rel.base_table_key(table_key) in rel.stock_source_specs()
144
- and not datastore.entity_live("stock_quant")):
145
  raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
146
- return datastore.ro_con()
147
  if table_key is not None:
148
  import odoo_relational as rel
149
  import routes_odoo_tables as route
@@ -152,13 +153,13 @@ def mirror_cursor(rt, table_key=None):
152
  raise RuntimeError("Enable this database in connection settings before reading it.")
153
  source = route._odoo_sources().get(table_key)
154
  if source is None:
155
- return datastore.ro_con()
156
  entities = set(source.get("tables", {}).values())
157
  if source.get("table"):
158
  entities.add(source["table"])
159
  if source.get("needs_excluded"):
160
  entities.add("res_partner")
161
- if any(not datastore.entity_live(key) for key in entities):
162
  raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
163
  # An empty first selection has no readers to run and need not create a mirror file.
164
- return datastore.ro_cursor() if entities else None
 
136
  if rt is None:
137
  return datastore.ro_con()
138
  rt.assert_datastore_matches()
139
+ mirror = rt.mirror_handle()
140
  entities = selected_mirror_entities(rt)
141
  if entities is None:
142
  import odoo_relational as rel
143
  if (table_key is not None
144
  and rel.base_table_key(table_key) in rel.stock_source_specs()
145
+ and not mirror.entity_live("stock_quant")):
146
  raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
147
+ return mirror.ro_con()
148
  if table_key is not None:
149
  import odoo_relational as rel
150
  import routes_odoo_tables as route
 
153
  raise RuntimeError("Enable this database in connection settings before reading it.")
154
  source = route._odoo_sources().get(table_key)
155
  if source is None:
156
+ return mirror.ro_con()
157
  entities = set(source.get("tables", {}).values())
158
  if source.get("table"):
159
  entities.add(source["table"])
160
  if source.get("needs_excluded"):
161
  entities.add("res_partner")
162
+ if any(not mirror.entity_live(key) for key in entities):
163
  raise RuntimeError("The selected database is completing its first sync. Try again shortly.")
164
  # An empty first selection has no readers to run and need not create a mirror file.
165
+ return mirror.ro_cursor() if entities else None
api/deps.py CHANGED
@@ -180,21 +180,173 @@ def _assistant_refuse(status, code, message):
180
  raise err(status, code, message)
181
 
182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  def assistant_unknown_field_message(requested, visible_keys):
184
- """Suggest at most five close keys from the caller's resolved visible fields only."""
 
 
 
 
185
  candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
186
  requested = [requested] if isinstance(requested, str) else requested
187
- matches = []
188
  for key in requested if isinstance(requested, (list, tuple)) else ():
189
  if isinstance(key, str) and key not in candidates:
 
 
 
190
  for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
191
  if match not in matches:
192
  matches.append(match)
193
- message = ("Every requested field must be readable in this database. "
194
- "Use a field key returned by describe_database.")
 
 
 
 
195
  if matches:
196
  message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
197
- return message
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
 
199
 
200
  def _assistant_tenant_allows(session: Session, database: str) -> bool:
@@ -217,13 +369,18 @@ def _assistant_preflight_request(session: Session, database: str, canonical_fiel
217
  "your current permissions do not allow this database")
218
  visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
219
  if grant is not None else list(canonical_fields or ()))
 
 
 
 
 
 
 
 
220
  try:
221
- perm_scope.validate_assistant_filter((grant or {}).get("filter"),
222
- canonical_fields if permission_fields is None else permission_fields)
223
- _assistant_requested_fields(requested_fields, visible)
224
  perm_scope.validate_assistant_filter(filters, visible)
225
  except ValueError as exc:
226
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
227
  return grant, visible
228
 
229
 
@@ -263,14 +420,16 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
263
  if field["key"] not in have)]
264
  fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
265
  internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
266
- _assistant_preflight_request(session, database, fields, requested_fields, filters,
267
- require_explicit_grant=True, permission_fields=internal["fields"])
 
268
  for member in members:
 
269
  try:
270
- perm_scope.validate_assistant_filter(
271
- (perm_scope.assistant_entry(member, database) or {}).get("filter"), internal["fields"])
272
  except ValueError as exc:
273
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
 
274
  if mode in ("schema", "query_schema"):
275
  if permission_scope is not None and "_permission_scope" in grid:
276
  permission_scope.update(internal)
@@ -299,15 +458,13 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
299
  except HTTPException:
300
  raise
301
  except Exception as exc: # noqa: BLE001
302
- _assistant_refuse(409, "assistant_snapshot_unavailable",
303
- f"this database's snapshot could not be built for your current "
304
- f"scope ({type(exc).__name__}); try again shortly")
305
  entry = cache.get(cache_key)
306
  if (not isinstance(entry, tuple) or len(entry) < 2
307
  or not isinstance(entry[1], (list, tuple))
308
  or any(not isinstance(row, dict) for row in entry[1])):
309
- _assistant_refuse(409, "assistant_snapshot_unavailable",
310
- "this database has no app-stored snapshot for your current scope")
311
  grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
312
  if permission_scope is not None and "_permission_scope" in grid:
313
  permission_scope.update(grid["_permission_scope"])
@@ -389,12 +546,11 @@ def _assistant_table_defn(session: Session, database: str):
389
 
390
  try:
391
  definitions = session.runtime.get_projection('user_tables', drop=('rows',))
392
- except Exception:
393
- _assistant_refuse(409, "assistant_snapshot_unavailable",
394
- "this database has no readable definition snapshot")
395
  if not isinstance(definitions, dict):
396
- _assistant_refuse(409, "assistant_snapshot_unavailable",
397
- "this database has no readable definition snapshot")
398
  if database == "ut_activity":
399
  definitions = user_tables._with_activity(definitions)
400
  definition = definitions.get(database)
@@ -428,8 +584,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
428
  # upstream fallback: it is the tenant's materialised document already held by the runtime.
429
  table = user_tables.get(database, st=session.runtime)
430
  if not isinstance(table, dict):
431
- _assistant_refuse(409, "assistant_snapshot_unavailable",
432
- "this database changed before its stored snapshot could be read")
433
  if not user_tables.materialises(database, st=session.runtime, defn=table):
434
  # ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
435
  # seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
@@ -442,9 +597,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
442
  # ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
443
  # is separately wrong; changing the flag to make a read pass would hide that defect and
444
  # move ten grids onto the wrong reader.
445
- _assistant_refuse(409, "assistant_snapshot_unavailable",
446
- "this database stopped storing its rows while it was being read; ask "
447
- "again and it will be read from its connector mirror instead")
448
 
449
  # A table's field contract was authorized from the rows-free definition. If it changed while
450
  # the materialised document was fetched, fail rather than projecting a schema that was never
@@ -452,12 +605,10 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
452
  actual_fields = [dict(field) for field in (table.get("fields") or [])
453
  if isinstance(field, dict) and field.get("key")]
454
  if actual_fields != fields:
455
- _assistant_refuse(409, "assistant_snapshot_unavailable",
456
- "this database changed before its stored snapshot could be read")
457
  raw_rows = table.get("rows") or {}
458
  if not isinstance(raw_rows, dict):
459
- _assistant_refuse(409, "assistant_snapshot_unavailable",
460
- "this database does not have a readable stored snapshot")
461
  keys = {field["key"] for field in fields}
462
  rows = []
463
  for record_id, raw in raw_rows.items():
@@ -539,9 +690,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
539
  # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
540
  declared = assistant_mirror.declared_fields(database, definition)
541
  if declared is None:
542
- _assistant_refuse(409, "assistant_snapshot_unavailable",
543
- "this database does not store its rows and it is not one of the "
544
- "connected databases the assistant can read from the mirror")
545
  # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
546
  # first sync. Passed only for an Amazon key, which is the one family it changes.
547
  cur, refusal = (assistant_mirror.open_cursor(session, database)
@@ -554,9 +703,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
554
  assistant_mirror.unavailable_reason(database, definition))
555
  fields = assistant_mirror.answerable_fields(database, spec, declared)
556
  if not fields:
557
- _assistant_refuse(409, "assistant_snapshot_unavailable",
558
- "this database's connector mirror carries none of its declared columns "
559
- "on this deployment, so there is nothing to answer from")
560
 
561
  grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
562
  filters, require_explicit_grant=False)
@@ -567,11 +714,17 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
567
  # Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
568
  # predicates that happen to agree, which is this repo's recorded defect class. Same input to
569
  # both compilers, same answer by construction.
 
570
  try:
571
  wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
 
 
 
 
 
572
  filter_tree = perm_scope.validate_assistant_filter(filters, visible)
573
  except ValueError as exc:
574
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
575
  raise # unreachable; keeps the names bound for a reader
576
  # ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
577
  # applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
@@ -589,10 +742,11 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
589
  hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
590
  tree = ((perm_scope.entry(member, database) or {}).get("filter")
591
  if not perms.is_admin(member) else None)
 
592
  try:
593
  tree = perm_scope.validate_assistant_filter(tree, fields)
594
  except ValueError as exc:
595
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
596
  if tree is not None:
597
  by_key = {field["key"]: field for field in fields}
598
  for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
@@ -602,13 +756,15 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
602
  verdict = perm_scope.wall_leaf_verdict(
603
  database, {"colId": leaf_key}, session.runtime)
604
  if not verdict["enforceable"]:
605
- _assistant_refuse(400, "assistant_bad_filter", verdict["reason"])
 
 
606
  trees.append(tree)
607
  visible = [field for field in visible if field.get("key") not in hidden]
608
  try:
609
  filter_tree = perm_scope.validate_assistant_filter(filters, visible)
610
  except ValueError as exc:
611
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
612
  wall_tree = {"conj": "and", "nodes": [
613
  {"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
614
  ]} if trees else None
@@ -788,7 +944,7 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
788
  try:
789
  perm_scope.validate_assistant_filter(filters, schema["fields"])
790
  except ValueError as exc:
791
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
792
  rows = []
793
  if mode not in ("schema", "query_schema"):
794
  cursor = None
@@ -804,16 +960,20 @@ def _assistant_activity_table(session, definition, fields, filters, mode, member
804
  if cursor in seen:
805
  break
806
  seen.add(cursor)
807
- _assistant_refuse(409, "assistant_snapshot_unavailable",
808
- "This activity history is too large for an Assistant snapshot. Read it in Table by page.")
809
  return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
810
 
811
 
 
 
 
 
812
  def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
813
  """Separate one opaque identity equality from visible field operands."""
814
  if database not in _ASSISTANT_BUILT_INS:
815
  if identity_pid is not None:
816
- _assistant_refuse(400, "assistant_bad_filter", "Opaque identity reads require a built-in database.")
 
817
  return filters, identity_pid
818
  if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
819
  and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
@@ -824,14 +984,17 @@ def _assistant_identity_request(database, filters, identity_pid, *, trusted_filt
824
  and not set(leaf) - {"id", "colId", "op", "value"}):
825
  value = leaf["value"]
826
  if value is None:
827
- _assistant_refuse(400, "assistant_bad_filter", "Use one non-null opaque record identity.")
 
828
  if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
829
- _assistant_refuse(400, "assistant_bad_filter", "Opaque identity pins must agree.")
 
830
  identity_pid, filters = value, None
831
  if identity_pid is not None and (type(identity_pid) not in (int, str)
832
  or isinstance(identity_pid, str)
833
  and (not identity_pid.strip() or len(identity_pid) > 200)):
834
- _assistant_refuse(400, "assistant_bad_filter", "Use one opaque integer or string record identity.")
 
835
  return filters, identity_pid
836
 
837
 
@@ -885,8 +1048,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
885
  try:
886
  access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
887
  except (perm_scope.UnknownTable, perm_scope.Unresolvable):
888
- _assistant_refuse(409, "assistant_snapshot_unavailable",
889
- "The database field contract could not be read. Try again in a moment.")
890
  if not access["readable"]:
891
  blocked = access["blockedBy"]
892
  names = dash_norm.plain(", ".join(blocked))
@@ -906,7 +1068,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
906
  try:
907
  perm_scope.validate_assistant_filter(filters, shared_fields)
908
  except ValueError as exc:
909
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
910
  if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
911
  operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
912
  if len(mode) > 2:
@@ -970,9 +1132,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
970
  if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
971
  # C16 on a mirror-served table would have to run inside the SQL before counting
972
  # and paging; filtering the returned page afterwards gives a wrong total. Refuse.
973
- _assistant_refuse(409, "assistant_snapshot_unavailable",
974
- "This database limits each record to its listed recipients, "
975
- "and that limit cannot be applied to this kind of read yet.")
976
  if members:
977
  rows, canonical_fields, source_version, label = _assistant_mirror_table(
978
  session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
@@ -1012,7 +1172,17 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
1012
  try:
1013
  perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
1014
  except ValueError as exc:
1015
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
 
 
 
 
 
 
 
 
 
 
1016
  # ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
1017
  # the same thing in the Analyst's answer as it does on the grid (see
1018
  # `perm_scope._enrich_for_wall`).
@@ -1079,7 +1249,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None,
1079
  try:
1080
  requested_filter = perm_scope.validate_assistant_filter(filters, visible)
1081
  except ValueError as exc:
1082
- _assistant_refuse(400, "assistant_bad_filter", str(exc))
1083
  if requested_filter is not None:
1084
  visible_keys = {field["key"] for field in visible}
1085
  evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
 
180
  raise err(status, code, message)
181
 
182
 
183
+ def _assistant_source_causes():
184
+ """Return each unavailable cause's sentence and whether waiting can make the read succeed."""
185
+ import assistant_mirror
186
+ return {
187
+ "loading": ("The copy of this database is still loading on this server.", True),
188
+ "syncing": ("A sync of this database's source is running.", True),
189
+ "store_silent": ("The store did not answer.", True),
190
+ "changed": ("This database changed while it was being read.", True),
191
+ "not_built": ("The copy of this database for your access could not be built.", False),
192
+ "contract_unread": ("The field list of this database could not be read.", False),
193
+ "no_binding": (assistant_mirror.NO_BINDING, False),
194
+ "no_copy": ("This database stores no rows and has no connector copy to read them from.", False),
195
+ "no_columns": ("The connector copy of this database holds none of its fields on this server.", False),
196
+ "unreadable": ("The stored copy of this database cannot be read.", False),
197
+ "audience": ("This database limits each record to its listed recipients, and that limit "
198
+ "cannot be applied to this kind of read yet.", False),
199
+ "too_large": ("This activity history is too large for an Assistant snapshot. "
200
+ "Read it in Table by page.", False),
201
+ "no_connector": ("This database names no connector, so its completed sync cannot be checked.", False),
202
+ "no_activity_store": ("Activity storage is not set up for this workspace.", False),
203
+ "invalid_publication": ("The completed sync identity of this database is unreadable.", False),
204
+ "restore_blocked": ("The saved copy of this database could not be restored. "
205
+ "Ask an Administrator to repair store access and restart the server.", False),
206
+ "mirror_open": ("The connector copy of this database could not be opened on this server.", False),
207
+ "not_supported": ("Selected record reads are not available on this deployment until it is updated.",
208
+ False),
209
+ }
210
+
211
+
212
+ def _assistant_source_message(cause, verb="read", *, message=None):
213
+ """The refusal text for one cause: its sentence, then a retry line only where a retry can succeed."""
214
+ if cause in {"amazon_restore", "amazon_sync"} and _assistant_source_cause_of(message) == cause:
215
+ return message
216
+ sentence, retry = _assistant_source_causes()[cause]
217
+ return f"{sentence} Try the same {verb} again shortly." if retry else sentence
218
+
219
+
220
+ def _assistant_source_cause_of(message):
221
+ """The cause a refusal message names, or None for text outside the closed set."""
222
+ text = " ".join(str(message or "").split())
223
+ for cause, (sentence, _retry) in _assistant_source_causes().items():
224
+ if text.startswith(sentence):
225
+ return cause
226
+ if text.startswith("Amazon "):
227
+ import amazon_relational as amazon
228
+ # Recognize native instructions without another pending-state read.
229
+ for key in amazon.SOURCE:
230
+ for cause, pending in (("amazon_restore", True), ("amazon_sync", False)):
231
+ if text == amazon.missing_source(key, pending=pending)["missing"]:
232
+ return cause
233
+ return None
234
+
235
+
236
+ def _assistant_refuse_source(cause, status=409, code="assistant_snapshot_unavailable"):
237
+ """Refuse a read whose snapshot or source is unavailable, naming its cause."""
238
+ _assistant_refuse(status, code, _assistant_source_message(cause))
239
+
240
+
241
+ def _assistant_log_cause(where, exc):
242
+ """Log an exception's class, never its text, beside the cause the caller was told."""
243
+ import logging
244
+ logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
245
+
246
+
247
+ def _assistant_display_key(key):
248
+ """A caller-supplied key or operator as message text: one line, at most 80 characters."""
249
+ return " ".join(str(key).split())[:80]
250
+
251
+
252
  def assistant_unknown_field_message(requested, visible_keys):
253
+ """Name the requested keys that are not readable, then suggest at most five close visible keys.
254
+
255
+ S82-T124: only keys the caller sent are echoed, and suggestions come from `visible_keys` only.
256
+ A hidden key and an unknown key therefore read the same, so the message is no existence oracle.
257
+ """
258
  candidates = sorted({key for key in visible_keys if isinstance(key, str) and key})
259
  requested = [requested] if isinstance(requested, str) else requested
260
+ matches, missing = [], []
261
  for key in requested if isinstance(requested, (list, tuple)) else ():
262
  if isinstance(key, str) and key not in candidates:
263
+ name = _assistant_display_key(key)
264
+ if name and name not in missing:
265
+ missing.append(name)
266
  for match in difflib.get_close_matches(key, candidates, n=5, cutoff=0.6):
267
  if match not in matches:
268
  matches.append(match)
269
+ message = ""
270
+ if missing:
271
+ message = (("No readable field is named " if len(missing) == 1
272
+ else "No readable fields are named ") + ", ".join(missing[:5]) + ". ")
273
+ message += ("Every requested field must be readable in this database. "
274
+ "Use a field key returned by describe_database.")
275
  if matches:
276
  message += " Close visible field keys: " + ", ".join(matches[:5]) + "."
277
+ return dash_norm.plain(message)
278
+
279
+
280
+ def _assistant_filter_leaves(tree, depth=1):
281
+ """Every condition of a raw filter tree, in the order the strict validator reads them."""
282
+ nodes = tree if isinstance(tree, list) else (tree.get("nodes") if isinstance(tree, dict) else None)
283
+ out = []
284
+ for node in nodes if isinstance(nodes, list) else ():
285
+ if isinstance(node, dict) and "children" in node:
286
+ if depth < 3:
287
+ out.extend(_assistant_filter_leaves(node.get("children"), depth + 1))
288
+ else:
289
+ out.append(node)
290
+ return out
291
+
292
+
293
+ def _assistant_rejected_condition(tree, fields):
294
+ """The first condition the strict validator refuses on its own, or None for a shape fault."""
295
+ import core.perm_scope as perm_scope
296
+ for leaf in _assistant_filter_leaves(tree):
297
+ try:
298
+ perm_scope.validate_assistant_filter([leaf], fields)
299
+ except ValueError:
300
+ return leaf
301
+ except Exception: # noqa: BLE001 - a leaf the tree never reached
302
+ return None
303
+ return None
304
+
305
+
306
+ def _assistant_bad_filter_message(reason, condition, *, visible_keys=(), wall=False):
307
+ """The `assistant_bad_filter` text: the validator's reason plus the rejected field and operator.
308
+
309
+ S82-T124: every refusal names its field. A wall's field is named only when the caller may read
310
+ it, so the restriction an administrator set never names a hidden field.
311
+ """
312
+ text = " ".join(str(reason or "").split()).rstrip(".") or "assistant filter cannot be read"
313
+ where = " in the restriction an administrator set on this database" if wall else ""
314
+ field = condition.get("colId") if isinstance(condition, dict) else None
315
+ name = _assistant_display_key(field) if isinstance(field, str) else ""
316
+ if condition is None:
317
+ text += ". The filter's structure is at fault, not one field"
318
+ elif not name:
319
+ text += f". The rejected condition{where} names no field"
320
+ elif wall and field not in set(visible_keys or ()):
321
+ text += f". The rejected condition{where} is on a field you cannot read"
322
+ else:
323
+ text += f'. The rejected condition{where} is on field "{name}"'
324
+ if "op" in condition:
325
+ op = condition.get("op")
326
+ text += (f' with operator "{_assistant_display_key(op)}"'
327
+ if isinstance(op, str) and op.strip() else " with no operator")
328
+ return dash_norm.plain(text + ".")
329
+
330
+
331
+ def _assistant_refuse_filter(reason, tree, fields, *, visible_keys=None, wall=False):
332
+ """Refuse a filter the strict validator rejected, naming the condition it rejected.
333
+
334
+ A caller's condition on a key outside its readable `fields` answers `assistant_unknown_field`,
335
+ as a field selection does, so hidden and unknown keys read alike. `pid` is the record identity
336
+ no filter may name, and a wall is the administrator's: both stay `assistant_bad_filter`.
337
+ """
338
+ keys = {str(field.get("key")) for field in fields or ()
339
+ if isinstance(field, dict) and field.get("key")}
340
+ visible = keys if visible_keys is None else set(visible_keys)
341
+ condition = _assistant_rejected_condition(tree, fields)
342
+ if not wall and isinstance(condition, dict):
343
+ rhs = condition.get("rhs")
344
+ for key in (condition.get("colId"), rhs.get("colId") if isinstance(rhs, dict) else None):
345
+ if isinstance(key, str) and key.strip() and key not in keys and key != "pid":
346
+ _assistant_refuse(400, "assistant_unknown_field",
347
+ assistant_unknown_field_message(key, visible))
348
+ _assistant_refuse(400, "assistant_bad_filter",
349
+ _assistant_bad_filter_message(reason, condition, visible_keys=visible, wall=wall))
350
 
351
 
352
  def _assistant_tenant_allows(session: Session, database: str) -> bool:
 
369
  "your current permissions do not allow this database")
370
  visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database)
371
  if grant is not None else list(canonical_fields or ()))
372
+ wall_fields = canonical_fields if permission_fields is None else permission_fields
373
+ visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
374
+ try:
375
+ perm_scope.validate_assistant_filter((grant or {}).get("filter"), wall_fields)
376
+ except ValueError as exc:
377
+ _assistant_refuse_filter(exc, (grant or {}).get("filter"), wall_fields,
378
+ visible_keys=visible_keys, wall=True)
379
+ _assistant_requested_fields(requested_fields, visible)
380
  try:
 
 
 
381
  perm_scope.validate_assistant_filter(filters, visible)
382
  except ValueError as exc:
383
+ _assistant_refuse_filter(exc, filters, visible)
384
  return grant, visible
385
 
386
 
 
420
  if field["key"] not in have)]
421
  fields, _ = schema_overlay.for_grid(database, fields, st=session.runtime, contract=fields)
422
  internal = grid.get("_permission_scope") or {"fields": fields, "rows": [], "row_fields": fields}
423
+ _grant, visible = _assistant_preflight_request(
424
+ session, database, fields, requested_fields, filters,
425
+ require_explicit_grant=True, permission_fields=internal["fields"])
426
  for member in members:
427
+ wall = (perm_scope.assistant_entry(member, database) or {}).get("filter")
428
  try:
429
+ perm_scope.validate_assistant_filter(wall, internal["fields"])
 
430
  except ValueError as exc:
431
+ _assistant_refuse_filter(exc, wall, internal["fields"], wall=True, visible_keys=[
432
+ field.get("key") for field in visible if isinstance(field, dict)])
433
  if mode in ("schema", "query_schema"):
434
  if permission_scope is not None and "_permission_scope" in grid:
435
  permission_scope.update(internal)
 
458
  except HTTPException:
459
  raise
460
  except Exception as exc: # noqa: BLE001
461
+ _assistant_log_cause("assistant pool build", exc)
462
+ _assistant_refuse_source("not_built")
 
463
  entry = cache.get(cache_key)
464
  if (not isinstance(entry, tuple) or len(entry) < 2
465
  or not isinstance(entry[1], (list, tuple))
466
  or any(not isinstance(row, dict) for row in entry[1])):
467
+ _assistant_refuse_source("not_built")
 
468
  grid = grid_assembly(session, scope=grid_scope, consume_corrections=False, with_rows=True)
469
  if permission_scope is not None and "_permission_scope" in grid:
470
  permission_scope.update(grid["_permission_scope"])
 
546
 
547
  try:
548
  definitions = session.runtime.get_projection('user_tables', drop=('rows',))
549
+ except Exception as exc:
550
+ _assistant_log_cause("assistant definition read", exc)
551
+ _assistant_refuse_source("store_silent")
552
  if not isinstance(definitions, dict):
553
+ _assistant_refuse_source("unreadable")
 
554
  if database == "ut_activity":
555
  definitions = user_tables._with_activity(definitions)
556
  definition = definitions.get(database)
 
584
  # upstream fallback: it is the tenant's materialised document already held by the runtime.
585
  table = user_tables.get(database, st=session.runtime)
586
  if not isinstance(table, dict):
587
+ _assistant_refuse_source("changed")
 
588
  if not user_tables.materialises(database, st=session.runtime, defn=table):
589
  # ⭐⭐ W47-T23 / R11 - THIS IS NO LONGER THE ANSWER FOR A MIRROR-SERVED DATABASE, and the
590
  # seam this comment used to ask for now exists: `assistant_mirror.py`, reached through
 
597
  # ⛔ DO NOT "FIX" THIS BY FLIPPING `readThrough`. D-378 says these tables' classification
598
  # is separately wrong; changing the flag to make a read pass would hide that defect and
599
  # move ten grids onto the wrong reader.
600
+ _assistant_refuse_source("changed")
 
 
601
 
602
  # A table's field contract was authorized from the rows-free definition. If it changed while
603
  # the materialised document was fetched, fail rather than projecting a schema that was never
 
605
  actual_fields = [dict(field) for field in (table.get("fields") or [])
606
  if isinstance(field, dict) and field.get("key")]
607
  if actual_fields != fields:
608
+ _assistant_refuse_source("changed")
 
609
  raw_rows = table.get("rows") or {}
610
  if not isinstance(raw_rows, dict):
611
+ _assistant_refuse_source("unreadable")
 
612
  keys = {field["key"] for field in fields}
613
  rows = []
614
  for record_id, raw in raw_rows.items():
 
690
  # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
691
  declared = assistant_mirror.declared_fields(database, definition)
692
  if declared is None:
693
+ _assistant_refuse_source("no_copy")
 
 
694
  # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
695
  # first sync. Passed only for an Amazon key, which is the one family it changes.
696
  cur, refusal = (assistant_mirror.open_cursor(session, database)
 
703
  assistant_mirror.unavailable_reason(database, definition))
704
  fields = assistant_mirror.answerable_fields(database, spec, declared)
705
  if not fields:
706
+ _assistant_refuse_source("no_columns")
 
 
707
 
708
  grant, visible = _assistant_preflight_request(session, database, fields, requested_fields,
709
  filters, require_explicit_grant=False)
 
714
  # Pushing the raw input to SQL instead would mean the pushdown and the Python pass are two
715
  # predicates that happen to agree, which is this repo's recorded defect class. Same input to
716
  # both compilers, same answer by construction.
717
+ visible_keys = [field.get("key") for field in visible if isinstance(field, dict)]
718
  try:
719
  wall_tree = perm_scope.validate_assistant_filter((grant or {}).get("filter"), fields)
720
+ except ValueError as exc:
721
+ _assistant_refuse_filter(exc, (grant or {}).get("filter"), fields,
722
+ visible_keys=visible_keys, wall=True)
723
+ raise # unreachable; keeps the names bound for a reader
724
+ try:
725
  filter_tree = perm_scope.validate_assistant_filter(filters, visible)
726
  except ValueError as exc:
727
+ _assistant_refuse_filter(exc, filters, visible)
728
  raise # unreachable; keeps the names bound for a reader
729
  # ⚠ NO GRANT MEANS NO ROW WALL ON THIS PATH, matching `assistant_read_scope` exactly: it
730
  # applies `assistant_apply_row_scope` only when a grant resolved, because `may_open` above is
 
742
  hidden.update(perm_scope.hidden_keys(member, database, fields, st=session.runtime))
743
  tree = ((perm_scope.entry(member, database) or {}).get("filter")
744
  if not perms.is_admin(member) else None)
745
+ raw_wall = tree
746
  try:
747
  tree = perm_scope.validate_assistant_filter(tree, fields)
748
  except ValueError as exc:
749
+ _assistant_refuse_filter(exc, raw_wall, fields, visible_keys=visible_keys, wall=True)
750
  if tree is not None:
751
  by_key = {field["key"]: field for field in fields}
752
  for leaf_key in perm_scope._wall_leaf_keys(tree["nodes"]):
 
756
  verdict = perm_scope.wall_leaf_verdict(
757
  database, {"colId": leaf_key}, session.runtime)
758
  if not verdict["enforceable"]:
759
+ _assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
760
+ verdict["reason"], {"colId": leaf_key},
761
+ visible_keys=visible_keys, wall=True))
762
  trees.append(tree)
763
  visible = [field for field in visible if field.get("key") not in hidden]
764
  try:
765
  filter_tree = perm_scope.validate_assistant_filter(filters, visible)
766
  except ValueError as exc:
767
+ _assistant_refuse_filter(exc, filters, visible)
768
  wall_tree = {"conj": "and", "nodes": [
769
  {"conj": tree["conj"], "children": tree["nodes"]} for tree in trees
770
  ]} if trees else None
 
944
  try:
945
  perm_scope.validate_assistant_filter(filters, schema["fields"])
946
  except ValueError as exc:
947
+ _assistant_refuse_filter(exc, filters, schema["fields"])
948
  rows = []
949
  if mode not in ("schema", "query_schema"):
950
  cursor = None
 
960
  if cursor in seen:
961
  break
962
  seen.add(cursor)
963
+ _assistant_refuse_source("too_large")
 
964
  return rows, canonical, {"read": "field contract"}, definition.get("label") or "Activity"
965
 
966
 
967
+ #: The one condition an opaque identity read carries; its refusals name this field and operator.
968
+ _IDENTITY_CONDITION = {"colId": "pid", "op": "eq"}
969
+
970
+
971
  def _assistant_identity_request(database, filters, identity_pid, *, trusted_filters=False):
972
  """Separate one opaque identity equality from visible field operands."""
973
  if database not in _ASSISTANT_BUILT_INS:
974
  if identity_pid is not None:
975
+ _assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
976
+ "Opaque identity reads require a built-in database", _IDENTITY_CONDITION))
977
  return filters, identity_pid
978
  if (trusted_filters and isinstance(filters, dict) and set(filters) == {"conj", "nodes"}
979
  and filters.get("conj") == "and" and isinstance(filters.get("nodes"), list)
 
984
  and not set(leaf) - {"id", "colId", "op", "value"}):
985
  value = leaf["value"]
986
  if value is None:
987
+ _assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
988
+ "Use one non-null opaque record identity", _IDENTITY_CONDITION))
989
  if identity_pid is not None and (type(value) is not type(identity_pid) or value != identity_pid):
990
+ _assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
991
+ "Opaque identity pins must agree", _IDENTITY_CONDITION))
992
  identity_pid, filters = value, None
993
  if identity_pid is not None and (type(identity_pid) not in (int, str)
994
  or isinstance(identity_pid, str)
995
  and (not identity_pid.strip() or len(identity_pid) > 200)):
996
+ _assistant_refuse(400, "assistant_bad_filter", _assistant_bad_filter_message(
997
+ "Use one opaque integer or string record identity", _IDENTITY_CONDITION))
998
  return filters, identity_pid
999
 
1000
 
 
1048
  try:
1049
  access = perm_scope.min_access([session.user, *members], key, st=session.runtime)
1050
  except (perm_scope.UnknownTable, perm_scope.Unresolvable):
1051
+ _assistant_refuse_source("contract_unread")
 
1052
  if not access["readable"]:
1053
  blocked = access["blockedBy"]
1054
  names = dash_norm.plain(", ".join(blocked))
 
1068
  try:
1069
  perm_scope.validate_assistant_filter(filters, shared_fields)
1070
  except ValueError as exc:
1071
+ _assistant_refuse_filter(exc, filters, shared_fields)
1072
  if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
1073
  operands = [spec.get("field") for spec in mode[1] if isinstance(spec, dict)]
1074
  if len(mode) > 2:
 
1132
  if perm_scope.audience_declared(key, st=session.runtime, defn=_defn):
1133
  # C16 on a mirror-served table would have to run inside the SQL before counting
1134
  # and paging; filtering the returned page afterwards gives a wrong total. Refuse.
1135
+ _assistant_refuse_source("audience")
 
 
1136
  if members:
1137
  rows, canonical_fields, source_version, label = _assistant_mirror_table(
1138
  session, key, fields, filters, defn=_defn, lent=_lent, mode=mode,
 
1172
  try:
1173
  perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields)
1174
  except ValueError as exc:
1175
+ # Only on the refusal path: the caller's readable keys decide whether the wall's field
1176
+ # is named. An unreadable list names no field, so the refusal stays a 400.
1177
+ try:
1178
+ withheld = set(perm_scope.hidden_keys(session.user, key, canonical_fields,
1179
+ st=session.runtime))
1180
+ readable = [field.get("key") for field in perm_scope.assistant_visible_fields(
1181
+ canonical_fields, session.user, key) if field.get("key") not in withheld]
1182
+ except Exception: # noqa: BLE001 - the refusal must still answer
1183
+ readable = []
1184
+ _assistant_refuse_filter(exc, (grant or {}).get("filter"), canonical_fields,
1185
+ visible_keys=readable, wall=True)
1186
  # ⭐ OWNER I16 — the tenant handle rides into the row wall here too, so one stored rule means
1187
  # the same thing in the Analyst's answer as it does on the grid (see
1188
  # `perm_scope._enrich_for_wall`).
 
1249
  try:
1250
  requested_filter = perm_scope.validate_assistant_filter(filters, visible)
1251
  except ValueError as exc:
1252
+ _assistant_refuse_filter(exc, filters, visible)
1253
  if requested_filter is not None:
1254
  visible_keys = {field["key"] for field in visible}
1255
  evaluation_visible = [field for field in evaluation_fields if field["key"] in visible_keys]
api/diag_aging.py CHANGED
@@ -26,22 +26,17 @@ def validate(session, rows, restrictions, *, today, documents=None):
26
  scoped = ScopedOdoo(O.get_odoo(), restrictions)
27
  helpers = scoped.helpers()
28
  module = bind_functions(ar, {"O": helpers})
29
- docs = module._open_docs(today)
 
30
  exposures = {}
31
  for doc in docs:
32
  pid = helpers.m2o_id(doc["partner_id"])
33
- row = exposures.setdefault(pid, {"open": 0.0, "overdue": 0.0,
34
- **{f"aged_{b}": 0.0 for b in ar.OVERDUE_BUCKETS}})
35
- if doc["days_overdue"] > 5:
36
- row["overdue"] += doc["open"]
37
- row[f'aged_{doc["bucket"]}'] += doc["open"]
38
- else:
39
- row["open"] += doc["open"]
40
- # The unrelated settlement check remains false and is not part of this diagnostic.
41
- existing = module.validate(today, exposure={"_all_rows": list(exposures.values())},
42
- behavior={"violations": 0, "n_customers": 0})[0]
43
- existing = {"check": "AR aging buckets equal open documents",
44
- **{key: existing[key] for key in ("a", "b", "ok")}}
45
  canonical = module.aging(today)
46
  checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
47
  "b": len(canonical), "ok": rows == [
 
26
  scoped = ScopedOdoo(O.get_odoo(), restrictions)
27
  helpers = scoped.helpers()
28
  module = bind_functions(ar, {"O": helpers})
29
+ docs = list(module._open_docs(today))
30
+ # Tie in place: ar.validate reads account.move.line, which the scoped client refuses.
31
  exposures = {}
32
  for doc in docs:
33
  pid = helpers.m2o_id(doc["partner_id"])
34
+ row = exposures.setdefault(pid, {"open": Decimal(0), "overdue": Decimal(0)})
35
+ row["overdue" if doc["days_overdue"] > 5 else "open"] += amount(doc["open"])
36
+ exposed = sum((row["open"] + row["overdue"] for row in exposures.values()), Decimal(0))
37
+ population = sum((amount(doc["open"]) for doc in docs), Decimal(0))
38
+ existing = {"check": "AR aging buckets equal open documents", "a": float(exposed),
39
+ "b": float(population), "ok": abs(exposed - population) < Decimal("0.01")}
 
 
 
 
 
 
40
  canonical = module.aging(today)
41
  checks = [existing, {"check": "AR aging matches module buckets", "a": len(rows),
42
  "b": len(canonical), "ok": rows == [
api/external_freshness.py CHANGED
@@ -82,8 +82,18 @@ def _amazon_raw(table_key, source=None):
82
  return None
83
 
84
 
85
- def amazon_sync_state(cur):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
 
 
 
 
 
 
 
 
 
 
87
  return {str(entity): at for entity, at in cur.execute(
88
  "SELECT entity, updated_at FROM _sync_state "
89
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
@@ -138,7 +148,7 @@ def _amazon_refreshes(session, keys):
138
  cur = None
139
  continue
140
  if state is None:
141
- state = amazon_sync_state(cur)
142
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
143
  except Exception:
144
  continue
@@ -167,7 +177,7 @@ def amazon_nav(session, pages):
167
  cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
168
  if refusal is not None or cur is None:
169
  return pages
170
- state = amazon_sync_state(cur)
171
  if not state:
172
  # An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
173
  return pages
@@ -247,14 +257,17 @@ def snapshot(session, databases):
247
  edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
248
  edits = {}
249
  if edited:
 
 
 
250
  try:
251
- history = rt.get(user_tables.VALUE_HISTORY_KEY) or []
 
252
  except Exception:
253
- history = []
254
- for event in history if isinstance(history, list) else ():
255
- if isinstance(event, dict) and event.get("table") in edited:
256
- key = event["table"]
257
- edits[key] = _latest(edits.get(key), event.get("at"))
258
  receipts = {}
259
  if connected:
260
  try:
 
82
  return None
83
 
84
 
85
+ def amazon_sync_state(cur, rt=None):
86
  """`{entity: updated_at}` for every `done` Amazon `_sync_state` row, in one read."""
87
+ if rt is not None:
88
+ from core import connector_sync, connector_spec
89
+ entry = connector_sync.status(rt, "amazon_sp", fresh=True)
90
+ if entry is not None:
91
+ last_pass = entry.get("pass") or {}
92
+ held = last_pass.get("held") or {}
93
+ stamps = last_pass.get("tableSync") or {}
94
+ tables = {row.source.removeprefix("amzn_") for row in connector_spec.AMAZON_TABLES}
95
+ return {"amzn_sync_mark_" + name: stamp for name in tables
96
+ if (stamp := stamps.get(name) or (entry.get("lastGoodSyncAt") if name not in held else None))}
97
  return {str(entity): at for entity, at in cur.execute(
98
  "SELECT entity, updated_at FROM _sync_state "
99
  "WHERE phase = 'done' AND starts_with(entity, 'amzn_')").fetchall()}
 
148
  cur = None
149
  continue
150
  if state is None:
151
+ state = amazon_sync_state(cur, session.runtime)
152
  refreshed[key] = amazon_sync_stamp(cur, key, state=state)
153
  except Exception:
154
  continue
 
177
  cur, refusal = assistant_mirror.open_cursor(session, rows[0]["key"])
178
  if refusal is not None or cur is None:
179
  return pages
180
+ state = amazon_sync_state(cur, session.runtime)
181
  if not state:
182
  # An unrestored or absent mirror holds no receipt at all: unknown, not never synced.
183
  return pages
 
257
  edited = {key for key in wanted if key in definitions and key not in connected and key != "ut_activity"}
258
  edits = {}
259
  if edited:
260
+ # D-11777: edits append only to the history rows, so one SQL aggregate gives each table's
261
+ # newest edit. A store with no history backend (TypeError) or an unreachable one keeps the
262
+ # definition's `updated` stamp below and never fails discovery.
263
  try:
264
+ from core import history_rows
265
+ latest = history_rows.backend(rt).latest(sorted(edited))
266
  except Exception:
267
+ latest = {}
268
+ for key, at in latest.items() if isinstance(latest, dict) else ():
269
+ if key in edited:
270
+ edits[key] = _latest(edits.get(key), at)
 
271
  receipts = {}
272
  if connected:
273
  try:
api/loopy_file_tools.py CHANGED
@@ -55,7 +55,9 @@ def read_all(session, database, *, fields=None, filters=None, sorts=None, max_ro
55
  or type(page.get("returned")) is not int or page["returned"] != len(rows)):
56
  raise ValueError("The database did not return a complete readable page.")
57
  page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
58
- if page_keys != [key for key in by_key if key in selected]:
 
 
59
  raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
60
  if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
61
  raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
 
55
  or type(page.get("returned")) is not int or page["returned"] != len(rows)):
56
  raise ValueError("The database did not return a complete readable page.")
57
  page_keys = [field.get("key") for field in result.get("fields", []) if isinstance(field, dict)]
58
+ # Field set and count, not order: query pages and the schema list fields in different orders (D-8381).
59
+ wanted = [key for key in by_key if key in selected]
60
+ if len(page_keys) != len(wanted) or set(page_keys) != set(wanted):
61
  raise err(409, "query_cursor_stale", "The readable fields changed. Ask again to read the complete list.")
62
  if "offset" in page and (type(page["offset"]) is not int or page["offset"] != len(records)):
63
  raise ValueError("The database skipped rows. Read the complete result again. No file was stored.")
api/loopy_tools.py CHANGED
@@ -4,7 +4,7 @@ import logging
4
 
5
  MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
6
  "loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
7
- "loopy_reminder_tools", "tool_groups.apps")
8
  TOOLS = []
9
  _LOG = logging.getLogger(__name__)
10
 
 
4
 
5
  MODULES = ("loopy_file_tools", "loopy_chat_link_tools", "loopy_skill_tools",
6
  "loopy_catalog_tools", "loopy_diagnostic_tools", "loopy_worklist_tools",
7
+ "loopy_reminder_tools", "tool_groups.apps", "tool_groups.live")
8
  TOOLS = []
9
  _LOG = logging.getLogger(__name__)
10
 
api/main.py CHANGED
@@ -412,10 +412,11 @@ def _mirror_answers(rt=None):
412
 
413
  def _tenant_readiness(rt):
414
  """Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
415
- checks = {"mirror": False, "pools": False, "first_reads": False}
416
  detail = {"mirror": "The tenant mirror check did not answer.",
417
  "pools": "The tenant store did not answer.",
418
- "first_reads": "The tenant's first read did not answer."}
 
419
  stage = "mirror"
420
  try:
421
  if rt.odoo_source() is not None:
@@ -432,6 +433,11 @@ def _tenant_readiness(rt):
432
  # HF display reads hide outages, so readiness uses the tenant's strict reader.
433
  rt._store()._read_strict(rt.store_key("sync_settings"))
434
  checks["first_reads"] = True
 
 
 
 
 
435
  except Exception as exc: # noqa: BLE001
436
  detail[stage] += f" ({type(exc).__name__})"
437
  failing = [name for name in checks if not checks[name]]
@@ -441,21 +447,52 @@ def _tenant_readiness(rt):
441
 
442
  @app.get("/api/v1/ready")
443
  def readiness_check(request: Request, response: Response):
444
- """Report tenant readiness publicly and reveal the version only after session verification."""
 
445
  session = (deps_require_session(request, response)
446
  if request.cookies.get(aios_session.COOKIE_NAME) else None)
 
 
 
 
 
447
  try:
448
  from harness import runtime as _ready_runtime
449
  if session is not None:
450
  report = _tenant_readiness(session.runtime)
451
  else:
452
- names = ("mirror", "pools", "first_reads")
453
- reports = []
454
- for key in _ready_runtime.known_tenants():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
455
  try:
456
- reports.append(_tenant_readiness(_ready_runtime.get_runtime(key)))
457
  except Exception:
458
  reports.append({"checks": dict.fromkeys(names, False)})
 
 
 
 
 
 
 
 
459
  checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
460
  for name in names}
461
  failing = [name for name in names if not checks[name]]
@@ -465,7 +502,9 @@ def readiness_check(request: Request, response: Response):
465
  report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
466
  "detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
467
  body = {"ready": report["ready"], "checks": report["checks"]}
468
- if session is not None:
 
 
469
  body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
470
  response.headers["Cache-Control"] = "no-store"
471
  if report["ready"]:
@@ -704,6 +743,8 @@ app.include_router(routes_external.router)
704
  import app_routes
705
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
706
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
 
 
707
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
708
  _mount_router("routes_sku_actions")
709
  _mount_router("notify_destinations")
@@ -977,6 +1018,131 @@ def _migrate_env_odoo(why):
977
  print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
978
 
979
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
980
  def _pull_meta(why, tenant_key=None):
981
  """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
982
 
@@ -996,22 +1162,35 @@ def _pull_meta(why, tenant_key=None):
996
  outcomes = {}
997
  try:
998
  import process_role as _meta_role
 
999
  # Each tenant is pulled by exactly one worker: its own, else the primary.
1000
  tenants = [tenant_key] if tenant_key else [
1001
  key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
1002
- for key in tenants:
1003
- outcomes[key] = False
 
 
 
 
 
 
1004
  try:
1005
- rt = runtime.get_runtime(key)
1006
  if not _meta._connection_rows(rt, key, {"problems": []}):
1007
  continue
 
 
 
1008
  rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
1009
  log=lambda *_a: None, insights_days=7)
1010
  outcomes[key] = mirror_store._sync_result("meta", rep)[1]
 
1011
  for problem in rep.get("problems") or []:
1012
  print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
1013
  except Exception as exc:
1014
  print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
 
 
 
1015
  except Exception as exc:
1016
  print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
1017
  return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
@@ -1101,6 +1280,9 @@ def _pull_amazon(why, vendor=True):
1101
  print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
1102
  f"for {tenant_key}; the connector is idle, not broken")
1103
  return False
 
 
 
1104
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
1105
  # bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
1106
  # ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
@@ -1118,17 +1300,28 @@ def _pull_amazon(why, vendor=True):
1118
  _rot.invalidate_sources()
1119
  _rot._sources()
1120
  if first_load_needed:
1121
- from core import connector_sync as _connector_sync
1122
- receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
1123
- print(f"[aios-api] amazon sync ({why}): first load "
1124
- + ("queued on the sync desk" if receipt.get("accepted") else "already queued or running"))
1125
  if not vendor:
1126
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
1127
  return True
1128
  from harness import mirror_store
1129
- rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1130
- refresh_loaded=first_load_needed, slot_pass=bool(RESYNC_STATE.get("slotPass")),
1131
- log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
 
 
 
 
 
 
 
 
 
 
 
1132
  _rot.invalidate_sources()
1133
  _rot._sources()
1134
  for problem in rep.get("problems") or []:
@@ -1143,7 +1336,7 @@ def _pull_amazon(why, vendor=True):
1143
  store=rt, quiet=_sa.amazon_standing(rep))
1144
  except Exception: # noqa: BLE001
1145
  pass
1146
- return not bool(rep.get("problems"))
1147
  except Exception as exc: # noqa: BLE001
1148
  print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
1149
  return False
@@ -1295,6 +1488,17 @@ def _sync_now_cause(cause):
1295
  """Classify actual provider failures without returning vendor text or exception reprs."""
1296
  import re
1297
  if isinstance(cause, dict):
 
 
 
 
 
 
 
 
 
 
 
1298
  sentence = "The connector could not read an expected value in the sync response."
1299
  if cause.get("classifier") == "unreadable_value":
1300
  table = cause.get("table")
@@ -1311,6 +1515,8 @@ def _sync_now_cause(cause):
1311
  raw = str(cause)
1312
  text = raw.lower()
1313
  causes = (
 
 
1314
  ("response_too_large", ("reduce the amount of data", "response too large"),
1315
  "The provider refused the pull because the response contains too much data."),
1316
  ("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
@@ -1389,7 +1595,7 @@ def _sync_now_failure(cause):
1389
  return _sync_now_cause(cause)[1]
1390
 
1391
 
1392
- def _amazon_sync_receipt(report):
1393
  """Project table windows and resync counts without vendor diagnostics or exception values."""
1394
  import re
1395
  details = []
@@ -1397,6 +1603,8 @@ def _amazon_sync_receipt(report):
1397
  if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
1398
  continue
1399
  parts = []
 
 
1400
  progress = row.get("resync")
1401
  if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
1402
  for k in ("landed", "left")):
@@ -1405,8 +1613,27 @@ def _amazon_sync_receipt(report):
1405
  if isinstance(window, str):
1406
  # Window descriptions are authored by the loader, never copied from its diagnostics.
1407
  parts.append(" ".join(window.split()))
 
 
 
 
 
 
 
 
1408
  if parts:
1409
- details.append(f"{table}: " + "; ".join(parts))
 
 
 
 
 
 
 
 
 
 
 
1410
  return " | ".join(details)
1411
 
1412
 
@@ -1415,6 +1642,10 @@ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
1415
  if ok:
1416
  return detail
1417
  classifier, sentence = _sync_now_cause(cause)
 
 
 
 
1418
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
1419
  if isinstance(cause, BaseException):
1420
  import traceback
@@ -1512,6 +1743,31 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1512
 
1513
  return log, lines
1514
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1515
  odoo_requested = False
1516
  # A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
1517
  hold = during_odoo if hold_first_load is None else hold_first_load
@@ -1531,8 +1787,10 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1531
  ok, detail = False, ""
1532
  first_load_progress, continuing = None, False
1533
  floor_wait = None
 
1534
  progress = ()
1535
  amazon_detail, amazon_failure = "", None
 
1536
  try:
1537
  from harness import runtime, datastore, mirror_store
1538
  rt = runtime.get_runtime(tenant)
@@ -1544,6 +1802,12 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1544
  datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
1545
  raise _SyncNowRefused(
1546
  "This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
 
 
 
 
 
 
1547
  connector_sync.poll_remote_requests(rt)
1548
  if provider == "odoo":
1549
  import core.odoo as odoo
@@ -1574,6 +1838,8 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1574
  problems = []
1575
  elif provider == "amazon_sp":
1576
  from harness import amazon_store as amazon
 
 
1577
  _drain_amazon_connection_revocations(rt)
1578
  if not amazon.credential(tenant, rt=rt):
1579
  raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
@@ -1581,13 +1847,14 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1581
  target_options = notification_sync_options(receipt)
1582
  targeted = bool(target_options)
1583
  first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
1584
- options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS,
1585
- "refresh_loaded": True} if first_load else {}
1586
  _log, progress = progress_log(provider, tenant, receipt, rt)
1587
- report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1588
- scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1589
- **target_options, **options,
1590
- log=_log)
 
 
1591
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1592
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1593
  + (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
@@ -1599,11 +1866,17 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1599
  # HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
1600
  if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
1601
  floor_wait = float(report["nextDueSeconds"])
1602
- problems = list(report.get("problems") or []) + list(report.get("audit_problems") or [])
 
1603
  tables = report.get("tables") or {}
1604
- amazon_detail = _amazon_sync_receipt(report)
1605
  amazon_failure = next((row["failure"] for row in tables.values()
1606
  if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
 
 
 
 
 
1607
  if not problems and not tables and not continuing:
1608
  raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
1609
  if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
@@ -1621,7 +1894,15 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1621
  raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
1622
  else:
1623
  raise _SyncNowRefused("This connector has no Sync now runner.")
1624
- if continuing:
 
 
 
 
 
 
 
 
1625
  ok = True
1626
  detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
1627
  f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
@@ -1652,9 +1933,22 @@ def _run_sync_requests(outcome=None, *, during_odoo=False, hold_first_load=None)
1652
  first_load=first_load_progress, request_id=receipt.get("id"))
1653
  else:
1654
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
 
 
 
 
 
 
1655
  if outcome is not None:
1656
- outcome[provider] = "continuing" if (continuing and ok) else ("done" if ok else "failed")
1657
- if during_odoo and ok and not continuing:
 
 
 
 
 
 
 
1658
  _project_desk({provider: "done"}, tenant)
1659
  if continuing and ok:
1660
  # The continuation takes this run's desk slot: publish the finish to its web hand-off first.
@@ -1706,7 +2000,7 @@ def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations
1706
  if state["present"] and (state["changed"] or force_relations):
1707
  try:
1708
  import automation_engine as _engine
1709
- _engine.refresh_relations(rt, log=lambda *_a: None)
1710
  except Exception as e: # noqa: BLE001
1711
  print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
1712
  return state
@@ -1959,7 +2253,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1959
  # refusal silently cancel a relational pass that had already succeeded.
1960
  try:
1961
  import automation_engine as _engine
1962
- _engine.refresh_relations(_rt, log=lambda *_a: None)
1963
  except Exception as e: # noqa: BLE001
1964
  rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
1965
  print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
@@ -2938,6 +3232,12 @@ def _sheet_warm_after_pass(tenant, good, generation_before, rebuilt):
2938
  print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
2939
 
2940
 
 
 
 
 
 
 
2941
  def _store_resync_loop():
2942
  """Keep the analytical mirror current — the API-process stand-in for the re-sync the
2943
  Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
@@ -2984,15 +3284,18 @@ def _store_resync_loop():
2984
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
2985
  slot_resume_checked = False
2986
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
 
2987
  _slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
2988
  _slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
2989
- # H1: a held Odoo backfill that moved queues a bounded Odoo-only pass, at most 24 per slot day.
2990
  _backfill_read = globals().get("_odoo_backfill_checkpoints")
2991
  backfill_next_at = None
2992
  backfill_continuations = 0
2993
  backfill_spacing_seconds = 60.0
2994
- backfill_cap = 24
2995
  while True:
 
 
2996
  # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
2997
  #
2998
  # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
@@ -3078,7 +3381,7 @@ def _store_resync_loop():
3078
  # full pass too, so passes start a day apart however long the pass ran (a daily pass
3079
  # that also refreshes every Amazon table can run an hour, and a wait counted from its
3080
  # END drifted later every day and past the 26-hour stale line).
3081
- _woken = bool(connector_sync.sync_wake_event().wait(min(
3082
  max(0.0, _every - (_t.time() - last_full_pass_at)),
3083
  float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
3084
  connector_sync.sync_wake_event().clear()
@@ -3090,7 +3393,7 @@ def _store_resync_loop():
3090
  if passes or slot_restored:
3091
  # Sleep to one second past the next slot; Sync now still interrupts the wait.
3092
  _slot_now = _t.time()
3093
- _woken = bool(connector_sync.sync_wake_event().wait(min(
3094
  0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
3095
  else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
3096
  float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
@@ -3145,7 +3448,7 @@ def _store_resync_loop():
3145
  if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
3146
  _desk = {}
3147
  _desk_started = _t.time()
3148
- if not _run_sync_requests(_desk, hold_first_load=_near_slot()):
3149
  _project_desk(_desk, _cad["tenant"])
3150
  if "continuing" in _desk.values():
3151
  # A floor between continuations, so a chunk that returns at once cannot spin.
@@ -3156,12 +3459,13 @@ def _store_resync_loop():
3156
  _odoo_requested = None
3157
  # H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
3158
  _continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
 
3159
  and _odoo_requested is None and not _slot_due
3160
  and (_slot is not None or _t.time() - last_full_pass_at < _every))
3161
  if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
3162
  # H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
3163
  _desk = {}
3164
- if _run_sync_requests(_desk, hold_first_load=_near_slot()):
3165
  _odoo_requested = True
3166
  else:
3167
  _project_desk(_desk, _cad["tenant"])
@@ -3184,10 +3488,20 @@ def _store_resync_loop():
3184
  # H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
3185
  last_slot_pass_at = last_full_pass_at
3186
  _slot_write(last_full_pass_at)
3187
- if _odoo_requested is None:
3188
- _odoo_requested = _run_sync_requests(hold_first_load=True)
 
 
 
 
 
 
3189
  _warm_generation = _sheet_warm_generation()
3190
  _backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
 
 
 
 
3191
  try:
3192
  selected_entities = _configured_mirror_entities(_cad["tenant"])
3193
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
@@ -3225,6 +3539,10 @@ def _store_resync_loop():
3225
  print(f"[aios-api] store resync failed ({_err['kind']}, "
3226
  f"{RESYNC_STATE['consecutiveFailures']} in a row): "
3227
  f"{_err['type']}: {_err['message']}")
 
 
 
 
3228
  if _publication_held:
3229
  print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
3230
  "the next admitted pass resumes the unfinished Odoo cohort")
@@ -3234,8 +3552,8 @@ def _store_resync_loop():
3234
  _backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
3235
  _moved = ([row for row in _backfill_after if row not in _backfill_before]
3236
  if _err is None and _backfill_after and _backfill_before is not None else [])
3237
- if _moved and backfill_continuations >= backfill_cap:
3238
- print(f"[aios-api] odoo backfill stops for today: {backfill_cap} bounded passes ran; "
3239
  f"the next slot pass or Sync now resumes it")
3240
  elif _moved:
3241
  backfill_continuations += 1
@@ -3243,7 +3561,7 @@ def _store_resync_loop():
3243
  _row = max(_moved, key=lambda row: row[4] or 0)
3244
  print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
3245
  f"{_row[4] or 0:,} rows; next bounded pass queued "
3246
- f"({backfill_continuations} of {backfill_cap} today)")
3247
  continue
3248
  if _boot_catch_up and not _odoo_requested:
3249
  # Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
@@ -3476,6 +3794,28 @@ def _boot_app_migrations(why):
3476
  print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
3477
  except Exception as exc:
3478
  print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3479
  try:
3480
  result = application_registry.ensure_route_application(rt, slug)
3481
  reason = f": {result['reason']}" if result.get("reason") else ""
@@ -3493,6 +3833,97 @@ def _boot_app_migrations(why):
3493
  print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
3494
 
3495
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3496
  def _prewarm():
3497
  if _freeze_holds("prewarm"):
3498
  return
@@ -3565,17 +3996,35 @@ def _seed_mirror_if_absent():
3565
  from harness import runtime as _runtime
3566
  MIRROR_SEED["attempted"] = True
3567
  try:
3568
- for tenant_key in _runtime.known_tenants():
 
 
 
 
 
 
 
 
 
3569
  try:
3570
- rt = _runtime.get_runtime(tenant_key)
3571
  result = _ds.restore_odoo_mirror(tenant_key, rt)
3572
  except Exception as exc:
3573
- result = {"state": "refused", "problems": [
3574
- f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
3575
- _ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
3576
  if result["state"] == "refused":
3577
  MIRROR_SEED["cause"] = "; ".join(result["problems"])
3578
  MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
 
 
 
 
 
 
 
 
 
 
 
 
3579
  if _ds.ensure_seed():
3580
  MIRROR_SEED["seeded"] = True
3581
  print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
@@ -3681,7 +4130,7 @@ def _await_mirror_restore():
3681
 
3682
  @app.on_event("startup")
3683
  def _start_process_role():
3684
- """Web role: spawn and supervise the sync worker, forward Sync now, install its snapshots."""
3685
  if data_binding.gate_process():
3686
  return
3687
  import process_role as _process_role
@@ -3694,12 +4143,22 @@ def _restore_sync_request_desks():
3694
  import process_role as _process_role
3695
  # Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
3696
  requeue = _process_role.may_write()
3697
- for tenant_key in runtime.known_tenants():
 
 
 
3698
  try:
3699
- connector_sync.restore_sync_requests(runtime.get_runtime(tenant_key), requeue=requeue)
3700
  except Exception as exc:
3701
  print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
3702
 
 
 
 
 
 
 
 
3703
 
3704
  def _boot_mirrors():
3705
  import process_role as _role_boot
@@ -3737,6 +4196,53 @@ _READ_WARM_ATTEMPTS = 40
3737
  _READ_WARM_RETRY_SECONDS = 15
3738
 
3739
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3740
  def _probe_read_pools(rt):
3741
  """Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
3742
  from harness import datastore as _probe_ds
@@ -3745,11 +4251,18 @@ def _probe_read_pools(rt):
3745
  physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
3746
  if not physical.exists():
3747
  return "The mirror file is missing."
3748
- cur = handle.ro_cursor() if handle is not None else _probe_ds.ro_cursor()
3749
- try:
3750
- cur.execute("SELECT 1").fetchone()
3751
- finally:
3752
- cur.close()
 
 
 
 
 
 
 
3753
  if rt.available() is False:
3754
  return "The tenant store did not answer."
3755
  return ""
@@ -3764,6 +4277,7 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
3764
  import process_role as _warm_role
3765
  attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
3766
  retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
 
3767
  for attempt in range(1, attempts + 1):
3768
  t0 = _t.time()
3769
  stage = "pools"
@@ -3776,13 +4290,23 @@ def _warm_read_pools(attempts=None, retry_seconds=None):
3776
  raise RuntimeError(problem)
3777
  _warm_role.mark_ready("pools", True)
3778
  stage = "first_reads"
3779
- routes_customers.warm_default(rt)
3780
- routes_products._pool_for(rt, None)
 
 
 
 
 
 
 
3781
  _warm_role.mark_ready("first_reads", True)
3782
  routes_customers.warm_bu_pools(rt)
3783
  print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
3784
  return True
3785
  except Exception as exc: # noqa: BLE001
 
 
 
3786
  if not _warm_role._READINESS.get(stage, {}).get("ok"):
3787
  _warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
3788
  f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
@@ -3797,14 +4321,16 @@ def _restore_declared_mirrors(why):
3797
  """Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
3798
  from core import connector_spec
3799
  from harness import mirror_store, runtime
 
 
3800
  errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
3801
- for tenant_key in runtime.known_tenants():
 
3802
  for connector in tuple(connector_spec.MIRRORS):
3803
  if connector in {"odoo", "amazon"}:
3804
  continue # Their existing adapters also enforce seed and deletion policies.
3805
  identity = f"{tenant_key}/{connector}"
3806
  try:
3807
- rt = runtime.get_runtime(tenant_key)
3808
  result = mirror_store.restore_mirror(connector, tenant_key, rt)
3809
  if result["problems"]:
3810
  errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
@@ -3813,9 +4339,25 @@ def _restore_declared_mirrors(why):
3813
  print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
3814
  except Exception as exc:
3815
  errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
 
 
 
 
 
 
 
 
 
 
 
 
 
3816
  return not errors
3817
 
3818
 
 
 
 
3819
  try:
3820
  from harness import datastore as _ds_boot
3821
  if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
@@ -3852,9 +4394,13 @@ if os.environ.get("AIOS_BOOT_SYNC") == "1" and freeze_notice.enabled():
3852
  if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
3853
  import marketing_stream_loop
3854
  import ads_schedule_loop
 
 
3855
  import routes_returns_refunds
3856
  marketing_stream_loop.start()
3857
  ads_schedule_loop.start()
 
 
3858
  routes_returns_refunds.start_auto_approve_worker()
3859
 
3860
 
@@ -3862,8 +4408,10 @@ if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
3862
  def _stop_amazon_app_workers():
3863
  import marketing_stream_loop
3864
  import ads_schedule_loop
 
3865
  marketing_stream_loop.stop()
3866
  ads_schedule_loop.stop()
 
3867
 
3868
 
3869
  if (not data_binding.gate_process()
 
412
 
413
  def _tenant_readiness(rt):
414
  """Check this tenant's reads without requiring an Odoo mirror for tenants without Odoo."""
415
+ checks = {"mirror": False, "pools": False, "first_reads": False, "tenant_keys": False}
416
  detail = {"mirror": "The tenant mirror check did not answer.",
417
  "pools": "The tenant store did not answer.",
418
+ "first_reads": "The tenant's first read did not answer.",
419
+ "tenant_keys": "The tenant data keys did not answer."}
420
  stage = "mirror"
421
  try:
422
  if rt.odoo_source() is not None:
 
433
  # HF display reads hide outages, so readiness uses the tenant's strict reader.
434
  rt._store()._read_strict(rt.store_key("sync_settings"))
435
  checks["first_reads"] = True
436
+ stage = "tenant_keys"
437
+ from core import tenant_keys as _tenant_keys
438
+ checks["tenant_keys"], _keys_detail = _tenant_keys.readiness_check([rt.key])
439
+ if not checks["tenant_keys"]:
440
+ detail["tenant_keys"] = _keys_detail
441
  except Exception as exc: # noqa: BLE001
442
  detail[stage] += f" ({type(exc).__name__})"
443
  failing = [name for name in checks if not checks[name]]
 
447
 
448
  @app.get("/api/v1/ready")
449
  def readiness_check(request: Request, response: Response):
450
+ """Report tenant readiness publicly; only a verified session or the deploy nonce sees the version."""
451
+ import hmac as _ready_hmac
452
  session = (deps_require_session(request, response)
453
  if request.cookies.get(aios_session.COOKIE_NAME) else None)
454
+ # A deploy probe proves itself with the deploy nonce; it sees the version, never a tenant's data.
455
+ nonce = os.environ.get("AIOS_DEPLOY_NONCE") or ""
456
+ probe = bool(nonce) and _ready_hmac.compare_digest(
457
+ request.headers.get("x-aios-deploy-nonce") or "", nonce)
458
+ skipped = {}
459
  try:
460
  from harness import runtime as _ready_runtime
461
  if session is not None:
462
  report = _tenant_readiness(session.runtime)
463
  else:
464
+ import tenant_scan as _ready_scan
465
+ names = ("mirror", "pools", "first_reads", "tenant_keys")
466
+ reports, unmirrored, resolved = [], [], set()
467
+
468
+ def failed(slug, _exc):
469
+ # An active tenant whose runtime fails is judged, and fails closed.
470
+ resolved.add(slug)
471
+ reports.append({"checks": dict.fromkeys(names, False)})
472
+
473
+ # R70: only active tenants with a declared mirror are judged; the others are named.
474
+ for key, tenant_rt in _ready_scan.live_tenants(_ready_runtime, on_error=failed):
475
+ resolved.add(key)
476
+ try:
477
+ mirrored = tenant_rt.odoo_source() is not None
478
+ except Exception:
479
+ mirrored = True
480
+ if not mirrored:
481
+ unmirrored.append((key, tenant_rt))
482
+ skipped[key] = "no mirror"
483
+ continue
484
  try:
485
+ reports.append(_tenant_readiness(tenant_rt))
486
  except Exception:
487
  reports.append({"checks": dict.fromkeys(names, False)})
488
+ for key in _ready_runtime.known_tenants():
489
+ if key not in resolved:
490
+ skipped[key] = "inactive"
491
+ if not reports:
492
+ # No tenant declares a mirror, so the active tenants' own stores answer for the process.
493
+ for key, tenant_rt in unmirrored:
494
+ skipped.pop(key, None)
495
+ reports.append(_tenant_readiness(tenant_rt))
496
  checks = {name: bool(reports) and all(r["checks"].get(name, False) for r in reports)
497
  for name in names}
498
  failing = [name for name in names if not checks[name]]
 
502
  report = {"ready": False, "checks": {"tenant": False}, "failing": ["tenant"],
503
  "detail": {"tenant": f"The tenant could not be resolved ({type(exc).__name__})."}}
504
  body = {"ready": report["ready"], "checks": report["checks"]}
505
+ if skipped:
506
+ body["skipped"] = skipped
507
+ if session is not None or probe:
508
  body["version"] = os.environ.get("AIOS_VERSION") or "unknown"
509
  response.headers["Cache-Control"] = "no-store"
510
  if report["ready"]:
 
743
  import app_routes
744
  _mount_router("routes_security") # W59-T43: lane F declares the router; keep before static mount.
745
  _mount_router("routes_amazon_profit") # S78-T03: the Command Center refresh and settings doors.
746
+ _mount_router("routes_amazon_projects")
747
+ _mount_router("routes_competitors")
748
  _mount_router("routes_odoo_attachments") # S78-T14 (lane G, C5): delivery photo doors; before static mount.
749
  _mount_router("routes_sku_actions")
750
  _mount_router("notify_destinations")
 
1018
  print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}")
1019
 
1020
 
1021
+ def _publish_connector_status(rt, provider, outcome=None, *, detail="", connection_id=None, report=None):
1022
+ """C-status: write a pass end and the mirror's completed sync to the tenant's status document.
1023
+
1024
+ `outcome=None` writes only the completed sync, so a restored mirror seeds the store. Odoo's
1025
+ good sync is the pass end itself. Never raises: a status write cannot fail a pass.
1026
+ """
1027
+ try:
1028
+ from core import connector_sync as _cs
1029
+ good = None
1030
+ if provider == "odoo":
1031
+ good = _cs._utc_stamp() if outcome == "done" else None
1032
+ else:
1033
+ import routes_connectors as _rc
1034
+ stamp, known = _rc._mirror_completed_sync(rt, provider)
1035
+ good = stamp if known else None
1036
+ if outcome is None:
1037
+ return _cs.record_good_sync(rt, provider, good, connection_id=connection_id) if good else False
1038
+ extra = {}
1039
+ if (provider == "odoo" or provider == "amazon_sp" and RESYNC_STATE.get("slotPass")) and RESYNC_STATE.get("slotDate"):
1040
+ extra["slotDate"] = RESYNC_STATE["slotDate"]
1041
+ if provider == "amazon_sp" and isinstance(report, dict):
1042
+ from harness import amazon_store as _az
1043
+ holds = _az._MIRROR_STORE._publication_holds(report, ()) or {}
1044
+ extra["notOffered"] = len(holds.get("standing") or ())
1045
+ extra["notOfferedTables"] = sorted(holds.get("standing") or ())
1046
+ extra["held"] = {name: {"reason": "Amazon does not offer this report to this seller."
1047
+ if name in (holds.get("standing") or ())
1048
+ else "This table did not finish. Its last published rows are kept."}
1049
+ for name in holds.get("held") or {}}
1050
+ previous = _cs.status(rt, provider, connection_id, fresh=True) or {}
1051
+ stamps = dict((previous.get("pass") or {}).get("tableSync") or {})
1052
+ for name, row in (report.get("tables") or {}).items():
1053
+ if name not in extra["held"] and isinstance(row, dict) and row.get("state", "done") == "done":
1054
+ stamps[name] = good or _cs._utc_stamp()
1055
+ extra["tableSync"] = stamps
1056
+ if extra["held"]:
1057
+ from harness.runtime import mirror_cursor
1058
+ cur = None
1059
+ try:
1060
+ cur = mirror_cursor(rt, require_ready=False)
1061
+ for name, held in extra["held"].items():
1062
+ table = _az.table_name(name)
1063
+ if _az._MIRROR_STORE._valid_table(table):
1064
+ try:
1065
+ held["rows"] = int(cur.execute(f'SELECT count(*) FROM "{table}"').fetchone()[0])
1066
+ except Exception:
1067
+ held["rows"] = None
1068
+ except Exception:
1069
+ for held in extra["held"].values():
1070
+ held["rows"] = None
1071
+ finally:
1072
+ if cur is not None:
1073
+ cur.close()
1074
+ return _cs.record_pass_end(rt, provider, outcome, detail=detail, last_good_sync_at=good,
1075
+ connection_id=connection_id, extra=extra)
1076
+ except Exception as exc: # noqa: BLE001
1077
+ print(f"[aios-api] connector status not written for {provider} ({type(exc).__name__})", flush=True)
1078
+ return False
1079
+
1080
+
1081
+ #: The last resync snapshot written to the store, so an unchanged loop state is not rewritten.
1082
+ _RESYNC_PUBLISHED = [None]
1083
+
1084
+
1085
+ def _publish_resync_state(tenant_key, *, started=False, ended=False, outcome=None):
1086
+ """C-status: copy RESYNC_STATE, and the Odoo pass it ran, to the tenant's status document."""
1087
+ try:
1088
+ tenant_key = str(tenant_key or "")
1089
+ if not tenant_key:
1090
+ return False
1091
+ from core import connector_sync as _cs
1092
+ from harness import runtime as _status_runtime
1093
+ rt = _status_runtime.get_runtime(tenant_key)
1094
+ cid = str(RESYNC_STATE.get("connectionId") or "default")
1095
+ if started:
1096
+ _cs.record_pass_start(rt, "odoo", connection_id=cid)
1097
+ if ended:
1098
+ error = RESYNC_STATE.get("lastError")
1099
+ _publish_connector_status(rt, "odoo", outcome, connection_id=cid,
1100
+ detail=str(error.get("message") or "") if isinstance(error, dict) else "")
1101
+ import json as _json
1102
+ snapshot = _json.dumps(RESYNC_STATE, sort_keys=True, default=str)
1103
+ if snapshot != _RESYNC_PUBLISHED[0] and _cs.record_resync_state(
1104
+ rt, "odoo", RESYNC_STATE, connection_id=cid):
1105
+ _RESYNC_PUBLISHED[0] = snapshot
1106
+ return True
1107
+ except Exception as exc: # noqa: BLE001
1108
+ print(f"[aios-api] resync status not written ({type(exc).__name__})", flush=True)
1109
+ return False
1110
+
1111
+
1112
+ def _connector_publication_status(connector, tenant_key, manifest, _rows, _paths):
1113
+ """Close a running pass at publication, before Amazon queues its App callbacks."""
1114
+ if not process_role.may_write() or not manifest.get("snapshot_id"):
1115
+ return
1116
+ provider = {"odoo": "odoo", "amazon": "amazon_sp", "meta": "meta_ads"}.get(connector)
1117
+ if provider is None:
1118
+ return
1119
+ from core import connector_sync
1120
+ from harness import runtime
1121
+ rt = runtime.get_runtime(tenant_key)
1122
+ cid = None
1123
+ if provider == "odoo":
1124
+ import odoo_relational
1125
+ cid = str(odoo_relational.default_connection_id(rt) or "default")
1126
+ entry = connector_sync.status(rt, provider, cid, fresh=True) or {}
1127
+ last_pass = entry.get("pass") or {}
1128
+ if last_pass.get("outcome") != "running":
1129
+ return
1130
+ from datetime import datetime
1131
+ started = connector_sync._utc_stamp(last_pass["startedAt"]) if last_pass.get("startedAt") else None
1132
+ at = connector_sync._utc_stamp(manifest["completed_at"]) if manifest.get("completed_at") else None
1133
+ if not started or not at or datetime.fromisoformat(at) < datetime.fromisoformat(started):
1134
+ return
1135
+ extra = {"publishedAt": at}
1136
+ if RESYNC_STATE.get("slotDate") and (provider == "odoo" or RESYNC_STATE.get("slotPass")):
1137
+ extra["slotDate"] = RESYNC_STATE["slotDate"]
1138
+ connector_sync.record_pass_end(rt, provider, "done", at=at, last_good_sync_at=at,
1139
+ connection_id=cid, extra=extra)
1140
+
1141
+
1142
+ from harness import mirror_store as _status_mirrors
1143
+ _status_mirrors.register_publish_listener(_connector_publication_status)
1144
+
1145
+
1146
  def _pull_meta(why, tenant_key=None):
1147
  """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it.
1148
 
 
1162
  outcomes = {}
1163
  try:
1164
  import process_role as _meta_role
1165
+ import tenant_scan as _meta_scan
1166
  # Each tenant is pulled by exactly one worker: its own, else the primary.
1167
  tenants = [tenant_key] if tenant_key else [
1168
  key for key in runtime.known_tenants() if _meta_role.syncs_here(key)]
1169
+ outcomes.update(dict.fromkeys(tenants, False))
1170
+
1171
+ def runtime_failed(key, exc):
1172
+ print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
1173
+
1174
+ # S82-T18: an inactive tenant is skipped with one named line, never a KeyError per pass.
1175
+ for key, rt in _meta_scan.live_tenants(runtime, tenants=tenants, name="meta-pull",
1176
+ on_error=runtime_failed):
1177
  try:
 
1178
  if not _meta._connection_rows(rt, key, {"problems": []}):
1179
  continue
1180
+ from core import connector_sync as _meta_status
1181
+ _meta_status.record_pass_start(rt, "meta_ads")
1182
+ _status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
1183
  rep = mirror_store.tracked_sync("meta", rt, _meta.sync, key,
1184
  log=lambda *_a: None, insights_days=7)
1185
  outcomes[key] = mirror_store._sync_result("meta", rep)[1]
1186
+ _status(rt, "meta_ads", "done" if outcomes[key] else "failed")
1187
  for problem in rep.get("problems") or []:
1188
  print(f"[aios-api] meta sync PROBLEM ({why}, {key}): {problem}")
1189
  except Exception as exc:
1190
  print(f"[aios-api] meta sync FAILED ({why}, {key}): {type(exc).__name__}")
1191
+ _failed = globals().get("_publish_connector_status")
1192
+ if _failed:
1193
+ _failed(rt, "meta_ads", "failed", detail="The Meta Ads pull failed.")
1194
  except Exception as exc:
1195
  print(f"[aios-api] meta scheduling FAILED ({why}): {type(exc).__name__}")
1196
  return outcomes.get(tenant_key or runtime.tenant_for_store_path(), False)
 
1280
  print(f"[aios-api] amazon sync skipped ({why}): no stored amazon_sp credential "
1281
  f"for {tenant_key}; the connector is idle, not broken")
1282
  return False
1283
+ if _amazon.first_load_job_live(rt):
1284
+ print("[aios-api] the first-load job holds the Amazon writer lease", flush=True)
1285
+ return False
1286
  # Hotfix 2026-09-24: an unfinished first load goes through the Sync now desk, whose
1287
  # bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
1288
  # ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
 
1300
  _rot.invalidate_sources()
1301
  _rot._sources()
1302
  if first_load_needed:
1303
+ print(f"[aios-api] amazon sync ({why}): history belongs to the independent first-load job")
1304
+ # C-status: the restored mirror seeds the store's last good sync; an older stamp never moves it back.
1305
+ _status = globals().get("_publish_connector_status") or (lambda *_a, **_k: False)
1306
+ _status(rt, "amazon_sp")
1307
  if not vendor:
1308
  print(f"[aios-api] amazon mirror restored ({why}); the daily sync is not due yet")
1309
  return True
1310
  from harness import mirror_store
1311
+ from core import connector_sync as _amazon_status
1312
+ _amazon_status.record_pass_start(rt, "amazon_sp")
1313
+ try:
1314
+ with _amazon_role.connector_writer(rt, "amazon"):
1315
+ rep = mirror_store.tracked_sync("amazon", rt, lambda: _amazon.sync(tenant_key, rt=rt, scheduled=True,
1316
+ slot_pass=bool(RESYNC_STATE.get("slotPass")),
1317
+ log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}")))
1318
+ except Exception:
1319
+ _status(rt, "amazon_sp", "failed", detail="The Amazon pull failed.")
1320
+ raise
1321
+ _holds = _amazon._MIRROR_STORE._publication_holds(rep, ()) or {}
1322
+ _problems = _holds.get("unsettled", rep.get("problems") or [])
1323
+ _status(rt, "amazon_sp", "failed" if _problems or rep.get("audit_problems") else
1324
+ "continuing" if rep.get("state") == "partial" else "done", report=rep)
1325
  _rot.invalidate_sources()
1326
  _rot._sources()
1327
  for problem in rep.get("problems") or []:
 
1336
  store=rt, quiet=_sa.amazon_standing(rep))
1337
  except Exception: # noqa: BLE001
1338
  pass
1339
+ return not bool(_problems or rep.get("audit_problems"))
1340
  except Exception as exc: # noqa: BLE001
1341
  print(f"[aios-api] amazon sync FAILED ({why}): {type(exc).__name__}: {exc}")
1342
  return False
 
1488
  """Classify actual provider failures without returning vendor text or exception reprs."""
1489
  import re
1490
  if isinstance(cause, dict):
1491
+ if cause.get("amazonStatus") == "NOT_OFFERED" or cause.get("classifier") == "not_offered":
1492
+ return "not_offered", "Amazon does not offer this report to this seller. Its last published rows are kept."
1493
+ # S82-T04 zero-row class: a pass that moved no rows because nothing was due names when it reopens.
1494
+ if cause.get("classifier") == "nothing_due":
1495
+ import datetime as _dt
1496
+ at = cause.get("nextAt")
1497
+ sentence = "Amazon had nothing due in this pass, so no rows moved. Each table waits for its next report window."
1498
+ if isinstance(at, (int, float)) and not isinstance(at, bool) and 0 < at < 4102444800:
1499
+ when = _dt.datetime.fromtimestamp(at, _dt.timezone.utc)
1500
+ sentence += f" The next window opens at {when:%H:%M} UTC on {when.day} {when:%b}."
1501
+ return "nothing_due", sentence
1502
  sentence = "The connector could not read an expected value in the sync response."
1503
  if cause.get("classifier") == "unreadable_value":
1504
  table = cause.get("table")
 
1515
  raw = str(cause)
1516
  text = raw.lower()
1517
  causes = (
1518
+ ("not_offered", ("not_offered", "not offered", "does not offer this report"),
1519
+ "Amazon does not offer this report to this seller. Its last published rows are kept."),
1520
  ("response_too_large", ("reduce the amount of data", "response too large"),
1521
  "The provider refused the pull because the response contains too much data."),
1522
  ("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
 
1595
  return _sync_now_cause(cause)[1]
1596
 
1597
 
1598
+ def _amazon_sync_receipt(report, rt=None):
1599
  """Project table windows and resync counts without vendor diagnostics or exception values."""
1600
  import re
1601
  details = []
 
1603
  if not isinstance(row, dict) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", str(table)):
1604
  continue
1605
  parts = []
1606
+ if row.get("state") == "refused" and row.get("amazonStatus") == "NOT_OFFERED":
1607
+ parts.append("report not offered; last published rows kept")
1608
  progress = row.get("resync")
1609
  if isinstance(progress, dict) and all(type(progress.get(k)) is int and progress[k] >= 0
1610
  for k in ("landed", "left")):
 
1613
  if isinstance(window, str):
1614
  # Window descriptions are authored by the loader, never copied from its diagnostics.
1615
  parts.append(" ".join(window.split()))
1616
+ operation, resume_at = row.get("operation"), row.get("resume_at")
1617
+ if isinstance(operation, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,79}", operation):
1618
+ parts.append(f"operation {operation}")
1619
+ if isinstance(resume_at, str):
1620
+ from core.connector_sync import _utc_stamp
1621
+ stamp = _utc_stamp(resume_at)
1622
+ if stamp:
1623
+ parts.append(f"resume_at {stamp}")
1624
  if parts:
1625
+ line = f"{table}: " + "; ".join(parts)
1626
+ if operation or resume_at:
1627
+ details.insert(0, line)
1628
+ else:
1629
+ details.append(line)
1630
+ if rt is not None and not any("operation " in line or "resume_at " in line for line in details):
1631
+ from harness import amazon_store
1632
+ job = amazon_store.first_load_job_summary(rt)
1633
+ if job:
1634
+ tail = _amazon_sync_receipt({"tables": {"first_load": job}})
1635
+ if tail:
1636
+ details.append(tail)
1637
  return " | ".join(details)
1638
 
1639
 
 
1642
  if ok:
1643
  return detail
1644
  classifier, sentence = _sync_now_cause(cause)
1645
+ if classifier == "nothing_due":
1646
+ # A wait with a named cause is not a failure: no FAILED line, no failure location.
1647
+ print(f"[aios-api] sync WAITS provider={provider} tenant={tenant} classifier={classifier}")
1648
+ return sentence
1649
  print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
1650
  if isinstance(cause, BaseException):
1651
  import traceback
 
1743
 
1744
  return log, lines
1745
 
1746
+ def nothing_due_at(report, problems, failure, tenant):
1747
+ """S82-T04: the epoch the next Amazon window opens if a pass moved no rows and every table waited.
1748
+
1749
+ An unclassified problem on such a pass is a wait, not a failure. A named failure class keeps its FAILED.
1750
+ Nested on purpose: several gates compile this runner with a fixed set of main.py names.
1751
+ """
1752
+ import time
1753
+ tables = report.get("tables") or {}
1754
+ rows = [row for row in tables.values() if isinstance(row, dict)]
1755
+ if (not rows or len(rows) != len(tables) or report.get("landedTasks")
1756
+ or any(row.get("state") != "deferred" or row.get("rows") for row in rows)):
1757
+ return None
1758
+ classify = globals().get("_sync_now_cause")
1759
+ if problems and classify and classify(failure or problems[0])[0] != "unclassified":
1760
+ return None
1761
+ now, idle = time.time(), 1800.0
1762
+ wait, held = report.get("nextDueSeconds"), connector_sync.sync_not_before("amazon_sp", tenant)
1763
+ if isinstance(wait, (int, float)) and not isinstance(wait, bool) and wait > 0:
1764
+ at = now + float(wait)
1765
+ elif held and held > now:
1766
+ at = float(held)
1767
+ else:
1768
+ at = now + idle # No floor is known: wait one desk cadence, never requeue at once.
1769
+ return min(max(at, now + 60.0), now + 86400.0)
1770
+
1771
  odoo_requested = False
1772
  # A first-load continuation never runs inside an Odoo pass: it requeues itself without end.
1773
  hold = during_odoo if hold_first_load is None else hold_first_load
 
1787
  ok, detail = False, ""
1788
  first_load_progress, continuing = None, False
1789
  floor_wait = None
1790
+ nothing_due, first_load = None, False
1791
  progress = ()
1792
  amazon_detail, amazon_failure = "", None
1793
+ status_rt, status_cid = None, None
1794
  try:
1795
  from harness import runtime, datastore, mirror_store
1796
  rt = runtime.get_runtime(tenant)
 
1802
  datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
1803
  raise _SyncNowRefused(
1804
  "This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
1805
+ # C-status: the pass starts on the resident writer; its end is written after the finish below.
1806
+ status_rt = rt
1807
+ if provider == "odoo":
1808
+ import routes_connectors as _status_routes
1809
+ status_cid = _status_routes._odoo_connection_slot(rt, None)
1810
+ connector_sync.record_pass_start(rt, provider, connection_id=status_cid)
1811
  connector_sync.poll_remote_requests(rt)
1812
  if provider == "odoo":
1813
  import core.odoo as odoo
 
1838
  problems = []
1839
  elif provider == "amazon_sp":
1840
  from harness import amazon_store as amazon
1841
+ if amazon.first_load_job_live(rt):
1842
+ raise _SyncNowRefused("The first-load job holds the Amazon writer lease.")
1843
  _drain_amazon_connection_revocations(rt)
1844
  if not amazon.credential(tenant, rt=rt):
1845
  raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
 
1847
  target_options = notification_sync_options(receipt)
1848
  targeted = bool(target_options)
1849
  first_load = False if targeted else amazon.needs_first_load(tenant, rt=rt)
1850
+ options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
 
1851
  _log, progress = progress_log(provider, tenant, receipt, rt)
1852
+ import process_role as _desk_role
1853
+ with _desk_role.connector_writer(rt, "amazon"):
1854
+ report = mirror_store.tracked_sync("amazon", rt, amazon.sync, tenant, rt=rt,
1855
+ scheduled=False if targeted else receipt.get("requestedBy") == "first load",
1856
+ **target_options, **options,
1857
+ log=_log)
1858
  print("[aios-api] amazon sync done (sync desk): " + ", ".join(
1859
  f"{table}={row.get('state', 'done')}" + (f", {row['rows']} rows" if 'rows' in row else "")
1860
  + (f", resync {row['resync']['landed']} windows landed, {row['resync']['left']} left"
 
1866
  # HOTFIX 2026-09-29: a continuation that landed nothing only met refresh floors.
1867
  if continuing and report.get("landedTasks") == 0 and report.get("nextDueSeconds"):
1868
  floor_wait = float(report["nextDueSeconds"])
1869
+ _holds = amazon._MIRROR_STORE._publication_holds(report, ()) or {}
1870
+ problems = list(_holds.get("unsettled", report.get("problems") or [])) + list(report.get("audit_problems") or [])
1871
  tables = report.get("tables") or {}
1872
+ amazon_detail = _amazon_sync_receipt(report, rt)
1873
  amazon_failure = next((row["failure"] for row in tables.values()
1874
  if isinstance(row, dict) and isinstance(row.get("failure"), dict)), None)
1875
+ # The floor timer below already serves a clean 0-row continuation that names its floor.
1876
+ nothing_due = (None if targeted or (continuing and floor_wait and not problems)
1877
+ else nothing_due_at(report, problems, amazon_failure, tenant))
1878
+ if nothing_due:
1879
+ continuing, floor_wait = False, None
1880
  if not problems and not tables and not continuing:
1881
  raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
1882
  if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
 
1894
  raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
1895
  else:
1896
  raise _SyncNowRefused("This connector has no Sync now runner.")
1897
+ if nothing_due:
1898
+ # S82-T04: nothing was due. Finish with the named cause, then wait for the floor.
1899
+ if problems:
1900
+ print(f"[aios-api] amazon sync: {len(problems)} unclassified problem(s) on a pass "
1901
+ f"with nothing due; see the connector sync report", flush=True)
1902
+ ok, problems, amazon_detail = True, [], ""
1903
+ detail = _sync_now_detail(provider, tenant, ok=False,
1904
+ cause={"classifier": "nothing_due", "nextAt": nothing_due})
1905
+ elif continuing:
1906
  ok = True
1907
  detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
1908
  f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
 
1933
  first_load=first_load_progress, request_id=receipt.get("id"))
1934
  else:
1935
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail, request_id=receipt.get("id"))
1936
+ pass_outcome = ("waiting" if nothing_due and ok else "continuing" if (continuing and ok)
1937
+ else ("done" if ok else "failed"))
1938
+ _status_write = globals().get("_publish_connector_status")
1939
+ if status_rt is not None and _status_write:
1940
+ _status_write(status_rt, provider, pass_outcome, detail=detail, connection_id=status_cid,
1941
+ **({"report": report} if provider == "amazon_sp" else {}))
1942
  if outcome is not None:
1943
+ outcome[provider] = pass_outcome
1944
+ if nothing_due and ok and first_load:
1945
+ # The next continuation waits on the desk for the floor; requeueing at once spins 0-row passes.
1946
+ connector_sync.request_sync("amazon_sp", tenant, requested_by="first load",
1947
+ not_before=nothing_due, wait_cause=detail)
1948
+ import time as _time
1949
+ print(f"[aios-api] amazon first load had nothing due; it waits "
1950
+ f"{int(nothing_due - _time.time())}s for the next report window", flush=True)
1951
+ if during_odoo and ok and not continuing and not nothing_due:
1952
  _project_desk({provider: "done"}, tenant)
1953
  if continuing and ok:
1954
  # The continuation takes this run's desk slot: publish the finish to its web hand-off first.
 
2000
  if state["present"] and (state["changed"] or force_relations):
2001
  try:
2002
  import automation_engine as _engine
2003
+ _engine.refresh_relations(rt, log=print)
2004
  except Exception as e: # noqa: BLE001
2005
  print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}")
2006
  return state
 
2253
  # refusal silently cancel a relational pass that had already succeeded.
2254
  try:
2255
  import automation_engine as _engine
2256
+ _engine.refresh_relations(_rt, log=print)
2257
  except Exception as e: # noqa: BLE001
2258
  rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
2259
  print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
 
3232
  print(f"[sheet-warm] request failed: {type(exc).__name__}: {exc}")
3233
 
3234
 
3235
+ def _sync_idle_wait(event, seconds):
3236
+ """The resync loop's wait. A `sync_worker.py --once` job exits here when it is idle (S82-T10)."""
3237
+ import sync_runtime
3238
+ return sync_runtime.idle_wait(event, seconds)
3239
+
3240
+
3241
  def _store_resync_loop():
3242
  """Keep the analytical mirror current — the API-process stand-in for the re-sync the
3243
  Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
 
3284
  # H2: the first slot cycle checks once whether a restart killed the newest slot's pass.
3285
  slot_resume_checked = False
3286
  _slot_write = globals().get("_resync_slot_pass_write") or (lambda *_a, **_k: None)
3287
+ _idle_wait = globals().get("_sync_idle_wait") or (lambda event, seconds: event.wait(seconds))
3288
  _slot_exhausted_write = globals().get("_resync_slot_exhausted_write") or (lambda *_a, **_k: None)
3289
  _slot_interrupted_clear = globals().get("_resync_slot_interrupted_clear") or (lambda *_a, **_k: False)
3290
+ # Column fills resume only before this slot's load window ends.
3291
  _backfill_read = globals().get("_odoo_backfill_checkpoints")
3292
  backfill_next_at = None
3293
  backfill_continuations = 0
3294
  backfill_spacing_seconds = 60.0
3295
+ backfill_cap = None
3296
  while True:
3297
+ if backfill_next_at is not None and backfill_cap is not None and _t.time() >= backfill_cap:
3298
+ backfill_next_at = None
3299
  # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800.
3300
  #
3301
  # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 —
 
3381
  # full pass too, so passes start a day apart however long the pass ran (a daily pass
3382
  # that also refreshes every Amazon table can run an hour, and a wait counted from its
3383
  # END drifted later every day and past the 26-hour stale line).
3384
+ _woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
3385
  max(0.0, _every - (_t.time() - last_full_pass_at)),
3386
  float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _t.time()))))
3387
  connector_sync.sync_wake_event().clear()
 
3393
  if passes or slot_restored:
3394
  # Sleep to one second past the next slot; Sync now still interrupts the wait.
3395
  _slot_now = _t.time()
3396
+ _woken = bool(_idle_wait(connector_sync.sync_wake_event(), min(
3397
  0.0 if _rel_slot.slot_due(_slot_now, _slot, last_slot_pass_at)
3398
  else max(0.0, _rel_slot.slot_after(_slot_now, _slot) + 1.0 - _slot_now),
3399
  float("inf") if backfill_next_at is None else max(0.0, backfill_next_at - _slot_now))))
 
3448
  if _woken and (_t.time() - last_full_pass_at < _every if _slot is None else not _slot_due):
3449
  _desk = {}
3450
  _desk_started = _t.time()
3451
+ if not _run_sync_requests(_desk, hold_first_load=True):
3452
  _project_desk(_desk, _cad["tenant"])
3453
  if "continuing" in _desk.values():
3454
  # A floor between continuations, so a chunk that returns at once cannot spin.
 
3459
  _odoo_requested = None
3460
  # H1: a queued backfill pass that is due, unless the slot, the cadence or a Sync now runs.
3461
  _continuation = (backfill_next_at is not None and _t.time() >= backfill_next_at - 1.0
3462
+ and backfill_cap is not None and _t.time() < backfill_cap
3463
  and _odoo_requested is None and not _slot_due
3464
  and (_slot is not None or _t.time() - last_full_pass_at < _every))
3465
  if _slot is not None and not _slot_due and _odoo_requested is None and not _continuation:
3466
  # H73: outside the slot no Odoo call is made. Serve the stored mirror, drain the desk.
3467
  _desk = {}
3468
+ if _run_sync_requests(_desk, hold_first_load=True):
3469
  _odoo_requested = True
3470
  else:
3471
  _project_desk(_desk, _cad["tenant"])
 
3488
  # H73: stamp the slot BEFORE any Odoo read. H2: `slotDoneAt` marks its end, any outcome.
3489
  last_slot_pass_at = last_full_pass_at
3490
  _slot_write(last_full_pass_at)
3491
+ from harness import datastore as _fill_ds
3492
+ if not _continuation:
3493
+ _slot_epoch = (_rel_slot.slot_before(last_full_pass_at, _slot)
3494
+ if _slot is not None else last_full_pass_at)
3495
+ backfill_cap = _slot_epoch + _fill_ds.COL_BACKFILL_WINDOW_SECONDS
3496
+ RESYNC_STATE["slotDate"] = _t.strftime("%Y-%m-%d", _t.gmtime(_slot_epoch))
3497
+ _fill_ds.COL_BACKFILL_STOP_AT = backfill_cap
3498
+ _odoo_requested = bool(_odoo_requested)
3499
  _warm_generation = _sheet_warm_generation()
3500
  _backfill_before = _backfill_read(_cad["tenant"]) if _backfill_read else None
3501
+ # C-status: the loop's state and its Odoo pass reach the store at each pass start and end.
3502
+ _resync_status = globals().get("_publish_resync_state")
3503
+ if _resync_status:
3504
+ _resync_status(_cad["tenant"], started=True)
3505
  try:
3506
  selected_entities = _configured_mirror_entities(_cad["tenant"])
3507
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes, selected_entities)
 
3539
  print(f"[aios-api] store resync failed ({_err['kind']}, "
3540
  f"{RESYNC_STATE['consecutiveFailures']} in a row): "
3541
  f"{_err['type']}: {_err['message']}")
3542
+ if _resync_status:
3543
+ _resync_status(_cad["tenant"], ended=True,
3544
+ outcome=("failed" if _err is not None else "waiting" if _manual
3545
+ else "held" if _publication_held else "done"))
3546
  if _publication_held:
3547
  print("[aios-api] store resync: publication held; no vendor pulls or relational rebuild; "
3548
  "the next admitted pass resumes the unfinished Odoo cohort")
 
3552
  _backfill_after = _backfill_read(_cad["tenant"]) if _backfill_read else None
3553
  _moved = ([row for row in _backfill_after if row not in _backfill_before]
3554
  if _err is None and _backfill_after and _backfill_before is not None else [])
3555
+ if _moved and _t.time() + backfill_spacing_seconds >= backfill_cap:
3556
+ print(f"[aios-api] odoo backfill stops for today: the load window ended; "
3557
  f"the next slot pass or Sync now resumes it")
3558
  elif _moved:
3559
  backfill_continuations += 1
 
3561
  _row = max(_moved, key=lambda row: row[4] or 0)
3562
  print(f"[aios-api] odoo backfill continues: {str(_row[0]).split('.', 1)[0]} at "
3563
  f"{_row[4] or 0:,} rows; next bounded pass queued "
3564
+ f"({backfill_continuations} continuations today)")
3565
  continue
3566
  if _boot_catch_up and not _odoo_requested:
3567
  # Restore only: the saved Amazon mirror and its read-through registry, no vendor call.
 
3794
  print(f"[aios-api] Returns split ({why}) {slug}: {split['outcome']}")
3795
  except Exception as exc:
3796
  print(f"[aios-api] Returns split ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
3797
+ try:
3798
+ result = application_registry.remove_template_doubles(rt, slug)
3799
+ reason = f": {result['reason']}" if result.get("reason") else ""
3800
+ print(f"[aios-api] one template copy ({why}) {slug}: {result['outcome']}{reason}")
3801
+ except Exception as exc:
3802
+ print(f"[aios-api] one template copy ({why}) {slug}: {type(exc).__name__}")
3803
+ try:
3804
+ from modules import amazon_profit as _amazon_profit
3805
+ moved = _amazon_profit.migrate_projects(rt)
3806
+ print(f"[aios-api] Projects move ({why}) {slug}: copied {moved['copied']} of {moved['rows']}")
3807
+ except Exception as exc:
3808
+ print(f"[aios-api] Projects move ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
3809
+ try:
3810
+ import core.user_tables as _ut
3811
+ from modules import amazon_profit as _amazon_profit
3812
+ import routes_amazon_projects as _projects_routes
3813
+ if _ut.get_one(_amazon_profit.TABLE_KEY, rt) is not None:
3814
+ _amazon_profit.ensure_table(rt, _amazon_profit.WRITER)
3815
+ painted = _projects_routes.paint(rt)
3816
+ print(f"[aios-api] Projects cells ({why}) {slug}: {sum(len(ids) for ids in painted.values())} rows")
3817
+ except Exception as exc:
3818
+ print(f"[aios-api] Projects cells ({why}) {slug}: {getattr(exc, 'code', type(exc).__name__)}")
3819
  try:
3820
  result = application_registry.ensure_route_application(rt, slug)
3821
  reason = f": {result['reason']}" if result.get("reason") else ""
 
3833
  print(f"[aios-api] Command Center App ({why}) {slug}: skipped: {type(exc).__name__}")
3834
 
3835
 
3836
+ def _rows_layouts():
3837
+ """The rows layouts the boot step migrates, each `(name, ready(rt, slug), migrate(rt, slug))`.
3838
+
3839
+ Every module marker reader needs an entry here: verify_account_rows fails a new one without it.
3840
+ """
3841
+ from core import account_rows, history_rows, sheets
3842
+
3843
+ def users_ready(_rt, slug):
3844
+ state = account_rows.verify(slug)
3845
+ return state["mode"] == "rows" or not state["records"]
3846
+
3847
+ def history_migrate(_rt, slug):
3848
+ # R58: history_rows refuses a production store unless it is isolated staging. Never widen it.
3849
+ result = history_rows.migrate(slug)
3850
+ return dict(result, records=(result.get("copied") or {}).get("events", 0))
3851
+
3852
+ return [*[(bucket,
3853
+ lambda _rt, slug, b=bucket: account_rows.record_rows_ready(slug, b),
3854
+ lambda _rt, slug, b=bucket: account_rows.records_action("migrate", slug, b))
3855
+ for bucket in account_rows.RECORD_BUCKETS],
3856
+ ("users", users_ready, lambda _rt, slug: account_rows.migrate(slug)),
3857
+ ("value_history", lambda _rt, slug: history_rows.rows_ready(slug), history_migrate),
3858
+ # Bounded per boot by sheets.BOOT_MIGRATION_LIMIT and BOOT_MIGRATION_SECONDS.
3859
+ ("workbooks", lambda _rt, slug: sheets.rows_ready(slug),
3860
+ lambda _rt, slug: sheets.migrate_rows_batch(slug))]
3861
+
3862
+
3863
+ def _boot_rows_migrations(why):
3864
+ """Migrate each live tenant's rows layouts before their writers run. A migrated layout is a no-op."""
3865
+ if _freeze_holds("rows migrations"):
3866
+ return
3867
+ import sync_alerts
3868
+
3869
+ def report(tenant, name, outcome):
3870
+ print(f"[aios-api] rows migration ({why}) {tenant} {name}: {outcome}")
3871
+
3872
+ def failure(exc):
3873
+ # Scrubbed, so a connection error never prints a credential.
3874
+ return f"failed: {type(exc).__name__}: {sync_alerts.scrub(exc)}"
3875
+
3876
+ def migrated(result):
3877
+ # A bounded layout names, by count, what it left for the next boot and what it could not copy.
3878
+ extra = [f"remaining={result['remaining']} for the next boot"] if result.get("remaining") else []
3879
+ extra += [f"failed {code}={count}" for code, count in sorted((result.get("failed") or {}).items())]
3880
+ if not (result.get("changed") or extra):
3881
+ return "already in rows mode"
3882
+ verb = "migrated" if result.get("changed") else "not migrated"
3883
+ return f"{verb} (" + ", ".join([f"records={result.get('records', 0)}", *extra]) + ")"
3884
+
3885
+ try:
3886
+ import tenant_scan
3887
+ from core import account_rows
3888
+ from harness import runtime as _runtime
3889
+ layouts = _rows_layouts()
3890
+ unresolved = []
3891
+ tenants = list(tenant_scan.live_tenants(
3892
+ _runtime, name="rows-migrations", on_error=lambda slug, exc: unresolved.append((slug, exc))))
3893
+ except Exception as exc:
3894
+ print(f"[aios-api] rows migrations ({why}) skipped: {failure(exc)}")
3895
+ return
3896
+ for tenant, exc in unresolved:
3897
+ for name, _ready, _migrate in layouts:
3898
+ report(tenant, name, failure(exc))
3899
+ for tenant, rt in tenants:
3900
+ try:
3901
+ backend = account_rows.records_backend(rt)
3902
+ except Exception as exc:
3903
+ for name, _ready, _migrate in layouts:
3904
+ report(tenant, name, failure(exc))
3905
+ continue
3906
+ for name, ready, migrate in layouts:
3907
+ # One tenant or layout failing never stops the others.
3908
+ try:
3909
+ if backend is None:
3910
+ report(tenant, name, "skipped, not a PostgreSQL rows store")
3911
+ elif ready(rt, backend[1]):
3912
+ report(tenant, name, "already in rows mode")
3913
+ else:
3914
+ report(tenant, name, migrated(migrate(rt, backend[1])))
3915
+ except Exception as exc:
3916
+ report(tenant, name, failure(exc))
3917
+
3918
+
3919
+ def _boot_rows_before_writers():
3920
+ """Run the rows migration synchronously at import, before this web or worker process starts a writer."""
3921
+ flag = os.environ.get("AIOS_BOOT_ROWS_MIGRATIONS")
3922
+ if data_binding.gate_process() or flag == "0" or not (os.environ.get("SPACE_ID") or flag == "1"):
3923
+ return
3924
+ _boot_rows_migrations("boot")
3925
+
3926
+
3927
  def _prewarm():
3928
  if _freeze_holds("prewarm"):
3929
  return
 
3996
  from harness import runtime as _runtime
3997
  MIRROR_SEED["attempted"] = True
3998
  try:
3999
+ import tenant_loops
4000
+ import tenant_scan
4001
+
4002
+ def refused(tenant_key, exc):
4003
+ result = {"state": "refused", "problems": [
4004
+ f"Odoo mirror restore failed for {tenant_key} ({type(exc).__name__})"]}
4005
+ _ds.ODOO_MIRROR_STATUS[str(_ds.path_for(tenant_key).resolve())] = result
4006
+ return result
4007
+
4008
+ def restore(tenant_key, rt):
4009
  try:
 
4010
  result = _ds.restore_odoo_mirror(tenant_key, rt)
4011
  except Exception as exc:
4012
+ result = refused(tenant_key, exc)
 
 
4013
  if result["state"] == "refused":
4014
  MIRROR_SEED["cause"] = "; ".join(result["problems"])
4015
  MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
4016
+
4017
+ def runtime_failed(tenant_key, exc):
4018
+ MIRROR_SEED["cause"] = "; ".join(refused(tenant_key, exc)["problems"])
4019
+ MIRROR_SEED["recommendation"] = "repair the saved mirror or its store access and restart"
4020
+
4021
+ # S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
4022
+ jobs = tenant_loops.for_each_tenant(
4023
+ restore, name="odoo-mirror-restore", tenants=_runtime.known_tenants(),
4024
+ get_runtime=tenant_scan.runtime_lookup(_runtime, "odoo-mirror-restore",
4025
+ on_error=runtime_failed), kind="light")
4026
+ for job in jobs.values():
4027
+ job.join() # The legacy seed below decides on the restored manifests.
4028
  if _ds.ensure_seed():
4029
  MIRROR_SEED["seeded"] = True
4030
  print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)")
 
4130
 
4131
  @app.on_event("startup")
4132
  def _start_process_role():
4133
+ """Web role: tick the sync runtime, forward Sync now, install the worker's snapshots."""
4134
  if data_binding.gate_process():
4135
  return
4136
  import process_role as _process_role
 
4143
  import process_role as _process_role
4144
  # Only the sync writer requeues in-flight claims: never the web role, never a worker without the lease.
4145
  requeue = _process_role.may_write()
4146
+ import tenant_loops
4147
+ import tenant_scan
4148
+
4149
+ def restore(tenant_key, rt):
4150
  try:
4151
+ connector_sync.restore_sync_requests(rt, requeue=requeue)
4152
  except Exception as exc:
4153
  print(f"[aios-api] sync request restore refused for {tenant_key} ({type(exc).__name__})")
4154
 
4155
+ # S82-T18: an inactive tenant is skipped with one named line; one failure leaves the others restored.
4156
+ jobs = tenant_loops.for_each_tenant(
4157
+ restore, name="sync-request-restore", tenants=runtime.known_tenants(),
4158
+ get_runtime=tenant_scan.runtime_lookup(runtime, "sync-request-restore"), kind="light")
4159
+ for job in jobs.values():
4160
+ job.join() # The desks are restored before the boot goes on.
4161
+
4162
 
4163
  def _boot_mirrors():
4164
  import process_role as _role_boot
 
4196
  _READ_WARM_RETRY_SECONDS = 15
4197
 
4198
 
4199
+ def _read_pool_state(rt):
4200
+ """The D-12159 settling line: which connection the read warm holds, and whether it is shared."""
4201
+ from harness import datastore as _state_ds
4202
+ try:
4203
+ handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
4204
+ shared = _state_ds._INSTANCE.get("con")
4205
+ return (f"instance={id(shared) if shared is not None else None} "
4206
+ f"handle_con_none={None if handle is None else handle._con is None} "
4207
+ f"handle_path_is_db_path={None if handle is None else handle.path == _state_ds.DB_PATH.resolve()}")
4208
+ except Exception as exc: # noqa: BLE001
4209
+ return f"unreadable ({type(exc).__name__})"
4210
+
4211
+
4212
+ def _connection_closed(con):
4213
+ try:
4214
+ con.cursor().close()
4215
+ return False
4216
+ except Exception: # noqa: BLE001
4217
+ return True
4218
+
4219
+
4220
+ def _reopen_read_pool(rt):
4221
+ """Drop this tenant's closed mirror read connection so the next cursor opens a new one. True if dropped."""
4222
+ from harness import datastore as _reopen_ds
4223
+ try:
4224
+ handle = rt.mirror_handle() if callable(getattr(rt, "mirror_handle", None)) else None
4225
+ if handle is None or handle.path == _reopen_ds.DB_PATH.resolve():
4226
+ with _reopen_ds._INSTANCE_LOCK:
4227
+ con = _reopen_ds._INSTANCE.get("con")
4228
+ if con is None or not _connection_closed(con):
4229
+ return False
4230
+ _reopen_ds._INSTANCE["con"] = None
4231
+ _reopen_ds._close_instance(con)
4232
+ else:
4233
+ with _reopen_ds._HANDLES_LOCK:
4234
+ con = handle._con
4235
+ if con is None or not _connection_closed(con):
4236
+ return False
4237
+ handle._con = None
4238
+ _reopen_ds._close_instance(con)
4239
+ except Exception as exc: # noqa: BLE001
4240
+ print(f"[aios-api] read pool re-open refused ({type(exc).__name__})")
4241
+ return False
4242
+ print("[aios-api] read pool connection was closed; the next read opens a new one")
4243
+ return True
4244
+
4245
+
4246
  def _probe_read_pools(rt):
4247
  """Open the mirror read cursor and the tenant store once. '' when both answer, else the cause."""
4248
  from harness import datastore as _probe_ds
 
4251
  physical = _probe_ds.serving_file(path)[0] if _probe_ds.versioned_mode() else path
4252
  if not physical.exists():
4253
  return "The mirror file is missing."
4254
+ for retry in (False, True):
4255
+ try:
4256
+ cur = handle.ro_cursor() if handle is not None else _probe_ds.ro_cursor()
4257
+ try:
4258
+ cur.execute("SELECT 1").fetchone()
4259
+ finally:
4260
+ cur.close()
4261
+ break
4262
+ except Exception: # noqa: BLE001
4263
+ # D-12159: a closed pool connection stays closed until something drops it.
4264
+ if retry or not _reopen_read_pool(rt):
4265
+ raise
4266
  if rt.available() is False:
4267
  return "The tenant store did not answer."
4268
  return ""
 
4277
  import process_role as _warm_role
4278
  attempts = _READ_WARM_ATTEMPTS if attempts is None else max(1, int(attempts))
4279
  retry_seconds = _READ_WARM_RETRY_SECONDS if retry_seconds is None else retry_seconds
4280
+ rt, diagnosed = None, False
4281
  for attempt in range(1, attempts + 1):
4282
  t0 = _t.time()
4283
  stage = "pools"
 
4290
  raise RuntimeError(problem)
4291
  _warm_role.mark_ready("pools", True)
4292
  stage = "first_reads"
4293
+ for retry in (False, True):
4294
+ try:
4295
+ routes_customers.warm_default(rt)
4296
+ routes_products._pool_for(rt, None)
4297
+ break
4298
+ except Exception: # noqa: BLE001
4299
+ # D-12159: a pool connection closed mid-warm is dropped, then the reads run once more.
4300
+ if retry or not _reopen_read_pool(rt):
4301
+ raise
4302
  _warm_role.mark_ready("first_reads", True)
4303
  routes_customers.warm_bu_pools(rt)
4304
  print(f"[aios-api] read warm done in {_t.time() - t0:.1f}s")
4305
  return True
4306
  except Exception as exc: # noqa: BLE001
4307
+ if not diagnosed and rt is not None:
4308
+ diagnosed = True
4309
+ print(f"[aios-api] read warm pool state at the first failure: {_read_pool_state(rt)}")
4310
  if not _warm_role._READINESS.get(stage, {}).get("ok"):
4311
  _warm_role.mark_ready(stage, False, f"The {'read pools' if stage == 'pools' else 'first reads'} "
4312
  f"failed ({type(exc).__name__}); attempt {attempt} of {attempts}.")
 
4321
  """Restore each tenant's declarations after the Odoo seed and Amazon policy adapters."""
4322
  from core import connector_spec
4323
  from harness import mirror_store, runtime
4324
+ import tenant_loops
4325
+ import tenant_scan
4326
  errors = RESYNC_STATE.setdefault("mirrorRestoreErrors", {})
4327
+
4328
+ def restore(tenant_key, rt):
4329
  for connector in tuple(connector_spec.MIRRORS):
4330
  if connector in {"odoo", "amazon"}:
4331
  continue # Their existing adapters also enforce seed and deletion policies.
4332
  identity = f"{tenant_key}/{connector}"
4333
  try:
 
4334
  result = mirror_store.restore_mirror(connector, tenant_key, rt)
4335
  if result["problems"]:
4336
  errors[identity] = {"state": result["state"], "problems": list(result["problems"])}
 
4339
  print(f"[aios-api] {connector} mirror {tenant_key} {result['state']} ({why})")
4340
  except Exception as exc:
4341
  errors[identity] = {"state": "refused", "problems": [type(exc).__name__]}
4342
+
4343
+ def runtime_failed(tenant_key, exc):
4344
+ for connector in tuple(connector_spec.MIRRORS):
4345
+ if connector not in {"odoo", "amazon"}:
4346
+ errors[f"{tenant_key}/{connector}"] = {"state": "refused", "problems": [type(exc).__name__]}
4347
+
4348
+ # S82-T18: each tenant restores in its own job; an inactive one is skipped with one named line.
4349
+ jobs = tenant_loops.for_each_tenant(
4350
+ restore, name="declared-mirror-restore", tenants=runtime.known_tenants(),
4351
+ get_runtime=tenant_scan.runtime_lookup(runtime, "declared-mirror-restore",
4352
+ on_error=runtime_failed), kind="light")
4353
+ for job in jobs.values():
4354
+ job.join() # The caller reads the errors of a finished restore.
4355
  return not errors
4356
 
4357
 
4358
+ # D-12554: the worker imports main and runs no startup hook, so the migration runs here, before any thread.
4359
+ _boot_rows_before_writers()
4360
+
4361
  try:
4362
  from harness import datastore as _ds_boot
4363
  if not data_binding.gate_process() and (data_binding.is_space() or not _ds_boot.DB_PATH.exists()):
 
4394
  if not data_binding.gate_process() and not _freeze_holds("Amazon App workers"):
4395
  import marketing_stream_loop
4396
  import ads_schedule_loop
4397
+ import amazon_quotes_loop
4398
+ from harness import amazon_store
4399
  import routes_returns_refunds
4400
  marketing_stream_loop.start()
4401
  ads_schedule_loop.start()
4402
+ amazon_quotes_loop.start()
4403
+ process_role.start("amazon-after-sync", "heavy", amazon_store.run_after_sync_loop)
4404
  routes_returns_refunds.start_auto_approve_worker()
4405
 
4406
 
 
4408
  def _stop_amazon_app_workers():
4409
  import marketing_stream_loop
4410
  import ads_schedule_loop
4411
+ import amazon_quotes_loop
4412
  marketing_stream_loop.stop()
4413
  ads_schedule_loop.stop()
4414
+ amazon_quotes_loop.stop()
4415
 
4416
 
4417
  if (not data_binding.gate_process()
api/odoo_relational.py CHANGED
@@ -49,6 +49,7 @@ the oracle's predicate as an explicit `payment_state` condition pair.
49
  """
50
  import datetime as _dt
51
  import json as _json
 
52
  import time as _time
53
 
54
  #: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
@@ -159,6 +160,13 @@ def _preset(field, flow="odoo_relational"):
159
  grid renders it grey and the preset walls refuse a rename or a delete."""
160
  field = dict(field)
161
  field["automation"] = {"flowId": flow, "preset": True}
 
 
 
 
 
 
 
162
  return field
163
 
164
 
@@ -773,6 +781,8 @@ def audit_mirror_additions():
773
  # Customer tags migrate before their order projections and presets are mounted.
774
  if "category_id" not in out["res_partner"]:
775
  out["res_partner"].append("category_id")
 
 
776
  return out
777
 
778
 
@@ -1124,6 +1134,14 @@ def invoice_fields():
1124
  )] + audit_fields(INVOICES_KEY)
1125
 
1126
 
 
 
 
 
 
 
 
 
1127
  def order_fields():
1128
  """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
1129
  return [_preset(f) for f in (
@@ -1151,8 +1169,8 @@ def order_fields():
1151
  "description": "Odoo's fulfilment status: pending, started, partial, or full."},
1152
  {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
1153
  "default": True, "agg": "sum"},
1154
- {"key": "team", "label": "Business unit", "type": "select", "source": "overlay",
1155
- "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]},
1156
  {"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
1157
  "default": True, "description": "The pricelist on this Odoo sale order."},
1158
  {"key": "state", "label": "State", "type": "select", "source": "overlay",
@@ -1235,8 +1253,8 @@ def order_line_fields():
1235
  "default": True},
1236
  {"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
1237
  "default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
1238
- {"key": "team", "label": "Business unit", "type": "select", "source": "odoo",
1239
- "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"],
1240
  "description": "The sales team on this line's Odoo order."},
1241
  {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
1242
  "default": False},
@@ -1428,6 +1446,9 @@ def landed_cost_line_fields():
1428
  "format": {"decimals": 2}},
1429
  {"key": "date", "label": "Date", "type": "date", "default": True},
1430
  {"key": "state", "label": "State", "type": "text", "default": True},
 
 
 
1431
  {"key": "write_date", "label": "Updated", "type": "date", "default": False},
1432
  ]] + system_link_fields(LANDED_COST_LINES_KEY)
1433
 
@@ -1532,7 +1553,8 @@ def customer_fields():
1532
  {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
1533
  "default": False},
1534
  {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
1535
- {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True},
 
1536
  {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
1537
  {"key": "country", "label": "Country", "type": "text", "source": "overlay",
1538
  "default": False},
@@ -2130,7 +2152,9 @@ def _as_date(value):
2130
  STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
2131
  if not value:
2132
  return ""
2133
- return str(value)[:10]
 
 
2134
 
2135
 
2136
  def _in_scope(pid, excluded):
@@ -4420,7 +4444,8 @@ def landed_cost_line_source_spec():
4420
  "quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
4421
  "additional_landed_cost": ("va.additional_landed_cost", _stock_number),
4422
  "final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
4423
- "state": ("lc.state", _stock_text), "write_date": ("va.write_date", _as_date),
 
4424
  },
4425
  }
4426
 
@@ -4438,11 +4463,11 @@ def require_landed_cost_lines_mirror(cur):
4438
  if not fields <= columns(cur, entity):
4439
  raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
4440
  try:
4441
- phase = cur.execute("SELECT phase FROM _sync_state WHERE entity=?", [entity]).fetchone()
4442
  except Exception:
4443
- phase = None
4444
  spec = datastore.ENTITIES.get(entity)
4445
- if (not phase or phase[0] != "live" or not spec
4446
  or (spec.get("backfill_fields")
4447
  and not datastore._columns_backfilled(cur, entity, spec))):
4448
  raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
@@ -4591,7 +4616,7 @@ def validate_customer_invoice_cadence(client, cur, today=None):
4591
  observed = dict(cur.execute(
4592
  f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
4593
  f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
4594
- f"WHERE {sem.render_scope(store['scope_sql'])} GROUP BY {partner}").fetchall())
4595
  pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
4596
  # The oracle stays on invoice_date when a mutant redirects the served binding to orders.
4597
  independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
@@ -4623,7 +4648,7 @@ def validate_customer_rollups(client, cur):
4623
  value = cur.execute(
4624
  f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
4625
  f"{store['alias']} {store.get('join', '')} "
4626
- f"WHERE {sem.render_scope(store['scope_sql'])}").fetchone()[0]
4627
  independent = sem.customer_rollup_oracle(bag['measure'], client)
4628
  ok = (str(value or '')[:10] == str(independent or '')[:10]
4629
  if metric.get('format') == 'date' else
@@ -5873,8 +5898,9 @@ def slot_after(now, slot):
5873
  #: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
5874
  FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
5875
 
5876
- #: A slot whose pass has not ended good this long after it is LATE, and the watchdog alerts.
5877
  SLOT_LATE_SECONDS = 90 * 60
 
5878
 
5879
 
5880
  def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
@@ -6234,6 +6260,82 @@ def _effective_sync_seconds(preset):
6234
  return max(secs, env)
6235
 
6236
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6237
  def sync_freshness(rt, now=None):
6238
  """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
6239
  fail. PURE over the store reads.
@@ -6242,6 +6344,7 @@ def sync_freshness(rt, now=None):
6242
  `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
6243
  (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
6244
  the last run's own reason, its code prefix removed, when that run ended in error.
 
6245
  """
6246
  now = now or _utc_now()
6247
  if isinstance(now, str):
@@ -6261,10 +6364,15 @@ def sync_freshness(rt, now=None):
6261
  except Exception: # noqa: BLE001
6262
  status = None
6263
  status = status if isinstance(status, dict) else {}
 
 
 
 
 
6264
  phase = str(status.get("phase") or "unknown")
6265
- last_good = raw.get("lastGoodSyncAt") or None
6266
  # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
6267
- if not last_good and phase == "ok":
6268
  last_good = status.get("lastRun") or None
6269
  every = _effective_sync_seconds(cfg["syncEvery"])
6270
  allowed = None if every is None else every + STALE_GRACE_SECONDS
@@ -6272,18 +6380,6 @@ def sync_freshness(rt, now=None):
6272
  if allowed is not None and not cfg["frozen"]:
6273
  good_at = _parse_iso(last_good)
6274
  stale = good_at is None or (now - good_at).total_seconds() > allowed
6275
- # A daily slot that passed SLOT_LATE_SECONDS ago with no good sync since is late, not fresh.
6276
- late_slot = None
6277
- slot = sync_slot(cfg)
6278
- if slot is not None and not cfg["frozen"] and not stale:
6279
- try:
6280
- due = slot_before(now.timestamp(), slot)
6281
- except Exception: # noqa: BLE001
6282
- due = None
6283
- good_at = _parse_iso(last_good)
6284
- if (due is not None and now.timestamp() - due > SLOT_LATE_SECONDS
6285
- and (good_at is None or good_at.timestamp() < due)):
6286
- late_slot = slot_iso(due)
6287
  failure = None
6288
  errors = status.get("errors") if isinstance(status.get("errors"), list) else []
6289
  if phase == "error" and errors:
@@ -6293,6 +6389,15 @@ def sync_freshness(rt, now=None):
6293
  code, message = "sync_failed", text
6294
  failure = {"code": code.strip()[:80], "message": message.strip()[:300],
6295
  "at": status.get("lastRun") or None}
 
 
 
 
 
 
 
 
 
6296
  # D-2679: a database that failed while its siblings synced is reported by itself.
6297
  failed_databases = []
6298
  for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
@@ -6303,10 +6408,11 @@ def sync_freshness(rt, now=None):
6303
  for name in ("key", "label", "code", "message", "stage", "at")
6304
  + (("cause",) if item.get("cause") else ())})
6305
  return {"stale": bool(stale), "lastGoodSyncAt": last_good,
6306
- "late": late_slot is not None, "lateSlotAt": late_slot,
6307
  "failedDatabases": failed_databases,
 
6308
  "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
6309
- "lastRunAt": status.get("lastRun") or None, "phase": phase,
6310
  "frozen": cfg["frozen"], "failure": failure,
6311
  "syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
6312
  "nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
@@ -6628,7 +6734,7 @@ def shipment_source_spec():
6628
  "state": ("sp.state", _stock_text),
6629
  "scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
6630
  "date_done": (date_sql("sp.date_done"), _as_date),
6631
- "order_date": (date_sql("so.date_order"), _as_date),
6632
  "delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
6633
  "TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
6634
  "refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
@@ -6965,6 +7071,11 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
6965
  # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
6966
  # column a user added to a locked database is never rewritten.
6967
  stored = have[fkey]
 
 
 
 
 
6968
  # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
6969
  # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
6970
  # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
@@ -6973,7 +7084,7 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
6973
  continue
6974
  if (stored.get("automation") or {}).get("preset"):
6975
  for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
6976
- "agg", "pinned", "default"):
6977
  # A subsequent row-only refresh must not replace authoritative selections with
6978
  # an older compiled vocabulary while its metadata catalogue is unavailable.
6979
  if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
@@ -7598,10 +7709,8 @@ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automatio
7598
 
7599
 
7600
  def _canonical_mirror_ready(rt):
7601
- # Initial source selection does not make unrelated canonical totals authoritative.
7602
- from connector_onboarding import selected_mirror_entities
7603
- from harness import datastore
7604
- return rt is None or selected_mirror_entities(rt) is None or datastore.ready()
7605
 
7606
 
7607
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
 
49
  """
50
  import datetime as _dt
51
  import json as _json
52
+ import math as _math
53
  import time as _time
54
 
55
  #: ⛔ ROUTE ENTITLEMENT, NOT THE SPAWN GATE — `refresh()` stopped reading this at W53-T39 / R3,
 
160
  grid renders it grey and the preset walls refuse a rename or a delete."""
161
  field = dict(field)
162
  field["automation"] = {"flowId": flow, "preset": True}
163
+ if field.get('key') in {'customer', 'vendor', 'email', 'phone', 'mobile', 'street',
164
+ 'street2', 'city', 'zip', 'country', 'contact', 'contact_name'}:
165
+ field['personal'] = True
166
+ if field.get('key') == 'team':
167
+ field['label'] = 'Sales team'
168
+ field['options'] = _team_options()
169
+ field['automation']['renamable'] = True
170
  return field
171
 
172
 
 
781
  # Customer tags migrate before their order projections and presets are mounted.
782
  if "category_id" not in out["res_partner"]:
783
  out["res_partner"].append("category_id")
784
+ out["stock_landed_cost"] = list(LANDED_COST_HEADER_ADDITIONS)
785
+ out[BANK_LINE_TABLE] = list(BANK_LINE_ADDITIONS)
786
  return out
787
 
788
 
 
1134
  )] + audit_fields(INVOICES_KEY)
1135
 
1136
 
1137
+ def _team_options():
1138
+ from core import odoo
1139
+ try:
1140
+ return list(odoo.active_team_names().values())
1141
+ except odoo.TeamsUnknown:
1142
+ return []
1143
+
1144
+
1145
  def order_fields():
1146
  """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope."""
1147
  return [_preset(f) for f in (
 
1169
  "description": "Odoo's fulfilment status: pending, started, partial, or full."},
1170
  {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
1171
  "default": True, "agg": "sum"},
1172
+ {"key": "team", "label": "Sales team", "type": "select", "source": "overlay",
1173
+ "default": True},
1174
  {"key": "pricelist", "label": "Pricelist", "type": "text", "source": "overlay",
1175
  "default": True, "description": "The pricelist on this Odoo sale order."},
1176
  {"key": "state", "label": "State", "type": "select", "source": "overlay",
 
1253
  "default": True},
1254
  {"key": "customer_type", "label": "Customer type", "type": "text", "source": "odoo",
1255
  "default": True, "description": "The customer's Odoo tags. Blank when no tag is set."},
1256
+ {"key": "team", "label": "Sales team", "type": "select", "source": "odoo",
1257
+ "default": True,
1258
  "description": "The sales team on this line's Odoo order."},
1259
  {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
1260
  "default": False},
 
1446
  "format": {"decimals": 2}},
1447
  {"key": "date", "label": "Date", "type": "date", "default": True},
1448
  {"key": "state", "label": "State", "type": "text", "default": True},
1449
+ {"key": "amount_total", "label": "Landed cost total", "type": "currency", "default": False,
1450
+ "format": {"decimals": 2},
1451
+ "description": "The header total, repeated on each adjustment line."},
1452
  {"key": "write_date", "label": "Updated", "type": "date", "default": False},
1453
  ]] + system_link_fields(LANDED_COST_LINES_KEY)
1454
 
 
1553
  {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay",
1554
  "default": False},
1555
  {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
1556
+ {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True,
1557
+ "personal": True},
1558
  {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False},
1559
  {"key": "country", "label": "Country", "type": "text", "source": "overlay",
1560
  "default": False},
 
2152
  STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort."""
2153
  if not value:
2154
  return ""
2155
+ from core import periods
2156
+ day = periods.business_day(value)
2157
+ return day.isoformat() if day else str(value)[:10]
2158
 
2159
 
2160
  def _in_scope(pid, excluded):
 
4444
  "quantity": ("va.quantity", _stock_number), "former_cost": ("va.former_cost", _stock_number),
4445
  "additional_landed_cost": ("va.additional_landed_cost", _stock_number),
4446
  "final_cost": ("va.final_cost", _stock_number), "date": ("lc.date", _as_date),
4447
+ "state": ("lc.state", _stock_text), "amount_total": ("lc.amount_total", _stock_number),
4448
+ "write_date": ("va.write_date", _as_date),
4449
  },
4450
  }
4451
 
 
4463
  if not fields <= columns(cur, entity):
4464
  raise Refused("The landed cost lines mirror is incomplete. Sync Odoo before reading it.")
4465
  try:
4466
+ state = datastore._state(cur, entity)
4467
  except Exception:
4468
+ state = None
4469
  spec = datastore.ENTITIES.get(entity)
4470
+ if (not state or state["phase"] != "live" or not spec
4471
  or (spec.get("backfill_fields")
4472
  and not datastore._columns_backfilled(cur, entity, spec))):
4473
  raise Refused("The landed cost lines mirror is catching up. Try again shortly.")
 
4616
  observed = dict(cur.execute(
4617
  f"SELECT {partner}, {sem._measure_sql(metric, store['alias'], today=today)} "
4618
  f"FROM {store['table']} {store['alias']} {store.get('join', '')} "
4619
+ f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)} GROUP BY {partner}").fetchall())
4620
  pids = [int(row[JOIN_KEY]) for row in read_customers(cur)]
4621
  # The oracle stays on invoice_date when a mutant redirects the served binding to orders.
4622
  independent = sem.customer_rollup_oracle('customer_days_since_invoice', client,
 
4648
  value = cur.execute(
4649
  f"SELECT {sem._measure_sql(metric, store['alias'])} FROM {store['table']} "
4650
  f"{store['alias']} {store.get('join', '')} "
4651
+ f"WHERE {sem.render_scope(store['scope_sql'], cur=cur)}").fetchone()[0]
4652
  independent = sem.customer_rollup_oracle(bag['measure'], client)
4653
  ok = (str(value or '')[:10] == str(independent or '')[:10]
4654
  if metric.get('format') == 'date' else
 
5898
  #: A first-load continuation does not start this close to the next slot, so it cannot delay the slot.
5899
  FIRST_LOAD_SLOT_QUIET_SECONDS = 2 * 3600
5900
 
5901
+ #: Retained for older scheduling callers. Freshness uses the stored pass budget.
5902
  SLOT_LATE_SECONDS = 90 * 60
5903
+ PASS_BUDGET_SECONDS = 30 * 60
5904
 
5905
 
5906
  def slot_near(now, slot, seconds=FIRST_LOAD_SLOT_QUIET_SECONDS):
 
6260
  return max(secs, env)
6261
 
6262
 
6263
+ def _filling_databases(rt, cfg, now):
6264
+ """Read column-fill progress from the tenant store, including after a worker restart."""
6265
+ from harness import datastore
6266
+ try:
6267
+ saved = rt.get(datastore.ODOO_FILL_STATUS_KEY, fresh=True) if rt is not None else None
6268
+ except Exception: # noqa: BLE001
6269
+ saved = None
6270
+ entities = saved.get("entities", {}) if isinstance(saved, dict) else {}
6271
+ if not isinstance(entities, dict):
6272
+ return []
6273
+ result = []
6274
+ for entity, connections in entities.items():
6275
+ if not isinstance(connections, dict):
6276
+ continue
6277
+ parts = [part for part in connections.values()
6278
+ if isinstance(part, dict) and part.get("state") in ("filling", "done")]
6279
+ if not any(part.get("state") == "filling" for part in parts):
6280
+ continue
6281
+ filling = [part for part in parts if part.get("state") == "filling"]
6282
+ active = any(part.get("active") is True for part in filling)
6283
+ resumes = [at for part in filling if (at := _parse_iso(part.get("resumeAt"))) is not None]
6284
+ resume_at = None
6285
+ if not cfg["frozen"] and not active:
6286
+ resume_at = (min(resumes).astimezone(_dt.timezone.utc).isoformat().replace("+00:00", "Z")
6287
+ if resumes else next_sync_at(cfg, now.timestamp()))
6288
+ model = (datastore.ENTITIES.get(entity) or {}).get("model")
6289
+ tables = [(key, label) for _bucket, key, label, _fields in TABLES
6290
+ if model and ODOO_TABLE_MODELS.get(key) == model]
6291
+ tables.extend((key, spec["topicKey"].removeprefix("odoo_").replace("_", " ").capitalize())
6292
+ for key, spec in CANONICAL_RECEIPT_SPECS.items() if model and spec["model"] == model)
6293
+ for key, label in tables or [(entity, entity.replace("_", " "))]:
6294
+ result.append({"key": key, "label": label, "entity": entity, "state": "filling",
6295
+ "rowsDone": sum(max(0, int(part.get("rowsDone") or 0)) for part in parts),
6296
+ "rowsTotal": sum(max(0, int(part.get("rowsTotal") or 0)) for part in parts),
6297
+ "active": active, "resumeAt": resume_at})
6298
+ return result
6299
+
6300
+
6301
+ def _sync_pass_freshness(entry, cfg, now):
6302
+ """Read duration, daily receipts and resync state from one stored C-status entry."""
6303
+ entry = entry if isinstance(entry, dict) else {}
6304
+ saved = entry.get("pass")
6305
+ saved = dict(saved) if isinstance(saved, dict) else {}
6306
+
6307
+ def number(value):
6308
+ if isinstance(value, (int, float)) and not isinstance(value, bool) and _math.isfinite(value) and value >= 0:
6309
+ return float(value)
6310
+ return None
6311
+
6312
+ budget = number(saved.get("budgetSeconds")) or PASS_BUDGET_SECONDS
6313
+ duration = number(saved.get("durationSeconds"))
6314
+ started = _parse_iso(saved.get("startedAt"))
6315
+ ended = _parse_iso(saved.get("endedAt"))
6316
+ if duration is None and started is not None:
6317
+ end = now if saved.get("outcome") == "running" else ended
6318
+ if end is not None:
6319
+ duration = max(0.0, (end - started).total_seconds())
6320
+ late = not cfg["frozen"] and (saved.get("outcome") == "failed"
6321
+ or duration is not None and duration > budget)
6322
+ late_slot = None
6323
+ slot = sync_slot(cfg)
6324
+ if late and slot is not None:
6325
+ late_slot = slot_iso(slot_before((started or ended or now).timestamp(), slot))
6326
+ receipts = entry.get("dailyPasses")
6327
+ receipts = dict(receipts) if isinstance(receipts, dict) else {}
6328
+ day = (slot_iso(slot_before(now.timestamp(), slot))[:10] if slot is not None
6329
+ else now.astimezone(_dt.timezone.utc).date().isoformat())
6330
+ daily = receipts.get(day)
6331
+ return {"pass": saved or None,
6332
+ "passMinutes": round(duration / 60, 3) if duration is not None else None,
6333
+ "budgetMinutes": budget / 60, "dailyPass": dict(daily) if isinstance(daily, dict) else None,
6334
+ "dailyPasses": receipts,
6335
+ "resync": dict(entry["resync"]) if isinstance(entry.get("resync"), dict) else {},
6336
+ "late": bool(late), "lateSlotAt": late_slot}
6337
+
6338
+
6339
  def sync_freshness(rt, now=None):
6340
  """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
6341
  fail. PURE over the store reads.
 
6344
  `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
6345
  (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
6346
  the last run's own reason, its code prefix removed, when that run ended in error.
6347
+ `late` means a failed pass or a pass that exceeds its stored duration budget.
6348
  """
6349
  now = now or _utc_now()
6350
  if isinstance(now, str):
 
6364
  except Exception: # noqa: BLE001
6365
  status = None
6366
  status = status if isinstance(status, dict) else {}
6367
+ from core import connector_sync
6368
+ reader = getattr(connector_sync, "status", None)
6369
+ canonical = reader(rt, "odoo", fresh=True) if reader is not None and rt is not None else None
6370
+ canonical = canonical if isinstance(canonical, dict) else None
6371
+ pass_status = _sync_pass_freshness(canonical, cfg, now)
6372
  phase = str(status.get("phase") or "unknown")
6373
+ last_good = (canonical.get("lastGoodSyncAt") if canonical is not None else raw.get("lastGoodSyncAt")) or None
6374
  # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
6375
+ if canonical is None and not last_good and phase == "ok":
6376
  last_good = status.get("lastRun") or None
6377
  every = _effective_sync_seconds(cfg["syncEvery"])
6378
  allowed = None if every is None else every + STALE_GRACE_SECONDS
 
6380
  if allowed is not None and not cfg["frozen"]:
6381
  good_at = _parse_iso(last_good)
6382
  stale = good_at is None or (now - good_at).total_seconds() > allowed
 
 
 
 
 
 
 
 
 
 
 
 
6383
  failure = None
6384
  errors = status.get("errors") if isinstance(status.get("errors"), list) else []
6385
  if phase == "error" and errors:
 
6389
  code, message = "sync_failed", text
6390
  failure = {"code": code.strip()[:80], "message": message.strip()[:300],
6391
  "at": status.get("lastRun") or None}
6392
+ last_run = status.get("lastRun") or None
6393
+ if canonical is not None:
6394
+ saved_pass = pass_status["pass"] or {}
6395
+ outcome = saved_pass.get("outcome")
6396
+ phase = {"done": "ok", "failed": "error", "running": "syncing"}.get(outcome, outcome or "unknown")
6397
+ last_run = saved_pass.get("endedAt") or saved_pass.get("startedAt")
6398
+ detail = " ".join(str(saved_pass.get("detail") or "Sync failed.").replace("—", ",").replace("–", ",").split())
6399
+ failure = ({"code": "sync_failed", "message": detail[:300],
6400
+ "at": last_run} if outcome == "failed" else None)
6401
  # D-2679: a database that failed while its siblings synced is reported by itself.
6402
  failed_databases = []
6403
  for item in raw.get("failedDatabases") if isinstance(raw.get("failedDatabases"), list) else ():
 
6408
  for name in ("key", "label", "code", "message", "stage", "at")
6409
  + (("cause",) if item.get("cause") else ())})
6410
  return {"stale": bool(stale), "lastGoodSyncAt": last_good,
6411
+ **pass_status,
6412
  "failedDatabases": failed_databases,
6413
+ "fillingDatabases": _filling_databases(rt, cfg, now),
6414
  "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
6415
+ "lastRunAt": last_run, "phase": phase,
6416
  "frozen": cfg["frozen"], "failure": failure,
6417
  "syncAt": cfg["syncAt"], "syncTz": cfg["syncTz"],
6418
  "nextSyncAt": None if cfg["frozen"] else next_sync_at(cfg, now.timestamp())}
 
6734
  "state": ("sp.state", _stock_text),
6735
  "scheduled_date": (date_sql("sp.scheduled_date"), _as_date),
6736
  "date_done": (date_sql("sp.date_done"), _as_date),
6737
+ "order_date": (__import__('routes_odoo_tables')._business_day_sql("so.date_order"), _as_date),
6738
  "delivery_days": ("epoch(TRY_CAST(sp.date_done AS TIMESTAMPTZ) - "
6739
  "TRY_CAST(so.date_order AS TIMESTAMPTZ)) / 86400.0", number),
6740
  "refreshed": ("(SELECT substr(updated_at, 1, 10) FROM _sync_state "
 
7071
  # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a
7072
  # column a user added to a locked database is never rewritten.
7073
  stored = have[fkey]
7074
+ if (field.get('automation') or {}).get('renamable'):
7075
+ repaired = ut.keep_user_edits(stored, {**stored, **field})
7076
+ stored.clear()
7077
+ stored.update(repaired)
7078
+ continue
7079
  # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp,
7080
  # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the
7081
  # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on
 
7084
  continue
7085
  if (stored.get("automation") or {}).get("preset"):
7086
  for prop in ("label", "type", "options", "choicesFrom", "link", "rollup", "description",
7087
+ "agg", "pinned", "default", "personal"):
7088
  # A subsequent row-only refresh must not replace authoritative selections with
7089
  # an older compiled vocabulary while its metadata catalogue is unavailable.
7090
  if (stored.get("choicesFrom") == "odoo" and field.get("choicesFrom") != "odoo"
 
7709
 
7710
 
7711
  def _canonical_mirror_ready(rt):
7712
+ # Every tenant's canonical totals require its own ready mirror, regardless of selection.
7713
+ return rt is None or rt.mirror_handle().ready()
 
 
7714
 
7715
 
7716
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
api/pages.py CHANGED
@@ -261,12 +261,13 @@ def bu_control(session: Session, team_id):
261
  """The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
262
  is assembled per request and never cached on a scope key."""
263
  labels = perms.allowed_bu_labels(session.user)
 
264
  options = []
265
  for lab in labels:
266
  if lab == "All":
267
  options.append({"value": "all", "label": "All"})
268
  continue
269
- tid = ctxlib.BRANDS.get(lab)
270
  if tid is not None:
271
  options.append({"value": tid, "label": lab})
272
  return {"key": "bu", "kind": "bu", "label": "Business Unit",
@@ -279,10 +280,10 @@ def choice_control(key, label, value, options):
279
 
280
 
281
  def bu_label(team_id):
282
- """'All Business Units' / 'Fisch' / 'Royal' — for prose, never for a permission decision."""
283
  if team_id is None:
284
  return "All Business Units"
285
- for lab, tid in ctxlib.BRANDS.items():
286
  if tid == team_id:
287
  return lab
288
  return str(team_id)
@@ -371,7 +372,8 @@ def _metrics_for(rt, page_key, team_id, granularity, builder):
371
  """
372
  import scope_cache
373
 
374
- today = time.strftime("%Y-%m-%d")
 
375
  key = ("page", page_key, team_id, granularity, today)
376
 
377
  def _build():
 
261
  """The BU picker, built from the user's OWN permitted labels — which is exactly why `controls`
262
  is assembled per request and never cached on a scope key."""
263
  labels = perms.allowed_bu_labels(session.user)
264
+ brands = ctxlib.brands()
265
  options = []
266
  for lab in labels:
267
  if lab == "All":
268
  options.append({"value": "all", "label": "All"})
269
  continue
270
+ tid = brands.get(lab)
271
  if tid is not None:
272
  options.append({"value": tid, "label": lab})
273
  return {"key": "bu", "kind": "bu", "label": "Business Unit",
 
280
 
281
 
282
  def bu_label(team_id):
283
+ """'All Business Units' or the team's label, for prose, never for a permission decision."""
284
  if team_id is None:
285
  return "All Business Units"
286
+ for lab, tid in ctxlib.brands().items():
287
  if tid == team_id:
288
  return lab
289
  return str(team_id)
 
372
  """
373
  import scope_cache
374
 
375
+ from core import periods
376
+ today = periods.today().isoformat()
377
  key = ("page", page_key, team_id, granularity, today)
378
 
379
  def _build():
api/process_role.py CHANGED
@@ -5,16 +5,24 @@ Roles: `web` serves reads and installs finished mirror snapshots, `worker` runs
5
  never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
6
  store in the background; `light` serves this process's own reads.
7
 
8
- One sync writer per store: a worker runs its heavy jobs only while it holds the store's writer
9
- lease, so a second Space's worker waits instead of becoming a second writer.
10
 
11
- Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock and
12
- supervises the sync worker and installs hand-offs. The others only read the mirror read-only.
 
 
 
 
 
 
 
13
 
14
  One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
15
  unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
16
  gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
17
- that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the total; a tenant past the cap is reported.
 
18
  """
19
  from __future__ import annotations
20
 
@@ -25,9 +33,11 @@ import shutil
25
  import socket
26
  import subprocess
27
  import sys
 
28
  import threading
29
  import time
30
  import uuid
 
31
  from pathlib import Path
32
 
33
  ROLES = ("web", "worker", "all", "reader")
@@ -47,6 +57,10 @@ PRIMARY_LOCK = "web-primary.lock"
47
  PRIMARY_FILE = "web-primary.json"
48
  INSTALL_ATTEMPTS = 3
49
  INCOMPLETE_GRACE_SECONDS = 3600
 
 
 
 
50
 
51
 
52
  def resolve_role(env=None):
@@ -78,6 +92,9 @@ _INSTALLED = {}
78
  _INSTALL_FAILURES = {}
79
  LAST_INSTALL = {}
80
  LAST_INSTALL_ERROR = {}
 
 
 
81
 
82
 
83
  def role():
@@ -126,7 +143,11 @@ def heartbeat_name(env=None):
126
 
127
 
128
  def fleet_tenants(root=None):
129
- """Tenants with their own tenant worker, from the web supervisor's fleet file."""
 
 
 
 
130
  state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
131
  tenants = state.get("tenants") if isinstance(state, dict) else None
132
  return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
@@ -143,7 +164,7 @@ def syncs_here(slug, root=None):
143
 
144
 
145
  def sync_worker_cap(env=None):
146
- """AIOS_SYNC_WORKERS_MAX: sync worker processes per container, the primary included. Default 4."""
147
  env = os.environ if env is None else env
148
  try:
149
  return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
@@ -249,7 +270,7 @@ def writer_holder(env=None):
249
  return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
250
 
251
 
252
- def acquire_writer(rt, holder=None, *, now=None, ttl=None):
253
  """Take or renew the store's writer lease. True when `holder` holds it after the call.
254
 
255
  The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
@@ -268,13 +289,14 @@ def acquire_writer(rt, holder=None, *, now=None, ttl=None):
268
  doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
269
  return doc
270
 
271
- doc = rt.update(WRITER_LEASE_DOC, take)
 
272
  if not isinstance(doc, dict):
273
- doc = rt.get(WRITER_LEASE_DOC, fresh=True)
274
  return isinstance(doc, dict) and doc.get("holder") == holder
275
 
276
 
277
- def release_writer(rt, holder=None):
278
  """Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
279
  holder = holder or writer_holder()
280
 
@@ -284,7 +306,39 @@ def release_writer(rt, holder=None):
284
  doc.update(holder=None, renewedAt=0, previous=holder)
285
  return doc
286
 
287
- rt.update(WRITER_LEASE_DOC, drop)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
 
289
 
290
  def holds_writer():
@@ -312,7 +366,7 @@ def writer_lease_tick(rt_factory, holder, *, now=None, ttl=None):
312
  stamp = time.time() if now is None else float(now)
313
  ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
314
  try:
315
- held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl)
316
  except Exception as exc: # noqa: BLE001
317
  held = None
318
  if _WRITER.get("error") != type(exc).__name__:
@@ -378,10 +432,11 @@ def arm_writer_lease(rt_factory=None, holder=None, exit_fn=None):
378
 
379
 
380
  # ----------------------------------------------------------------------------- readiness (C9)
381
- READY_CHECKS = ("mirror", "pools", "first_reads")
382
  _READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
383
  "pools": "The read pools are not open yet.",
384
- "first_reads": "The first reads have not answered yet."}
 
385
  _READINESS = {}
386
  _BOOT_RESTORE = {}
387
  # The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
@@ -415,8 +470,43 @@ def note_boot_restore(state="done"):
415
  print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
416
 
417
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
  def worker_boot_restored(root=None):
419
- """Web side: the current worker's heartbeat says its boot restore finished."""
 
 
 
 
 
 
 
420
  beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
421
  restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
422
  # A secondary web worker supervises nothing: the primary names the current sync worker.
@@ -432,10 +522,15 @@ def _installed_map(root=None):
432
 
433
 
434
  def pending_handoffs(root=None):
435
- """`tenant/connector` for each newest READY hand-off not installed yet (by the primary)."""
 
 
 
436
  installed = _installed_map(root)
437
- return sorted(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
438
- if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
 
 
439
 
440
 
441
  # ----------------------------------------------------------------------------- several web workers
@@ -476,7 +571,7 @@ def claim_primary(root=None):
476
  os.close(fd)
477
  return False
478
  _PRIMARY["fd"] = fd
479
- print(f"[process-role] web worker pid={os.getpid()} is the primary: it supervises the sync worker "
480
  f"and installs hand-offs", flush=True)
481
  write_primary_state(root)
482
  return True
@@ -493,7 +588,9 @@ def write_primary_state(root=None):
493
  version = None
494
  _write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
495
  "pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
496
- "installed": {f"{key[0]}/{key[1]}": snap for key, snap in _INSTALLED.items()},
 
 
497
  "mirrorVersion": version})
498
  return True
499
 
@@ -515,13 +612,21 @@ def readiness(mirror_live=None, role_name=None, root=None):
515
  role_name = role_name or ROLE
516
  checks, detail = {}, {}
517
  for name in READY_CHECKS:
 
 
 
 
 
 
 
 
518
  row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
519
  checks[name], detail[name] = bool(row["ok"]), row["detail"]
520
  boot = role_name == "web" and _READY_LATCH["at"] is None
521
  if "mirror" in checks and checks["mirror"]:
522
  if mirror_live is False:
523
  checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
524
- elif boot and not worker_boot_restored(root):
525
  checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
526
  elif boot and pending_handoffs(root):
527
  checks["mirror"] = False
@@ -556,7 +661,11 @@ def _read_json(path):
556
 
557
 
558
  def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
559
- """Keep one published snapshot's parquet for the web process. READY.json is written last."""
 
 
 
 
560
  root = Path(root or handoff_dir())
561
  snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
562
  target = root / connector / snapshot
@@ -600,9 +709,9 @@ def ready_handoffs(root=None):
600
  return groups
601
 
602
 
603
- def _install_default(connector, tenant, ready):
604
  from harness import mirror_store, runtime
605
- rt = runtime.get_runtime(tenant)
606
  from harness import datastore
607
  paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
608
  resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
@@ -633,7 +742,10 @@ def _remove_dir(path):
633
 
634
 
635
  def install_ready(root=None, installer=None, *, now=None):
636
- """Install the newest READY snapshot per tenant and connector, then delete older hand-offs."""
 
 
 
637
  root = Path(root or handoff_dir())
638
  installer = installer or _install_default
639
  stamp = time.time() if now is None else now
@@ -678,6 +790,72 @@ def install_ready(root=None, installer=None, *, now=None):
678
  return installed
679
 
680
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
681
  # ----------------------------------------------------------------------------- cross-process desk
682
  def _forward_request(receipt, connection_id):
683
  """Web side: hand an accepted Sync now to the worker through the store desk."""
@@ -693,34 +871,55 @@ def _forward_request(receipt, connection_id):
693
  "The sync request could not be passed to the sync worker. Try again.")
694
 
695
 
696
- def _tenants():
697
- import tenant_scan
698
  from harness import runtime
699
- return [(slug, rt) for slug, rt in tenant_scan.live_tenants(runtime) if rt is not None]
700
 
701
 
702
- def poll_worker_desk():
703
- """Worker side: pick up event receipts and Sync now hand-offs from the store desk."""
704
  from core import connector_sync
705
- for slug, rt in _tenants():
706
- if not syncs_here(slug):
707
- continue
708
- try:
709
- connector_sync.restore_sync_requests(rt)
710
- admitted = connector_sync.poll_remote_requests(rt)
711
- except Exception as exc: # noqa: BLE001
712
- print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
713
- continue
714
- for entry in admitted:
715
- print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
716
- import routes_connectors
717
- key = (entry["provider"], slug)
718
- with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
719
- # No connection id means the default connection, never the previous request's.
720
- if entry.get("connectionId"):
721
- routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
722
- else:
723
- routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
724
 
725
 
726
  def _worker_desk_loop():
@@ -731,43 +930,60 @@ def _worker_desk_loop():
731
  print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
732
 
733
 
734
- def _apply_worker_state():
735
- """Web side: mirror the worker's request progress and catch-up state into this process."""
 
 
 
 
 
 
 
 
 
 
736
  from core import connector_sync
737
  from harness import mirror_store
738
- beat = _read_json(handoff_dir() / HEARTBEAT_FILE) or {}
739
- catch_up = dict(beat.get("catchUp")) if isinstance(beat.get("catchUp"), dict) else {}
740
- for slug in list(_FLEET):
741
- tenant_beat = _read_json(handoff_dir() / heartbeat_name(
742
- {"AIOS_WORKER_SCOPE": "tenant", "AIOS_SYNC_WRITER_TENANT": slug})) or {}
743
- if isinstance(tenant_beat.get("catchUp"), dict) and slug in tenant_beat["catchUp"]:
744
- # A tenant worker's own catch-up wins for its tenant.
745
- catch_up[slug] = tenant_beat["catchUp"][slug]
746
- resync = beat.get("resync")
 
 
 
 
 
 
 
 
 
 
 
 
 
747
  app_main = _running_main()
748
- if isinstance(resync, dict) and app_main is not None:
749
- # The status routes read RESYNC_STATE here. Restore errors stay this process's own.
750
- app_main.RESYNC_STATE.update({k: v for k, v in resync.items() if "estore" not in k})
751
- for slug, rt in _tenants():
752
- try:
753
- connector_sync.apply_remote_progress(rt)
754
- except Exception as exc: # noqa: BLE001
755
- print(f"[process-role] sync progress read refused for {slug} ({type(exc).__name__})", flush=True)
756
- if slug in catch_up:
757
- mirror_store.apply_catch_up(rt, catch_up[slug])
758
 
759
 
760
  def _web_watch_loop():
761
  while not _STOP.wait(POLL_SECONDS):
762
- try:
763
- install_ready()
764
- write_primary_state()
765
- except Exception as exc: # noqa: BLE001
766
- print(f"[process-role] hand-off scan failed ({type(exc).__name__}: {exc})", flush=True)
767
  try:
768
  _apply_worker_state()
769
  except Exception as exc: # noqa: BLE001
770
  print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
 
 
 
 
771
 
772
 
773
  # ----------------------------------------------------------------------------- the worker process
@@ -775,17 +991,52 @@ def _running_main():
775
  return sys.modules.get("api.main") or sys.modules.get("main")
776
 
777
 
778
- def write_heartbeat(now=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
779
  try:
780
  from harness import mirror_store
781
  catch_up = mirror_store.catch_up_snapshot()
782
  except Exception: # noqa: BLE001
783
  catch_up = {}
 
 
 
 
 
 
 
 
 
784
  app_main = _running_main()
785
- resync = dict(getattr(app_main, "RESYNC_STATE", None) or {})
786
- _write_json(handoff_dir() / heartbeat_name(),
787
- {"pid": os.getpid(), "tenant": writer_tenant(), "scope": worker_scope(), "at": time.time() if now is None else now,
788
- "catchUp": catch_up, "resync": resync, "bootRestore": dict(_BOOT_RESTORE)})
 
 
 
 
 
 
 
 
 
 
 
 
789
 
790
 
791
  def _heartbeat_loop(parent_pid):
@@ -803,9 +1054,9 @@ def _heartbeat_loop(parent_pid):
803
 
804
 
805
  def start_worker_services():
806
- """Called by sync_worker.py: heartbeat, hand-off writer and the store desk poll."""
807
  from harness import mirror_store
808
- mirror_store.register_publish_listener(write_handoff)
809
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
810
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
811
  # Before the first heavy job: each one now waits until this worker holds the writer lease.
@@ -814,6 +1065,7 @@ def start_worker_services():
814
 
815
 
816
  def _worker_env(tenant=None):
 
817
  env = dict(os.environ)
818
  env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
819
  AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
@@ -828,84 +1080,35 @@ def _worker_env(tenant=None):
828
 
829
 
830
  def _supervise(tenant=None):
831
- """Keep one sync worker running: the primary when `tenant` is None, else that tenant's."""
832
- record = _WORKER if tenant is None else _FLEET[tenant]
833
- label = "sync worker" if tenant is None else f"sync worker for {tenant}"
834
- backoff = 5.0
835
- while not _STOP.is_set():
836
- started = time.time()
837
- try:
838
- proc = subprocess.Popen([sys.executable, "sync_worker.py"], cwd=str(API_DIR),
839
- env=_worker_env(tenant))
840
- except OSError as exc:
841
- print(f"[process-role] {label} could not start ({type(exc).__name__}: {exc})", flush=True)
842
- proc = None
843
- if proc is not None:
844
- record.update(proc=proc, pid=proc.pid, startedAt=started)
845
- print(f"[process-role] {label} started pid={proc.pid}", flush=True)
846
- if tenant is None:
847
- write_primary_state()
848
- code = proc.wait()
849
- record["lastExit"] = code
850
- else:
851
- code = None
852
- if _STOP.is_set():
853
- return
854
- if time.time() - started > 600:
855
- backoff = 5.0
856
- record["restarts"] += 1
857
- print(f"[process-role] {label} exited code={code}; restart {record['restarts']} "
858
- f"in {int(backoff)}s", flush=True)
859
- if _STOP.wait(backoff):
860
- return
861
- backoff = min(backoff * 2, 300.0)
862
 
863
 
 
864
  _FLEET = {}
865
  _FLEET_STATE = {"overCap": [], "error": None, "at": None}
866
 
867
 
868
- def _write_fleet(root=None):
869
- _write_json(Path(root or handoff_dir()) / FLEET_FILE, {
870
- "at": time.time(), "primary": writer_tenant(),
871
- "tenants": {slug: {"pid": row.get("pid")} for slug, row in _FLEET.items()},
872
- "overCap": list(_FLEET_STATE["overCap"])})
873
-
874
-
875
  def _report_over_cap(over, cap):
876
  """Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
877
- cause = (f"its daily Odoo and Amazon sync cannot run: this container allows {cap} sync worker(s) "
878
- f"(AIOS_SYNC_WORKERS_MAX) and they are all in use. Raise AIOS_SYNC_WORKERS_MAX or move "
879
- f"to a larger box.")
880
  for slug in over:
881
- print(f"[process-role] sync worker for {slug} NOT started: {cause}", flush=True)
882
  try:
883
  import sync_alerts
884
  from harness import runtime
885
  sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
886
  "code": "over_cap", "cause": cause}],
887
- store=runtime.get_runtime(slug))
888
  except Exception as exc: # noqa: BLE001
889
  print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
890
 
891
 
892
- def fleet_tick(wanted=None, *, starter=None, root=None):
893
- """Start a tenant worker for each wanted tenant inside the cap; report the rest. Returns the started slugs."""
894
- wanted = fleet_wanted() if wanted is None else list(wanted)
895
- cap = sync_worker_cap()
896
- new = [slug for slug in wanted if slug not in _FLEET]
897
- room = max(0, cap - 1 - len(_FLEET))
898
- starting, over = new[:room], new[room:]
899
- for slug in starting:
900
- _FLEET[slug] = {"proc": None, "pid": None, "restarts": 0, "startedAt": None, "lastExit": None}
901
- _FLEET_STATE.update(overCap=over, at=time.time(), error=None)
902
- # The fleet file names a tenant before its worker starts, so the primary stops serving it first.
903
- _write_fleet(root)
904
- for slug in starting:
905
- (starter or (lambda s: start(f"sync-worker-supervisor-{s}", "light", _supervise, args=(s,))))(slug)
906
- if over:
907
- _report_over_cap(over, cap)
908
- return starting
909
 
910
 
911
  def _fleet_loop():
@@ -914,7 +1117,7 @@ def _fleet_loop():
914
  fleet_tick()
915
  except Exception as exc: # noqa: BLE001
916
  _FLEET_STATE.update(error=type(exc).__name__, at=time.time())
917
- print(f"[process-role] tenant worker scan failed ({type(exc).__name__}: {exc})", flush=True)
918
  if _STOP.wait(FLEET_SCAN_SECONDS):
919
  return
920
 
@@ -933,7 +1136,7 @@ def stop_worker():
933
 
934
 
935
  def start_web_services():
936
- """Web role only: forward Sync now, supervise the worker, install its snapshots.
937
 
938
  With several uvicorn workers only the primary does this. The others stand by and read.
939
  """
@@ -946,35 +1149,37 @@ def start_web_services():
946
  if _WORKER.get("standby") is None:
947
  _WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
948
  print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
949
- f"the primary supervises the sync worker", flush=True)
950
  return False
951
- _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _supervise)
952
- start("sync-worker-fleet", "light", _fleet_loop)
953
  start("mirror-handoff-watcher", "light", _web_watch_loop)
954
  atexit.register(stop_worker)
955
  return True
956
 
957
 
958
  def status():
959
- """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response."""
960
- beat = _read_json(handoff_dir() / HEARTBEAT_FILE) if ROLE != "all" else None
961
- if ROLE == "web":
962
- proc = _WORKER.get("proc")
963
- pid = _WORKER.get("pid")
964
- alive = proc is not None and proc.poll() is None
965
- elif ROLE == "worker":
966
  pid, alive = os.getpid(), True
 
 
 
 
967
  else:
968
  pid, alive = None, False
969
- age = None
970
- if isinstance(beat, dict) and beat.get("pid") == pid and beat.get("at"):
971
- age = round(max(0.0, time.time() - float(beat["at"])), 1)
972
  fleet = {}
973
  for slug, row in _FLEET.items():
974
- tproc = row.get("proc")
975
- fleet[slug] = {"pid": row.get("pid"), "alive": tproc is not None and tproc.poll() is None,
976
- "restarts": row.get("restarts", 0), "lastExit": row.get("lastExit")}
977
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
 
978
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
979
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
980
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
 
5
  never syncs (a second Space on the same store). Kinds: `heavy` moves source data or writes the
6
  store in the background; `light` serves this process's own reads.
7
 
8
+ One writer per connector: resident jobs hold the Odoo lease, while each Amazon pass claims and
9
+ renews its own lease independently of the first-load job.
10
 
11
+ Several uvicorn workers (AIOS_UVICORN_WORKERS > 1): one web worker holds the primary lock, asks
12
+ the sync runtime for passes and installs mirrors. The others only read the mirror read-only.
13
+
14
+ The web never starts a sync worker child. sync_runtime picks where the worker runs: one HF job
15
+ per pass, per tenant (`hf-job`), a separately run resident worker (`external`), or nothing (`off`).
16
+
17
+ The web reads the worker's heartbeat, resync state and published mirror ids from the store's
18
+ C-status document (connector_sync.STATUS_KEY, provider WORKER_STATUS) and installs each mirror from
19
+ its stored manifest. No web read depends on a local child process or a hand-off file.
20
 
21
  One sync worker per tenant: the primary worker serves AIOS_SYNC_WRITER_TENANT (royal-imports when
22
  unset) and runs every job that loops all tenants. Each other tenant that holds a sync connector
23
  gets a tenant worker (AIOS_WORKER_SCOPE=tenant) with its own DuckDB file and lease, which runs only
24
+ that tenant's sync jobs. AIOS_SYNC_WORKERS_MAX caps the sync jobs that run at once; a tenant past
25
+ the cap is reported.
26
  """
27
  from __future__ import annotations
28
 
 
33
  import socket
34
  import subprocess
35
  import sys
36
+ import tempfile
37
  import threading
38
  import time
39
  import uuid
40
+ from contextlib import contextmanager
41
  from pathlib import Path
42
 
43
  ROLES = ("web", "worker", "all", "reader")
 
57
  PRIMARY_FILE = "web-primary.json"
58
  INSTALL_ATTEMPTS = 3
59
  INCOMPLETE_GRACE_SECONDS = 3600
60
+ #: The C-status provider key the sync worker writes its heartbeat, catch-up and mirror ids under.
61
+ WORKER_STATUS = "sync_worker"
62
+ #: A stored heartbeat older than this reads as a dead worker. verify_live allows 120 s.
63
+ HEARTBEAT_STALE_SECONDS = 120
64
 
65
 
66
  def resolve_role(env=None):
 
92
  _INSTALL_FAILURES = {}
93
  LAST_INSTALL = {}
94
  LAST_INSTALL_ERROR = {}
95
+ # The web's last read of the store: the worker heartbeat and each tenant's published mirror ids.
96
+ _STORED = {"beat": None, "mirrors": {}}
97
+ _RESYNC_APPLIED = {"at": ""}
98
 
99
 
100
  def role():
 
143
 
144
 
145
  def fleet_tenants(root=None):
146
+ """Tenants with their own tenant worker: AIOS_SYNC_FLEET_TENANTS in a job, else the fleet file."""
147
+ named = os.environ.get("AIOS_SYNC_FLEET_TENANTS")
148
+ if named is not None:
149
+ # The web names them when it submits the primary's job: a job has no shared fleet file.
150
+ return {slug.strip() for slug in named.split(",") if slug.strip()}
151
  state = _read_json(Path(root or handoff_dir()) / FLEET_FILE)
152
  tenants = state.get("tenants") if isinstance(state, dict) else None
153
  return {str(slug) for slug in tenants} if isinstance(tenants, dict) else set()
 
164
 
165
 
166
  def sync_worker_cap(env=None):
167
+ """AIOS_SYNC_WORKERS_MAX: sync jobs that run at once, the primary's included. Default 4."""
168
  env = os.environ if env is None else env
169
  try:
170
  return max(1, int(env.get("AIOS_SYNC_WORKERS_MAX") or 4))
 
270
  return f"{env.get('SPACE_ID') or 'local'}@{socket.gethostname()}"
271
 
272
 
273
+ def acquire_writer(rt, holder=None, *, now=None, ttl=None, connector=None):
274
  """Take or renew the store's writer lease. True when `holder` holds it after the call.
275
 
276
  The store's update is the compare-and-set: Postgres locks the row for the read-modify-write.
 
289
  doc.update(holder=holder, renewedAt=stamp, pid=os.getpid())
290
  return doc
291
 
292
+ lease_key = f"{WRITER_LEASE_DOC}:{connector or 'odoo'}"
293
+ doc = rt.update(lease_key, take)
294
  if not isinstance(doc, dict):
295
+ doc = rt.get(lease_key, fresh=True)
296
  return isinstance(doc, dict) and doc.get("holder") == holder
297
 
298
 
299
+ def release_writer(rt, holder=None, *, connector=None):
300
  """Give the lease up at a clean exit, so the next writer need not wait for it to expire."""
301
  holder = holder or writer_holder()
302
 
 
306
  doc.update(holder=None, renewedAt=0, previous=holder)
307
  return doc
308
 
309
+ rt.update(f"{WRITER_LEASE_DOC}:{connector or 'odoo'}", drop)
310
+
311
+
312
+ @contextmanager
313
+ def connector_writer(rt, connector, *, exit_fn=None):
314
+ """Hold and renew one connector lease for a pass, refusing a competing first-load writer."""
315
+ holder = f"{writer_holder()}:{connector}:{uuid.uuid4().hex}"
316
+ if not acquire_writer(rt, holder, connector=connector):
317
+ raise RuntimeError(f"Another job holds the {connector} writer lease.")
318
+ stopped = threading.Event()
319
+
320
+ def renew():
321
+ while not stopped.wait(WRITER_LEASE_SECONDS / 3):
322
+ try:
323
+ held = acquire_writer(rt, holder, connector=connector)
324
+ except Exception:
325
+ held = False
326
+ if not held:
327
+ print(f"[process-role] {connector} writer lease lost; this worker stops writing", flush=True)
328
+ (exit_fn or os._exit)(3)
329
+ return
330
+
331
+ job = None
332
+ try:
333
+ job = start("connector-writer-renew", "light", renew)
334
+ if job is None:
335
+ raise RuntimeError("The connector writer lease could not be renewed.")
336
+ yield
337
+ finally:
338
+ stopped.set()
339
+ if job is not None:
340
+ job.join()
341
+ release_writer(rt, holder, connector=connector)
342
 
343
 
344
  def holds_writer():
 
366
  stamp = time.time() if now is None else float(now)
367
  ttl = WRITER_LEASE_SECONDS if ttl is None else float(ttl)
368
  try:
369
+ held = acquire_writer(rt_factory(), holder, now=stamp, ttl=ttl, connector="odoo")
370
  except Exception as exc: # noqa: BLE001
371
  held = None
372
  if _WRITER.get("error") != type(exc).__name__:
 
432
 
433
 
434
  # ----------------------------------------------------------------------------- readiness (C9)
435
+ READY_CHECKS = ("mirror", "pools", "first_reads", "tenant_keys")
436
  _READY_DEFAULT = {"mirror": "The mirror is not installed yet.",
437
  "pools": "The read pools are not open yet.",
438
+ "first_reads": "The first reads have not answered yet.",
439
+ "tenant_keys": "The tenant data keys have not been checked yet."}
440
  _READINESS = {}
441
  _BOOT_RESTORE = {}
442
  # The first full pass latches the boot conditions: a later worker restart or a fresh hand-off
 
470
  print(f"[sync-worker] heartbeat write failed ({type(exc).__name__})", flush=True)
471
 
472
 
473
+ def _known_beat(root=None):
474
+ """The stored worker heartbeat this web worker last read; a secondary takes the primary's read."""
475
+ beat = _STORED["beat"] if is_primary() else primary_state(root).get("workerBeat")
476
+ return beat if isinstance(beat, dict) else None
477
+
478
+
479
+ def _known_mirrors(root=None):
480
+ """{(tenant, connector): snapshotId} the store says the worker published, as last read."""
481
+ if is_primary():
482
+ return dict(_STORED["mirrors"])
483
+ stored = primary_state(root).get("storeMirrors") or {}
484
+ return {tuple(key.split("/", 1)): snap for key, snap in stored.items() if "/" in key}
485
+
486
+
487
+ def _waits_for_worker_boot():
488
+ """Only a resident worker reports a boot restore. A job per pass leaves the web to its own restore."""
489
+ import sync_runtime
490
+ return sync_runtime.runtime_name()[0] == "external"
491
+
492
+
493
+ def beat_age(beat, now=None):
494
+ """Seconds since a heartbeat's stamp, or None when it carries none."""
495
+ try:
496
+ return round(max(0.0, (time.time() if now is None else now) - float(beat["at"])), 1)
497
+ except (KeyError, TypeError, ValueError):
498
+ return None
499
+
500
+
501
  def worker_boot_restored(root=None):
502
+ """Web side: the worker's stored heartbeat is fresh and says its boot restore finished."""
503
+ beat = _known_beat(root)
504
+ if beat is not None:
505
+ restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
506
+ age = beat_age(beat)
507
+ return (bool(restore.get("state")) and restore.get("pid") == beat.get("pid")
508
+ and age is not None and age < HEARTBEAT_STALE_SECONDS)
509
+ # Fallback for hand-off dir fixtures only (verify_readiness, verify_box_sizing): the store outranks it.
510
  beat = _read_json(Path(root or handoff_dir()) / HEARTBEAT_FILE) or {}
511
  restore = beat.get("bootRestore") if isinstance(beat.get("bootRestore"), dict) else {}
512
  # A secondary web worker supervises nothing: the primary names the current sync worker.
 
522
 
523
 
524
  def pending_handoffs(root=None):
525
+ """`tenant/connector` for each stored publication the primary has not installed yet.
526
+
527
+ READY hand-offs count too, for the fixtures of verify_readiness and verify_box_sizing.
528
+ """
529
  installed = _installed_map(root)
530
+ pending = {f"{key[0]}/{key[1]}" for key, snap in _known_mirrors(root).items() if installed.get(key) != snap}
531
+ pending.update(f"{key[0]}/{key[1]}" for key, rows in ready_handoffs(root).items()
532
+ if installed.get(key) != (rows[-1][2].get("snapshotId") or rows[-1][1].name))
533
+ return sorted(pending)
534
 
535
 
536
  # ----------------------------------------------------------------------------- several web workers
 
571
  os.close(fd)
572
  return False
573
  _PRIMARY["fd"] = fd
574
+ print(f"[process-role] web worker pid={os.getpid()} is the primary: it ticks the sync runtime "
575
  f"and installs hand-offs", flush=True)
576
  write_primary_state(root)
577
  return True
 
588
  version = None
589
  _write_json(Path(root or handoff_dir()) / PRIMARY_FILE, {
590
  "pid": os.getpid(), "workerPid": _WORKER.get("pid"), "at": time.time(),
591
+ "installed": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_INSTALLED).items()},
592
+ "workerBeat": _STORED["beat"],
593
+ "storeMirrors": {f"{key[0]}/{key[1]}": snap for key, snap in dict(_STORED["mirrors"]).items()},
594
  "mirrorVersion": version})
595
  return True
596
 
 
612
  role_name = role_name or ROLE
613
  checks, detail = {}, {}
614
  for name in READY_CHECKS:
615
+ if name == "tenant_keys":
616
+ from core import tenant_keys
617
+ from harness import runtime
618
+ try:
619
+ checks[name], detail[name] = tenant_keys.readiness_check(runtime.known_tenants())
620
+ except Exception:
621
+ checks[name], detail[name] = False, "The tenant data keys did not answer."
622
+ continue
623
  row = _READINESS.get(name) or {"ok": False, "detail": _READY_DEFAULT.get(name, "")}
624
  checks[name], detail[name] = bool(row["ok"]), row["detail"]
625
  boot = role_name == "web" and _READY_LATCH["at"] is None
626
  if "mirror" in checks and checks["mirror"]:
627
  if mirror_live is False:
628
  checks["mirror"], detail["mirror"] = False, "The Odoo mirror has not finished loading."
629
+ elif boot and _waits_for_worker_boot() and not worker_boot_restored(root):
630
  checks["mirror"], detail["mirror"] = False, "The sync worker has not finished its boot restore."
631
  elif boot and pending_handoffs(root):
632
  checks["mirror"] = False
 
661
 
662
 
663
  def write_handoff(connector, tenant_key, manifest, rows, paths, *, root=None, now=None):
664
+ """Keep one snapshot's parquet in a hand-off dir. READY.json is written last.
665
+
666
+ No runtime caller: the web installs from the store (install_stored). verify_readiness and
667
+ verify_box_sizing build their fixtures with it.
668
+ """
669
  root = Path(root or handoff_dir())
670
  snapshot = str(manifest.get("snapshot_id") or uuid.uuid4().hex)
671
  target = root / connector / snapshot
 
709
  return groups
710
 
711
 
712
+ def _install_default(connector, tenant, ready, rt=None):
713
  from harness import mirror_store, runtime
714
+ rt = rt or runtime.get_runtime(tenant)
715
  from harness import datastore
716
  paths = {name: [Path(p) for p in files] for name, files in ready["paths"].items()}
717
  resident = Path(rt.datastore_path).resolve() == datastore.DB_PATH.resolve()
 
742
 
743
 
744
  def install_ready(root=None, installer=None, *, now=None):
745
+ """Install the newest READY snapshot per tenant and connector, then delete older hand-offs.
746
+
747
+ No runtime caller since the store install; kept for the verify_readiness fixture.
748
+ """
749
  root = Path(root or handoff_dir())
750
  installer = installer or _install_default
751
  stamp = time.time() if now is None else now
 
790
  return installed
791
 
792
 
793
+ def _resident_snapshot(connector, tenant):
794
+ """The snapshot id this process already serves, through its own read path. None when unreadable."""
795
+ from harness import datastore
796
+ try:
797
+ return datastore.snapshot_id(tenant, connector)
798
+ except Exception: # noqa: BLE001
799
+ return None
800
+
801
+
802
+ def _install_from_store(connector, tenant, rt, snapshot_id):
803
+ """Download the store's published manifest and parquet, then install them. False if resident."""
804
+ from harness import datastore, mirror_store
805
+ store = mirror_store.for_connector(connector)
806
+ if snapshot_id and _resident_snapshot(store.connector, tenant) == snapshot_id:
807
+ return False
808
+ raw = rt.download_bytes(store.prefix + mirror_store._MANIFEST_KEY)
809
+ if raw is None:
810
+ raise LookupError(f"no published {connector} manifest in the store")
811
+ with tempfile.TemporaryDirectory(prefix="mirror-store-install-") as tmp:
812
+ # An in-memory DuckDB counts the parquet rows; the tenant file stays closed until install.
813
+ con = datastore.duckdb.connect()
814
+ try:
815
+ manifest, rows, paths, _total, _refused = store._validated_snapshot(tenant, rt, con, raw, tmp)
816
+ finally:
817
+ con.close()
818
+ return _install_default(connector, tenant, {"manifest": manifest, "stateRows": rows,
819
+ "paths": {name: list(files) for name, files in paths.items()}},
820
+ rt)
821
+
822
+
823
+ def install_stored(tenant, rt, mirrors, installer=None, *, now=None):
824
+ """Install each mirror the store says the worker published and this process has not installed.
825
+
826
+ `mirrors` is the C-status `sync_worker.mirrors` section: {connector: {"snapshotId": ...}}.
827
+ """
828
+ installer = installer or _install_from_store
829
+ stamp = time.time() if now is None else now
830
+ installed = []
831
+ for connector, row in sorted((mirrors or {}).items()):
832
+ snapshot = row.get("snapshotId") if isinstance(row, dict) else None
833
+ if not snapshot:
834
+ continue
835
+ key = (str(tenant), str(connector))
836
+ _STORED["mirrors"][key] = snapshot
837
+ if _INSTALLED.get(key) == snapshot or _INSTALL_FAILURES.get((key, snapshot), 0) >= INSTALL_ATTEMPTS:
838
+ continue
839
+ try:
840
+ changed = installer(key[1], key[0], rt, snapshot)
841
+ except Exception as exc: # noqa: BLE001
842
+ _INSTALL_FAILURES[(key, snapshot)] = _INSTALL_FAILURES.get((key, snapshot), 0) + 1
843
+ LAST_INSTALL_ERROR.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp,
844
+ attempts=_INSTALL_FAILURES[(key, snapshot)], type=type(exc).__name__)
845
+ print(f"[process-role] store install {key[0]}/{key[1]} {snapshot} failed "
846
+ f"({type(exc).__name__}: {exc})", flush=True)
847
+ continue
848
+ _INSTALLED[key] = snapshot
849
+ if changed is False:
850
+ continue
851
+ LAST_INSTALL.update(tenant=key[0], connector=key[1], snapshotId=snapshot, at=stamp)
852
+ if (LAST_INSTALL_ERROR.get("tenant"), LAST_INSTALL_ERROR.get("connector")) == key:
853
+ LAST_INSTALL_ERROR.clear()
854
+ installed.append(key)
855
+ print(f"[process-role] installed {key[1]} snapshot {snapshot} for {key[0]} from the store", flush=True)
856
+ return installed
857
+
858
+
859
  # ----------------------------------------------------------------------------- cross-process desk
860
  def _forward_request(receipt, connection_id):
861
  """Web side: hand an accepted Sync now to the worker through the store desk."""
 
871
  "The sync request could not be passed to the sync worker. Try again.")
872
 
873
 
874
+ def _status_runtime(tenant=None):
875
+ """The runtime whose C-status document holds the sync worker's heartbeat."""
876
  from harness import runtime
877
+ return runtime.get_runtime(tenant or writer_tenant())
878
 
879
 
880
+ def stored_worker(rt):
881
+ """The `sync_worker` C-status entry of one tenant, read past this process's cache. {} when absent."""
882
  from core import connector_sync
883
+ return connector_sync.status(rt, WORKER_STATUS, fresh=True) or {}
884
+
885
+
886
+ def stored_heartbeat(tenant=None):
887
+ """The worker heartbeat in the store, or None. A store that does not answer reads as None."""
888
+ try:
889
+ beat = stored_worker(_status_runtime(tenant)).get("heartbeat")
890
+ except Exception: # noqa: BLE001
891
+ return None
892
+ return beat if isinstance(beat, dict) else None
893
+
894
+
895
+ def poll_worker_desk(tenants=None, get_runtime=None):
896
+ """Worker side: pick up event receipts and Sync now hand-offs, one isolated job per tenant."""
897
+ import tenant_loops
898
+ # Light: the heavy desk loop that calls this already holds the writer lease.
899
+ return tenant_loops.for_each_tenant(_poll_tenant_desk, name="sync-desk-tenant", tenants=tenants,
900
+ get_runtime=get_runtime, kind="light")
901
+
902
+
903
+ def _poll_tenant_desk(slug, rt):
904
+ from core import connector_sync
905
+ if not syncs_here(slug):
906
+ return
907
+ try:
908
+ connector_sync.restore_sync_requests(rt)
909
+ admitted = connector_sync.poll_remote_requests(rt)
910
+ except Exception as exc: # noqa: BLE001
911
+ print(f"[process-role] sync desk poll refused for {slug} ({type(exc).__name__})", flush=True)
912
+ raise
913
+ for entry in admitted:
914
+ print(f"[process-role] sync request {entry['provider']} for {slug} admitted from the web", flush=True)
915
+ import routes_connectors
916
+ key = (entry["provider"], slug)
917
+ with routes_connectors._SYNC_REQUEST_CONNECTION_LOCK:
918
+ # No connection id means the default connection, never the previous request's.
919
+ if entry.get("connectionId"):
920
+ routes_connectors._SYNC_REQUEST_CONNECTIONS[key] = entry["connectionId"]
921
+ else:
922
+ routes_connectors._SYNC_REQUEST_CONNECTIONS.pop(key, None)
923
 
924
 
925
  def _worker_desk_loop():
 
930
  print(f"[process-role] sync desk poll failed ({type(exc).__name__})", flush=True)
931
 
932
 
933
+ def _apply_worker_state(tenants=None, get_runtime=None):
934
+ """Web side: apply each tenant's stored worker state and install its mirrors, one job per tenant.
935
+
936
+ One tenant whose store read raises leaves the other tenants' state applied (D-11881).
937
+ """
938
+ import tenant_loops
939
+ return tenant_loops.for_each_tenant(_apply_tenant_state, name="worker-state", tenants=tenants,
940
+ get_runtime=get_runtime, kind="light")
941
+
942
+
943
+ def _apply_tenant_state(slug, rt):
944
+ """Read one tenant's C-status document once, then apply heartbeat, resync, progress and mirrors."""
945
  from core import connector_sync
946
  from harness import mirror_store
947
+ document = connector_sync.status_all(rt, fresh=True)
948
+ worker = (document.get(WORKER_STATUS) or {}).get("default") or {}
949
+ # The runtime tick reads this copy instead of a second store read.
950
+ _STORED.setdefault("workers", {})[slug] = (time.time(), dict(worker))
951
+ if slug == writer_tenant():
952
+ if isinstance(worker.get("heartbeat"), dict):
953
+ _STORED["beat"] = worker["heartbeat"]
954
+ _apply_resync(document.get("odoo") or {})
955
+ connector_sync.apply_remote_progress(rt)
956
+ if isinstance(worker.get("catchUp"), dict) and worker["catchUp"]:
957
+ mirror_store.apply_catch_up(rt, worker["catchUp"])
958
+ if isinstance(worker.get("mirrors"), dict):
959
+ install_stored(slug, rt, worker["mirrors"])
960
+
961
+
962
+ def _apply_resync(connections):
963
+ """Copy the newest stored Odoo resync state into this process's RESYNC_STATE for the status routes."""
964
+ rows = [entry["resync"] for entry in connections.values()
965
+ if isinstance(entry, dict) and isinstance(entry.get("resync"), dict)]
966
+ if not rows:
967
+ return
968
+ newest = max(rows, key=lambda row: str(row.get("at") or ""))
969
  app_main = _running_main()
970
+ if app_main is None or str(newest.get("at") or "") < _RESYNC_APPLIED["at"]:
971
+ return
972
+ _RESYNC_APPLIED["at"] = str(newest.get("at") or "")
973
+ # Restore errors stay this process's own.
974
+ app_main.RESYNC_STATE.update({k: v for k, v in newest.items() if "estore" not in k and k != "at"})
 
 
 
 
 
975
 
976
 
977
  def _web_watch_loop():
978
  while not _STOP.wait(POLL_SECONDS):
 
 
 
 
 
979
  try:
980
  _apply_worker_state()
981
  except Exception as exc: # noqa: BLE001
982
  print(f"[process-role] worker state read failed ({type(exc).__name__})", flush=True)
983
+ try:
984
+ write_primary_state()
985
+ except Exception as exc: # noqa: BLE001
986
+ print(f"[process-role] primary state write failed ({type(exc).__name__}: {exc})", flush=True)
987
 
988
 
989
  # ----------------------------------------------------------------------------- the worker process
 
991
  return sys.modules.get("api.main") or sys.modules.get("main")
992
 
993
 
994
+ _CATCH_UP_SENT = {}
995
+
996
+
997
+ def write_heartbeat(now=None, get_runtime=None):
998
+ """Worker side: write the heartbeat, each tenant's catch-up and RESYNC_STATE to the C-status store.
999
+
1000
+ Only the writer lease holder beats once the lease is armed, so a waiting worker never masks it.
1001
+ """
1002
+ from core import connector_sync
1003
+ if _WRITER["armed"] and not _WRITER_HELD.is_set():
1004
+ return False
1005
+ get_runtime = get_runtime or _status_runtime
1006
+ beat = {"pid": os.getpid(), "host": writer_holder(), "tenant": writer_tenant(), "scope": worker_scope(),
1007
+ "at": time.time() if now is None else now, "bootRestore": dict(_BOOT_RESTORE)}
1008
+ written = connector_sync.update_status(get_runtime(writer_tenant()), WORKER_STATUS, "heartbeat", beat)
1009
  try:
1010
  from harness import mirror_store
1011
  catch_up = mirror_store.catch_up_snapshot()
1012
  except Exception: # noqa: BLE001
1013
  catch_up = {}
1014
+ for tenant, sources in catch_up.items():
1015
+ text = json.dumps(sources, sort_keys=True, default=str)
1016
+ if _CATCH_UP_SENT.get(tenant) == text:
1017
+ continue
1018
+ try:
1019
+ if connector_sync.update_status(get_runtime(tenant), WORKER_STATUS, "catchUp", sources):
1020
+ _CATCH_UP_SENT[tenant] = text
1021
+ except Exception as exc: # noqa: BLE001
1022
+ print(f"[sync-worker] catch-up not stored for {tenant} ({type(exc).__name__})", flush=True)
1023
  app_main = _running_main()
1024
+ publish = getattr(app_main, "_publish_resync_state", None)
1025
+ tenant = str((getattr(app_main, "RESYNC_STATE", None) or {}).get("tenant") or "")
1026
+ if callable(publish) and tenant:
1027
+ # Writes only when RESYNC_STATE changed since its last copy.
1028
+ publish(tenant)
1029
+ return written
1030
+
1031
+
1032
+ def record_publication(connector, tenant_key, manifest, _rows=None, _paths=None):
1033
+ """Publish listener: name the stored snapshot in C-status, so every web process installs it."""
1034
+ from core import connector_sync
1035
+ snapshot = str((manifest or {}).get("snapshot_id") or "")
1036
+ if not snapshot:
1037
+ return False
1038
+ return connector_sync.update_status(_status_runtime(tenant_key), WORKER_STATUS, "mirrors", {
1039
+ str(connector): {"snapshotId": snapshot, "at": time.time()}})
1040
 
1041
 
1042
  def _heartbeat_loop(parent_pid):
 
1054
 
1055
 
1056
  def start_worker_services():
1057
+ """Called by sync_worker.py: heartbeat, the stored publication record and the store desk poll."""
1058
  from harness import mirror_store
1059
+ mirror_store.register_publish_listener(record_publication)
1060
  parent = int(os.environ.get("AIOS_PARENT_PID") or 0)
1061
  start("sync-worker-heartbeat", "light", _heartbeat_loop, args=(parent,))
1062
  # Before the first heavy job: each one now waits until this worker holds the writer lease.
 
1065
 
1066
 
1067
  def _worker_env(tenant=None):
1068
+ """A resident worker's env. No runtime caller since the web runs no child; gates read it."""
1069
  env = dict(os.environ)
1070
  env.update(AIOS_PROCESS_ROLE="worker", AIOS_PARENT_PID=str(os.getpid()),
1071
  AIOS_MIRROR_HANDOFF_DIR=str(handoff_dir()), PYTHONUNBUFFERED="1")
 
1080
 
1081
 
1082
  def _supervise(tenant=None):
1083
+ """One sync runtime tick, for `tenant` or every tenant. The web never starts a sync worker child."""
1084
+ return fleet_tick(None if tenant is None else [tenant])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1085
 
1086
 
1087
+ #: {tenant: {job, jobId, jobs, alive, aliveWhy}} from this web process's runtime ticks.
1088
  _FLEET = {}
1089
  _FLEET_STATE = {"overCap": [], "error": None, "at": None}
1090
 
1091
 
 
 
 
 
 
 
 
1092
  def _report_over_cap(over, cap):
1093
  """Rule 1: a tenant past the cap is told, with the cause and the fix, never skipped in silence."""
1094
+ cause = (f"its daily Odoo and Amazon sync waits: {cap} sync job(s) may run at once "
1095
+ f"(AIOS_SYNC_WORKERS_MAX) and they are all running. Raise AIOS_SYNC_WORKERS_MAX.")
 
1096
  for slug in over:
1097
+ print(f"[process-role] sync job for {slug} NOT submitted: {cause}", flush=True)
1098
  try:
1099
  import sync_alerts
1100
  from harness import runtime
1101
  sync_alerts.notify(slug, "sync-worker", [{"key": "worker", "label": "Sync worker",
1102
  "code": "over_cap", "cause": cause}],
1103
+ observed={"worker"}, store=runtime.get_runtime(slug))
1104
  except Exception as exc: # noqa: BLE001
1105
  print(f"[process-role] over-cap alert for {slug} failed ({type(exc).__name__})", flush=True)
1106
 
1107
 
1108
+ def fleet_tick(wanted=None, **kwargs):
1109
+ """Ask the sync runtime for each due tenant's pass inside the cap. Returns the tenants it submitted."""
1110
+ import sync_runtime
1111
+ return sync_runtime.tick(wanted, **kwargs)
 
 
 
 
 
 
 
 
 
 
 
 
 
1112
 
1113
 
1114
  def _fleet_loop():
 
1117
  fleet_tick()
1118
  except Exception as exc: # noqa: BLE001
1119
  _FLEET_STATE.update(error=type(exc).__name__, at=time.time())
1120
+ print(f"[process-role] sync runtime tick failed ({type(exc).__name__}: {exc})", flush=True)
1121
  if _STOP.wait(FLEET_SCAN_SECONDS):
1122
  return
1123
 
 
1136
 
1137
 
1138
  def start_web_services():
1139
+ """Web role only: forward Sync now, tick the sync runtime, install the worker's snapshots.
1140
 
1141
  With several uvicorn workers only the primary does this. The others stand by and read.
1142
  """
 
1149
  if _WORKER.get("standby") is None:
1150
  _WORKER["standby"] = start("web-primary-standby", "light", _standby_loop)
1151
  print(f"[process-role] web worker pid={os.getpid()} serves reads only; "
1152
+ f"the primary ticks the sync runtime", flush=True)
1153
  return False
1154
+ # No worker child: the supervisor thread only ticks the sync runtime.
1155
+ _WORKER["supervisor"] = start("sync-worker-supervisor", "light", _fleet_loop)
1156
  start("mirror-handoff-watcher", "light", _web_watch_loop)
1157
  atexit.register(stop_worker)
1158
  return True
1159
 
1160
 
1161
  def status():
1162
+ """{role, worker: {pid, alive, heartbeatAgeS, restarts, lastInstall}} for the settings response.
1163
+
1164
+ A serving role reads the worker from its stored heartbeat, never from a child process (D-11183).
1165
+ """
1166
+ beat = stored_heartbeat() if ROLE != "all" else None
1167
+ age = beat_age(beat) if beat is not None else None
1168
+ if ROLE == "worker":
1169
  pid, alive = os.getpid(), True
1170
+ age = age if beat is not None and beat.get("pid") == pid else None
1171
+ elif ROLE in ("web", "reader") and beat is not None:
1172
+ import sync_runtime
1173
+ pid, alive = beat.get("pid"), sync_runtime.beat_alive(beat)
1174
  else:
1175
  pid, alive = None, False
 
 
 
1176
  fleet = {}
1177
  for slug, row in _FLEET.items():
1178
+ fleet[slug] = {"pid": None, "alive": bool(row.get("alive")), "restarts": row.get("jobs", 0),
1179
+ "lastExit": None, "jobId": row.get("jobId")}
1180
+ import sync_runtime
1181
  return {"role": ROLE, "roleSource": ROLE_SOURCE, "webWorkers": web_workers(), "primary": is_primary(),
1182
+ "syncRuntime": sync_runtime.runtime_name()[0],
1183
  "tenant": writer_tenant(), "scope": worker_scope(), "tenantWorkers": fleet,
1184
  "tenantWorkersOverCap": list(_FLEET_STATE["overCap"]), "syncWorkersMax": sync_worker_cap(),
1185
  "worker": {"pid": pid, "alive": alive, "heartbeatAgeS": age,
api/registry_source.py CHANGED
@@ -1,6 +1,6 @@
1
  """Bounded stored-source reads before canonical grid projection."""
2
  import copy
3
- from contextlib import closing
4
  from dataclasses import is_dataclass, replace
5
 
6
  from deps import err
@@ -530,6 +530,198 @@ def canonical_metadata(session, table_key, *, metadata=None, cur=None):
530
  consume_corrections=False, with_rows=False, needed=frozenset())
531
 
532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
533
  def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
534
  """Refuse a failed compiled source by name without publishing an empty population."""
535
  from fastapi import HTTPException
 
1
  """Bounded stored-source reads before canonical grid projection."""
2
  import copy
3
+ from contextlib import closing, contextmanager
4
  from dataclasses import is_dataclass, replace
5
 
6
  from deps import err
 
530
  consume_corrections=False, with_rows=False, needed=frozenset())
531
 
532
 
533
+ def _selection_teams(session):
534
+ """Validate the explicit runtime identity and its configured teams before SQL."""
535
+ runtime = session.runtime
536
+ key = str(getattr(runtime, 'key', '') or '').strip().lower()
537
+ tenant = getattr(runtime, 'tenant', None)
538
+ owner = str(getattr(tenant, 'key', '') or '').strip().lower()
539
+ actor = str(session.user.get('tenant') or key).strip().lower()
540
+ if not key or key != owner or actor != key:
541
+ raise err(403, 'tenant_mismatch', 'Canonical selection requires a bound tenant identity.')
542
+ config = getattr(tenant, 'config', None) or {}
543
+ units = config.get('business_units')
544
+ if (not isinstance(units, dict) or not units
545
+ or any(not isinstance(label, str) or not label.strip()
546
+ or type(tid) is not int or tid <= 0 for label, tid in units.items())
547
+ or len(set(units.values())) != len(units)):
548
+ raise err(403, 'bu_scope_denied', 'Canonical selection requires known tenant teams.')
549
+ labels = {tid: label for label, tid in units.items()}
550
+ bus = session.user.get('bus', 'all')
551
+ if bus != 'all' and (not isinstance(bus, (list, tuple, set, frozenset)) or not bus
552
+ or any(type(tid) is not int or tid not in labels for tid in bus)):
553
+ raise err(403, 'bu_scope_denied', 'Account teams are empty or unknown for this tenant.')
554
+ return labels
555
+
556
+
557
+ @contextmanager
558
+ def canonical_selection(session, cur, table_key, metadata, needed, *, ids=None):
559
+ """Lend SQL identities and proven native operands before canonical page hydration."""
560
+ from core import perm_scope
561
+ from harness import canonical_source as cs
562
+ from modules import ar, customer_data, product_data
563
+ import routes_keychain
564
+ import routes_odoo_tables as rot
565
+ import time
566
+
567
+ team_labels = _selection_teams(session)
568
+ if (needed is None or getattr(session.runtime, 'key', None) != 'royal-imports'
569
+ or routes_keychain.odoo_paused(session.runtime)):
570
+ yield None
571
+ return
572
+ session.require(table_key)
573
+ team, agent = perm_scope.derive_pool_scope(session.user, table_key)
574
+ cache_key = ('product_pool', team) if table_key == 'product_data' else ('pool', team, agent)
575
+ cached = session.runtime.pool_cache.get(cache_key)
576
+ if cached and time.time() - cached[0] < 900:
577
+ yield None
578
+ return # The canonical page must read the same source as the selection.
579
+ fields = metadata['fields']
580
+ scope_fields = (metadata.get('_permission_scope') or {}).get('fields', fields)
581
+ tree = {} if session.admin else (perm_scope.entry(session.user, table_key) or {}).get('filter') or {}
582
+ from harness import filter_eval
583
+ nodes, _conj = filter_eval.tree_parts(tree)
584
+ required = set(needed) | rot._leaf_cols(nodes)
585
+ # Set-dependent and authored permission operands retain their canonical evaluator.
586
+ def complex_wall(nodes):
587
+ return any(isinstance(n, dict) and (n.get('window') is not None
588
+ or n.get('op') in rot._fs().RANK_OPS or complex_wall(n.get('children') or [])) for n in nodes)
589
+ if complex_wall(nodes) or perm_scope._row_audience_field(table_key, st=session.runtime) is not None:
590
+ yield None
591
+ return
592
+ whitespace = "'" + ''.join(chr(c) for c in (*range(9, 14), *range(28, 33), 133, 160,
593
+ 5760, *range(8192, 8203), 8232, 8233, 8239, 8287, 12288)) + "'"
594
+ if table_key == 'customer_data':
595
+ expressions = {
596
+ 'archived': "CASE WHEN NOT live THEN true WHEN p.id IS NULL THEN false ELSE NOT coalesce(p.active, false) END",
597
+ 'dba': "CASE WHEN live THEN coalesce(brands.brand, '') END",
598
+ 'agent': "CASE WHEN live THEN coalesce((SELECT string_agg(a.name, ', ' ORDER BY cast(member.key AS BIGINT)) "
599
+ "FROM json_each(p.agent_ids) member JOIN res_partner a ON a.id=try_cast(member.value AS BIGINT) "
600
+ "WHERE coalesce(a.active, false) AND a.name IS NOT NULL AND a.name<>''), '(none)') ELSE '(none)' END"}
601
+ # These source attributes have no authored display override or case transformation.
602
+ for key in ('street', 'street2', 'zip'):
603
+ expressions[key] = f"CASE WHEN live THEN coalesce(nullif(trim(p.{key}, {whitespace}), ''), '(none)') ELSE '(none)' END"
604
+ for key, column in (('state', 'state_name'), ('country', 'country_name'),
605
+ ('payment_terms', 'property_payment_term_name'),
606
+ ('pricelist', 'property_product_pricelist_name')):
607
+ expressions[key] = f"CASE WHEN live THEN coalesce(nullif(p.{column}, ''), '(none)') ELSE '(none)' END"
608
+ for key in ('customer_since', 'created_on'):
609
+ expressions[key] = "CASE WHEN live THEN coalesce(substr(p.create_date, 1, 10), '') ELSE '' END"
610
+ else:
611
+ expressions = {'code': 'code'}
612
+ vocabulary = {f['key']: f for f in scope_fields}
613
+ visible = {f['key'] for f in fields}
614
+ if (not set(needed) <= visible or not required <= expressions.keys()
615
+ or any((vocabulary.get(k) or {}).get('source') != 'odoo'
616
+ or (vocabulary.get(k) or {}).get('derived')
617
+ or (vocabulary.get(k) or {}).get('type') not in {
618
+ 'text', 'long_text', 'status', 'select', 'multiselect', 'date', 'checkbox'} for k in required)):
619
+ yield None
620
+ return
621
+ kind = 'product' if table_key == 'product_data' else 'customer'
622
+ missing = cs.mirror_missing(cur, kind)
623
+ if missing:
624
+ raise err(503, 'canonical_source_unavailable', cs._unready(missing))
625
+ hash_function = None
626
+ with cs.mirrored(cur, session.runtime, kind=kind) as source:
627
+ try:
628
+ if table_key == 'customer_data':
629
+ params, legs = [], []
630
+ domains = ([('sale.order', [('state', 'in', ('sale', 'done')), ('team_id', '=', team)], 'partner_id')]
631
+ if team else [
632
+ ('res.partner', [('customer_rank', '>', 0), ('active', 'in', [True, False])], 'id'),
633
+ ('sale.order', [('state', 'in', ('sale', 'done'))], 'partner_id'),
634
+ ('account.move', [('state', '=', 'posted'), ('move_type', 'in', ('out_invoice', 'out_refund'))], 'partner_id'),
635
+ ('account.move.line', list(ar.AGED_LINE_DOMAIN), 'partner_id')])
636
+ for model, domain, key in domains:
637
+ query = cs._Query(source, model, params)
638
+ value, field_kind, _ = query.field(key)
639
+ value = value[0] if field_kind == 'm2o' else value
640
+ where = query.where(domain if ids is None else [*domain, (key, 'in', list(ids))])
641
+ legs.append(f'SELECT {value} pid FROM {query.from_sql()} WHERE {where}')
642
+ cur.execute('CREATE TEMP TABLE canonical_live AS SELECT DISTINCT pid FROM ('
643
+ + ' UNION ALL '.join(legs) + ') source WHERE pid IS NOT NULL', params)
644
+ if agent and agent not in {'All', 'All agents'}:
645
+ members = source.agent_members(agent)
646
+ cur.execute('DELETE FROM canonical_live WHERE pid NOT IN '
647
+ f'(SELECT try_cast(json_extract_string(body, \'/id\') AS BIGINT) FROM "{members.table}")')
648
+ cur.execute('CREATE TEMP TABLE canonical_retained (pid BIGINT)')
649
+ if not agent and team is None and session.runtime.exists(customer_data.LEDGER_KEY):
650
+ with closing(items(session.runtime, customer_data.LEDGER_KEY, selected=ids)) as batches:
651
+ for batch in batches:
652
+ retained = []
653
+ for key, _entry in batch:
654
+ try:
655
+ retained.append(int(key))
656
+ except (TypeError, ValueError):
657
+ continue
658
+ if retained:
659
+ cur.execute('INSERT INTO canonical_retained SELECT unnest(?::BIGINT[])', [retained])
660
+ population = ('SELECT pid, true live FROM canonical_live UNION ALL '
661
+ 'SELECT DISTINCT pid, false live FROM canonical_retained '
662
+ 'WHERE pid NOT IN (SELECT pid FROM canonical_live)')
663
+ projected = ', '.join(f'{expressions[k]} AS "{k}"' for k in sorted(required))
664
+ brand_join = ''
665
+ if 'dba' in required:
666
+ import datetime
667
+ start = (datetime.date.fromisoformat(str(metadata['today'])) - datetime.timedelta(days=731)).isoformat()
668
+ query = cs._Query(source, 'sale.order')
669
+ where = query.where([('state', 'in', ('sale', 'done')), ('date_order', '>=', start),
670
+ ('team_id', 'in', list(team_labels))])
671
+ cases = ' '.join('WHEN ? THEN ?' for _ in team_labels)
672
+ label_params = [value for pair in team_labels.items() for value in pair]
673
+ cur.execute('CREATE TEMP TABLE canonical_brands AS SELECT b.partner_id pid, '
674
+ f'CASE WHEN count(DISTINCT b.team_id)>1 THEN ? ELSE CASE min(b.team_id) {cases} END END AS brand '
675
+ f'FROM {query.from_sql()} WHERE {where} AND b.partner_id<>0 GROUP BY b.partner_id',
676
+ [customer_data._multi_team_label(len(team_labels)), *label_params, *query.params])
677
+ brand_join = ' LEFT JOIN canonical_brands brands ON brands.pid=u.pid'
678
+ # Absent optional columns retain the canonical path's blank/refusal behavior.
679
+ try:
680
+ cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT u.pid'
681
+ + (', ' + projected if projected else '')
682
+ + f' FROM ({population}) u LEFT JOIN res_partner p ON p.id=u.pid' + brand_join)
683
+ except Exception as exc:
684
+ from core.request_timing import raise_if_resource_error
685
+ raise_if_resource_error(exc)
686
+ import duckdb
687
+ if not isinstance(exc, duckdb.BinderException):
688
+ raise
689
+ yield None
690
+ return
691
+ else:
692
+ import uuid
693
+ function_name = 'canonical_pid_' + uuid.uuid4().hex
694
+ cur.create_function(function_name, lambda code: product_data.sku_pid(code), ['VARCHAR'], 'BIGINT')
695
+ hash_function = function_name
696
+ cur.execute('CREATE TEMP TABLE canonical_selection AS SELECT '
697
+ f'{hash_function}(code) pid, code FROM (SELECT DISTINCT '
698
+ "CASE WHEN default_code IS NULL OR default_code='' THEN 'pid:'||CAST(id AS VARCHAR) "
699
+ f'ELSE trim(default_code, {whitespace}) END code FROM product_product WHERE active=true) codes')
700
+ if cur.execute('SELECT pid FROM canonical_selection GROUP BY pid HAVING count(*)>1 LIMIT 1').fetchone():
701
+ raise err(503, 'canonical_source_unavailable', 'product_data: pid collision. Refusing the catalogue.')
702
+ bindings = {k: (f'"{k}"', str) for k in required}
703
+ spec = {'table': 'canonical_selection', 'id': 'pid', 'cols': bindings}
704
+ wall_cols = rot._filter_columns(spec, scope_fields)
705
+ # Permission booleans retain their source spelling in the canonical payload.
706
+ wall, params = perm_scope._window_wall(session.user, table_key, scope_fields,
707
+ wall_cols, id_sql='pid', st=session.runtime)
708
+ if ids is not None:
709
+ wall = f'({wall}) AND pid IN (SELECT unnest(?::BIGINT[]))'
710
+ params.append(list(ids))
711
+ cur.execute(f'DELETE FROM canonical_selection WHERE NOT coalesce(({wall}), false)', params)
712
+ yield spec
713
+ except Exception as exc:
714
+ from fastapi import HTTPException
715
+ from core.request_timing import raise_if_resource_error
716
+ if isinstance(exc, HTTPException):
717
+ raise
718
+ raise_if_resource_error(exc)
719
+ raise err(503, 'canonical_source_unavailable', f'{table_key}: {type(exc).__name__}: {str(exc)[:180]}') from exc
720
+ finally:
721
+ if hash_function:
722
+ cur.remove_function(hash_function)
723
+
724
+
725
  def canonical_batches(session, cur, table_key, *, ids=None, needed=None, metric_window=None, related=None):
726
  """Refuse a failed compiled source by name without publishing an empty population."""
727
  from fastapi import HTTPException
api/rollup_sql.py CHANGED
@@ -59,7 +59,7 @@ def source_fields(table_def):
59
  return out
60
 
61
 
62
- def group_values(bag, today=None):
63
  """`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
64
 
65
  ⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
@@ -67,6 +67,7 @@ def group_values(bag, today=None):
67
  """
68
  from harness import semantic as sem
69
  from harness import windows as W
 
70
 
71
  topic = str(bag.get("topic") or "")
72
  measure = str(bag.get("measure") or "")
@@ -86,9 +87,20 @@ def group_values(bag, today=None):
86
  date_from, date_to = rng
87
 
88
  try:
89
- res = sem.store_query(topic, [measure], group_by=[dim],
90
- date_from=date_from, date_to=date_to,
91
- limit=sem.MAX_GROUPS, today=today)
 
 
 
 
 
 
 
 
 
 
 
92
  except Exception as e: # noqa: BLE001
93
  raise RollupSourceError(f"{type(e).__name__}: {e}") from e
94
 
@@ -121,8 +133,8 @@ def group_values(bag, today=None):
121
 
122
 
123
  def _today():
124
- import datetime as _dt
125
- return _dt.date.today().strftime("%Y-%m-%d")
126
 
127
 
128
  def compute(rt, table_key, today=None, tables=None, persist=False):
@@ -146,7 +158,7 @@ def compute(rt, table_key, today=None, tables=None, persist=False):
146
  bag = f["rollup"]["source"]
147
  # ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
148
  # was — a partially applied pass would mix two vintages of the same column.
149
- plans.append((f["key"], str(bag.get("on") or ""), group_values(bag, today=stamp)))
150
 
151
  written = {}
152
 
 
59
  return out
60
 
61
 
62
+ def group_values(rt, bag, today=None):
63
  """`{group_id: value}` for one source bag — ONE grouped query over the whole scope.
64
 
65
  ⚠ Keyed by the dim's `_id` column, never its label. Two customers can share a display name;
 
67
  """
68
  from harness import semantic as sem
69
  from harness import windows as W
70
+ from harness.runtime import mirror_cursor
71
 
72
  topic = str(bag.get("topic") or "")
73
  measure = str(bag.get("measure") or "")
 
87
  date_from, date_to = rng
88
 
89
  try:
90
+ if rt is None:
91
+ raise RollupSourceError("A tenant runtime is required for source rollups")
92
+ handle = rt.mirror_handle()
93
+ if handle is None:
94
+ raise RollupSourceError("The runtime has no tenant-owned mirror handle")
95
+ cur = mirror_cursor(rt)
96
+ try:
97
+ if getattr(cur, "_mirror_handle", None) is not handle:
98
+ raise RollupSourceError("The runtime has no tenant-owned mirror cursor")
99
+ res = sem.store_query(topic, [measure], group_by=[dim],
100
+ date_from=date_from, date_to=date_to,
101
+ limit=sem.MAX_GROUPS, today=today, runtime=rt, cur=cur)
102
+ finally:
103
+ cur.close()
104
  except Exception as e: # noqa: BLE001
105
  raise RollupSourceError(f"{type(e).__name__}: {e}") from e
106
 
 
133
 
134
 
135
  def _today():
136
+ from core import periods
137
+ return periods.today().isoformat()
138
 
139
 
140
  def compute(rt, table_key, today=None, tables=None, persist=False):
 
158
  bag = f["rollup"]["source"]
159
  # ⚠ Resolved BEFORE anything is written. A rollup that refuses must leave every cell as it
160
  # was — a partially applied pass would mix two vintages of the same column.
161
+ plans.append((f["key"], str(bag.get("on") or ""), group_values(rt, bag, today=stamp)))
162
 
163
  written = {}
164
 
api/routes_account_health.py CHANGED
@@ -14,13 +14,15 @@ def _module():
14
  return account_health
15
 
16
 
17
- def _reader(session: Session = Depends(require_session)) -> Session:
18
- """Reads need read access to the account health database."""
19
- from core import perm_scope
20
  if session.tenant != getattr(session.runtime, "key", None):
21
  raise err(403, "account_health_forbidden", "This workspace does not match your session.")
22
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
23
- raise err(403, "account_health_forbidden", "Your account cannot read Amazon account health.")
 
 
24
  return session
25
 
26
 
@@ -104,8 +106,12 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
104
  offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
105
  metric: str | None = Query(None), session: Session = Depends(_app_reader)):
106
  ah = _module()
107
- out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit,
108
- metric=metric, st=session.runtime)
 
 
 
 
109
  return {**out, "canEdit": _may_edit(session, _app(session, app))}
110
 
111
 
@@ -113,6 +119,17 @@ def read_summary(app: str = Query(..., min_length=1), search: str = Query(""),
113
  def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
114
  limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
115
  ah = _module()
 
 
 
 
 
 
 
 
 
 
 
116
  return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
117
 
118
 
@@ -122,8 +139,13 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
122
  cursor: str | None = Query(None), limit: int = Query(50, ge=1),
123
  session: Session = Depends(_app_reader)):
124
  ah = _module()
125
- listed = _call(ah.items, _tenant(session), session.user, status=status, search=search, offset=offset,
126
- cursor=cursor, limit=limit, metric=metric, st=session.runtime)
 
 
 
 
 
127
  return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
128
 
129
 
@@ -131,8 +153,10 @@ def list_items(app: str = Query(..., min_length=1), status: str | None = Query(N
131
  def read_item(item_id: str, app: str = Query(..., min_length=1),
132
  session: Session = Depends(_app_reader)):
133
  ah = _module()
 
134
  alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
135
- context = _call(_source_context, session.runtime, session.user)
 
136
  if not _source_allowed(alert, context):
137
  raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
138
  return {"item": ah._public(alert, session.user), "owners": owners(session),
@@ -144,6 +168,10 @@ def patch_item(item_id: str, owner: str | None = Body(None), status: str | None
144
  due: str | None = Body(None), note: str | None = Body(None),
145
  session: Session = Depends(_app_writer)):
146
  ah = _module()
 
 
 
 
147
  return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
148
  due=due, note=note, st=session.runtime)
149
 
@@ -168,6 +196,10 @@ def list_risks(app: str = Query(..., min_length=1), status: str | None = Query(N
168
  search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
169
  session: Session = Depends(_app_reader)):
170
  ah = _module()
 
 
 
 
171
  out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
172
  st=session.runtime)
173
  return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
@@ -179,10 +211,109 @@ def patch_risk(item_id: str, owner: str | None = Body(None), status: str | None
179
  outcome: str | None = Body(None), note: str | None = Body(None),
180
  session: Session = Depends(_app_writer)):
181
  ah = _module()
 
 
 
182
  return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
183
  case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
184
 
185
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  # --------------------------------------------------------------------------- C5 worker bridge
187
 
188
  def _source_context(rt, user):
 
14
  return account_health
15
 
16
 
17
+ def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
18
+ """R62: reads need the App grant plus the shared source check, never the source open grant."""
19
+ import application_registry
20
  if session.tenant != getattr(session.runtime, "key", None):
21
  raise err(403, "account_health_forbidden", "This workspace does not match your session.")
22
+ record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
23
+ _tenant(session), session.uname, session.admin, TEMPLATE_KIND)
24
+ application_registry.admit_app(session, record, (SOURCE,), code="account_health")
25
+ session.source_app = record
26
  return session
27
 
28
 
 
106
  offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
107
  metric: str | None = Query(None), session: Session = Depends(_app_reader)):
108
  ah = _module()
109
+ context = _call(_narrowed, session)
110
+ if context is None:
111
+ out = _call(ah.summary_page, _tenant(session), search=search, offset=offset, limit=limit,
112
+ metric=metric, st=session.runtime)
113
+ else:
114
+ out = _narrow_summary(session, context, search=search, offset=offset, limit=limit, metric=metric)
115
  return {**out, "canEdit": _may_edit(session, _app(session, app))}
116
 
117
 
 
119
  def read_history(metric: str = Query(..., min_length=1), cursor: str | None = Query(None),
120
  limit: int = Query(400, ge=1), session: Session = Depends(_app_reader)):
121
  ah = _module()
122
+ context = _call(_narrowed, session)
123
+ if context is not None:
124
+ kept = _allowed_metrics(session, context)["metrics"]
125
+ row = next((row for row in kept if row["id"] == metric), None) or next(
126
+ (row for row in kept if str(row["id"]).split("@", 1)[0] == metric), None)
127
+ if row is None:
128
+ raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
129
+ out = _call(ah.history, _tenant(session), row["id"], cursor=cursor, limit=limit, st=session.runtime)
130
+ out["points"] = [point for point in out["points"] if _source_allowed({"sourceEvidence": _call(
131
+ ah._alert_source, _tenant(session), session.runtime, row, point["day"])}, context)]
132
+ return out
133
  return _call(ah.history, _tenant(session), metric, cursor=cursor, limit=limit, st=session.runtime)
134
 
135
 
 
139
  cursor: str | None = Query(None), limit: int = Query(50, ge=1),
140
  session: Session = Depends(_app_reader)):
141
  ah = _module()
142
+ context = _call(_narrowed, session)
143
+ if context is None:
144
+ listed = _call(ah.items, _tenant(session), session.user, status=status, search=search,
145
+ offset=offset, cursor=cursor, limit=limit, metric=metric, st=session.runtime)
146
+ else:
147
+ listed = _narrow_items(session, context, offset, limit, status=status, search=search,
148
+ cursor=cursor, metric=metric)
149
  return {**listed, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
150
 
151
 
 
153
  def read_item(item_id: str, app: str = Query(..., min_length=1),
154
  session: Session = Depends(_app_reader)):
155
  ah = _module()
156
+ import application_registry
157
  alert = _call(ah.alert_record, _tenant(session), item_id, st=session.runtime)
158
+ context = _call(application_registry.app_source_scope, session.user, session.runtime, SOURCE,
159
+ app=session.source_app)
160
  if not _source_allowed(alert, context):
161
  raise err(403, "account_health_source_forbidden", "Your account cannot read this alert's source records and fields.")
162
  return {"item": ah._public(alert, session.user), "owners": owners(session),
 
168
  due: str | None = Body(None), note: str | None = Body(None),
169
  session: Session = Depends(_app_writer)):
170
  ah = _module()
171
+ context = _call(_narrowed, session)
172
+ if context is not None and not _source_allowed(
173
+ _call(ah.alert_record, _tenant(session), item_id, st=session.runtime), context):
174
+ raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
175
  return _call(ah.update_item, _tenant(session), session.user, item_id, owner=owner, status=status,
176
  due=due, note=note, st=session.runtime)
177
 
 
196
  search: str = Query(""), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1),
197
  session: Session = Depends(_app_reader)):
198
  ah = _module()
199
+ context = _call(_narrowed, session)
200
+ if context is not None:
201
+ out = _narrow_risks(session, context, status=status, search=search, offset=offset, limit=limit)
202
+ return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
203
  out = _call(ah.risks, _tenant(session), status=status, search=search, offset=offset, limit=limit,
204
  st=session.runtime)
205
  return {**out, "owners": owners(session), "canEdit": _may_edit(session, _app(session, app))}
 
211
  outcome: str | None = Body(None), note: str | None = Body(None),
212
  session: Session = Depends(_app_writer)):
213
  ah = _module()
214
+ context = _call(_narrowed, session)
215
+ if context is not None:
216
+ raise err(403, "account_health_source_forbidden", "Risk changes need unrestricted source access.")
217
  return _call(ah.update_risk, _tenant(session), session.user, item_id, owner=owner, status=status,
218
  case_id=caseId, appeal_date=appealDate, outcome=outcome, note=note, st=session.runtime)
219
 
220
 
221
+ # --------------------------------------------------------------------------- R62 source narrowing
222
+
223
+ def _narrowed(session):
224
+ """The caller's (visible fields, permitted row ids) on the source, or None when no wall applies."""
225
+ import application_registry
226
+ wall = application_registry.app_access(session.user, session.runtime, None, (SOURCE,))["narrow"][SOURCE]
227
+ if not wall["rows"] and not wall["hidden"]:
228
+ return None
229
+ return application_registry.app_source_scope(session.user, session.runtime, SOURCE, app=session.source_app)
230
+
231
+
232
+ def _allowed_metrics(session, context, *, search="", metric=None):
233
+ """Every metric row whose source row and fields the caller's source wall admits."""
234
+ ah = _module()
235
+ tenant, rt = _tenant(session), session.runtime
236
+ first = _call(ah.summary_page, tenant, search=search, offset=0, limit=1, metric=metric, st=rt)
237
+ full = first if first["metricTotal"] <= 1 else _call(
238
+ ah.summary_page, tenant, search=search, offset=0, limit=first["metricTotal"], metric=metric, st=rt)
239
+ kept = [row for row in full["metrics"] if _source_allowed(
240
+ {"sourceEvidence": _call(ah._alert_source, tenant, rt, row, full["snapshotDate"])}, context)]
241
+ return {**full, "metrics": kept}
242
+
243
+
244
+ def _page(rows, offset, limit):
245
+ start = min(max(0, offset), max(0, (len(rows) - 1) // limit * limit))
246
+ return start, rows[start:start + limit]
247
+
248
+
249
+ def _narrow_summary(session, context, *, search, offset, limit, metric):
250
+ """A source row filter narrows the metric rows; a hidden source field hides its metric."""
251
+ ah = _module()
252
+ out = _allowed_metrics(session, context, search=search, metric=metric)
253
+ kept = out["metrics"]
254
+ if metric is not None and not kept:
255
+ raise err(404, "account_health_not_found", "That alert, risk or metric does not exist.")
256
+ markets = {row.get("marketplaceId") for row in kept}
257
+ if "groups" in out:
258
+ scoped = _allowed_metrics(session, context)["metrics"] if search or metric is not None else kept
259
+ groups, gauge = ah.group_scores(ah.count_groups(scoped))
260
+ out.update(groups=groups, gauge=gauge,
261
+ foldedCount=sum(row.get("status") in ah.FOLDED for row in scoped))
262
+ start, page = _page(kept, offset, limit)
263
+ out.update(metrics=page, metricTotal=len(kept), metricOffset=start, metricLimit=limit,
264
+ marketCount=len(markets), cursor=str(start + limit) if start + limit < len(kept) else None)
265
+ if out.get("marketplaceId") not in markets or "account_health_rating_ahr_score" not in context[0]:
266
+ out.update(rating=None, ratingStatus=None, marketplaceId=None)
267
+ elif "account_health_rating_ahr_status" not in context[0]:
268
+ out["ratingStatus"] = None
269
+ if not kept:
270
+ out.update(state="empty", message=ah.EMPTY, snapshotDate=None)
271
+ return out
272
+
273
+
274
+ def _narrow_items(session, context, offset, limit, **options):
275
+ """Keep the alerts whose source rows and fields the caller's source wall admits."""
276
+ ah = _module()
277
+ rows, scan = [], 0
278
+ while True:
279
+ listed = _call(ah.items, _tenant(session), session.user, offset=scan, limit=200,
280
+ st=session.runtime, **options)
281
+ for row in listed["items"]:
282
+ alert = _call(ah.alert_record, _tenant(session), row["id"], st=session.runtime)
283
+ if _source_allowed(alert, context):
284
+ rows.append(row)
285
+ scan += len(listed["items"])
286
+ if not listed["items"] or scan >= listed["total"]:
287
+ break
288
+ start, page = _page(rows, offset, limit)
289
+ more = start + len(page) < len(rows)
290
+ return {**listed, "items": page, "cursor": page[-1]["id"] if more and page else None,
291
+ "total": len(rows), "offset": start, "limit": limit}
292
+
293
+
294
+ def _narrow_risks(session, context, *, status, search, offset, limit):
295
+ """Serve only risks whose source evidence fits the Account health wall."""
296
+ ah = _module()
297
+ metrics = {row["id"]: row for row in _allowed_metrics(session, context)["metrics"]}
298
+ kept, scan = [], 0
299
+ while True:
300
+ out = _call(ah.risks, _tenant(session), status=status, search=search,
301
+ offset=scan, limit=200, st=session.runtime)
302
+ for risk in out["items"]:
303
+ metric = metrics.get(risk.get("metricId"))
304
+ if metric is not None and _source_allowed({"sourceEvidence": _call(
305
+ ah._alert_source, _tenant(session), session.runtime, metric, risk.get("snapshotDate"))}, context):
306
+ kept.append(risk)
307
+ scan += len(out["items"])
308
+ if not out["items"] or scan >= out["total"]:
309
+ break
310
+ start, page = _page(kept, offset, limit)
311
+ return {**out, "items": page, "total": len(kept), "offset": start, "limit": limit,
312
+ "cursor": str(start + limit) if start + limit < len(kept) else None,
313
+ "reserveSummary": {"available": False, "total": None, "byCurrency": {}, "lineCount": 0,
314
+ "unknownReasons": ["Reserve figures need unrestricted source access."]}}
315
+
316
+
317
  # --------------------------------------------------------------------------- C5 worker bridge
318
 
319
  def _source_context(rt, user):
api/routes_admin.py CHANGED
@@ -1464,6 +1464,8 @@ _VOCAB_ARM_CACHE = {}
1464
  #: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
1465
  #: stamp order can refuse it rather than each restating the rule.
1466
  _TENANT_OWNED_ARM = "_product_shared_choices"
 
 
1467
 
1468
 
1469
  def _server_side_vocabularies(session=None):
@@ -1560,15 +1562,31 @@ def _server_side_vocabularies(session=None):
1560
  "occassion", "shape_style", "size", "sub_category")
1561
  return {key: (stored.get(key) or {}).get("options") or () for key in keys}
1562
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1563
  if _VOCAB_CACHE is not None:
1564
  out = dict(_VOCAB_CACHE)
1565
- try:
1566
- for key, values in _product_shared_choices().items():
1567
- clean = sorted({str(v) for v in values if str(v or "").strip()})
1568
- if clean:
1569
- out[key] = clean
1570
- except Exception: # noqa: BLE001
1571
- pass
 
1572
  return out
1573
 
1574
  out = {}
@@ -1582,7 +1600,7 @@ def _server_side_vocabularies(session=None):
1582
  normalisation rather than two that can drift.
1583
  """
1584
  name = getattr(fn, "__name__", "")
1585
- memoable = name and name != _TENANT_OWNED_ARM
1586
  try:
1587
  got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
1588
  for key, values in (got or {}).items():
@@ -1708,6 +1726,7 @@ def _server_side_vocabularies(session=None):
1708
  # them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
1709
  # another tenant's permission editor for the life of the worker.
1710
  arm(_product_shared_choices)
 
1711
  return out
1712
 
1713
 
@@ -2543,6 +2562,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2543
  return {"username": uname,
2544
  "perms": perms_out,
2545
  "stored_perms": deepcopy(stored),
 
 
2546
  # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
2547
  # migrated record and LEGACY on an un-migrated one: opposite meanings for identical
2548
  # JSON, so the client cannot render honestly without knowing which world it is in.
@@ -2867,6 +2888,84 @@ def grant_database_access(session, username, database, role):
2867
  return {'username': uname, 'database': key, 'role': role}
2868
 
2869
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2870
  @router.put("/admin/users/{username}/perms")
2871
  def put_perms(username: str, body: dict = Body(default=None),
2872
  session: Session = Depends(admin_gate)):
 
1464
  #: The one arm whose answer belongs to the CALLER's tenant, named once so both the memo and the
1465
  #: stamp order can refuse it rather than each restating the rule.
1466
  _TENANT_OWNED_ARM = "_product_shared_choices"
1467
+ #: Every arm whose answer depends on the session's tenant: never memoed, never in the process cache.
1468
+ _TENANT_OWNED_ARMS = frozenset({_TENANT_OWNED_ARM, "_dba_choices"})
1469
 
1470
 
1471
  def _server_side_vocabularies(session=None):
 
1562
  "occassion", "shape_style", "size", "sub_category")
1563
  return {key: (stored.get(key) or {}).get("options") or () for key in keys}
1564
 
1565
+ def _dba_choices():
1566
+ """The customer `dba` choices: the session tenant's team labels plus one multi-team value.
1567
+
1568
+ Read under the tenant's own Odoo binding, so its configured teams answer. A tenant with
1569
+ no Odoo source supplies nothing rather than the process default tenant's teams."""
1570
+ if session is None:
1571
+ return {}
1572
+ import modules.customer_data as customer_data
1573
+
1574
+ src = session.runtime.odoo_source()
1575
+ if src is None:
1576
+ return {}
1577
+ with src._bound():
1578
+ return {"dba": customer_data.dba_options()}
1579
+
1580
  if _VOCAB_CACHE is not None:
1581
  out = dict(_VOCAB_CACHE)
1582
+ for tenant_arm in (_product_shared_choices, _dba_choices):
1583
+ try:
1584
+ for key, values in tenant_arm().items():
1585
+ clean = sorted({str(v) for v in values if str(v or "").strip()})
1586
+ if clean:
1587
+ out[key] = clean
1588
+ except Exception: # noqa: BLE001
1589
+ pass
1590
  return out
1591
 
1592
  out = {}
 
1600
  normalisation rather than two that can drift.
1601
  """
1602
  name = getattr(fn, "__name__", "")
1603
+ memoable = name and name not in _TENANT_OWNED_ARMS
1604
  try:
1605
  got = _VOCAB_ARM_CACHE[name] if memoable and name in _VOCAB_ARM_CACHE else fn()
1606
  for key, values in (got or {}).items():
 
1726
  # them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into
1727
  # another tenant's permission editor for the life of the worker.
1728
  arm(_product_shared_choices)
1729
+ arm(_dba_choices)
1730
  return out
1731
 
1732
 
 
2562
  return {"username": uname,
2563
  "perms": perms_out,
2564
  "stored_perms": deepcopy(stored),
2565
+ # The `expect` that `PUT /admin/users/{u}/perms/{database}` compares before writing.
2566
+ "access_state": database_access_token(rec),
2567
  # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a
2568
  # migrated record and LEGACY on an un-migrated one: opposite meanings for identical
2569
  # JSON, so the client cannot render honestly without knowing which world it is in.
 
2888
  return {'username': uname, 'database': key, 'role': role}
2889
 
2890
 
2891
+ def database_access_token(record):
2892
+ """A stable digest of `users.database_access_state(record)` that a client sends back as `expect`."""
2893
+ import hashlib
2894
+ state = users.database_access_state(record or {})
2895
+ raw = json.dumps(state, sort_keys=True, separators=(",", ":"), default=str)
2896
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
2897
+
2898
+
2899
+ def set_database_block(session, username, database, entry, *, expect, confirm_widen=False):
2900
+ """Write one database's access, row filter and hidden fields in one account compare-and-set.
2901
+
2902
+ `expect` is the `users.database_access_state` dict, or its `database_access_token`, the caller
2903
+ read. A changed account answers 409 and writes nothing.
2904
+ """
2905
+ import copy
2906
+ from core import user_tables
2907
+
2908
+ admin_gate(session)
2909
+ uname = str(username or "").strip().lower()
2910
+ key = str(database or "").strip()
2911
+ if not isinstance(entry, dict) or type(entry.get("access")) is not bool:
2912
+ raise err(400, "bad_perms", "The entry needs access set to true or false.")
2913
+ if not isinstance(expect, (dict, str)) or not expect:
2914
+ raise err(400, "missing_expect", "Read the account permissions first, then send its access state.")
2915
+ tenant = str(session.runtime.tenant.slug).strip().lower()
2916
+ if _tenant_of(session.user) != tenant:
2917
+ raise err(403, "forbidden", "Database access could not be changed.")
2918
+ # Read the account store strictly rather than the directory's fail-silent projection.
2919
+ reg = store.get("users", fresh=True)
2920
+ rec = (reg or {}).get(uname)
2921
+ if not isinstance(rec, dict) or not rec.get("active", True) or _tenant_of(rec) != tenant:
2922
+ raise err(404, "no_such_user", "That account is not active in this workspace.")
2923
+ if uname == session.uname:
2924
+ raise err(400, "self_perms", "Ask another administrator to change your access.")
2925
+ governed = {m["key"] for m in _perm_modules(session)}
2926
+ if key not in governed:
2927
+ raise err(404, "no_database", "That database is not available in this workspace.")
2928
+ state = users.database_access_state(rec)
2929
+ if (state if isinstance(expect, dict) else database_access_token(rec)) != expect:
2930
+ raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
2931
+ submitted = {field: copy.deepcopy(entry[field]) for field in users.DATABASE_ENTRY_KEYS
2932
+ if field in entry}
2933
+ cleaned = _clean_perms({key: submitted}, governed_keys=governed, session=session,
2934
+ stored=rec.get("perms"))
2935
+ folded = _fold_legacy_scope(rec, cleaned)
2936
+ _guard_bu_widening(rec, folded, bool(confirm_widen))
2937
+ candidate = {field: copy.deepcopy(folded[key][field]) for field in users.DATABASE_ENTRY_KEYS}
2938
+ prior = copy.deepcopy((rec.get("perms") or {}).get(key))
2939
+ result = users.set_database_access(uname, key, candidate, actor=session.uname,
2940
+ tenant=tenant, expect=state)
2941
+ status = result.get("status")
2942
+ if status == "conflict":
2943
+ raise err(409, "permissions_changed", "Account access changed. Read it again and retry.")
2944
+ if status == "missing":
2945
+ raise err(404, "no_such_user", "That account is not active in this workspace.")
2946
+ if status != "updated":
2947
+ raise err(403, "forbidden", "Database access could not be changed.")
2948
+ written = ((result.get("after") or {}).get("perms") or {}).get(key) or candidate
2949
+ if key.startswith(user_tables.KEY_PREFIX):
2950
+ _sync_database_shares(session, uname, {key: prior} if prior else {}, {key: written})
2951
+ return {"username": uname, "database": key, "entry": copy.deepcopy(written),
2952
+ "access_state": database_access_token(result.get("after"))}
2953
+
2954
+
2955
+ @router.put("/admin/users/{username}/perms/{database}")
2956
+ def put_database_perms(username: str, database: str, body: dict = Body(default=None),
2957
+ session: Session = Depends(admin_gate)):
2958
+ """Replace one database's access, row filter and hidden fields if the account is unchanged.
2959
+
2960
+ Body: `{entry: {access, filter?, hiddenFields?}, expect: <access_state from GET perms>,
2961
+ confirm_widen?}`. A stale `expect` answers 409 and writes nothing.
2962
+ """
2963
+ body = body if isinstance(body, dict) else {}
2964
+ return set_database_block(session, username, database, body.get("entry"),
2965
+ expect=body.get("expect"),
2966
+ confirm_widen=bool(body.get("confirm_widen")))
2967
+
2968
+
2969
  @router.put("/admin/users/{username}/perms")
2970
  def put_perms(username: str, body: dict = Body(default=None),
2971
  session: Session = Depends(admin_gate)):
api/routes_aged_stock.py CHANGED
@@ -19,10 +19,9 @@ def _module():
19
 
20
 
21
  def _reader(session: Session = Depends(require_session)) -> Session:
22
- """Reads need read access to the Amazon storage fees database."""
23
- from core import perm_scope
24
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
25
- raise err(403, "aged_stock_forbidden", "Your account cannot read Amazon storage fees.")
26
  return session
27
 
28
 
@@ -60,6 +59,8 @@ def _app(session, app_id):
60
  if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
61
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
62
  raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
 
 
63
  return app
64
 
65
 
@@ -140,9 +141,11 @@ def read_item(item_id: str, app: str = Query(..., min_length=1), offset: int = Q
140
  @router.patch("/items/{item_id}")
141
  def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
142
  due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
 
143
  session: Session = Depends(_app_writer)):
 
144
  return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
145
- action=action, note=note, user=session.uname or "unknown", st=session.runtime)
146
 
147
 
148
  @router.get("/settings")
 
19
 
20
 
21
  def _reader(session: Session = Depends(require_session)) -> Session:
22
+ """Admit a live App holder and refuse unsupported source row or field limits."""
23
+ from routes_app_dashboard import admit_reader
24
+ admit_reader(session, 'aged_stock')
 
25
  return session
26
 
27
 
 
59
  if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
60
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
61
  raise err(404, "aged_stock_app_not_found", "That Aged stock App does not exist.")
62
+ from routes_app_dashboard import admit_reader
63
+ admit_reader(session, 'aged_stock', app)
64
  return app
65
 
66
 
 
141
  @router.patch("/items/{item_id}")
142
  def update_item(item_id: str, owner: str | None = Body(None), status: str | None = Body(None),
143
  due: str | None = Body(None), action: str | None = Body(None), note: str | None = Body(None),
144
+ price: StrictFloat | StrictInt | StrictStr | None = Body(None),
145
  session: Session = Depends(_app_writer)):
146
+ """A price is stored as a proposal on the item and never sent to Amazon. A blank price clears it."""
147
  return _call(_module().update_item, _tenant(session), item_id, owner=owner, status=status, due=due,
148
+ action=action, note=note, price=price, user=session.uname or "unknown", st=session.runtime)
149
 
150
 
151
  @router.get("/settings")
api/routes_amazon_dashboard.py CHANGED
@@ -13,57 +13,28 @@ from modules import amazon_dashboard as dashboard_module
13
  router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
14
 
15
 
16
- def _unrestricted_bu(user):
17
- from core import perms, users
18
- raw = user.get("bus", "all")
19
- if raw == "all":
20
- return "All" in perms.allowed_bu_labels(user)
21
- if not isinstance(raw, (list, tuple)) or not raw:
22
- return False
23
- by_label = {label: key for key, label in users.BU_LABELS.items()}
24
- identities = []
25
- for value in raw:
26
- if type(value) is int and value in users.BU_LABELS:
27
- identities.append(value)
28
- elif isinstance(value, str) and value in by_label:
29
- identities.append(by_label[value])
30
- elif value != "All":
31
- return False
32
- return bool(identities) and "All" in perms.allowed_bu_labels(dict(user, bus=identities))
33
-
34
-
35
  def admit_sources(session):
36
  """Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
37
- from core import perm_scope, perms, route_system, user_tables
 
38
  runtime, user = session.runtime, session.user
39
  if (not getattr(runtime, "key", None) or session.tenant != runtime.key
40
  or user.get("tenant", session.tenant) != runtime.key):
41
  raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
42
- if not _unrestricted_bu(user) or perms.scope_agent(user) is not None:
43
- raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply your business unit or agent scope.")
44
  allowed, refused = {}, {}
45
- config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
46
- tenant_modules = config.get("modules", "all")
47
  for table, key in dashboard_module.SOURCE_DATABASES.items():
48
  # No production cost mapping is available, so this door never opens the Odoo cost mirror.
49
  if table == "product_product":
50
  refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
51
  continue
52
- if ((tenant_modules != "all" and key not in (tenant_modules or ()))
53
- or not perm_scope.may_read(user, key, st=runtime)):
54
- refused[table] = "Your account cannot read a required source database."
 
 
 
55
  continue
56
  definition = user_tables.get_definition(key, st=runtime)
57
- if not isinstance(definition, dict) or not definition.get("fields"):
58
- refused[table] = "A required source database has no readable field definition."
59
- continue
60
- wall = perm_scope.entry(user, key) or {}
61
- # Explicit source restrictions remain binding on aggregate reads, including admin records.
62
- if (wall.get("access") is False or wall.get("filter") or wall.get("hiddenFields")
63
- or perm_scope.row_scope_applies(user, key, st=runtime, defn=definition)
64
- or perm_scope.hidden_keys(user, key, definition["fields"], st=runtime)
65
- or route_system.is_managed(definition) or route_system.is_managed(key)):
66
- raise err(403, "amazon_dashboard_scope_unsupported", "This dashboard cannot apply a source row or field access limit.")
67
  allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
68
  if isinstance(field, dict) and field.get("key"))
69
  if "amzn_order_metrics" not in allowed:
@@ -86,14 +57,16 @@ class AdmittedReader(dashboard_module.MirrorReader):
86
 
87
 
88
  def _app(session, app_id):
89
- if not app_id:
90
- if not session.admin:
91
- raise err(403, "amazon_dashboard_app_required", "Open this dashboard from an App you can read.")
92
- return
93
  import application_registry
94
- opening = application_registry.opening(session, app_id)
95
- if opening["application"]["templateKind"] != dashboard_module.KIND:
 
 
 
 
 
96
  raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
 
97
 
98
 
99
  @router.get("/dashboard")
@@ -101,5 +74,7 @@ def read_dashboard(start: str | None = None, end: str | None = None, compare: st
101
  app: str | None = None, session: Session = Depends(require_session)):
102
  """Serve the published Amazon route through canonical admission and figure validation."""
103
  from routes_app_dashboard import read_dashboard as canonical_dashboard
 
 
104
  return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
105
  compare=compare, session=session)
 
13
  router = APIRouter(prefix="/api/v1/app-kinds/amazon_dashboard", tags=["amazon-dashboard"])
14
 
15
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  def admit_sources(session):
17
  """Resolve source, recipient, field and BU walls before any mirror metadata or row I/O."""
18
+ import application_registry
19
+ from core import user_tables
20
  runtime, user = session.runtime, session.user
21
  if (not getattr(runtime, "key", None) or session.tenant != runtime.key
22
  or user.get("tenant", session.tenant) != runtime.key):
23
  raise err(403, "amazon_dashboard_forbidden", "The account does not belong to this workspace.")
 
 
24
  allowed, refused = {}, {}
 
 
25
  for table, key in dashboard_module.SOURCE_DATABASES.items():
26
  # No production cost mapping is available, so this door never opens the Odoo cost mirror.
27
  if table == "product_product":
28
  refused[table] = "No proven Amazon SKU and Odoo unit mapping is available for this period."
29
  continue
30
+ # R46: the shared check; explicit source restrictions bind aggregate reads, admin records too.
31
+ access = application_registry.app_access(user, runtime, None, (key,))
32
+ if access["code"] == "app_dashboard_scope":
33
+ raise err(403, "amazon_dashboard_scope_unsupported", access["message"])
34
+ if access["dashboard"] != "ready":
35
+ refused[table] = access["message"]
36
  continue
37
  definition = user_tables.get_definition(key, st=runtime)
 
 
 
 
 
 
 
 
 
 
38
  allowed[table] = frozenset(str(field["key"]) for field in definition["fields"]
39
  if isinstance(field, dict) and field.get("key"))
40
  if "amzn_order_metrics" not in allowed:
 
57
 
58
 
59
  def _app(session, app_id):
 
 
 
 
60
  import application_registry
61
+ record = (application_registry.load(session.runtime, strict=True)["applications"].get(app_id) if app_id else
62
+ application_registry.readable_app(session.runtime, session.tenant, session.uname,
63
+ session.admin, dashboard_module.KIND))
64
+ if (session.tenant != getattr(session.runtime, "key", None) or not isinstance(record, dict)
65
+ or record.get("tenant") != session.tenant or record.get("retired") is True
66
+ or record.get("templateKind") != dashboard_module.KIND
67
+ or not application_registry.may_read(record, session.uname, session.admin)):
68
  raise err(404, "amazon_dashboard_app_not_found", "That Amazon dashboard App does not exist.")
69
+ return record["id"]
70
 
71
 
72
  @router.get("/dashboard")
 
74
  app: str | None = None, session: Session = Depends(require_session)):
75
  """Serve the published Amazon route through canonical admission and figure validation."""
76
  from routes_app_dashboard import read_dashboard as canonical_dashboard
77
+ if not app:
78
+ app = _app(session, app)
79
  return canonical_dashboard(kind="amazon_dashboard", app=app, start=start, end=end,
80
  compare=compare, session=session)
api/routes_amazon_operator.py CHANGED
@@ -1,7 +1,7 @@
1
  """Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
2
 
3
- Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`, the
4
- same path an MCP approval takes. The router never calls the Ads client.
5
  """
6
  from fastapi import APIRouter, Body, Depends, Query
7
 
@@ -77,13 +77,52 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
77
  return session
78
 
79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  @router.get("/today")
81
  def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
82
  app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
83
  limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
84
  op = _module()
85
  out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
86
- app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q)
 
87
  return {**out, "canEdit": _may_edit(session, _app(session, app))}
88
 
89
 
@@ -110,7 +149,7 @@ def read_history(status: str | None = Query(None), cursor: str | None = Query(No
110
  limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
111
  op = _module()
112
  return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
113
- st=session.runtime)
114
 
115
 
116
  @router.get("/by-hour")
@@ -241,12 +280,16 @@ def approve_wasted(group_id: str, digest: str = Body(..., embed=True, min_length
241
  @router.get("/results")
242
  def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
243
  op = _module()
244
- return _call(op.results, _tenant(session), proposal_id, st=session.runtime)
 
245
 
246
 
247
  @router.post("/undo")
248
  def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
249
  op = _module()
 
 
 
250
  return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
251
 
252
 
@@ -286,12 +329,20 @@ def _weekly_context(st):
286
  config = getattr(getattr(st, "tenant", None), "config", {}) or {}
287
  timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
288
 
 
 
289
  def may_read(app, owner):
290
- user = _module().account_get(owner, st=st) or {}
291
- return bool(user and user.get("active", True)
 
 
 
 
 
292
  and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
293
  and application_registry.may_read(
294
- app, owner, is_admin=str(user.get("role") or "").lower() == "admin"))
 
295
  return {"apps": apps, "may_read": may_read, "timezone": timezone,
296
  "base_url": os.environ.get("APP_BASE_URL") or ""}
297
 
@@ -302,7 +353,20 @@ def _weekly_delivery():
302
  from notify_destinations import deliver
303
  except ImportError:
304
  return None
305
- return deliver
 
 
 
 
 
 
 
 
 
 
 
 
 
306
 
307
 
308
  def _weekly_destinations(session):
@@ -352,6 +416,7 @@ def read_weekly(session: Session = Depends(_app_reader)):
352
  if row.get("source_module") in op.PROPOSAL_SOURCES})}
353
  summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
354
  source_state=source_state, **context)
 
355
  return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
356
  "destinations": _weekly_destinations(session)}
357
 
@@ -362,15 +427,38 @@ def send_weekly_now(owner: str | None = Body(None, embed=True), session: Session
362
  if not session.admin:
363
  raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
364
  from modules import amazon_operator_weekly as weekly
365
- return _call(weekly.send_tenant, _tenant(session), session.runtime, force=True, owner=owner,
366
- deliver=_weekly_delivery(), **_call(_weekly_context, session.runtime))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
367
 
368
 
369
  @router.post("/approve")
370
- def approve(item_ids: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
371
- """Approve the ticked proposals as one batch. Returns each row's outcome."""
 
 
 
 
 
 
 
372
  op = _module()
373
- return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime)
 
374
 
375
 
376
  @router.post("/dismiss")
@@ -378,7 +466,7 @@ def dismiss(item_ids: list[str] = Body(...), reason: str = Body(..., min_length=
378
  session: Session = Depends(_app_writer)):
379
  op = _module()
380
  return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
381
- st=session.runtime)
382
 
383
 
384
  def _refresh_operator_tenant(tenant, st, *, chat=None):
 
1
  """Amazon operator App routes (C3): today's batch, history, settings, and the batch approve and dismiss.
2
 
3
+ Every route names its App with `?app=<id>`. Approve and dismiss call `amazon_operator.set_status`.
4
+ MCP approvals are refused. The router never calls the Ads client.
5
  """
6
  from fastapi import APIRouter, Body, Depends, Query
7
 
 
77
  return session
78
 
79
 
80
+ def _source_reader(user, st, *, weekly=False):
81
+ """Cache Dashboard admission per source kind for one viewer and request."""
82
+ import application_registry
83
+ import app_templates
84
+ cache = {}
85
+
86
+ def allowed(source):
87
+ if str(source or "").startswith("modules.") and source not in _module().PROPOSAL_SOURCES:
88
+ return False
89
+ kind = str(source or "").removeprefix("modules.")
90
+ if kind not in cache:
91
+ cache[kind] = False
92
+ try:
93
+ template = app_templates.BY_KIND.get(kind)
94
+ if not isinstance(template, dict):
95
+ return False
96
+ required = template.get("sources")
97
+ optional = template.get("optional_sources", ())
98
+ if not isinstance(required, (list, tuple)) or not required or not isinstance(optional, (list, tuple)):
99
+ return False
100
+ sources = list(dict.fromkeys((*required, *optional)))
101
+ # Removal prose includes ship-to details even when these sources become optional.
102
+ if kind == "returns_refunds":
103
+ sources = list(dict.fromkeys((*sources, "ut_amzn_removal_orders", "ut_amzn_removal_shipments")))
104
+ if weekly:
105
+ sources = list(dict.fromkeys((*sources, "ut_amazon_sku_profit", "ut_amzn_inventory_planning")))
106
+ if any(not isinstance(key, str) or not key for key in sources):
107
+ return False
108
+ access = application_registry.app_access(user, st, {"templateKind": kind}, sources)
109
+ cache[kind] = access.get("dashboard") == "ready"
110
+ except Exception:
111
+ import logging
112
+ logging.getLogger(__name__).exception("Operator source access check failed for %s", kind)
113
+ return cache[kind]
114
+
115
+ return allowed
116
+
117
+
118
  @router.get("/today")
119
  def read_today(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
120
  app_cursor: str | None = Query(None), held_cursor: str | None = Query(None),
121
  limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
122
  op = _module()
123
  out = _call(op.screen_today, _tenant(session), st=session.runtime, cursor=cursor,
124
+ app_cursor=app_cursor, held_cursor=held_cursor, limit=limit, q=q,
125
+ source_allowed=_source_reader(session.user, session.runtime))
126
  return {**out, "canEdit": _may_edit(session, _app(session, app))}
127
 
128
 
 
149
  limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
150
  op = _module()
151
  return _call(op.screen_history, _tenant(session), status=status, cursor=cursor, limit=limit, q=q,
152
+ st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
153
 
154
 
155
  @router.get("/by-hour")
 
280
  @router.get("/results")
281
  def read_results(proposal_id: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
282
  op = _module()
283
+ return _call(op.results, _tenant(session), proposal_id, st=session.runtime,
284
+ source_allowed=_source_reader(session.user, session.runtime))
285
 
286
 
287
  @router.post("/undo")
288
  def undo(proposal_id: str = Body(..., embed=True, min_length=1), session: Session = Depends(_app_writer)):
289
  op = _module()
290
+ allowed = _source_reader(session.user, session.runtime)
291
+ if not allowed(TEMPLATE_KIND):
292
+ return op.withheld(1)
293
  return _call(op.propose_undo, _tenant(session), session.user, proposal_id, st=session.runtime)
294
 
295
 
 
329
  config = getattr(getattr(st, "tenant", None), "config", {}) or {}
330
  timezone = config.get("timezone") or _module()._doc(st, "tenant_settings").get("timezone")
331
 
332
+ readers = {}
333
+
334
  def may_read(app, owner):
335
+ account = _module().account_get(owner, st=st) or {}
336
+ user = {**account, "username": owner} if account else {}
337
+ if owner not in readers:
338
+ readers[owner] = _source_reader(user, st, weekly=True)
339
+ return bool(isinstance(app, dict) and not app.get("retired")
340
+ and app.get("tenant") == str(getattr(st, "key", ""))
341
+ and user and user.get("active", True)
342
  and str(user.get("tenant") or getattr(st, "key", "")) == str(getattr(st, "key", ""))
343
  and application_registry.may_read(
344
+ app, owner, is_admin=str(user.get("role") or "").lower() == "admin")
345
+ and readers[owner](app.get("templateKind")))
346
  return {"apps": apps, "may_read": may_read, "timezone": timezone,
347
  "base_url": os.environ.get("APP_BASE_URL") or ""}
348
 
 
353
  from notify_destinations import deliver
354
  except ImportError:
355
  return None
356
+ def guarded(st, kind, items, *, user, **kwargs):
357
+ context = _weekly_context(st)
358
+ apps = {str(app.get("id")): app for app in context["apps"]}
359
+ kept = []
360
+ for item in items:
361
+ source = item.get("source") or {}
362
+ ids = source.get("rowIds") or []
363
+ if (source.get("table") == "applications" and ids
364
+ and all(str(key) in apps and context["may_read"](apps[str(key)], user) for key in ids)):
365
+ kept.append(item)
366
+ if not kept:
367
+ return {"state": "empty", "withheldCount": len(items), "results": []}
368
+ return deliver(st, kind, kept, user=user, **kwargs)
369
+ return guarded
370
 
371
 
372
  def _weekly_destinations(session):
 
416
  if row.get("source_module") in op.PROPOSAL_SOURCES})}
417
  summary = _call(weekly.compose, _tenant(session), session.uname, week, st=st, now=now,
418
  source_state=source_state, **context)
419
+ # Compose independently validates the count for this recipient's owned items.
420
  return {**summary, "inbox_basis": "Open items in the saved Operator inbox.",
421
  "destinations": _weekly_destinations(session)}
422
 
 
427
  if not session.admin:
428
  raise err(403, "amazon_operator_admin_only", "Only an Administrator can send a weekly summary now.")
429
  from modules import amazon_operator_weekly as weekly
430
+ result = _call(weekly.send_tenant, _tenant(session), session.runtime, force=True, owner=owner,
431
+ deliver=_weekly_delivery(), **_call(_weekly_context, session.runtime))
432
+ allowed = _source_reader(session.user, session.runtime, weekly=True)
433
+ rows = []
434
+ for row in result.get("rows", []):
435
+ row = dict(row)
436
+ row.pop("delivery", None)
437
+ if isinstance(row.get("summary"), dict):
438
+ summary = dict(row["summary"])
439
+ entries = summary.get("apps", [])
440
+ summary["apps"] = [app for app in entries if allowed(app.get("kind"))]
441
+ # Validation contains App names and figures too.
442
+ summary.pop("validation", None)
443
+ row["summary"] = summary
444
+ rows.append(row)
445
+ # Recipient counts survive caller text filtering without an all-owner recount.
446
+ return {**result, "rows": rows}
447
 
448
 
449
  @router.post("/approve")
450
+ def approve(item_ids: list[str] = Body(..., embed=True), reviews: dict | None = Body(None, embed=True),
451
+ session: Session = Depends(_app_writer)):
452
+ """Approve the ticked proposals as one batch. Returns each row's outcome.
453
+
454
+ `reviews` maps each removal's inbox id to the `review.reviewed` value shown in its drawer. Only a person
455
+ signed in to the App approves: an API key or OAuth session is refused before anything is stored.
456
+ """
457
+ if (session.claims or {}).get("external"):
458
+ raise err(403, "amazon_operator_internal_approval", "Approve these inside the Operator inbox.")
459
  op = _module()
460
+ return _call(op.set_status, _tenant(session), session.user, item_ids, "approved", st=session.runtime,
461
+ reviews=reviews, source_allowed=_source_reader(session.user, session.runtime))
462
 
463
 
464
  @router.post("/dismiss")
 
466
  session: Session = Depends(_app_writer)):
467
  op = _module()
468
  return _call(op.set_status, _tenant(session), session.user, item_ids, "dismissed", note=reason,
469
+ st=session.runtime, source_allowed=_source_reader(session.user, session.runtime))
470
 
471
 
472
  def _refresh_operator_tenant(tenant, st, *, chat=None):
api/routes_amazon_profit.py CHANGED
@@ -1,5 +1,5 @@
1
  """Command Center doors: read the settings, switch the period, recompute, set the owner,
2
- confirm or reject a suggested Odoo match, and relink projects.
3
 
4
  The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
5
  A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
@@ -12,7 +12,7 @@ from typing import Any
12
  from fastapi import APIRouter, Body, Depends, HTTPException
13
 
14
  from deps import Session, err, require_session
15
- from modules import amazon_profit, amazon_profit_account
16
 
17
  router = APIRouter(prefix="/api/v1")
18
 
@@ -96,6 +96,8 @@ def profit_status(session: Session = Depends(require_session)):
96
  "linked": settings.get("linked"),
97
  "verifiedMonth": settings.get("verifiedMonth"),
98
  "refusal": settings.get("refusal"),
 
 
99
  "defaultOwner": settings["defaultOwner"],
100
  "periods": amazon_profit.period_choices(),
101
  "statuses": list(amazon_profit.STATUSES),
@@ -109,6 +111,26 @@ def profit_status(session: Session = Depends(require_session)):
109
  "canAdmin": bool(session.admin)}
110
 
111
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
  @router.get("/amazon-profit/account")
113
  def profit_account(period: str | None = None, session: Session = Depends(require_session)):
114
  """Read this tenant's account P&L without refreshing costs, caches or history."""
@@ -158,7 +180,8 @@ def profit_what_if(sku: str, price: float | None = None, landed_cost: float | No
158
 
159
 
160
  def _recompute(session, period=None, thin=None):
161
- """Run one refresh through the tenant's mirror receipts and return 409 on a failed tie."""
 
162
  try:
163
  if period is not None:
164
  amazon_profit.period_bounds(period)
@@ -207,16 +230,6 @@ def profit_period(body: Any = Body(default=None), session: Session = Depends(req
207
  return _recompute(session, period)
208
 
209
 
210
- @router.post("/amazon-profit/projects/relink")
211
- def projects_relink(session: Session = Depends(require_session)):
212
- """Write every project's row id onto its SKU, ASIN and parent ASIN rows; editors only."""
213
- if not _may_edit(session):
214
- raise err(403, "forbidden", "Only an editor of Command Center can relink projects.")
215
- with amazon_profit._LOCK:
216
- linked = amazon_profit.relink_projects(session.runtime)
217
- return {"ok": True, "linked": linked}
218
-
219
-
220
  @router.put("/amazon-profit/settings")
221
  def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
222
  """Administrator settings: the default owner of new items and the thin margin."""
@@ -238,12 +251,13 @@ def profit_settings(body: Any = Body(default=None), session: Session = Depends(r
238
 
239
  @router.post("/amazon-profit/matches/{sku:path}/{decision}")
240
  def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
241
- """Confirm or reject the suggested Odoo product for one SKU; editors only.
242
 
243
- Confirm copies the suggestion into the link and recomputes the period, so COGS and profit
244
- fill at once. Reject clears the suggestion, keeps the SKU unlinked and remembers the code."""
 
245
  import core.user_tables as ut
246
- states = {"confirm": amazon_profit.CONFIRMED, "reject": amazon_profit.REJECTED}
247
  if decision not in states:
248
  raise err(404, "not_found", "Choose confirm or reject.")
249
  if not _may_edit(session):
@@ -266,12 +280,142 @@ def profit_match(sku: str, decision: str, session: Session = Depends(require_ses
266
  "unitsPerSku": (result["row"] or {}).get("units_per_sku")}
267
  if decision == "confirm":
268
  try:
269
- report = _recompute(session)
270
- out["refresh"] = {k: report.get(k) for k in ("skus", "linked", "refreshedAt")}
 
271
  except HTTPException as exc:
272
  out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
 
 
273
  current = next((r for r in amazon_profit.table_rows(session.runtime).values()
274
  if str(r.get("sku") or "").strip() == sku), {})
275
  out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
276
  margin=current.get("margin"), band=current.get("band"))
277
  return out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """Command Center doors: read the settings, switch the period, recompute, set the owner,
2
+ link or reject a suggested Odoo match at once.
3
 
4
  The table itself (`ut_amazon_sku_profit`) is read and edited through the ordinary table doors.
5
  A refresh rewrites only computed columns, so owner, status and due date survive a period switch.
 
12
  from fastapi import APIRouter, Body, Depends, HTTPException
13
 
14
  from deps import Session, err, require_session
15
+ from modules import amazon_data_entry, amazon_profit, amazon_profit_account
16
 
17
  router = APIRouter(prefix="/api/v1")
18
 
 
96
  "linked": settings.get("linked"),
97
  "verifiedMonth": settings.get("verifiedMonth"),
98
  "refusal": settings.get("refusal"),
99
+ "verified": settings.get("verified"),
100
+ "lastRefusal": settings.get("lastRefusal"),
101
  "defaultOwner": settings["defaultOwner"],
102
  "periods": amazon_profit.period_choices(),
103
  "statuses": list(amazon_profit.STATUSES),
 
111
  "canAdmin": bool(session.admin)}
112
 
113
 
114
+ COMMAND_CENTER_KEYS = (amazon_profit.TABLE_KEY, amazon_profit.ASIN_KEY, amazon_profit.PARENT_KEY,
115
+ amazon_profit.PROJECTS_KEY, amazon_data_entry.TABLE_KEY)
116
+
117
+
118
+ @router.get("/amazon-profit/tables")
119
+ def profit_tables(session: Session = Depends(require_session)):
120
+ """Whether each Command Center table this session may open exists.
121
+
122
+ Each table is walled by its own `session.require`, never by `_account_access`, and no row is read."""
123
+ import core.user_tables as ut
124
+ tables = {}
125
+ for key in COMMAND_CENTER_KEYS:
126
+ try:
127
+ session.require(key)
128
+ except HTTPException:
129
+ continue
130
+ tables[key] = ut.get_definition(key, session.runtime) is not None
131
+ return {"tables": tables}
132
+
133
+
134
  @router.get("/amazon-profit/account")
135
  def profit_account(period: str | None = None, session: Session = Depends(require_session)):
136
  """Read this tenant's account P&L without refreshing costs, caches or history."""
 
180
 
181
 
182
  def _recompute(session, period=None, thin=None):
183
+ """Run one refresh through the tenant's mirror receipts and return 409 on a failed tie.
184
+ A 409 left the stored rows and period as they were."""
185
  try:
186
  if period is not None:
187
  amazon_profit.period_bounds(period)
 
230
  return _recompute(session, period)
231
 
232
 
 
 
 
 
 
 
 
 
 
 
233
  @router.put("/amazon-profit/settings")
234
  def profit_settings(body: Any = Body(default=None), session: Session = Depends(require_session)):
235
  """Administrator settings: the default owner of new items and the thin margin."""
 
251
 
252
  @router.post("/amazon-profit/matches/{sku:path}/{decision}")
253
  def profit_match(sku: str, decision: str, session: Session = Depends(require_session)):
254
+ """Link or reject the suggested Odoo product for one SKU; editors only.
255
 
256
+ Confirm copies the suggestion into the link and recomputes that SKU's cost and profit in the
257
+ same request (`recompute_sku`), so a refused refresh never blocks it. Reject clears the
258
+ suggestion, keeps the SKU unlinked and remembers the code."""
259
  import core.user_tables as ut
260
+ states = {"confirm": amazon_profit.LINKED, "reject": amazon_profit.WRONG_MATCH}
261
  if decision not in states:
262
  raise err(404, "not_found", "Choose confirm or reject.")
263
  if not _may_edit(session):
 
280
  "unitsPerSku": (result["row"] or {}).get("units_per_sku")}
281
  if decision == "confirm":
282
  try:
283
+ linked = link_now(session.runtime, sku, by=session.uname)
284
+ if linked.get("refusal"):
285
+ out["refreshError"] = linked["refusal"]
286
  except HTTPException as exc:
287
  out["refreshError"] = ((exc.detail or {}).get("error") or {}).get("message")
288
+ except (amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
289
+ out["refreshError"] = str(exc)
290
  current = next((r for r in amazon_profit.table_rows(session.runtime).values()
291
  if str(r.get("sku") or "").strip() == sku), {})
292
  out.update(cogsPerUnit=current.get("cogs_per_unit"), netProfit=current.get("net_profit"),
293
  margin=current.get("margin"), band=current.get("band"))
294
  return out
295
+
296
+
297
+ def _runtime_of(st):
298
+ """The tenant runtime behind a lent store snapshot; the mirror and the writes need it."""
299
+ import core.user_tables as ut
300
+ lent = getattr(ut, "_Lent", None)
301
+ while lent is not None and isinstance(st, lent):
302
+ st = st._runtime
303
+ return st
304
+
305
+
306
+ def link_now(runtime, sku, *, by=""):
307
+ """Recompute one SKU's cost and profit from its Odoo link on this tenant's mirror."""
308
+ con = _mirror(runtime)
309
+ try:
310
+ result = amazon_profit.recompute_sku(runtime, con, sku, by=by, cost_reader=_cost_reader(runtime))
311
+ finally:
312
+ try:
313
+ con.close()
314
+ except Exception: # noqa: BLE001
315
+ pass
316
+ # S82-T71: a linked SKU that is now complete leaves Data Entry in the same request.
317
+ amazon_data_entry.sync(runtime, skus=[sku])
318
+ return result
319
+
320
+
321
+ #: Cell edits on the SKU table that change its Odoo link.
322
+ LINK_FIELDS = ("match_state", "odoo_product", "units_per_sku", "amazon_cogs")
323
+
324
+
325
+ def link_hook(evt):
326
+ """A person's Link status, Odoo product or Units per SKU edit links that SKU at once.
327
+
328
+ Registered on `core.user_tables.ROW_HOOKS`, which only the human write doors emit into.
329
+ Its own writes go through `update_with_activity`, so it never fires itself."""
330
+ if str(evt.get("type") or "") != "event_field" or str(evt.get("table") or "") != amazon_profit.TABLE_KEY:
331
+ return
332
+ field = str(evt.get("field") or "")
333
+ if field not in LINK_FIELDS or evt.get("st") is None:
334
+ return
335
+ runtime = _runtime_of(evt["st"])
336
+ row = amazon_profit.table_rows(runtime).get(str(evt.get("rowId") or "")) or {}
337
+ sku = str(row.get("sku") or "").strip()
338
+ if not sku:
339
+ return
340
+ by = str(evt.get("user") or "") or amazon_profit.WRITER
341
+ if field == "match_state":
342
+ state = str(evt.get("after") or "").strip()
343
+ state = amazon_profit.LEGACY_MATCH_STATES.get(state, state)
344
+ if state not in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
345
+ return
346
+ with amazon_profit._LOCK:
347
+ result = amazon_profit.apply_decisions(runtime, by=by, sku=sku, state=state)
348
+ if state != amazon_profit.LINKED or not result.get("confirmed"):
349
+ return
350
+ link_now(runtime, sku, by=by)
351
+
352
+
353
+ # One registration per process: a reimport replaces the hook rather than adding a second one.
354
+ import core.user_tables as _ut_hooks # noqa: E402
355
+
356
+ _ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
357
+ if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "link_hook")]
358
+ _ut_hooks.ROW_HOOKS.append(link_hook)
359
+
360
+
361
+ def data_entry_hook(evt):
362
+ """A person's Data Entry cell edit writes that value to the SKU table in the same request.
363
+
364
+ Registered on `core.user_tables.ROW_HOOKS` and filtered on the Data Entry key. The write-back
365
+ goes through `patch_cells`, which emits no row event, so it never fires itself or `link_hook`.
366
+ A link edit then links the SKU at once, and the row is rebuilt: a complete SKU leaves (R8)."""
367
+ if (str(evt.get("type") or "") != "event_field"
368
+ or str(evt.get("table") or "") != amazon_data_entry.TABLE_KEY or evt.get("st") is None):
369
+ return
370
+ field = str(evt.get("field") or "")
371
+ if field not in amazon_data_entry.ENTRY_KEYS:
372
+ return
373
+ runtime = _runtime_of(evt["st"])
374
+ user = str(evt.get("user") or "")
375
+ result = amazon_data_entry.write_back(runtime, str(evt.get("rowId") or ""), field, evt.get("after"), user=user)
376
+ sku = result.get("sku")
377
+ if not sku:
378
+ return
379
+ try:
380
+ if result["written"] and field in (*amazon_data_entry.LINK_KEYS, "amazon_cogs"):
381
+ state = result["changes"].get("match_state")
382
+ if state in (amazon_profit.LINKED, amazon_profit.WRONG_MATCH):
383
+ with amazon_profit._LOCK:
384
+ amazon_profit.apply_decisions(runtime, by=user or amazon_profit.WRITER, sku=sku, state=state)
385
+ row = next((r for r in amazon_profit.table_rows(runtime).values()
386
+ if str(r.get("sku") or "").strip() == sku), {})
387
+ if str(row.get("odoo_product") or "").strip() or field == "amazon_cogs":
388
+ try:
389
+ link_now(runtime, sku, by=user)
390
+ except (HTTPException, amazon_profit.SourceUnavailable, amazon_profit.TableRefused) as exc:
391
+ print(f"[amazon-profit] Data Entry link for {sku} not recomputed: {exc}")
392
+ finally:
393
+ amazon_data_entry.sync(runtime, skus=[sku])
394
+
395
+
396
+ _ut_hooks.ROW_HOOKS[:] = [h for h in _ut_hooks.ROW_HOOKS
397
+ if (getattr(h, "__module__", None), getattr(h, "__name__", None)) != (__name__, "data_entry_hook")]
398
+ _ut_hooks.ROW_HOOKS.append(data_entry_hook)
399
+
400
+
401
+ @router.get("/amazon-profit/fba-capacity")
402
+ def fba_capacity_read(session: Session = Depends(require_session)):
403
+ """The one typed FBA capacity Command Center and Restock share."""
404
+ from core import amazon_inputs
405
+ session.require(amazon_profit.TABLE_KEY)
406
+ return {"fbaCapacity": amazon_inputs.fba_capacity(session.runtime), "units": list(amazon_inputs.UNITS),
407
+ "canEdit": _may_edit(session)}
408
+
409
+
410
+ @router.put("/amazon-profit/fba-capacity")
411
+ def fba_capacity_write(body: Any = Body(default=None), session: Session = Depends(require_session)):
412
+ """Save the typed FBA capacity `{value, unit}`, or clear it with null; editors only."""
413
+ from core import amazon_inputs
414
+ if not _may_edit(session):
415
+ raise err(403, "forbidden", "Only an editor of Command Center can change FBA capacity.")
416
+ body = body if isinstance(body, dict) else {}
417
+ try:
418
+ stored = amazon_inputs.set_fba_capacity(session.runtime, body.get("fbaCapacity"), by=session.uname)
419
+ except ValueError as exc:
420
+ raise err(400, "invalid_setting", str(exc)) from None
421
+ return {"fbaCapacity": stored}
api/routes_amazon_projects.py ADDED
@@ -0,0 +1,389 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Projects doors: a row's projects, one project with its targets and activity, edits, and target search.
2
+
3
+ Editors of Command Center write and a viewer reads (R16, R17). Project rows live in the per-tenant
4
+ store (C6). Search and target details read the Amazon mirror, never `user_tables`. Each write
5
+ repaints the `projects` and `projects_label` cells of only the Command Center rows it touches.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from contextlib import contextmanager
11
+ from pathlib import Path
12
+ import sys
13
+ from typing import Any
14
+
15
+ from fastapi import APIRouter, Body, Depends, HTTPException
16
+
17
+ from core import amazon_projects as projects
18
+ from deps import Session, err, require_session
19
+ from modules import amazon_profit
20
+ import routes_amazon_profit
21
+
22
+ router = APIRouter(prefix="/api/v1/amazon-projects")
23
+
24
+ FORBIDDEN = "Only an editor of Command Center can change projects."
25
+ NO_MIRROR = "This workspace has no Amazon data yet."
26
+ UNREADABLE = "Amazon data could not be read. Try again shortly."
27
+ NOT_FOUND = "That project does not exist."
28
+ #: Contract field name to store field.
29
+ FIELDS = {"name": "name", "status": "status", "priority": "priority", "owner": "owner",
30
+ "startOn": "start_on", "targetOn": "target_on", "progress": "progress", "notes": "notes"}
31
+ #: Project JSON keys a client may echo back. They are ignored, never written.
32
+ READ_ONLY = frozenset(("id", "health", "targetCount", "createdAt", "updatedAt", "createdBy",
33
+ "completedOn", "rollup", "impact", "via"))
34
+
35
+
36
+ def _call(fn, *args, **kwargs):
37
+ try:
38
+ return fn(*args, **kwargs)
39
+ except HTTPException:
40
+ raise
41
+ except KeyError:
42
+ raise err(404, "project_not_found", NOT_FOUND) from None
43
+ except projects.BackendUnavailable:
44
+ raise err(503, "projects_store_unavailable", "Projects need the workspace database.") from None
45
+ except ValueError as exc:
46
+ raise err(400, "invalid_project", str(exc)) from None
47
+ except Exception as exc: # noqa: BLE001
48
+ print(f"[amazon-projects] {getattr(fn, '__name__', fn)} failed: {type(exc).__name__}: {exc}",
49
+ file=sys.stderr)
50
+ raise err(503, "projects_unavailable", "Projects are unavailable. Try again shortly.") from None
51
+
52
+
53
+ def _reader(session):
54
+ session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
55
+ return session.runtime
56
+
57
+
58
+ def _writer(session):
59
+ """Editors and Administrators of Command Center, through the Command Center edit check."""
60
+ if not routes_amazon_profit._may_edit(session):
61
+ raise err(403, "forbidden", FORBIDDEN)
62
+ return session.runtime
63
+
64
+
65
+ def _by(session):
66
+ return session.uname or "unknown"
67
+
68
+
69
+ @contextmanager
70
+ def _open_mirror(runtime):
71
+ """A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
72
+ from harness import runtime as R
73
+ if not Path(runtime.datastore_path).is_file():
74
+ yield None
75
+ return
76
+ cur = R.mirror_cursor(runtime, require_ready=False)
77
+ try:
78
+ yield cur
79
+ finally:
80
+ cur.close()
81
+
82
+
83
+ def _mirror(runtime, read, empty):
84
+ """(read(cursor), None), or (empty, the cause) when the mirror is absent or unreadable."""
85
+ try:
86
+ with _open_mirror(runtime) as cur:
87
+ if cur is None:
88
+ return empty, NO_MIRROR
89
+ return read(cur), None
90
+ except Exception as exc: # noqa: BLE001
91
+ print(f"[amazon-projects] mirror read failed: {type(exc).__name__}: {exc}", file=sys.stderr)
92
+ return empty, UNREADABLE
93
+
94
+
95
+ def _pictures(runtime):
96
+ """`{ASIN: picture ref}`; empty when no picture is stored or the index is unreadable."""
97
+ try:
98
+ from harness import amazon_pictures
99
+ return amazon_pictures.picture_refs(runtime)
100
+ except Exception: # noqa: BLE001
101
+ return {}
102
+
103
+
104
+ def _project(record, today):
105
+ return {"id": record["id"], "name": record["name"], "status": record["status"],
106
+ "priority": record["priority"], "owner": record["owner"], "startOn": record["start_on"],
107
+ "targetOn": record["target_on"], "progress": record["progress"], "notes": record["notes"],
108
+ "completedOn": record["completed_on"], "health": projects.health(record, today),
109
+ "targetCount": record.get("target_count", 0), "createdAt": record["created_at"],
110
+ "updatedAt": record["updated_at"], "createdBy": record["created_by"],
111
+ "rollup": record.get("rollup"), "impact": record.get("impact")}
112
+
113
+
114
+ def _item(grain, key, listing, pictures):
115
+ asin, parent = listing.get("asin"), listing.get("parent_asin")
116
+ return {"grain": grain, "key": key, "sku": listing.get("sku"), "asin": asin, "parentAsin": parent,
117
+ "title": listing.get("title"), "picture": pictures.get(asin) or pictures.get(parent)}
118
+
119
+
120
+ def _pair(grain, key):
121
+ try:
122
+ return projects.target(grain, key)
123
+ except (TypeError, ValueError):
124
+ raise err(400, "invalid_target", "Name a grain of sku, asin or parent_asin and its key.") from None
125
+
126
+
127
+ def _pairs(values):
128
+ if not isinstance(values, list) or not values or not all(isinstance(v, dict) for v in values):
129
+ raise err(400, "invalid_target", "Send targets as a list of grain and key.")
130
+ return [_pair(v.get("grain"), v.get("key")) for v in values]
131
+
132
+
133
+ def _fields(body, *, partial):
134
+ if not isinstance(body, dict):
135
+ raise err(400, "invalid_project", "Send the project fields as an object.")
136
+ unknown = sorted(set(body) - set(FIELDS) - READ_ONLY - {"targets"})
137
+ if unknown:
138
+ raise err(400, "invalid_project", f"Unknown project fields: {', '.join(unknown)}.")
139
+ values = {FIELDS[name]: body[name] for name in FIELDS if name in body}
140
+ if not partial and "name" not in values:
141
+ raise err(400, "invalid_project", "Name the project.")
142
+ return values
143
+
144
+
145
+ def _targets(runtime, project_id):
146
+ """The project's targets with their listing details, and a cause when the mirror is out."""
147
+ listed = _call(projects.targets, runtime, project_id)
148
+ pairs = [(t["grain"], t["key"]) for t in listed]
149
+ details, cause = _mirror(runtime, lambda cur: projects.describe(cur, pairs), {})
150
+ pictures = _pictures(runtime) if pairs else {}
151
+ out = [_item(grain, key, details.get((grain, key)) or {"sku": key if grain == "sku" else None,
152
+ "asin": key if grain == "asin" else None,
153
+ "parent_asin": key if grain == "parent_asin" else None}, pictures)
154
+ for grain, key in pairs]
155
+ return out, cause
156
+
157
+
158
+ def _with_cause(body, cause):
159
+ if cause:
160
+ body["cause"] = cause
161
+ return body
162
+
163
+
164
+ # --------------------------------------------------------------------------- Command Center cells
165
+
166
+ PROGRESS_KEY, LABEL_KEY = "projects", "projects_label"
167
+ #: (table, the grains one row counts, in order): a SKU row counts its ASIN's and parent's projects too.
168
+ CELL_TABLES = ((amazon_profit.TABLE_KEY, ("sku", "asin", "parent_asin")),
169
+ (amazon_profit.ASIN_KEY, ("asin", "parent_asin")),
170
+ (amazon_profit.PARENT_KEY, ("parent_asin",)))
171
+
172
+
173
+ def _distinct(found, pairs):
174
+ """`[(via pair, project)]` with each project once, in the order of `pairs`."""
175
+ seen, out = set(), []
176
+ for via in pairs:
177
+ for record in found.get(via, []):
178
+ if record["id"] not in seen:
179
+ seen.add(record["id"])
180
+ out.append((via, record))
181
+ return out
182
+
183
+
184
+ def cell_values(records):
185
+ """R19 and R67: the average progress as a stored pct and "N of M completed", blank with no counted project."""
186
+ counted = projects.summary(records)
187
+ if not counted["total"]:
188
+ return {PROGRESS_KEY: None, LABEL_KEY: None}
189
+ return {PROGRESS_KEY: amazon_profit.stored_cell({"type": "pct"}, counted["average"] / 100),
190
+ LABEL_KEY: f"{counted['completed']} of {counted['total']} completed"}
191
+
192
+
193
+ def _row_pairs(row, grains):
194
+ out = []
195
+ for grain in grains:
196
+ value = str(row.get(grain) or "").strip()
197
+ if value:
198
+ pair = projects.target(grain, value)
199
+ if pair not in out:
200
+ out.append(pair)
201
+ return out
202
+
203
+
204
+ def _cells_store(runtime):
205
+ """The store that holds the Command Center tables: the tenant runtime itself."""
206
+ return runtime
207
+
208
+
209
+ def _row_ids(cells):
210
+ """The write names only the rows whose cells change."""
211
+ return sorted(cells)
212
+
213
+
214
+ def _write_cells(st, table_key, cells):
215
+ import core.user_tables as ut
216
+
217
+ def _set(cur):
218
+ rows = ((cur or {}).get(table_key) or {}).get("rows") or {}
219
+ for rid, values in cells.items():
220
+ row = rows.get(rid)
221
+ if row is None:
222
+ continue
223
+ for key, value in values.items():
224
+ if value is None:
225
+ row.pop(key, None)
226
+ else:
227
+ row[key] = value
228
+ return cur
229
+
230
+ ut.update_with_activity(table_key, _set, st=st, by=amazon_profit.WRITER, via=amazon_profit.VIA,
231
+ row_ids=_row_ids(cells))
232
+
233
+
234
+ def paint(runtime, pairs=None):
235
+ """Rewrite the progress cells of the rows that count a touched pair; every row when `pairs` is None.
236
+
237
+ Returns `{table: [row id]}` for the rows whose cells changed. Never runs under `amazon_profit._LOCK`."""
238
+ touched = None if pairs is None else {projects.target(*pair) for pair in pairs}
239
+ if touched is not None and not touched:
240
+ return {}
241
+ st = _cells_store(runtime)
242
+ source = {grains[0]: amazon_profit.table_rows(st, key) for key, grains in CELL_TABLES}
243
+ own = {g: {rid: {g: row.get(g)} for rid, row in rows.items()} for g, rows in source.items()}
244
+ lineage, _cause = _mirror(runtime, lambda cur: amazon_profit.M.lineage_rows(cur, source), own)
245
+ plan = []
246
+ for table_key, grains in CELL_TABLES:
247
+ rows = {}
248
+ for rid, row in source[grains[0]].items():
249
+ chain = _row_pairs(lineage[grains[0]][rid], grains)
250
+ if touched is None or touched.intersection(chain + _row_pairs(row, grains)):
251
+ rows[str(rid)] = (chain, row)
252
+ plan.append((table_key, rows))
253
+ wanted = sorted({pair for _key, rows in plan for chain, _row in rows.values() for pair in chain})
254
+ found = projects.projects_for(runtime, wanted) if wanted else {}
255
+ written = {}
256
+ for table_key, rows in plan:
257
+ cells = {}
258
+ for rid, (chain, row) in rows.items():
259
+ values = cell_values([record for _via, record in _distinct(found, chain)])
260
+ if any(row.get(key) != value for key, value in values.items()):
261
+ cells[rid] = values
262
+ if cells:
263
+ _write_cells(st, table_key, cells)
264
+ written[table_key] = sorted(cells)
265
+ return written
266
+
267
+
268
+ def _repaint(runtime, pairs):
269
+ """Paint after a committed project write. A failure is logged: the write itself stands."""
270
+ try:
271
+ return paint(runtime, pairs)
272
+ except Exception as exc: # noqa: BLE001
273
+ print(f"[amazon-projects] cell repaint failed: {type(exc).__name__}: {exc}", file=sys.stderr)
274
+ return None
275
+
276
+
277
+ def _target_pairs(runtime, project_id):
278
+ return [(t["grain"], t["key"]) for t in _call(projects.targets, runtime, project_id)]
279
+
280
+
281
+ @router.get("")
282
+ def row_projects(grain: str | None = None, key: str | None = None,
283
+ session: Session = Depends(require_session)):
284
+ """A row's projects with its ASIN's and Parent ASIN's, each once; every project without a row."""
285
+ runtime = _reader(session)
286
+ can_edit = bool(routes_amazon_profit._may_edit(session))
287
+ today = projects._today()
288
+ if grain is None and key is None:
289
+ records = _call(projects.list_projects, runtime)
290
+ return {"grain": None, "key": None, "canEdit": can_edit,
291
+ "projects": [_project(r, today) for r in records], "summary": projects.summary(records)}
292
+ pair = _pair(grain, key)
293
+ above, cause = _mirror(runtime, lambda cur: projects.lineage(cur, *pair), [])
294
+ pairs = [pair] + [p for p in above if p != pair]
295
+ listed = _distinct(_call(projects.projects_for, runtime, pairs), pairs)
296
+ out = [{**_project(record, today), "via": {"grain": via[0], "key": via[1]}} for via, record in listed]
297
+ return _with_cause({"grain": pair[0], "key": pair[1], "canEdit": can_edit, "projects": out,
298
+ "summary": projects.summary([record for _via, record in listed])}, cause)
299
+
300
+
301
+ @router.get("/search")
302
+ def search_targets(q: str = "", limit: int = 25, session: Session = Depends(require_session), offset: int = 0):
303
+ """A SKU-ordered page from the Amazon mirror; total counts every match, including past the end."""
304
+ runtime = _reader(session)
305
+ if not 1 <= limit <= projects.SEARCH_MAX:
306
+ raise err(400, "invalid_limit", f"Limit must be from 1 to {projects.SEARCH_MAX}.")
307
+ if type(offset) is not int or offset < 0:
308
+ raise err(400, "invalid_offset", "Offset must be a nonnegative whole number.")
309
+ (listings, total), cause = _mirror(runtime, lambda cur: projects.search(cur, q, limit, offset), ([], 0))
310
+ pictures = _pictures(runtime) if listings else {}
311
+ results = [_item("sku", item["sku"], item, pictures) for item in listings]
312
+ return _with_cause({"results": results, "total": total}, cause)
313
+
314
+
315
+ @router.post("")
316
+ def create_project(body: Any = Body(default=None), session: Session = Depends(require_session)):
317
+ """Create one project, with targets when the body names them."""
318
+ runtime = _writer(session)
319
+ values = _fields(body, partial=False)
320
+ targets = _pairs(body["targets"]) if body.get("targets") else []
321
+ record = _call(projects.create, runtime, targets=targets, by=_by(session), **values)
322
+ _repaint(runtime, targets)
323
+ return {"project": _project(record, projects._today())}
324
+
325
+
326
+ @router.get("/{project_id}")
327
+ def one_project(project_id: str, session: Session = Depends(require_session)):
328
+ """One project with its targets and its activity, newest first."""
329
+ runtime = _reader(session)
330
+ record = _call(projects.get, runtime, project_id)
331
+ targets, cause = _targets(runtime, project_id)
332
+ activity = _call(projects.activity, runtime, project_id)
333
+ return _with_cause({"canEdit": bool(routes_amazon_profit._may_edit(session)),
334
+ "project": _project(record, projects._today()), "targets": targets,
335
+ "activity": activity}, cause)
336
+
337
+
338
+ @router.patch("/{project_id}")
339
+ def edit_project(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
340
+ """Change the named fields. Targets named here are added."""
341
+ runtime = _writer(session)
342
+ values = _fields(body, partial=True)
343
+ targets = _pairs(body["targets"]) if body.get("targets") else []
344
+ if not values and not targets:
345
+ raise err(400, "invalid_project", "Change at least one project field.")
346
+ if values:
347
+ record = _call(projects.update, runtime, project_id, values, by=_by(session))
348
+ if targets:
349
+ record = _call(projects.add_targets, runtime, project_id, targets, by=_by(session))["project"]
350
+ # Only status and progress move a cell: a name or note edit repaints only the targets it adds.
351
+ _repaint(runtime, _target_pairs(runtime, project_id) if {"status", "progress"} & set(values) else targets)
352
+ return {"project": _project(record, projects._today())}
353
+
354
+
355
+ @router.delete("/{project_id}")
356
+ def delete_project(project_id: str, session: Session = Depends(require_session)):
357
+ """Delete one project and its targets."""
358
+ runtime = _writer(session)
359
+ pairs = _target_pairs(runtime, project_id)
360
+ _call(projects.delete, runtime, project_id, by=_by(session))
361
+ _repaint(runtime, pairs)
362
+ return {"ok": True, "id": project_id}
363
+
364
+
365
+ def _target_answer(runtime, project_id, result):
366
+ targets, cause = _targets(runtime, project_id)
367
+ return _with_cause({"project": _project(result["project"], projects._today()), "targets": targets}, cause)
368
+
369
+
370
+ @router.post("/{project_id}/targets")
371
+ def add_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
372
+ """Assign the project to more SKUs, ASINs or Parent ASINs."""
373
+ runtime = _writer(session)
374
+ pairs = _pairs((body or {}).get("targets") if isinstance(body, dict) else None)
375
+ result = _call(projects.add_targets, runtime, project_id, pairs, by=_by(session))
376
+ _repaint(runtime, pairs)
377
+ return _target_answer(runtime, project_id, result)
378
+
379
+
380
+ @router.post("/{project_id}/targets/remove")
381
+ def remove_targets(project_id: str, body: Any = Body(default=None), session: Session = Depends(require_session)):
382
+ """Unassign one SKU, ASIN or Parent ASIN from the project."""
383
+ runtime = _writer(session)
384
+ if not isinstance(body, dict):
385
+ raise err(400, "invalid_target", "Name the grain and key to remove.")
386
+ pair = _pair(body.get("grain"), body.get("key"))
387
+ result = _call(projects.remove_targets, runtime, project_id, [pair], by=_by(session))
388
+ _repaint(runtime, [pair])
389
+ return _target_answer(runtime, project_id, result)
api/routes_app_dashboard.py CHANGED
@@ -3,21 +3,83 @@ from __future__ import annotations
3
 
4
  from datetime import date, timedelta
5
  from importlib import import_module
 
6
 
7
  from fastapi import APIRouter, Depends, HTTPException, Query
8
 
9
  from deps import Session, err, require_session
10
- from core.app_dashboard import checked_dashboard
11
  from core.request_timing import raise_if_resource_error, remaining_budget
12
 
13
  router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
 
15
 
16
  def admit(session, kind, app_id):
17
  """Refuse global saved aggregates when any source has a row or field restriction."""
18
  import app_templates
19
  import application_registry
20
- from core import perm_scope, perms, route_system, user_tables
21
 
22
  runtime, user = session.runtime, session.user
23
  if (not session.tenant or getattr(runtime, "key", None) != session.tenant
@@ -32,28 +94,10 @@ def admit(session, kind, app_id):
32
  or app.get("retired") is True or app.get("templateKind") != kind
33
  or not application_registry.may_read(app, session.uname, session.admin)):
34
  raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
35
- if "All" not in perms.allowed_bu_labels(user) or perms.scope_agent(user) is not None:
36
- raise err(403, "app_dashboard_scope", "This dashboard needs unrestricted source access.")
37
- sources = template.get("sources")
38
- if not isinstance(sources, (tuple, list)) or not sources:
39
- raise err(503, "app_dashboard_sources", "The dashboard source descriptor is unavailable.")
40
- config = getattr(getattr(runtime, "tenant", None), "config", None) or {}
41
- enabled = config.get("modules", "all")
42
- for source in sources:
43
- remaining_budget()
44
- if ((enabled != "all" and source not in (enabled or ()))
45
- or not perm_scope.may_read(user, source, st=runtime)):
46
- raise err(403, "app_dashboard_source", "Your account cannot read a required dashboard source.")
47
- definition = user_tables.get_definition(source, st=runtime)
48
- if not isinstance(definition, dict) or not definition.get("fields"):
49
- raise err(503, "app_dashboard_source", "A dashboard source is unavailable.")
50
- declared = perm_scope.entry(user, source) or {}
51
- if (declared.get("access") is False or declared.get("filter") or declared.get("hiddenFields")
52
- or perm_scope.wall_declared(user, source, st=runtime, defn=definition)
53
- or perm_scope.row_scope_applies(user, source, st=runtime, defn=definition)
54
- or perm_scope.hidden_keys(user, source, definition["fields"], st=runtime)
55
- or route_system.is_managed(definition) or route_system.is_managed(source)):
56
- raise err(403, "app_dashboard_scope", "This dashboard needs unrestricted source access.")
57
  return dict(template)
58
 
59
 
@@ -78,6 +122,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
78
  raise ValueError
79
  except ValueError:
80
  raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
 
81
  try:
82
  remaining_budget()
83
  descriptor = admit(session, kind, app)
@@ -86,7 +131,7 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
86
  options = {}
87
  if kind == "amazon_dashboard":
88
  options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
89
- if kind in ("payouts", "reimbursements"):
90
  options["user"] = dict(session.user)
91
  if kind == "opportunity_finder":
92
  options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
@@ -99,4 +144,5 @@ def read_dashboard(kind: str, app: str = Query(..., min_length=1), start: str |
99
  raise_if_resource_error(exc)
100
  if isinstance(exc, HTTPException):
101
  raise
102
- raise err(503, "app_dashboard_unavailable", "The dashboard could not be read. Try again shortly.") from None
 
 
3
 
4
  from datetime import date, timedelta
5
  from importlib import import_module
6
+ import logging
7
 
8
  from fastapi import APIRouter, Depends, HTTPException, Query
9
 
10
  from deps import Session, err, require_session
11
+ from core.app_dashboard import checked_dashboard, log_failure, plain_cause
12
  from core.request_timing import raise_if_resource_error, remaining_budget
13
 
14
  router = APIRouter(prefix="/api/v1/app-kinds", tags=["app-dashboard"])
15
+ _log = logging.getLogger(__name__)
16
+
17
+
18
+ def _app_source_admission(user, runtime, app_id, source):
19
+ """Bind core delegation to a fresh readable App and its declared sources."""
20
+ import application_registry
21
+ import app_templates
22
+ from core import perms
23
+ tenant = getattr(runtime, 'key', None)
24
+ if not tenant or user.get('tenant') != tenant or not user.get('username'):
25
+ return False
26
+ enabled = (getattr(getattr(runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
27
+ if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
28
+ return False
29
+ app = application_registry.load(runtime, strict=True)['applications'].get(app_id)
30
+ if (not isinstance(app, dict) or app.get('id') != app_id or app.get('tenant') != tenant
31
+ or app.get('retired') is True
32
+ or not application_registry.may_read(app, user['username'], perms.is_admin(user))):
33
+ return False
34
+ template = app_templates.BY_KIND.get(app.get('templateKind'))
35
+ return bool(template and not app_templates.held(app.get('templateKind'))
36
+ and source in (*template['sources'], *template.get('optional_sources', ())))
37
+
38
+
39
+ from core.perm_scope import register_app_source_admission
40
+ register_app_source_admission(_app_source_admission)
41
+
42
+
43
+ def source_scope_for(session, app):
44
+ """Bind a provider callback to the live App's tenant, principal and declared sources."""
45
+ import application_registry
46
+
47
+ def scope(source, records=None):
48
+ return application_registry.app_source_scope(session.user, session.runtime, source,
49
+ app=app, records=records)
50
+
51
+ return scope
52
+
53
+
54
+ def admit_reader(session, kind, app=None, *, sources=None):
55
+ """Admit an App holder while unsupported source narrowing remains refused."""
56
+ import application_registry
57
+ import app_templates
58
+ descriptor = app_templates.BY_KIND[kind]
59
+ if app is None:
60
+ app = application_registry.readable_app(session.runtime, session.tenant, session.uname,
61
+ session.admin, kind)
62
+ if not isinstance(app, dict) or app.get('templateKind') != kind:
63
+ raise err(404, kind + '_app_not_found', 'This App is unavailable or is not shared with you.')
64
+ required = tuple(descriptor['sources'] if sources is None else sources)
65
+ checked = tuple(dict.fromkeys((*required, *descriptor.get('optional_sources', ()))))
66
+ access = application_registry.admit_app(session, app, checked, code=kind)
67
+ from core import user_tables
68
+ enabled = (getattr(getattr(session.runtime, 'tenant', None), 'config', None) or {}).get('modules', 'all')
69
+ for source in required:
70
+ if not _app_source_admission(session.user, session.runtime, app.get('id'), source):
71
+ raise err(403, kind + '_source_unavailable', 'This App cannot read that source.')
72
+ if enabled != 'all' and (not isinstance(enabled, (list, tuple, set, frozenset)) or source not in enabled):
73
+ raise err(403, kind + '_source_unavailable', 'This App source is unavailable in this workspace.')
74
+ if not (user_tables.get_definition(source, st=session.runtime) or {}).get('fields'):
75
+ raise err(503, kind + '_source_contract', f'The field contract for {source} is unavailable. Sync its connector.')
76
+ return access
77
 
78
 
79
  def admit(session, kind, app_id):
80
  """Refuse global saved aggregates when any source has a row or field restriction."""
81
  import app_templates
82
  import application_registry
 
83
 
84
  runtime, user = session.runtime, session.user
85
  if (not session.tenant or getattr(runtime, "key", None) != session.tenant
 
94
  or app.get("retired") is True or app.get("templateKind") != kind
95
  or not application_registry.may_read(app, session.uname, session.admin)):
96
  raise err(404, "app_dashboard_app", "This App is unavailable or is not shared with you.")
97
+ # R46: the source walls live once, in the check availability also reads.
98
+ access = application_registry.app_access(user, runtime, app, template.get("sources"))
99
+ if access["dashboard"] != "ready":
100
+ raise err(access["status"], access["code"], access["message"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  return dict(template)
102
 
103
 
 
122
  raise ValueError
123
  except ValueError:
124
  raise err(400, "app_dashboard_period", "Choose valid dates and a supported comparison.") from None
125
+ descriptor = {}
126
  try:
127
  remaining_budget()
128
  descriptor = admit(session, kind, app)
 
131
  options = {}
132
  if kind == "amazon_dashboard":
133
  options.update(compare=compare, allowed_sources=tuple(descriptor["sources"]), cost_links=None)
134
+ if kind in ("payouts", "reimbursements", "returns_refunds", "repeat_buyers"):
135
  options["user"] = dict(session.user)
136
  if kind == "opportunity_finder":
137
  options.update(opener=_opener(session.runtime, "ut_amzn_brand_analytics"),
 
144
  raise_if_resource_error(exc)
145
  if isinstance(exc, HTTPException):
146
  raise
147
+ log_failure(_log, exc, "read", kind, session.tenant)
148
+ raise err(503, "app_dashboard_unavailable", plain_cause(exc, "read", descriptor.get("sources"))) from None
api/routes_applications.py CHANGED
@@ -311,7 +311,7 @@ def get_customer_card(app_id: str, customer_id: int,
311
  if section != "customer":
312
  _customer_card_admit(session, section, records)
313
 
314
- result = customer_card.card(customer_id, cursor, rt=session.runtime, team_id=team_id,
315
  invoice_limit=limit, invoice_offset=offset, authorize=authorize,
316
  include_agents=agent_field in visible)
317
  result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
 
311
  if section != "customer":
312
  _customer_card_admit(session, section, records)
313
 
314
+ result = customer_card.card(customer_id, cursor, rt=session.runtime, user=session.user, team_id=team_id,
315
  invoice_limit=limit, invoice_offset=offset, authorize=authorize,
316
  include_agents=agent_field in visible)
317
  result["customer"]["name"] = selected.get(app["fieldMap"]["name"])
api/routes_assets.py CHANGED
@@ -894,14 +894,32 @@ def amazon_picture_bytes(session, ref):
894
  return bytes(blob) if blob else None
895
 
896
 
 
 
 
 
 
 
 
 
 
 
 
 
897
  @router.get("/assets/amazon/{ref}")
898
  def amazon_asset(ref: str, q: str = "web",
899
  session: Session = Depends(require_session_or_bearer)):
900
- """Serve one stored Amazon listing picture for THIS session's tenant. `q` is accepted and
901
- ignored: one small JPEG is stored per ASIN. The page never loads an Amazon host."""
902
- blob = amazon_picture_bytes(session, ref)
 
 
 
 
 
 
903
  if blob is None:
904
- # One answer for a bad ref, a missing picture and another tenant's picture.
905
  raise err(404, "asset_not_found", "no image for that reference")
906
  from harness import amazon_pictures
907
  # Only a sha-stamped ref is immutable: the sha changes when the picture changes.
 
894
  return bytes(blob) if blob else None
895
 
896
 
897
+ def amazon_full_picture_bytes(session, ref):
898
+ """`(status, jpeg)` for the full-size copy of `<ASIN>.<sha>` in THIS session's tenant store."""
899
+ from harness import amazon_pictures
900
+ parsed = amazon_pictures.parse_ref(ref)
901
+ if parsed is None:
902
+ return "missing", None
903
+ try:
904
+ return amazon_pictures.full_picture(session.runtime, parsed[0])
905
+ except Exception: # noqa: BLE001
906
+ return "failed", None
907
+
908
+
909
  @router.get("/assets/amazon/{ref}")
910
  def amazon_asset(ref: str, q: str = "web",
911
  session: Session = Depends(require_session_or_bearer)):
912
+ """Serve one stored Amazon listing picture for THIS session's tenant: the 320 px thumbnail,
913
+ or with `q=full` the large copy the server fetched once and stored. The page never loads an
914
+ Amazon host."""
915
+ if q == "full":
916
+ status, blob = amazon_full_picture_bytes(session, ref)
917
+ if status == "failed":
918
+ raise err(502, "picture_unavailable", "the full picture could not be loaded, try again later")
919
+ else:
920
+ blob = amazon_picture_bytes(session, ref)
921
  if blob is None:
922
+ # One answer for a bad ref, a missing picture, a refused link and another tenant's picture.
923
  raise err(404, "asset_not_found", "no image for that reference")
924
  from harness import amazon_pictures
925
  # Only a sha-stamped ref is immutable: the sha changes when the picture changes.
api/routes_auth.py CHANGED
@@ -311,7 +311,8 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
311
  raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
312
  # TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
313
  # ordinary code already proves the factor, and never reset the guess budget on password.
314
- factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code)
 
315
  if not factor_valid:
316
  recovery_code_used = users.consume_recovery_code(user["username"], code)
317
  if not (factor_valid or recovery_code_used or remembered):
@@ -321,6 +322,9 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
321
  except audit.AuditRefused as exc:
322
  print(f"[aios-api] {exc}")
323
  raise err(401, "mfa_invalid", _MFA_INVALID)
 
 
 
324
 
325
  # ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
326
  # account already holds, so a forgotten sign-in on someone else's computer cannot lock the
 
311
  raise err(401, "mfa_required", "Enter the 6 digit code from your authenticator app.")
312
  # TOTP FIRST: redemption consumes a recovery code. Do not spend one when the app's
313
  # ordinary code already proves the factor, and never reset the guess budget on password.
314
+ factor_valid = bool(remembered) or users.totp_valid(rec.get("mfa_secret"), code,
315
+ username=user["username"])
316
  if not factor_valid:
317
  recovery_code_used = users.consume_recovery_code(user["username"], code)
318
  if not (factor_valid or recovery_code_used or remembered):
 
322
  except audit.AuditRefused as exc:
323
  print(f"[aios-api] {exc}")
324
  raise err(401, "mfa_invalid", _MFA_INVALID)
325
+ if factor_valid and not remembered:
326
+ # S82-T134: a verified plaintext secret is sealed in place. Fail silent, no epoch bump.
327
+ users.reseal_mfa(user["username"], rec.get("mfa_secret"))
328
 
329
  # ⭐ WAVE 40, R2 — SINGLE SESSION, NEWEST WINS. Signing in here signs out every session this
330
  # account already holds, so a forgotten sign-in on someone else's computer cannot lock the
api/routes_competitor_watch.py CHANGED
@@ -1,4 +1,6 @@
1
  """Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
 
 
2
  from fastapi import APIRouter, Body, Depends, Query
3
 
4
  from deps import Session, err, require_session
@@ -13,11 +15,11 @@ def _module():
13
  return competitor_watch
14
 
15
 
16
- def _reader(session: Session = Depends(require_session)) -> Session:
17
- """Reads need read access to the brand analytics database."""
18
- from core import perm_scope
19
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
- raise err(403, "competitor_watch_forbidden", "Your account cannot read Amazon brand analytics.")
21
  return session
22
 
23
 
@@ -33,19 +35,20 @@ def _tenant(session):
33
 
34
 
35
  def _seller_alert_access(session):
36
- """Cached seller alerts require unrestricted reads of their offer and ownership sources."""
37
- from core import perm_scope
38
  try:
39
- return all(perm_scope.may_read(session.user, source, st=session.runtime)
40
- and not perm_scope.wall_declared(session.user, source, st=session.runtime)
41
- for source in SELLER_SOURCES)
 
42
  except Exception: # noqa: BLE001
43
  return False
44
 
45
 
46
  def _call(fn, *args, **kwargs):
47
  try:
48
- return fn(*args, **kwargs)
49
  except ValueError as exc:
50
  raise err(400, "competitor_watch_invalid", str(exc)) from exc
51
  except KeyError as exc:
@@ -71,6 +74,21 @@ def _call(fn, *args, **kwargs):
71
 
72
  # --------------------------------------------------------------------------- S78-T17 App binding
73
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
  TEMPLATE_KIND = "competitor_watch"
75
 
76
 
@@ -81,9 +99,12 @@ def _app(session, app_id):
81
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
82
  except Exception as exc: # noqa: BLE001
83
  raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
84
- if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
 
85
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
86
  raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
 
 
87
  return app
88
 
89
 
@@ -108,9 +129,11 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
108
 
109
  @router.get("/terms")
110
  def list_terms(q: str = "", offset: int = 0, limit: int = 50,
 
111
  session: Session = Depends(_app_reader)):
112
  cw = _module()
113
- return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit)
 
114
 
115
 
116
  @router.get("/terms/detail")
@@ -122,13 +145,19 @@ def term_detail(term: str = Query(..., min_length=1), session: Session = Depends
122
  @router.post("/terms")
123
  def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
124
  cw = _module()
125
- return {"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
 
 
 
 
 
 
126
 
127
 
128
  @router.post("/terms/remove")
129
  def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
130
  cw = _module()
131
- return {"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)}
132
 
133
 
134
  @router.get("/settings")
@@ -188,6 +217,33 @@ def _not_ready_message(cw, periods):
188
  return cw.NOT_READY
189
 
190
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
192
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
193
  periods = list(detail.get("periods") or ())
@@ -231,7 +287,13 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
231
  comparison = detail.get("comparison") or {}
232
  ready = comparison.get("ready") is True
233
  return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
 
 
 
 
234
  "termsReady": (page or {}).get("termsReady", True),
 
 
235
  "week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
236
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
237
  "comparisonReady": ready,
@@ -244,13 +306,15 @@ def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
244
  @router.get("/screen")
245
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
246
  q: str = "", offset: int = 0, limit: int = 50,
 
247
  session: Session = Depends(_app_reader)):
248
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
249
  cw = _module()
250
  tenant = _tenant(session)
251
  record = _app(session, app)
252
  page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
253
- limit=limit, selected=term)
254
  terms, picked = page["terms"], page["term"]
 
255
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
256
  return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
 
1
  """Competitor watch App routes: watched terms, term detail, settings and owned alerts."""
2
+ from typing import Literal
3
+
4
  from fastapi import APIRouter, Body, Depends, Query
5
 
6
  from deps import Session, err, require_session
 
15
  return competitor_watch
16
 
17
 
18
+ def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
19
+ """Bind the requested App before admitting its declared source contracts and limits."""
20
+ from routes_app_dashboard import admit_reader
21
+ record = _app(session, app) if isinstance(app, str) and app else None
22
+ admit_reader(session, 'competitor_watch', record)
23
  return session
24
 
25
 
 
35
 
36
 
37
  def _seller_alert_access(session):
38
+ """Cached seller alerts use the App grant and refuse offer or ownership limits."""
39
+ import application_registry
40
  try:
41
+ record = application_registry.readable_app(session.runtime, _tenant(session), session.uname,
42
+ session.admin, TEMPLATE_KIND)
43
+ access = application_registry.admit_app(session, record, SELLER_SOURCES, code="competitor_watch")
44
+ return not any(wall["rows"] or wall["hidden"] for wall in access["narrow"].values())
45
  except Exception: # noqa: BLE001
46
  return False
47
 
48
 
49
  def _call(fn, *args, **kwargs):
50
  try:
51
+ return _term_labels(fn(*args, **kwargs))
52
  except ValueError as exc:
53
  raise err(400, "competitor_watch_invalid", str(exc)) from exc
54
  except KeyError as exc:
 
74
 
75
  # --------------------------------------------------------------------------- S78-T17 App binding
76
 
77
+ def _term_labels(value):
78
+ """Attach decoded labels while retaining stored term keys in every route payload."""
79
+ cw = _module()
80
+ if isinstance(value, list):
81
+ return [_term_labels(item) for item in value]
82
+ if isinstance(value, dict):
83
+ result = {key: _term_labels(item) for key, item in value.items()}
84
+ if isinstance(value.get("term"), str):
85
+ result["termLabel"] = cw.display_term(value["term"])
86
+ if isinstance(value.get("terms"), list) and all(isinstance(term, str) for term in value["terms"]):
87
+ result["termLabels"] = {term: cw.display_term(term) for term in value["terms"]}
88
+ return result
89
+ return value
90
+
91
+
92
  TEMPLATE_KIND = "competitor_watch"
93
 
94
 
 
99
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
100
  except Exception as exc: # noqa: BLE001
101
  raise err(503, "competitor_watch_unavailable", "The App could not be read. Try again shortly.") from exc
102
+ if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
103
+ or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
104
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
105
  raise err(404, "competitor_watch_app_not_found", "That Competitor watch App does not exist.")
106
+ from routes_app_dashboard import admit_reader
107
+ admit_reader(session, 'competitor_watch', app)
108
  return app
109
 
110
 
 
129
 
130
  @router.get("/terms")
131
  def list_terms(q: str = "", offset: int = 0, limit: int = 50,
132
+ tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
133
  session: Session = Depends(_app_reader)):
134
  cw = _module()
135
+ return _call(cw.term_page, _tenant(session), st=session.runtime, search=q, offset=offset, limit=limit,
136
+ tracked=tracked, termOrder=termOrder)
137
 
138
 
139
  @router.get("/terms/detail")
 
145
  @router.post("/terms")
146
  def add_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
147
  cw = _module()
148
+ return _term_labels({"terms": _call(cw.add_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
149
+
150
+
151
+ @router.patch("/terms/star")
152
+ def star_term(term: str = Body(...), starred: bool = Body(...), session: Session = Depends(_app_writer)):
153
+ cw = _module()
154
+ return _call(cw.star_term, _tenant(session), term, starred, st=session.runtime)
155
 
156
 
157
  @router.post("/terms/remove")
158
  def remove_terms(terms: list[str] = Body(..., embed=True), session: Session = Depends(_app_writer)):
159
  cw = _module()
160
+ return _term_labels({"terms": _call(cw.remove_terms, _tenant(session), terms, st=session.runtime, page_only=True)})
161
 
162
 
163
  @router.get("/settings")
 
217
  return cw.NOT_READY
218
 
219
 
220
+ def _term_rows(cw, tenant, st, page):
221
+ """Read dated-week counts for this server page without loading its ranking history."""
222
+ terms = page["terms"]
223
+ counts = {}
224
+ if terms:
225
+ try:
226
+ with cw._reader(tenant, st) as cur:
227
+ cols = cw._ba_columns(cur)
228
+ if {"search_term", "_period_start", "_period_end"} <= cols:
229
+ key = cw._term_sql("search_term")
230
+ rows = cur.execute(
231
+ f'SELECT {key}, COUNT(DISTINCT CAST("_period_start" AS DATE)) '
232
+ f'FROM {cw.BA_TABLE} WHERE {key} IN (' + ','.join('?' for _ in terms) + ') '
233
+ f'AND ({cw._weekly_period_sql()}) GROUP BY {key}', terms).fetchall()
234
+ counts = dict(rows)
235
+ except Exception as exc: # noqa: BLE001
236
+ cause = str(exc) if isinstance(exc, cw._ReadFailed) else type(exc).__name__
237
+ raise cw.SourceNotReady(cause, page=page) from exc
238
+ flags = page.get("termFlags") or {}
239
+ checked = page.get("trackingReady") is True
240
+ return [{"term": term, "termLabel": cw.display_term(term), "datedWeeks": counts.get(term, 0),
241
+ "starred": flags.get(term, {}).get("starred") is True,
242
+ "type": "Ours" if checked and flags.get(term, {}).get("ours") is True else
243
+ "Starred" if flags.get(term, {}).get("starred") is True else "Watched"}
244
+ for term in terms]
245
+
246
+
247
  def screen_payload(cw, detail, terms, term, can_edit, *, page=None):
248
  """Project `term_detail` to the screen: shares in percent points, previous values, causes, trend."""
249
  periods = list(detail.get("periods") or ())
 
287
  comparison = detail.get("comparison") or {}
288
  ready = comparison.get("ready") is True
289
  return {"canEdit": bool(can_edit), "terms": list(terms), "term": term,
290
+ "termLabel": cw.display_term(term) if term else None,
291
+ "termLabels": {key: cw.display_term(key) for key in [*terms, *([term] if term else [])]},
292
+ "termFlags": (page or {}).get("termFlags", {}),
293
+ "termRows": (page or {}).get("termRows", []),
294
  "termsReady": (page or {}).get("termsReady", True),
295
+ "trackingReady": (page or {}).get("trackingReady", False),
296
+ "trackingCause": (page or {}).get("trackingCause"),
297
  "week": {"start": periods[0], "end": _week_end(periods[0])} if periods else None,
298
  "previousWeek": {"start": periods[1], "end": _week_end(periods[1])} if len(periods) > 1 else None,
299
  "comparisonReady": ready,
 
306
  @router.get("/screen")
307
  def app_screen(app: str = Query(..., min_length=1), term: str | None = Query(None),
308
  q: str = "", offset: int = 0, limit: int = 50,
309
+ tracked: bool = False, termOrder: Literal["asc", "desc"] | None = None,
310
  session: Session = Depends(_app_reader)):
311
  """S78-T17: the Terms view in one read (watched terms, the picked term's top 3 and trend)."""
312
  cw = _module()
313
  tenant = _tenant(session)
314
  record = _app(session, app)
315
  page = _call(cw.term_page, tenant, st=session.runtime, search=q, offset=offset,
316
+ limit=limit, selected=term, tracked=tracked, termOrder=termOrder)
317
  terms, picked = page["terms"], page["term"]
318
+ page = {**page, "termRows": _call(_term_rows, cw, tenant, session.runtime, page)}
319
  detail = _call(cw.term_detail, tenant, picked, st=session.runtime) if picked else {}
320
  return screen_payload(cw, detail, terms, picked, _may_edit(session, record), page=page)
api/routes_competitors.py ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Competitors doors: list, add and remove an ASIN's rivals, with each rival's quote and the medians.
2
+
3
+ A SKU reads its ASIN's list. A Parent ASIN has no list (R74). Editors of Command Center write.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from contextlib import contextmanager
9
+ import logging
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from fastapi import APIRouter, Body, Depends
14
+
15
+ from core import amazon_rivals as rivals
16
+ from core import amazon_projects as projects
17
+ from deps import Session, err, require_session
18
+ import routes_amazon_profit
19
+
20
+ router = APIRouter(prefix="/api/v1")
21
+ _log = logging.getLogger(__name__)
22
+
23
+ GRAINS = ("sku", "asin", "parent_asin")
24
+ FORBIDDEN = "Only an editor of Command Center can change competitors."
25
+ NO_MIRROR = "This workspace has no Amazon data yet."
26
+
27
+
28
+ @contextmanager
29
+ def _open_mirror(runtime):
30
+ """A read cursor on this tenant's Amazon mirror, None when the file does not exist yet."""
31
+ from harness import runtime as R
32
+ if not Path(runtime.datastore_path).is_file():
33
+ yield None
34
+ return
35
+ cur = R.mirror_cursor(runtime, require_ready=False)
36
+ try:
37
+ yield cur
38
+ finally:
39
+ cur.close()
40
+
41
+
42
+ def _rt(session):
43
+ session.require(routes_amazon_profit.amazon_profit.TABLE_KEY)
44
+ try:
45
+ rivals._backend(session.runtime)
46
+ except rivals.BackendUnavailable:
47
+ raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
48
+ return session.runtime
49
+
50
+
51
+ def _grain(grain, key):
52
+ if grain not in GRAINS:
53
+ raise err(400, "bad_grain", "Grain must be sku, asin or parent_asin.")
54
+ if not isinstance(key, str) or not key.strip():
55
+ raise err(400, "bad_key", "Name the SKU, ASIN or Parent ASIN.")
56
+ return key.strip()
57
+
58
+
59
+ def _target(cur, grain, key):
60
+ """(asin, parent answer or None) for a grain and key."""
61
+ if grain == "sku":
62
+ try:
63
+ chain = projects.lineage(cur, "sku", key) if cur is not None else []
64
+ except projects.MirrorUnreadable as exc:
65
+ raise rivals.MirrorUnreadable(str(exc)) from exc
66
+ found = next((value for grain, value in chain if grain == "asin"), None)
67
+ if found is None:
68
+ raise err(404, "sku_not_listed", "This SKU has no ASIN in your Amazon listings.")
69
+ return found, None
70
+ try:
71
+ target = rivals.asin(key)
72
+ except ValueError as exc:
73
+ raise err(400, "bad_asin", str(exc)) from None
74
+ try:
75
+ kids = routes_amazon_profit.amazon_profit.M.parent_children(cur, target) if cur is not None else []
76
+ except projects.MirrorUnreadable as exc:
77
+ raise rivals.MirrorUnreadable(str(exc)) from exc
78
+ if grain == "parent_asin" or (kids and not rivals.our_asins(cur, [target])):
79
+ return target, {"children": kids}
80
+ return target, None
81
+
82
+
83
+ def _payload(runtime, grain, key, target, parent, can_edit, discovery_error=None):
84
+ if parent is not None:
85
+ # R74: the rollup and each child's standing, never a parent rival list or a parent status.
86
+ kids = parent["children"] or [target]
87
+ found = rivals.standing(runtime, kids)
88
+ return {"grain": grain, "key": key, "asin": target, "parent": True, "rivals": [],
89
+ "children": parent["children"], "cause": rivals.PARENT_ROLLS_UP, "canEdit": False,
90
+ "rollup": rivals.rollup([found.get(a) for a in kids]),
91
+ "childStanding": [found.get(a) or {"asin": a, "competitors_cause": rivals.NO_QUOTE} for a in kids]}
92
+ listed = rivals.rivals(runtime, target)
93
+ quotes = rivals.newest_quotes(runtime, [r["rival_asin"] for r in listed["rivals"]])
94
+ mine = rivals.standing(runtime, [target]).get(target)
95
+ group = (mine or {}).get("rank_group")
96
+ return {"grain": grain, "key": key, "asin": target, "parent": False,
97
+ "rivals": [{"asin": r["rival_asin"], "title": r["title"], "source": r["source"],
98
+ "clickShare": r["click_share"], "term": r["term"], "week": r["week"],
99
+ "addedBy": r["added_by"], "quote": quotes.get(r["rival_asin"]),
100
+ "sameGroup": bool(group) and (quotes.get(r["rival_asin"]) or {}).get("display_group") == group}
101
+ for r in listed["rivals"]],
102
+ "standing": mine,
103
+ "autoCount": sum(1 for r in listed["rivals"] if r["source"] == "auto"),
104
+ "removed": listed["removed"], "week": listed["week"], "discoveredAt": listed["discoveredAt"],
105
+ "cause": listed["cause"], "discoveryError": discovery_error, "canEdit": can_edit}
106
+
107
+
108
+ def _rival_set(runtime, target):
109
+ return {r["rival_asin"] for r in rivals.rivals(runtime, target)["rivals"]}
110
+
111
+
112
+ def _notify_rival_change(runtime, target, before):
113
+ """Repaint affected cells from stored quotes after a committed rival membership change."""
114
+ try:
115
+ if _rival_set(runtime, target) != before:
116
+ from amazon_quotes_loop import _after_quotes
117
+ _after_quotes(runtime.key, (target,))
118
+ except Exception as exc: # noqa: BLE001
119
+ _log.warning("tenant=%s rival-change notification failed (%s); the rival change remains saved.",
120
+ runtime.key, type(exc).__name__)
121
+
122
+
123
+ @router.get("/competitors")
124
+ def competitors_list(grain: str, key: str, session: Session = Depends(require_session)):
125
+ """One SKU's, ASIN's or Parent ASIN's rivals, rediscovered when the newest week moved on."""
126
+ runtime = _rt(session)
127
+ key = _grain(grain, key)
128
+ problem = None
129
+ before = None
130
+ try:
131
+ with _open_mirror(runtime) as cur:
132
+ target, parent = _target(cur, grain, key)
133
+ if cur is None:
134
+ problem = NO_MIRROR
135
+ elif parent is None and rivals.stale(runtime, cur, target):
136
+ previous = _rival_set(runtime, target)
137
+ rivals.discover(runtime, cur, [target])
138
+ before = previous
139
+ except rivals.MirrorUnreadable as exc:
140
+ if grain == "sku":
141
+ raise err(503, "amazon_mirror_unreadable",
142
+ f"The Amazon data could not be read ({exc}). Try again shortly.") from None
143
+ target, parent = rivals.asin(key), (None if grain == "asin" else {"children": []})
144
+ problem = f"The Amazon data could not be read ({exc}), so the stored rivals show."
145
+ except rivals.BackendUnavailable:
146
+ raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
147
+ except RuntimeError:
148
+ raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
149
+ if before is not None:
150
+ _notify_rival_change(runtime, target, before)
151
+ return _payload(runtime, grain, key, target, parent, routes_amazon_profit._may_edit(session), problem)
152
+
153
+
154
+ def _writable(session, body):
155
+ runtime = _rt(session)
156
+ if not routes_amazon_profit._may_edit(session):
157
+ raise err(403, "forbidden", FORBIDDEN)
158
+ body = body if isinstance(body, dict) else {}
159
+ grain = body.get("grain")
160
+ key = _grain(grain, body.get("key"))
161
+ try:
162
+ with _open_mirror(runtime) as cur:
163
+ target, parent = _target(cur, grain, key)
164
+ except rivals.MirrorUnreadable as exc:
165
+ raise err(503, "amazon_mirror_unreadable",
166
+ f"The Amazon data could not be read ({exc}). Try again shortly.") from None
167
+ except rivals.BackendUnavailable:
168
+ raise err(503, "competitors_store_unavailable", "Competitors need the workspace database.") from None
169
+ except RuntimeError:
170
+ raise err(503, "store_not_ready", "Amazon data is still loading. Try again shortly.") from None
171
+ if parent is not None:
172
+ raise err(400, "parent_has_no_rivals", rivals.PARENT_ROLLS_UP)
173
+ try:
174
+ rival = rivals.asin(body.get("rival"), "rival")
175
+ except ValueError as exc:
176
+ raise err(400, "bad_asin", str(exc)) from None
177
+ return runtime, grain, key, target, rival
178
+
179
+
180
+ @router.post("/competitors/rivals")
181
+ def competitors_add(body: Any = Body(default=None), session: Session = Depends(require_session)):
182
+ """Add a rival ASIN by hand. It sits above the auto-found rivals."""
183
+ runtime, grain, key, target, rival = _writable(session, body)
184
+ before = _rival_set(runtime, target)
185
+ try:
186
+ rivals.add(runtime, target, rival, user=session.uname)
187
+ except ValueError as exc:
188
+ raise err(400, "bad_rival", str(exc)) from None
189
+ _notify_rival_change(runtime, target, before)
190
+ return _payload(runtime, grain, key, target, None, True)
191
+
192
+
193
+ @router.post("/competitors/rivals/remove")
194
+ def competitors_remove(body: Any = Body(default=None), session: Session = Depends(require_session)):
195
+ """Remove a wrong rival. It stays removed after the next discovery."""
196
+ runtime, grain, key, target, rival = _writable(session, body)
197
+ before = _rival_set(runtime, target)
198
+ try:
199
+ rivals.remove(runtime, target, rival, user=session.uname)
200
+ except KeyError:
201
+ raise err(404, "rival_not_found", "This ASIN is not a rival of that product.") from None
202
+ _notify_rival_change(runtime, target, before)
203
+ return _payload(runtime, grain, key, target, None, True)
api/routes_connectors.py CHANGED
@@ -2069,8 +2069,13 @@ def odoo_health_receipt(rt, connection_id=None, *, now=None, jobs=None):
2069
  # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
2070
  cadence = stored.get("cadenceSeconds")
2071
  try:
2072
- _main = running_main() # D-781: resolve, never import
2073
- resident = dict(_main.RESYNC_STATE)
 
 
 
 
 
2074
  resident_cid = str(resident.get("connectionId") or "")
2075
  if resident_cid == cid:
2076
  cadence = resident.get("resolvedSeconds") or cadence
@@ -2150,21 +2155,66 @@ def _connection_jobs(rt, provider, connection_id):
2150
  return connector_sync.jobs(rt, provider, cid)
2151
 
2152
 
2153
- def _sync_payload(connector_key, connection_id, row=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2154
  """One wire shape for every registered connector, including one not yet run."""
2155
  meta = connector_sync.providers()[connector_key]
2156
  slot = str(connection_id or "").strip() or "default"
 
 
 
 
 
2157
  if row is None:
2158
- return _safe_connector_status(connector_key, {"provider": connector_key, "label": meta["label"],
2159
  "connectionId": slot, "recorded": False, "state": None,
2160
  "at": None, "stateAt": None, "lastSyncAt": None, "from": None,
2161
  "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
2162
- "databases": [], "retryable": False, "retryHint": meta["retryHint"]})
2163
  answer = dict(row)
 
2164
  answer.update({"provider": connector_key, "connectionId": slot,
2165
- "label": meta["label"], "recorded": True,
2166
  "stateAt": row.get("at"), "lastSyncAt": None,
2167
  "retryHint": meta["retryHint"]})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2168
  return _safe_connector_status(connector_key, answer)
2169
 
2170
 
@@ -2710,7 +2760,8 @@ def _start_database_jobs():
2710
  _materialization_wake.clear()
2711
  try:
2712
  for tenant, rt in tenant_scan.live_tenants(
2713
- runtime, on_error=lambda tenant, exc: _DATABASE_JOBS_LOG.error(
 
2714
  "connector database runtime failed: tenant=%s type=%s",
2715
  tenant, type(exc).__name__)):
2716
  if _materialization_stop.is_set():
@@ -2878,7 +2929,7 @@ def _odoo_status_reads(rt):
2878
  import odoo_relational
2879
  return _ConnectorStatusReads(rt, (
2880
  keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
2881
- odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY,
2882
  _ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
2883
 
2884
 
@@ -2958,7 +3009,22 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
2958
  "receipt": receipt, "lastSyncAt": last_sync_at}
2959
 
2960
 
 
 
 
 
 
 
 
 
 
2961
  def _amazon_completed_sync(rt):
 
 
 
 
 
 
2962
  """Project completion markers without counting business rows or building database detail."""
2963
  import duckdb
2964
  from harness import amazon_store as az, datastore
@@ -2991,9 +3057,19 @@ def _amazon_completed_sync(rt):
2991
 
2992
 
2993
  def _daily_completed_sync(rt, connector_key):
2994
- """Read completed mirror markers; never create a mirror or contact a provider."""
2995
  if connector_key == "amazon_sp":
2996
  return _amazon_completed_sync(rt)
 
 
 
 
 
 
 
 
 
 
2997
  if connector_key == "meta_ads":
2998
  from pathlib import Path
2999
  from harness import datastore
@@ -3038,7 +3114,10 @@ def _amazon_restore_pending(rt):
3038
 
3039
 
3040
  def _amazon_saved_completion(rt):
3041
- """The saved mirror manifest's completed_at as a public stamp, or None."""
 
 
 
3042
  key = str(getattr(rt, "key", "") or "")
3043
  now = time.monotonic()
3044
  cached = _SAVED_COMPLETION_CACHE.get(key)
@@ -3047,7 +3126,7 @@ def _amazon_saved_completion(rt):
3047
  stamp = None
3048
  try:
3049
  from harness import amazon_store as az
3050
- # Read the manifest object only: never rt.get, never a duckdb connection.
3051
  download = getattr(rt, "download_bytes", None)
3052
  raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
3053
  manifest = json.loads(raw) if raw else None
@@ -3076,19 +3155,23 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
3076
  raise err(404, "sync_provider_unknown",
3077
  f"{connector_key} is not registered for connector sync")
3078
  rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
3079
- else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY)
3080
  if connector_key == "amazon_sp" else ()))
3081
  resolved_connection = (_odoo_connection_slot(rt, connection_id)
3082
  if connector_key == "odoo" else connection_id)
3083
  jobs = _connection_jobs(rt, connector_key, resolved_connection)
3084
  row = (connector_sync.job_summary(jobs)
3085
  or connector_sync.read(rt, connector_key, resolved_connection))
3086
- answer = _sync_payload(connector_key, resolved_connection, row)
 
3087
  if connector_key == "odoo":
3088
  receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
3089
- answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
 
3090
  if connector_key in ("amazon_sp", "meta_ads"):
3091
- if connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
 
 
3092
  completed, known = None, False
3093
  else:
3094
  completed, known = _daily_completed_sync(rt, connector_key)
@@ -4019,7 +4102,9 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
4019
  refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
4020
  ((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
4021
  )
4022
- if refused:
 
 
4023
  state = "partial" if success else "failed"
4024
  error = {"code": "amazon_report_refused"}
4025
  elif raw.get("stuck") is True:
@@ -4034,9 +4119,11 @@ def _amazon_database_sync_health(raw, enabled=True, *, now=None, database_key=""
4034
  state, error = "queued", None
4035
  else:
4036
  state, error = "never_run", None
4037
- return _database_sync_health(
4038
  state=state, last_success=success, materialized_timestamp=success,
4039
  row_count=raw.get("rowsLoaded"), error=error, now=now)
 
 
4040
 
4041
 
4042
  def _stored_database_sync_health(raw, enabled=True, *, now=None):
@@ -4834,6 +4921,12 @@ def _amazon_databases(rt, is_admin, spec=None):
4834
  raise_if_resource_error(exc)
4835
  states = {}
4836
 
 
 
 
 
 
 
4837
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
4838
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
4839
  by_report = {}
@@ -4872,6 +4965,13 @@ def _amazon_databases(rt, is_admin, spec=None):
4872
  # what `native` carries: the vendor's own name for the thing, never one we invented. It is
4873
  # empty for a table whose report type is unknown, and the tab says so rather than guessing.
4874
  raw_sync = (states or {}).get(name) if is_admin else None
 
 
 
 
 
 
 
4875
  if isinstance(raw_sync, dict):
4876
  from harness.amazon_store import plain_refusal_reason, is_refusal_reason
4877
  raw_sync = dict(raw_sync)
@@ -5085,9 +5185,11 @@ def _databases_sync_health(databases):
5085
  "succeeded", "disabled") if state in states), "never_run")
5086
  successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
5087
  failure = next((row.get("error") for row in health if row.get("error")), None)
5088
- return _database_sync_health(state=state,
5089
  last_success=max((stamp for stamp in successes if stamp), default=None),
5090
  error=failure)
 
 
5091
 
5092
 
5093
  def _detail_overview(row, databases, graph, *, sync_health=None):
@@ -5650,16 +5752,12 @@ class _ReplacementConflict(Exception):
5650
  """The addressed keychain row changed between replacement preparation and its store update."""
5651
 
5652
 
5653
- def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
5654
- """Validate and encrypt through ``core.keychain.add_entry`` without writing a tenant row.
5655
-
5656
- ``add_entry`` is the keychain's one validation, preview and Fernet implementation. A tiny
5657
- memory-only runtime lets this route reuse it while discarding the temporary identity it mints;
5658
- the tenant store sees only the later, single-row payload swap.
5659
- """
5660
  class _Stage:
5661
  def __init__(self):
5662
  self.bucket = {}
 
5663
 
5664
  def get(self, name):
5665
  return self.bucket if name == kc.KEY else {}
@@ -5676,8 +5774,9 @@ def _encrypted_replacement_payload(kc, label, entry_type, fields, username):
5676
  row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
5677
  if not row.get("enc"):
5678
  raise RuntimeError("the keychain encryptor returned no encrypted payload")
5679
- payload = {"enc": str(row["enc"]), "preview": str(row.get("preview") or ""),
5680
- "envelope_version": row["envelope_version"]}
 
5681
  if entry_type == "amazon_sp":
5682
  payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
5683
  return payload
@@ -5768,7 +5867,7 @@ def _replace_connector_key(connector_key, entry_id, body, session):
5768
  raise err(400, "bad_replacement", refusal)
5769
  try:
5770
  encrypted = _encrypted_replacement_payload(
5771
- kc, entry.get("label"), entry_type, merged_fields, session.uname)
5772
  except kc.KeychainLocked as exc:
5773
  raise err(503, "keychain_locked",
5774
  f"the keychain is locked: {exc}. Nothing was changed.")
 
2069
  # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
2070
  cadence = stored.get("cadenceSeconds")
2071
  try:
2072
+ canonical = connector_sync.status(rt, "odoo", cid, fresh=True)
2073
+ if canonical and isinstance(canonical.get("resync"), dict):
2074
+ resident = dict(canonical["resync"])
2075
+ resident["connectionId"] = cid
2076
+ else:
2077
+ _main = running_main() # D-781: resolve, never import
2078
+ resident = dict(_main.RESYNC_STATE)
2079
  resident_cid = str(resident.get("connectionId") or "")
2080
  if resident_cid == cid:
2081
  cadence = resident.get("resolvedSeconds") or cadence
 
2155
  return connector_sync.jobs(rt, provider, cid)
2156
 
2157
 
2158
+ _PASS_STATES = {"running": "syncing", "continuing": "syncing", "done": "ready", "waiting": "ready",
2159
+ "held": "partial", "failed": "error"}
2160
+
2161
+
2162
+ def _stored_sync_row(rt, connector_key, slot):
2163
+ """C-status: a recorded lifecycle row from the store's last pass and last good sync, or None."""
2164
+ entry = connector_sync.status(rt, connector_key, slot, fresh=True) if rt is not None else None
2165
+ if not entry:
2166
+ return None
2167
+ last_pass = entry.get("pass") if isinstance(entry.get("pass"), dict) else {}
2168
+ good = _sync_stamp(entry.get("lastGoodSyncAt"))
2169
+ state = _PASS_STATES.get(last_pass.get("outcome")) or ("ready" if good else None)
2170
+ at = _sync_stamp(last_pass.get("endedAt") or last_pass.get("startedAt")) or good
2171
+ detail = str(last_pass.get("detail") or "") or ("Sync finished." if state == "ready" else "")
2172
+ return {"state": state, "at": at, "from": None, "detail": detail,
2173
+ "errors": [detail] if state == "error" and detail else [],
2174
+ "errorsTotal": 1 if state == "error" and detail else 0, "databases": [],
2175
+ "retryable": state in ("error", "partial"), "storedStatus": entry}
2176
+
2177
+
2178
+ def _sync_payload(connector_key, connection_id, row=None, *, rt=None):
2179
  """One wire shape for every registered connector, including one not yet run."""
2180
  meta = connector_sync.providers()[connector_key]
2181
  slot = str(connection_id or "").strip() or "default"
2182
+ stored_row = _stored_sync_row(rt, connector_key, slot)
2183
+ entry = (stored_row or {}).get("storedStatus") or {}
2184
+ if row is None:
2185
+ row = stored_row
2186
+ recorded = row is not None
2187
  if row is None:
2188
+ row = {"provider": connector_key, "label": meta["label"],
2189
  "connectionId": slot, "recorded": False, "state": None,
2190
  "at": None, "stateAt": None, "lastSyncAt": None, "from": None,
2191
  "detail": "no sync has been recorded", "errors": [], "errorsTotal": 0,
2192
+ "databases": [], "retryable": False, "retryHint": meta["retryHint"]}
2193
  answer = dict(row)
2194
+ answer.pop("storedStatus", None)
2195
  answer.update({"provider": connector_key, "connectionId": slot,
2196
+ "label": meta["label"], "recorded": recorded,
2197
  "stateAt": row.get("at"), "lastSyncAt": None,
2198
  "retryHint": meta["retryHint"]})
2199
+ if entry:
2200
+ last_pass = entry.get("pass") or {}
2201
+ duration, budget = last_pass.get("durationSeconds"), last_pass.get("budgetSeconds")
2202
+ import math
2203
+ def minutes(value, *, positive=False):
2204
+ return (round(value / 60, 3) if type(value) in (int, float) and math.isfinite(value)
2205
+ and (value > 0 if positive else value >= 0) else None)
2206
+ days = entry.get("dailyPasses") or {}
2207
+ answer.update(lastGoodSyncAt=entry.get("lastGoodSyncAt"), lastSyncAt=entry.get("lastGoodSyncAt"),
2208
+ lastCompletedKnown=True,
2209
+ **{"pass": last_pass}, passMinutes=minutes(duration), budgetMinutes=minutes(budget, positive=True),
2210
+ dailyPasses=days, dailyPass=days[max(days)] if days else None,
2211
+ resync=entry.get("resync"), held=last_pass.get("held") or {},
2212
+ notOffered=last_pass.get("notOffered") or 0)
2213
+ if connector_key == "amazon_sp" and rt is not None:
2214
+ from harness import amazon_store
2215
+ job = amazon_store.first_load_job_summary(rt)
2216
+ if job:
2217
+ answer["amazonFirstLoad"] = job
2218
  return _safe_connector_status(connector_key, answer)
2219
 
2220
 
 
2760
  _materialization_wake.clear()
2761
  try:
2762
  for tenant, rt in tenant_scan.live_tenants(
2763
+ runtime, name="connector-materialization",
2764
+ on_error=lambda tenant, exc: _DATABASE_JOBS_LOG.error(
2765
  "connector database runtime failed: tenant=%s type=%s",
2766
  tenant, type(exc).__name__)):
2767
  if _materialization_stop.is_set():
 
2929
  import odoo_relational
2930
  return _ConnectorStatusReads(rt, (
2931
  keychain.KEY, keychain.SCOPE_KEY, odoo_relational.CONFIG_KEY,
2932
+ odoo_relational.MODEL_CATALOG_KEY, connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY,
2933
  _ODOO_HEALTH_STATUS, _DATABASE_DISPATCH_STATUS, _ODOO_SYNC_STATUS))
2934
 
2935
 
 
3009
  "receipt": receipt, "lastSyncAt": last_sync_at}
3010
 
3011
 
3012
+ def _stored_completed_sync(rt, connector_key):
3013
+ """C-status: the store's lastGoodSyncAt as (stamp, True), or None when the store holds no entry."""
3014
+ entry = connector_sync.status(rt, connector_key, fresh=True)
3015
+ if entry is None:
3016
+ return None
3017
+ stamp = _sync_stamp(entry.get("lastGoodSyncAt"))
3018
+ return stamp, True
3019
+
3020
+
3021
  def _amazon_completed_sync(rt):
3022
+ """The store's last good Amazon sync; the mirror markers answer only before the store holds one."""
3023
+ stored = _stored_completed_sync(rt, "amazon_sp")
3024
+ return stored if stored is not None else _amazon_mirror_completion(rt)
3025
+
3026
+
3027
+ def _amazon_mirror_completion(rt):
3028
  """Project completion markers without counting business rows or building database detail."""
3029
  import duckdb
3030
  from harness import amazon_store as az, datastore
 
3057
 
3058
 
3059
  def _daily_completed_sync(rt, connector_key):
3060
+ """Read the stored last good sync; never create a mirror or contact a provider."""
3061
  if connector_key == "amazon_sp":
3062
  return _amazon_completed_sync(rt)
3063
+ if connector_key == "meta_ads":
3064
+ stored = _stored_completed_sync(rt, "meta_ads")
3065
+ return stored if stored is not None else _mirror_completed_sync(rt, "meta_ads")
3066
+ return None, False
3067
+
3068
+
3069
+ def _mirror_completed_sync(rt, connector_key):
3070
+ """The container mirror's completed markers. Writers seed C-status from it; cards read the store."""
3071
+ if connector_key == "amazon_sp":
3072
+ return _amazon_mirror_completion(rt)
3073
  if connector_key == "meta_ads":
3074
  from pathlib import Path
3075
  from harness import datastore
 
3114
 
3115
 
3116
  def _amazon_saved_completion(rt):
3117
+ """The stored last good sync, else the saved mirror manifest's completed_at, or None."""
3118
+ stored = _stored_completed_sync(rt, "amazon_sp")
3119
+ if stored is not None:
3120
+ return stored[0]
3121
  key = str(getattr(rt, "key", "") or "")
3122
  now = time.monotonic()
3123
  cached = _SAVED_COMPLETION_CACHE.get(key)
 
3126
  stamp = None
3127
  try:
3128
  from harness import amazon_store as az
3129
+ # Read the manifest object only: never a duckdb connection.
3130
  download = getattr(rt, "download_bytes", None)
3131
  raw = download(az.MIRROR_PREFIX + "manifest.json") if callable(download) else None
3132
  manifest = json.loads(raw) if raw else None
 
3155
  raise err(404, "sync_provider_unknown",
3156
  f"{connector_key} is not registered for connector sync")
3157
  rt = (_odoo_status_reads(session.runtime) if connector_key == "odoo"
3158
+ else _ConnectorStatusReads(session.runtime, (connector_sync.JOBS_KEY, connector_sync.STATE_KEY, connector_sync.STATUS_KEY)
3159
  if connector_key == "amazon_sp" else ()))
3160
  resolved_connection = (_odoo_connection_slot(rt, connection_id)
3161
  if connector_key == "odoo" else connection_id)
3162
  jobs = _connection_jobs(rt, connector_key, resolved_connection)
3163
  row = (connector_sync.job_summary(jobs)
3164
  or connector_sync.read(rt, connector_key, resolved_connection))
3165
+ # Odoo's lifecycle comes from its job receipts; the other connectors read C-status beside no row.
3166
+ answer = _sync_payload(connector_key, resolved_connection, row, rt=rt)
3167
  if connector_key == "odoo":
3168
  receipt = odoo_health_receipt(rt, resolved_connection, jobs=jobs)
3169
+ if not answer.get("lastGoodSyncAt"):
3170
+ answer["lastSyncAt"] = _odoo_last_sync_at(rt, receipt, connection_id)
3171
  if connector_key in ("amazon_sp", "meta_ads"):
3172
+ if answer.get("lastCompletedKnown") is True:
3173
+ completed, known = answer.get("lastGoodSyncAt"), True
3174
+ elif connector_key == "amazon_sp" and (str(resolved_connection or "").strip() or "default") != "default":
3175
  completed, known = None, False
3176
  else:
3177
  completed, known = _daily_completed_sync(rt, connector_key)
 
4102
  refused = (raw.get("refused") is True or is_refusal_reason(raw.get("unknownCause")) or any(item.get("table") == table for item in
4103
  ((raw.get("firstLoad") or {}).get("refused") or ()) if isinstance(item, dict))
4104
  )
4105
+ if raw.get("notOffered") is True:
4106
+ state, error = "succeeded", None
4107
+ elif refused:
4108
  state = "partial" if success else "failed"
4109
  error = {"code": "amazon_report_refused"}
4110
  elif raw.get("stuck") is True:
 
4119
  state, error = "queued", None
4120
  else:
4121
  state, error = "never_run", None
4122
+ answer = _database_sync_health(
4123
  state=state, last_success=success, materialized_timestamp=success,
4124
  row_count=raw.get("rowsLoaded"), error=error, now=now)
4125
+ answer["notOffered"] = int(raw.get("notOffered") is True)
4126
+ return answer
4127
 
4128
 
4129
  def _stored_database_sync_health(raw, enabled=True, *, now=None):
 
4921
  raise_if_resource_error(exc)
4922
  states = {}
4923
 
4924
+ canonical = connector_sync.status(rt, "amazon_sp", fresh=True) or {}
4925
+ canonical_pass = canonical.get("pass") or {}
4926
+ held_tables = canonical_pass.get("held") or {}
4927
+ from harness import amazon_store as _load_store
4928
+ first_load_job = _load_store.first_load_job_summary(rt) if is_admin else None
4929
+
4930
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
4931
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
4932
  by_report = {}
 
4965
  # what `native` carries: the vendor's own name for the thing, never one we invented. It is
4966
  # empty for a table whose report type is unknown, and the tab says so rather than guessing.
4967
  raw_sync = (states or {}).get(name) if is_admin else None
4968
+ if is_admin and (name in held_tables or first_load_job):
4969
+ raw_sync = dict(raw_sync or {})
4970
+ if name in held_tables:
4971
+ raw_sync["held"] = held_tables[name]
4972
+ raw_sync["notOffered"] = name in (canonical_pass.get("notOfferedTables") or ())
4973
+ if first_load_job:
4974
+ raw_sync["firstLoad"] = {**(raw_sync.get("firstLoad") or {}), **first_load_job}
4975
  if isinstance(raw_sync, dict):
4976
  from harness.amazon_store import plain_refusal_reason, is_refusal_reason
4977
  raw_sync = dict(raw_sync)
 
5185
  "succeeded", "disabled") if state in states), "never_run")
5186
  successes = [_sync_stamp(row.get("lastSuccess")) for row in health]
5187
  failure = next((row.get("error") for row in health if row.get("error")), None)
5188
+ answer = _database_sync_health(state=state,
5189
  last_success=max((stamp for stamp in successes if stamp), default=None),
5190
  error=failure)
5191
+ answer["notOffered"] = sum(row.get("notOffered") or 0 for row in health)
5192
+ return answer
5193
 
5194
 
5195
  def _detail_overview(row, databases, graph, *, sync_health=None):
 
5752
  """The addressed keychain row changed between replacement preparation and its store update."""
5753
 
5754
 
5755
+ def _encrypted_replacement_payload(kc, label, entry_type, fields, username, rt, entry_id):
5756
+ """Validate a temporary keychain row and reseal its fields for the retained tenant and entry."""
 
 
 
 
 
5757
  class _Stage:
5758
  def __init__(self):
5759
  self.bucket = {}
5760
+ self.key = rt.key
5761
 
5762
  def get(self, name):
5763
  return self.bucket if name == kc.KEY else {}
 
5774
  row = ((stage.bucket.get("entries") or {}).get(str(temporary.get("id") or "")) or {})
5775
  if not row.get("enc"):
5776
  raise RuntimeError("the keychain encryptor returned no encrypted payload")
5777
+ validated_fields = kc.read_fields(stage, temporary["id"])
5778
+ envelope = kc.seal_fields(rt.key, str(entry_id), validated_fields)
5779
+ payload = {**envelope, "preview": str(row.get("preview") or "")}
5780
  if entry_type == "amazon_sp":
5781
  payload[AMAZON_ROTATION_DEADLINE_FIELD] = row[AMAZON_ROTATION_DEADLINE_FIELD]
5782
  return payload
 
5867
  raise err(400, "bad_replacement", refusal)
5868
  try:
5869
  encrypted = _encrypted_replacement_payload(
5870
+ kc, entry.get("label"), entry_type, merged_fields, session.uname, session.runtime, entry_id)
5871
  except kc.KeychainLocked as exc:
5872
  raise err(503, "keychain_locked",
5873
  f"the keychain is locked: {exc}. Nothing was changed.")
api/routes_customers.py CHANGED
@@ -278,10 +278,11 @@ def warm_default(rt):
278
  def warm_bu_pools(rt):
279
  """Warm each permitted business-unit pool outside the request path."""
280
  from core import context, perms
 
281
  for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
282
- if label == "All" or label not in context.BRANDS:
283
  continue
284
- team = perms.scope_team_id({"bus": [context.BRANDS[label]], "role": "user"})
285
  if team is None:
286
  continue
287
  try:
@@ -1303,7 +1304,16 @@ def allowed_pids(session: Session):
1303
 
1304
  @router.get("/customers")
1305
  def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
1306
- return _payload(session, related=related)
 
 
 
 
 
 
 
 
 
1307
 
1308
 
1309
  @router.post("/customers/aggregates")
@@ -1339,10 +1349,10 @@ def assigned_view_rows(view_id: str, session: Session = Depends(module_gate(MODU
1339
  session, MODULE, rows, g['fields'], g['measures'], None)
1340
  rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
1341
  rows = _exclude_sourceless(rows, g['fields'])
1342
- return {"fields": g["fields"], "rows": rows, "viewId": view_id,
1343
  "locked": True, "today": g["today"],
1344
  "refusals": metric_refusals,
1345
- "limits": field_limits + linked_metric_limits(metric_refusals)}
1346
 
1347
 
1348
  # ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
@@ -1649,6 +1659,26 @@ def _visible_routes(session, records=None, application=None):
1649
  return [route for route, _mark in _visible_normalized_routes(session, records, application)]
1650
 
1651
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1652
  def _legacy_route_navigation(session):
1653
  """Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
1654
  try:
@@ -1668,7 +1698,7 @@ def _route_navigation(session, routes, application=None):
1668
  legacy_folders=folders, legacy_placements=placements, application=application)
1669
 
1670
 
1671
- def _route_dto(route, allowed, active, session, share_mark=None):
1672
  from core import route_system
1673
  allowed = {int(pid) for pid in allowed}
1674
  stops = [stop for stop in route_system.ordered_stops(route)
@@ -1696,6 +1726,9 @@ def _route_dto(route, allowed, active, session, share_mark=None):
1696
  # ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
1697
  # Only a projected Route ever carried that marker, and nothing mints one now.
1698
  can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
 
 
 
1699
  dto = {"id": route["id"], "name": route["name"], "color": route["color"],
1700
  "application": route.get("application"),
1701
  "number": route["number"], "visibleStopIds": visible,
@@ -1754,6 +1787,32 @@ def _route_or_404(session, route_id, application=None):
1754
  return route
1755
 
1756
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1757
  def _route_mutation_allowed(session, route, requested_source=None):
1758
  """W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
1759
 
@@ -1896,15 +1955,19 @@ def routes(session: Session = Depends(module_gate(MODULE)), application: str = N
1896
  50,000 visible records sends exactly the same request as an empty grid; a future annotation
1897
  door may accept a compact saved-View/filter reference without reopening that row-sized wire.
1898
  """
1899
- application = _entry_route_application(session, application)
1900
  allowed = allowed_pids(session)
1901
  records = _route_share_records(session)
1902
  navigation = _route_navigation(session, _visible_routes(session, records, application), application)
1903
- return {"routes": [_route_dto(
1904
- route, allowed, (), session,
1905
- _route_share_mark(session, route, records))
1906
- for route in navigation["routes"]],
1907
- "folders": navigation["folders"]}
 
 
 
 
1908
 
1909
 
1910
  @router.get("/customers/routes/delivery-recipients")
@@ -2079,10 +2142,13 @@ def remove_route(route_id: str, session: Session = Depends(module_gate(MODULE)),
2079
  def move_route_to_folder(route_id: str, body: dict = Body(default=None),
2080
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2081
  from core import route_system
2082
- application = _entry_route_application(session, application, write=True)
2083
  _route_application_body(body or {}, application)
2084
  routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
2085
  if str(route_id) not in {str(route.get("id")) for route in routes}:
 
 
 
2086
  raise err(404, "route_not_found", "that Route does not exist")
2087
  folder_id = (body or {}).get("folderId")
2088
  if folder_id is not None and not isinstance(folder_id, str):
@@ -2102,12 +2168,16 @@ def move_route_to_folder(route_id: str, body: dict = Body(default=None),
2102
  def reorder_route_navigation(body: dict = Body(default=None),
2103
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2104
  from core import route_system
2105
- application = _entry_route_application(session, application, write=True)
2106
  _route_application_body(body or {}, application)
2107
  routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
2108
  order = (body or {}).get("order")
2109
  if not isinstance(order, list):
2110
  raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
 
 
 
 
2111
  result = route_system.reorder_routes(
2112
  session.runtime, session.uname, order, routes,
2113
  legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
@@ -2715,8 +2785,8 @@ def route_delivery_list(session: Session = Depends(module_gate(MODULE)), applica
2715
  def normalized_route_delivery_list(route_id: str,
2716
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2717
  """Deliveries for the one normalized Route whose detail is open."""
2718
- application = _entry_route_application(session, application)
2719
- _route_or_404(session, route_id, application)
2720
  return {"deliveries": _route_delivery_definitions(session, route_id, application)}
2721
 
2722
 
 
278
  def warm_bu_pools(rt):
279
  """Warm each permitted business-unit pool outside the request path."""
280
  from core import context, perms
281
+ brands = context.brands()
282
  for label in perms.allowed_bu_labels({"bus": "all", "role": "admin"}):
283
+ if label == "All" or label not in brands:
284
  continue
285
+ team = perms.scope_team_id({"bus": [brands[label]], "role": "user"})
286
  if team is None:
287
  continue
288
  try:
 
1304
 
1305
  @router.get("/customers")
1306
  def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
1307
+ return _audit_customer_response(_payload(session, related=related), session)
1308
+
1309
+
1310
+ def _audit_customer_response(payload, session):
1311
+ if not isinstance(payload, dict) or payload.get('ok') is False:
1312
+ return payload
1313
+ from core import personal_reads
1314
+ personal_reads.record_rows(MODULE, payload.get('rows') or [], st=session.runtime,
1315
+ user=session.user, definition={'fields': payload.get('fields') or []})
1316
+ return payload
1317
 
1318
 
1319
  @router.post("/customers/aggregates")
 
1349
  session, MODULE, rows, g['fields'], g['measures'], None)
1350
  rows = _project_cell_notes(MODULE, rows, g['fields'], session.runtime)
1351
  rows = _exclude_sourceless(rows, g['fields'])
1352
+ return _audit_customer_response({"fields": g["fields"], "rows": rows, "viewId": view_id,
1353
  "locked": True, "today": g["today"],
1354
  "refusals": metric_refusals,
1355
+ "limits": field_limits + linked_metric_limits(metric_refusals)}, session)
1356
 
1357
 
1358
  # ── Wave 44: normalized, Loopable-owned Route/Zone APIs ─────────────────────
 
1659
  return [route for route, _mark in _visible_normalized_routes(session, records, application)]
1660
 
1661
 
1662
+ def _cross_partition_routes(session, records, application):
1663
+ """R11: the no-App list also reads Routes in other partitions shared to the caller, plus the
1664
+ no-App partition for an admin whose own Routing App took the request. Never writable here."""
1665
+ from core import route_system
1666
+ records = records if isinstance(records, dict) else _route_share_records(session)
1667
+ home = str(application or "").strip() or None
1668
+ out = []
1669
+ for route in route_system.read(session.runtime).get("routes") or []:
1670
+ if not isinstance(route, dict):
1671
+ continue
1672
+ partition = str(route.get("application") or "").strip() or None
1673
+ if partition == home:
1674
+ continue
1675
+ mark = _route_share_mark(session, route, records)
1676
+ if mark["shared"] or (session.admin and home is not None and partition is None):
1677
+ out.append((deepcopy(route), mark))
1678
+ out.sort(key=lambda item: (int(item[0].get("number") or 0), str(item[0].get("id") or "")))
1679
+ return out
1680
+
1681
+
1682
  def _legacy_route_navigation(session):
1683
  """Wave 42 per-reader Route folders, used only as a no-write migration fallback."""
1684
  try:
 
1698
  legacy_folders=folders, legacy_placements=placements, application=application)
1699
 
1700
 
1701
+ def _route_dto(route, allowed, active, session, share_mark=None, *, writable=True):
1702
  from core import route_system
1703
  allowed = {int(pid) for pid in allowed}
1704
  stops = [stop for stop in route_system.ordered_stops(route)
 
1726
  # ⭐ `not compatibility and ...` led this expression until the legacy projection was retired.
1727
  # Only a projected Route ever carried that marker, and nothing mints one now.
1728
  can_edit = bool(owns_route or mark.get("sharedRole") == "edit" or source_edit)
1729
+ if not writable:
1730
+ # R11: a Route listed from another partition offers no write; the client hides each one.
1731
+ can_edit = source_edit = False
1732
  dto = {"id": route["id"], "name": route["name"], "color": route["color"],
1733
  "application": route.get("application"),
1734
  "number": route["number"], "visibleStopIds": visible,
 
1787
  return route
1788
 
1789
 
1790
+ def _route_for_read_or_404(session, route_id, application=None, requested=None):
1791
+ """A read door's Route lookup. On the no-App door it also finds a Route the no-App list shows
1792
+ from another partition (R11). Every write door keeps `_route_or_404`."""
1793
+ try:
1794
+ return _route_or_404(session, route_id, application)
1795
+ except Exception as exc:
1796
+ if requested is not None or getattr(exc, "status_code", None) != 404:
1797
+ raise
1798
+ found = next((route for route, _mark in _cross_partition_routes(session, None, application)
1799
+ if route.get("id") == str(route_id)), None)
1800
+ if found is None:
1801
+ raise
1802
+ return found
1803
+
1804
+
1805
+ def _cross_partition_refusal(session, route_id, application):
1806
+ """R11: the 409 a no-App write door answers for a Route listed from another partition."""
1807
+ found = next((mark for route, mark in _cross_partition_routes(session, None, application)
1808
+ if route.get("id") == str(route_id)), None)
1809
+ if found is None:
1810
+ return None
1811
+ message = ("This route is shared with you. Only its owner can move it." if found["shared"]
1812
+ else "This route belongs to another Routing App. Only its owner can move it.")
1813
+ return err(409, "route_read_only_here", message)
1814
+
1815
+
1816
  def _route_mutation_allowed(session, route, requested_source=None):
1817
  """W47-T08 / RULING R6 — OWNERSHIP SHORT-CIRCUITS, and that is the whole of instruction 22.
1818
 
 
1955
  50,000 visible records sends exactly the same request as an empty grid; a future annotation
1956
  door may accept a compact saved-View/filter reference without reopening that row-sized wire.
1957
  """
1958
+ requested, application = application, _entry_route_application(session, application)
1959
  allowed = allowed_pids(session)
1960
  records = _route_share_records(session)
1961
  navigation = _route_navigation(session, _visible_routes(session, records, application), application)
1962
+ listed = [_route_dto(route, allowed, (), session, _route_share_mark(session, route, records))
1963
+ for route in navigation["routes"]]
1964
+ if requested is None:
1965
+ # R11: a no-App reader also sees what was shared to them from another partition.
1966
+ rank = len(listed)
1967
+ for index, (route, mark) in enumerate(_cross_partition_routes(session, records, application)):
1968
+ route.update(folderId=None, navOrder=rank + index)
1969
+ listed.append(_route_dto(route, allowed, (), session, mark, writable=False))
1970
+ return {"routes": listed, "folders": navigation["folders"]}
1971
 
1972
 
1973
  @router.get("/customers/routes/delivery-recipients")
 
2142
  def move_route_to_folder(route_id: str, body: dict = Body(default=None),
2143
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2144
  from core import route_system
2145
+ requested, application = application, _entry_route_application(session, application, write=True)
2146
  _route_application_body(body or {}, application)
2147
  routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
2148
  if str(route_id) not in {str(route.get("id")) for route in routes}:
2149
+ refusal = _cross_partition_refusal(session, route_id, application) if requested is None else None
2150
+ if refusal is not None:
2151
+ raise refusal
2152
  raise err(404, "route_not_found", "that Route does not exist")
2153
  folder_id = (body or {}).get("folderId")
2154
  if folder_id is not None and not isinstance(folder_id, str):
 
2168
  def reorder_route_navigation(body: dict = Body(default=None),
2169
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2170
  from core import route_system
2171
+ requested, application = application, _entry_route_application(session, application, write=True)
2172
  _route_application_body(body or {}, application)
2173
  routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session, **_route_scope(application))
2174
  order = (body or {}).get("order")
2175
  if not isinstance(order, list):
2176
  raise err(400, "invalid_route_order", "order must be a list of visible Route ids")
2177
+ if requested is None:
2178
+ # R11: the no-App rail lists other partitions' Routes read-only. They are not ranked here.
2179
+ foreign = {str(route.get("id")) for route, _mark in _cross_partition_routes(session, None, application)}
2180
+ order = [value for value in order if str(value) not in foreign]
2181
  result = route_system.reorder_routes(
2182
  session.runtime, session.uname, order, routes,
2183
  legacy_folders=legacy_folders, legacy_placements=legacy_placements, application=application)
 
2785
  def normalized_route_delivery_list(route_id: str,
2786
  session: Session = Depends(module_gate(MODULE)), application: str = None):
2787
  """Deliveries for the one normalized Route whose detail is open."""
2788
+ requested, application = application, _entry_route_application(session, application)
2789
+ _route_for_read_or_404(session, route_id, application, requested)
2790
  return {"deliveries": _route_delivery_definitions(session, route_id, application)}
2791
 
2792
 
api/routes_external.py CHANGED
@@ -198,6 +198,10 @@ class _PreviewRuntime:
198
  return self.get(name)
199
 
200
  def get_projection(self, name, drop=()):
 
 
 
 
201
  value = self.get(name)
202
  if isinstance(value, dict):
203
  for key in drop or ():
@@ -220,6 +224,12 @@ class _PreviewRuntime:
220
  return sorted(key for key in keys if key.startswith(prefix))
221
 
222
  def get_entry(self, name, entry):
 
 
 
 
 
 
223
  return copy.deepcopy((self.get(name) or {}).get(entry))
224
 
225
  def get_projections(self, names, drop=(), timeout_s=None):
@@ -487,6 +497,8 @@ def _database_preview_documents(session, before, after):
487
  pair[side] = None
488
  continue
489
  rt = _PreviewRuntime(session.runtime)
 
 
490
  rt.put(user_tables.STORE_KEY, bucket)
491
  reader = Session(session.tenant, session.user, session.claims, rt)
492
  try:
@@ -738,14 +750,33 @@ def _query_publication(session: Session, database: str):
738
  source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
739
  source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
740
  if not source or source in {"blank", "loopable"}:
741
- raise ValueError("Missing connector metadata")
742
  identity = provider(session.tenant, source)
743
  if not isinstance(identity, str) or not identity.strip():
744
- raise ValueError("Invalid snapshot identity")
745
  return {"source": source, "snapshot_id": identity}
 
 
746
  except Exception as exc:
747
- raise err(503, "query_source_unavailable",
748
- "This database is temporarily unavailable. Try the same query again shortly.") from exc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
749
 
750
 
751
  def _assert_query_publication(previous, current):
@@ -972,6 +1003,85 @@ def _assert_database_scope(session, database):
972
  {"database": named})
973
 
974
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
975
  def _external_read_scope(session: Session, database, *args, **kwargs):
976
  """Attach external refusal details at the shared Assistant read boundary."""
977
  try:
@@ -981,7 +1091,7 @@ def _external_read_scope(session: Session, database, *args, **kwargs):
981
  if database == "ut_activity":
982
  scoped = _native_activity_scope(session, *args, **kwargs)
983
  else:
984
- scoped = deps.assistant_read_scope(session, database, *args, **kwargs)
985
  if pin is not None and pin.source == database:
986
  mode = kwargs.get("mode")
987
  if scoped.get("source_kind") == "mirror_user_table":
@@ -1146,7 +1256,7 @@ def _native_activity_scope(session, fields=None, filters=None, *, mode=None):
1146
  try:
1147
  activity_log._backend(session.runtime)
1148
  except (TypeError, ValueError) as exc:
1149
- raise err(503, "query_source_unavailable", "Activity storage is unavailable for this workspace.") from exc
1150
  grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
1151
  visible = list(grid["fields"])
1152
  keys = {field["key"] for field in visible}
@@ -2569,9 +2679,10 @@ def query_as(rt, tenant, username, database, payload, scope=None) -> dict:
2569
  if not message.endswith((".", "!", "?")):
2570
  message += "."
2571
  return {"ok": False, "error": {"code": code, "message": message}}
2572
- except Exception:
2573
- return {"ok": False, "error": {"code": "query_source_unavailable",
2574
- "message": "This query could not be completed."}}
 
2575
 
2576
 
2577
  def _query_database(session: Session, database: str, payload: dict):
@@ -2590,10 +2701,11 @@ def _query_database(session: Session, database: str, payload: dict):
2590
  return result
2591
  except HTTPException as exc:
2592
  info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
2593
- if exc.status_code == 503 and info.get("code") in {
2594
- "assistant_store_not_ready", "store_not_ready", "amazon_mirror_not_loaded"}:
2595
- raise err(503, "query_source_unavailable",
2596
- "This database is temporarily unavailable. Try the same query again shortly.") from exc
 
2597
  if info.get("code") == "assistant_bad_filter":
2598
  # S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
2599
  info["message"] = bad_filter_message(info.get("message"))
@@ -2607,6 +2719,8 @@ def _query_database(session: Session, database: str, payload: dict):
2607
  info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
2608
  info["retry_query"].pop("cursor", None)
2609
  raise
 
 
2610
 
2611
 
2612
  def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
@@ -2741,8 +2855,7 @@ def _query_database_page(session: Session, database: str, payload: dict):
2741
  if identity is not None and (
2742
  scoped.get("source_kind") != "runtime_cache"
2743
  or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
2744
- raise err(409, "assistant_snapshot_unavailable",
2745
- "Selected record reads are unavailable in this deployment. Retry after an update.")
2746
  mirror = scoped.get("source_kind") == "mirror_user_table"
2747
  visible_fields = list(scoped.get("fields") or [])
2748
  visible_keys = {
@@ -3575,10 +3688,232 @@ def _database_record_write_guard(session, database, *, field_keys=None):
3575
  from core import user_tables
3576
 
3577
  guard = routes_tables._records_or_refuse
 
 
 
 
3578
  if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
3579
  # The shared wall admits a locked database's update only when every key is an authored field.
3580
- return guard(session, database, cell_updates=dict.fromkeys(field_keys))
3581
- return guard(session, database)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3582
 
3583
 
3584
  def _plan_database_write(session, args):
@@ -3595,19 +3930,18 @@ def _plan_database_write(session, args):
3595
  if action == "rename":
3596
  return _plan_database_rename(session, args)
3597
  routes_tables._managed_or_refuse(session, database, mutation=True)
3598
- bucket_before = session.runtime.get(user_tables.STORE_KEY)
3599
- if not isinstance(bucket_before, dict):
3600
- raise err(503, "store_unavailable", "The database is unavailable. Nothing was saved.")
3601
- table = copy.deepcopy(bucket_before.get(database) or {})
3602
  if not table or not user_tables.is_user_table(database, st=session.runtime):
3603
  raise err(403, "connected_source_read_only",
3604
  "Records from connected sources are read only.")
3605
- revision = _bucket_revision(session.runtime, user_tables.STORE_KEY, bucket_before)
 
3606
  if type(revision.get("rev")) is not int:
3607
  raise err(503, "confirmation_revision_unavailable",
3608
  "The database version could not be checked. Nothing was saved.")
3609
  warnings = []
3610
- bucket_after = copy.deepcopy(bucket_before)
3611
 
3612
  if action.startswith("record_"):
3613
  field_keys = None
@@ -3629,6 +3963,11 @@ def _plan_database_write(session, args):
3629
  record_id = str(args.get("record_id") or "").strip()
3630
  if not record_id.isdigit() or int(record_id) not in g["pids"]:
3631
  raise err(403, "record_unavailable", "That record is not available to this account.")
 
 
 
 
 
3632
  before = copy.deepcopy(rows.get(record_id))
3633
  if before is None:
3634
  raise err(403, "record_unavailable", "That record is not available to this account.")
@@ -3645,7 +3984,8 @@ def _plan_database_write(session, args):
3645
  after[field["key"]] = ""
3646
  else:
3647
  after = None
3648
- target = bucket_after[database]
 
3649
  if action == "record_delete":
3650
  (target.get("rows") or {}).pop(record_id, None)
3651
  else:
@@ -3659,13 +3999,12 @@ def _plan_database_write(session, args):
3659
  # This is part of the UI row writer's durable result, not an optional callback.
3660
  # Include newly encountered select values in the preview and the one guarded commit.
3661
  user_tables._hydrate_choice_options(target.get("fields") or [], [after])
3662
- changed, truncated = _diff_documents(_database_preview_documents(session, bucket_before, bucket_after))
3663
  if truncated:
3664
  warnings.append("The preview diff reached its display cap; split this change.")
3665
  return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
3666
  "warnings": warnings, "revision": revision,
3667
- "bucket_before_digest": _json_digest(bucket_before),
3668
- "bucket_after": bucket_after,
3669
  "planned": {"database": database, "action": action, "record_id": record_id}}
3670
 
3671
  field_key = str(args.get("field_key") or "").strip()
@@ -3719,8 +4058,11 @@ def _plan_database_write(session, args):
3719
  # Build the exact schema document through the core schema mechanics on an isolated runtime.
3720
  # This includes reciprocal-link cleanup/creation while keeping preview free of relation
3721
  # refreshes, audit calls, automation events and every other out-of-store side effect.
 
 
 
3722
  preview = _PreviewRuntime(session.runtime)
3723
- preview.update(user_tables.STORE_KEY, lambda _cur: copy.deepcopy(bucket_before))
3724
  if action == "field_create":
3725
  stored = user_tables.add_field(
3726
  database, after, st=preview, recurrence_validator=recurrence_validator)
@@ -3739,8 +4081,8 @@ def _plan_database_write(session, args):
3739
  warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
3740
  elif not user_tables.delete_field(database, field_key, st=preview):
3741
  raise err(403, "field_refused", "That field cannot be removed.")
3742
- bucket_after = preview.get(user_tables.STORE_KEY)
3743
- changed, truncated = _diff_documents(_database_preview_documents(session, bucket_before, bucket_after))
3744
  if truncated:
3745
  warnings.append("The preview diff reached its display cap; split this change.")
3746
  affected_ids = [field_key]
@@ -3752,8 +4094,7 @@ def _plan_database_write(session, args):
3752
  return {"diff": changed,
3753
  "affected": {"ids": affected_ids, "count": len(affected_ids)},
3754
  "warnings": warnings, "revision": revision,
3755
- "bucket_before_digest": _json_digest(bucket_before),
3756
- "bucket_after": bucket_after,
3757
  "planned": {"database": database, "action": action, "field_key": field_key}}
3758
 
3759
 
@@ -3782,28 +4123,47 @@ def _execute_database_object(session, args, *, plan=None):
3782
  "The database metadata changed after the preview, " + _STALE_FIX) from exc
3783
  return {"database": database, "action": action, "meta": result["meta"],
3784
  "affected": copy.deepcopy(plan.get("affected") or {})}
 
 
 
 
 
 
 
 
 
 
3785
  expected_rev = plan.get("revision", {}).get("rev")
3786
  expected_digest = plan.get("bucket_before_digest")
3787
  replacement = copy.deepcopy(plan.get("bucket_after"))
3788
- if type(expected_rev) is not int or not isinstance(replacement, dict):
3789
  raise err(503, "confirmation_revision_unavailable",
3790
  "The database version could not be checked. Nothing was saved.")
 
 
 
 
 
 
3791
 
3792
  def apply_exact(current):
3793
- if _json_digest(current) != expected_digest:
 
3794
  raise view_audit.ViewRevisionStale(
3795
  rev_sent=expected_rev, rev_current=expected_rev + 1,
3796
  message="The database changed after preview.")
3797
- return copy.deepcopy(replacement)
 
 
3798
 
3799
  try:
3800
- # One customer bucket, one guarded read-modify-write. The expected revision is checked
3801
  # inside the PG lock where available; the closure's digest check also protects a backend
3802
  # that can only provide the runtime's documented guarded fallback.
3803
  user_tables.update_with_activity(
3804
  database, apply_exact, st=session.runtime, by=session.uname,
3805
- via=_trusted_activity_via(session), row_ids=None, field_keys=None,
3806
- flush="sync", expect_rev=expected_rev)
3807
  except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
3808
  raise err(409, "confirmation_stale",
3809
  "The database changed after the preview, " + _STALE_FIX) from exc
@@ -4204,7 +4564,15 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
4204
  )
4205
 
4206
 
 
 
 
 
 
4207
  def _strong_confirmation_required(name, args):
 
 
 
4208
  if name not in external_tools.TOOLS:
4209
  import tool_groups
4210
  resolved = tool_groups.resolve(name, args)
@@ -4429,7 +4797,7 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
4429
  "Part of this change could not be accepted. Nothing was previewed.")
4430
 
4431
  remote_note = _is_website_note(name, normalised)
4432
- plan = result if name == "database_object" or remote_note else None
4433
  if plan is not None:
4434
  diff = copy.deepcopy(plan.get("diff") or [])
4435
  warnings = list(plan.get("warnings") or [])
@@ -4462,6 +4830,9 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
4462
  normalised["arguments"]["expect"] = [
4463
  {"user": row["user"], "role": row["role"]}
4464
  for row in result["current_entries"]]
 
 
 
4465
 
4466
  _canonical_json(diff, capped=False)
4467
  diff_digest = _json_digest(diff)
@@ -4520,6 +4891,8 @@ def _preview_external_write(session, name, arguments, *, issue_token=True, execu
4520
  response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
4521
  if name == "apps" and normalised.get("action") == "catalog_build":
4522
  response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
 
 
4523
  if payload["strong"]:
4524
  response["confirmation_phrase"] = _CONFIRMATION_PHRASE
4525
  if issue_token:
@@ -5055,6 +5428,9 @@ def _confirm_change(session, args):
5055
  if tool == "database_object":
5056
  result = _execute_database_object(
5057
  fresh_session, arguments, plan=recomputed.get("_plan"))
 
 
 
5058
  else:
5059
  from datetime import datetime, timezone
5060
  commit_started = datetime.now(timezone.utc)
@@ -5085,11 +5461,13 @@ def _confirm_change(session, args):
5085
  from core import user_tables
5086
  import routes_nav
5087
  bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
5088
- after_revision = _bucket_revision(
5089
- fresh_session.runtime, bucket, fresh_session.runtime.get(bucket))
5090
  produced_rev = before_revision["rev"] + 1
5091
  if after_revision.get("rev") != produced_rev:
5092
  after_revision = {"rev": produced_rev}
 
 
5093
  else:
5094
  after_revision = copy.deepcopy(guarded_session.runtime._expected)
5095
  after_revision_digest = _json_digest(after_revision)
@@ -5119,10 +5497,14 @@ def _confirm_change(session, args):
5119
  response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
5120
  if tool == "apps" and arguments.get("action") == "catalog_build":
5121
  response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
 
 
 
5122
  _confirmation_status(fresh_session, operation_id, "committed", response)
5123
  if isinstance(stored.get("arguments_ref"), dict):
5124
  _drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
5125
- if tool != "database_object" and receipt.get("persisted") is True:
 
5126
  try:
5127
  targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
5128
  build_kind=payload["buildKind"])
@@ -5254,7 +5636,7 @@ MCP_RESULT_CODES.update({
5254
  "query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
5255
  "query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
5256
  "query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
5257
- "query_source_unavailable": "This database is temporarily unavailable. Try the same query again shortly.",
5258
  "test_connection_production": "Test connections cannot write to production. Use a staging environment.",
5259
  "test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
5260
  "external_test_connection_invalid": "Set test_connection to true or false.",
@@ -5288,7 +5670,8 @@ _MCP_INTERNAL_PROSE = re.compile(
5288
  r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
5289
  r"\b(?:tenant store|dispatcher|document cleaner)\b|"
5290
  r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
5291
- r"(?:[\w.-]+[/\\])*[\w.-]+\.(?:py|tsx?|pyc|ps1|sh|json)(?::\d+)?|"
 
5292
  r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
5293
  _MCP_PROSE_REPLACEMENTS = (
5294
  (r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
@@ -5893,6 +6276,8 @@ def _mcp_capabilities(session, groups, rows):
5893
  required.append("a named non-production environment for this test connection")
5894
  if group == "websites":
5895
  required.extend(_website_requirements(session, note=action == "add_internal_note"))
 
 
5896
  if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
5897
  legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
5898
  if not rows:
@@ -5993,10 +6378,37 @@ def _guide_unread_count(session: Session) -> int:
5993
  return count if type(count) is int and count >= 0 else 0
5994
 
5995
 
 
 
 
 
 
 
 
 
 
 
 
5996
  def _mcp_guide(session: Session, group=None) -> str:
5997
  """Return the group index or one group's action schemas without reading records."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5998
  import tool_groups
5999
  groups = tool_groups.groups()
 
6000
  if isinstance(group, str) and group not in groups and group != "data":
6001
  # A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
6002
  key = group.strip().lower().partition(".")[0]
@@ -6004,9 +6416,10 @@ def _mcp_guide(session: Session, group=None) -> str:
6004
  if name in groups or name == "data"), group)
6005
  alias = group == "data"
6006
  group = tool_groups.canonical_name(group)
 
6007
  if group is not None and group not in groups:
6008
- raise sheets.SheetRefusal("tool_not_found", "Set group to one of: " + ", ".join(groups)
6009
- + ". Omit group to read the index.")
6010
  # One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
6011
  # The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
6012
  try:
@@ -6024,7 +6437,7 @@ def _mcp_guide(session: Session, group=None) -> str:
6024
  rows = [row for row in rows if row["permitted"]]
6025
  capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
6026
  unread = _guide_unread_count(session)
6027
- lines = ["# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
6028
  f"Scheduled results you have not read: {unread}.",
6029
  *(["Read schedules.inbox before answering."] if unread > 0 else []), "",
6030
  "Call a group with action and arguments. Each action keeps its own read or write rules.",
@@ -6176,7 +6589,7 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
6176
  import tool_groups
6177
  resolved = tool_groups.resolve(name, arguments)
6178
  if resolved is None:
6179
- raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
6180
  entry, args = resolved
6181
  if call is not None and call["tool"] == name:
6182
  external_activity.note_action(arguments["action"])
@@ -6191,6 +6604,30 @@ def _dispatch_external_tool(ctx, name, arguments, *, about=None):
6191
  return result
6192
 
6193
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6194
  def _record_mcp_method(session, req, reply):
6195
  """Retain bounded method names for admitted protocol requests without request values."""
6196
  try:
@@ -6249,8 +6686,9 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
6249
  code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
6250
  if code and not call["limitation_recorded"]:
6251
  from routes_feedback import record_limitation
6252
- record_limitation(session, tool=limitation_tool, action=limitation_action,
6253
- code=code, question=call["question"])
 
6254
  except Exception:
6255
  pass
6256
  finally:
@@ -6463,6 +6901,9 @@ def _mcp_next_call(code, message):
6463
  if code == "assistant_unknown_database":
6464
  # The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
6465
  return fix
 
 
 
6466
  return text if not fix or fix in text else (text + " " + fix).strip()
6467
 
6468
 
@@ -6680,9 +7121,11 @@ def _dispatch_mcp_call(req, session: Session, base_url: str = ""):
6680
  out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
6681
  except (TypeError, ValueError, KeyError, AttributeError):
6682
  out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
6683
- except Exception:
 
6684
  out = _tool_refusal(name, "tool_unavailable",
6685
  "The tool could not complete this operation, so try the same call again shortly.", 503)
 
6686
  return _mcp_content(rid, out, session)
6687
 
6688
 
@@ -6736,22 +7179,42 @@ _WRITE_TOOLS = frozenset({
6736
  "database_object", "view_object", "workbook_object", "worksheet_object",
6737
  "range_object", "calculation_object", "validated_batch",
6738
  "confirm_change",
 
 
 
 
 
 
 
 
 
6739
  })
6740
 
6741
 
6742
  def _assert_write_allow_list(*, check_groups=False) -> None:
6743
  declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
6744
- undeclared = declared - _WRITE_TOOLS
6745
- stale = _WRITE_TOOLS - declared
 
6746
  if undeclared or stale:
6747
  raise RuntimeError("External write allow-list mismatch: undeclared="
6748
  + repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
6749
  if check_groups:
6750
  import tool_groups
6751
- mapped = {entry.get("legacy_name") for group in tool_groups.groups().values()
 
6752
  for entry in group["actions"].values()}
6753
  if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
6754
  raise RuntimeError("Every legacy tool must have a grouped action")
 
 
 
 
 
 
 
 
 
6755
 
6756
 
6757
  _assert_write_allow_list()
 
198
  return self.get(name)
199
 
200
  def get_projection(self, name, drop=()):
201
+ project = getattr(self._real, "get_projection", None)
202
+ if str(name) not in self._docs and callable(project):
203
+ # D-11174: an unstaged document is projected by its source, never copied whole here.
204
+ return project(name, drop=drop)
205
  value = self.get(name)
206
  if isinstance(value, dict):
207
  for key in drop or ():
 
224
  return sorted(key for key in keys if key.startswith(prefix))
225
 
226
  def get_entry(self, name, entry):
227
+ if str(name) not in self._docs:
228
+ # D-11174: an unstaged document answers one entry from its source's entry door.
229
+ from core import user_tables
230
+ reader = user_tables._entry_reader(self._real)
231
+ if reader is not None:
232
+ return copy.deepcopy(reader(name, entry))
233
  return copy.deepcopy((self.get(name) or {}).get(entry))
234
 
235
  def get_projections(self, names, drop=(), timeout_s=None):
 
497
  pair[side] = None
498
  continue
499
  rt = _PreviewRuntime(session.runtime)
500
+ # Stage the planned bucket without reading the source's whole document first.
501
+ _seed_preview(rt, user_tables.STORE_KEY, {})
502
  rt.put(user_tables.STORE_KEY, bucket)
503
  reader = Session(session.tenant, session.user, session.claims, rt)
504
  try:
 
750
  source = {"meta ads": "meta", "amazon sp-api": "amazon"}.get(source, source)
751
  source = connector_spec.CONNECTOR_SPEC_KEYS.get(source, source)
752
  if not source or source in {"blank", "loopable"}:
753
+ raise _query_source_unavailable("no_connector")
754
  identity = provider(session.tenant, source)
755
  if not isinstance(identity, str) or not identity.strip():
756
+ raise _query_source_unavailable("invalid_publication")
757
  return {"source": source, "snapshot_id": identity}
758
+ except HTTPException:
759
+ raise
760
  except Exception as exc:
761
+ raise _query_failure(exc, "query publication") from exc
762
+
763
+
764
+ def _query_source_unavailable(cause, *, message=None):
765
+ """S82-T125: a 503 `query_source_unavailable` naming one cause from `deps`' closed set."""
766
+ return err(503, "query_source_unavailable", deps._assistant_source_message(cause, "query", message=message))
767
+
768
+
769
+ def _query_failure(exc, where):
770
+ """Name transport failures and log unexpected exceptions as unavailable tools."""
771
+ import logging
772
+ logging.getLogger(__name__).warning("%s failed (%s)", where, type(exc).__name__)
773
+ transport = isinstance(exc, (ConnectionError, TimeoutError, OSError)) or any(
774
+ cls.__module__.split(".")[0] in {"duckdb", "_duckdb", "psycopg", "psycopg2", "psycopg_pool"}
775
+ and cls.__name__ in {"IOException", "ConnectionException", "OperationalError", "InterfaceError", "PoolTimeout"}
776
+ for cls in type(exc).__mro__)
777
+ if transport:
778
+ return _query_source_unavailable("store_silent")
779
+ return err(503, "tool_unavailable", "This query could not be completed because of an error on the server.")
780
 
781
 
782
  def _assert_query_publication(previous, current):
 
1003
  {"database": named})
1004
 
1005
 
1006
+ def _external_shared_cells(session, database, rows):
1007
+ """Read overlay values for exactly the records returned by the permissioned source reader."""
1008
+ from core import shared_overlay
1009
+ return shared_overlay.cells(database, [row["pid"] for row in rows], st=session.runtime)
1010
+
1011
+
1012
+ def _external_shared_read_scope(session, database, fields=None, filters=None, *, mode=None, **kwargs):
1013
+ """Project shared user cells over permitted connector pages with the existing field walls."""
1014
+ from core import shared_overlay
1015
+
1016
+ shared = {}
1017
+ if str(database).startswith("ut_") and database != "ut_activity":
1018
+ definition, _ = deps._assistant_table_defn(session, database)
1019
+ declared = {field.get("key") for field in definition.get("fields") or ()}
1020
+ shared = {key: dict(field, key=key, source="overlay", shared=True)
1021
+ for key, field in shared_overlay.fields(database, st=session.runtime).items()
1022
+ if key not in declared and isinstance(field, dict)}
1023
+ if not shared:
1024
+ return deps.assistant_read_scope(session, database, fields=fields, filters=filters,
1025
+ mode=mode, **kwargs)
1026
+ # The source reader still resolves every row permission before any overlay cell is fetched.
1027
+ schema = deps.assistant_read_scope(session, database, fields=None, mode="schema", **kwargs)
1028
+ contract = list(definition.get("fields") or ()) + list(shared.values())
1029
+ available = list(schema["fields"]) + list(shared.values())
1030
+ withheld = set()
1031
+ for user in [session.user, *perm_scope._read_users(kwargs.get("also_as", ()))]:
1032
+ hidden = perm_scope.hidden_keys(user, database, contract, st=session.runtime)
1033
+ visible = (perm_scope.assistant_visible_fields(available, user, database)
1034
+ if perm_scope.assistant_entry(user, database) is not None else available)
1035
+ allowed = {field["key"] for field in visible if field.get("key") not in hidden}
1036
+ withheld.update(field["key"] for field in contract if field.get("key") not in allowed)
1037
+ available = [field for field in available if field.get("key") in allowed]
1038
+ selected = deps._assistant_requested_fields(fields, available)
1039
+ try:
1040
+ perm_scope.validate_assistant_filter(filters, available)
1041
+ except ValueError as exc:
1042
+ deps._assistant_refuse_filter(exc, filters, available)
1043
+
1044
+ def refuse(key, code, verb):
1045
+ raise err(400, code, f'Field "{deps._assistant_display_key(key)}" is stored beside the connector rows, '
1046
+ f"so it can be read but not {verb}.")
1047
+
1048
+ for leaf in deps._assistant_filter_leaves(filters):
1049
+ rhs = leaf.get("rhs") if isinstance(leaf, dict) else None
1050
+ for key in (leaf.get("colId") if isinstance(leaf, dict) else None,
1051
+ rhs.get("colId") if isinstance(rhs, dict) else None):
1052
+ if key in shared:
1053
+ refuse(key, "assistant_bad_filter", "filtered")
1054
+ if isinstance(mode, tuple) and mode:
1055
+ if mode[0] == "page" and len(mode) > 3:
1056
+ for rule in mode[3] or ():
1057
+ if rule.get("field") in shared:
1058
+ refuse(rule["field"], "query_sorts_invalid", "sorted")
1059
+ if mode[0] == "aggregate":
1060
+ for spec in mode[1] or ():
1061
+ if spec.get("field") in shared:
1062
+ refuse(spec["field"], "query_aggregates_invalid", "aggregated")
1063
+ for group in (mode[2].get("groupBy") or ()) if len(mode) > 2 else ():
1064
+ if group.get("field") in shared:
1065
+ refuse(group["field"], "query_groups_invalid", "grouped")
1066
+ source_fields = None if fields is None else [key for key in selected if key not in shared]
1067
+ scoped = deps.assistant_read_scope(session, database, fields=source_fields, filters=filters,
1068
+ mode=mode, **kwargs)
1069
+ # Query schemas may omit declared fields that have no mirror expression.
1070
+ source_keys = {field["key"] for field in scoped["fields"]}
1071
+ available = [field for field in available if field["key"] in shared or field["key"] in source_keys]
1072
+ selected = deps._assistant_requested_fields(fields, available)
1073
+ overlay_keys = set(selected) & shared.keys()
1074
+ rows = scoped.get("records") or []
1075
+ cells = _external_shared_cells(session, database, rows) if overlay_keys else {}
1076
+ scoped["records"] = [{"pid": row["pid"], **{
1077
+ key: copy.deepcopy((cells.get(str(row["pid"])) or {}).get(key) if key in overlay_keys else row.get(key))
1078
+ for key in selected}} for row in rows]
1079
+ from core import if_field
1080
+ scoped["fields"] = if_field.project_fields(
1081
+ [copy.deepcopy(field) for field in available if field["key"] in selected], withheld)
1082
+ return scoped
1083
+
1084
+
1085
  def _external_read_scope(session: Session, database, *args, **kwargs):
1086
  """Attach external refusal details at the shared Assistant read boundary."""
1087
  try:
 
1091
  if database == "ut_activity":
1092
  scoped = _native_activity_scope(session, *args, **kwargs)
1093
  else:
1094
+ scoped = _external_shared_read_scope(session, database, *args, **kwargs)
1095
  if pin is not None and pin.source == database:
1096
  mode = kwargs.get("mode")
1097
  if scoped.get("source_kind") == "mirror_user_table":
 
1256
  try:
1257
  activity_log._backend(session.runtime)
1258
  except (TypeError, ValueError) as exc:
1259
+ raise _query_source_unavailable("no_activity_store") from exc
1260
  grid = routes_tables._compute_ut_assembly(session, "ut_activity", with_rows=False, fields_only=True)
1261
  visible = list(grid["fields"])
1262
  keys = {field["key"] for field in visible}
 
2679
  if not message.endswith((".", "!", "?")):
2680
  message += "."
2681
  return {"ok": False, "error": {"code": code, "message": message}}
2682
+ except Exception as exc:
2683
+ refused = _query_failure(exc, "saved query").detail["error"]
2684
+ message = re.split(r"(?<=[.!?])\s+", refused["message"], maxsplit=1)[0]
2685
+ return {"ok": False, "error": {"code": refused["code"], "message": message}}
2686
 
2687
 
2688
  def _query_database(session: Session, database: str, payload: dict):
 
2701
  return result
2702
  except HTTPException as exc:
2703
  info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
2704
+ cause = {"assistant_store_not_ready": deps._assistant_source_cause_of(info.get("message")) or "syncing",
2705
+ "store_not_ready": "syncing",
2706
+ "amazon_mirror_not_loaded": deps._assistant_source_cause_of(info.get("message")) or "loading"}.get(info.get("code"))
2707
+ if exc.status_code == 503 and cause:
2708
+ raise _query_source_unavailable(cause, message=info.get("message")) from exc
2709
  if info.get("code") == "assistant_bad_filter":
2710
  # S79-T11: lim_4f0cea312bfb. The read path's filter refusal names its own rules.
2711
  info["message"] = bad_filter_message(info.get("message"))
 
2719
  info["retry_query"] = {"database": database, **copy.deepcopy(payload)}
2720
  info["retry_query"].pop("cursor", None)
2721
  raise
2722
+ except Exception as exc:
2723
+ raise _query_failure(exc, "database query") from exc
2724
 
2725
 
2726
  def _query_activity_page(session, payload, scoped, selected_keys, sorts, aggregates, groups, limit, cursor):
 
2855
  if identity is not None and (
2856
  scoped.get("source_kind") != "runtime_cache"
2857
  or "identity_pid" not in inspect.signature(deps.assistant_read_scope).parameters):
2858
+ raise err(409, "assistant_snapshot_unavailable", deps._assistant_source_message("not_supported"))
 
2859
  mirror = scoped.get("source_kind") == "mirror_user_table"
2860
  visible_fields = list(scoped.get("fields") or [])
2861
  visible_keys = {
 
3688
  from core import user_tables
3689
 
3690
  guard = routes_tables._records_or_refuse
3691
+ # D-11174: the wall reads this one definition, never a lend of the whole bucket.
3692
+ definition = user_tables.get_definition(database, st=session.runtime)
3693
+ lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: (
3694
+ {database: definition} if isinstance(definition, dict) else {})})
3695
  if field_keys is not None and not user_tables.records_mutable(database, st=session.runtime):
3696
  # The shared wall admits a locked database's update only when every key is an authored field.
3697
+ return guard(session, database, st=lent, cell_updates=dict.fromkeys(field_keys))
3698
+ return guard(session, database, st=lent)
3699
+
3700
+
3701
+ def _database_write_source(runtime):
3702
+ """The runtime that owns the unstaged `user_tables` document beneath any preview layers."""
3703
+ from core import user_tables
3704
+
3705
+ source = runtime
3706
+ while isinstance(source, _PreviewRuntime) and user_tables.STORE_KEY not in source._docs:
3707
+ source = source._real
3708
+ return source
3709
+
3710
+
3711
+ def _database_write_tables(runtime, keys):
3712
+ """D-11174: copy only the named tables of `user_tables`, never the whole bucket."""
3713
+ from core import user_tables
3714
+
3715
+ source = _database_write_source(runtime)
3716
+ reader = user_tables._entry_reader(source)
3717
+ if reader is None:
3718
+ # Only a store without an entry door reads the document; the plan still keeps the named tables.
3719
+ whole = source.get(user_tables.STORE_KEY)
3720
+ if not isinstance(whole, dict):
3721
+ raise err(503, "store_unavailable", "The database is unavailable. Nothing was saved.")
3722
+ reader = lambda _name, key: whole.get(key)
3723
+ tables = {}
3724
+ for key in dict.fromkeys(str(key) for key in keys if key):
3725
+ table = reader(user_tables.STORE_KEY, key)
3726
+ if isinstance(table, dict):
3727
+ tables[key] = copy.deepcopy(table)
3728
+ return tables
3729
+
3730
+
3731
+ def _seed_preview(preview, key, value, revision=None):
3732
+ """Stage the known part of a document in a preview without reading the rest from its source."""
3733
+ preview._initial[key] = copy.deepcopy(value)
3734
+ preview._docs[key] = copy.deepcopy(value)
3735
+ preview._base_revisions[key] = copy.deepcopy(revision)
3736
+ preview._local_revisions[key] = 0
3737
+
3738
+
3739
+ def _database_row_maps():
3740
+ from core import user_table_rows
3741
+ return ("rows",) + tuple(user_table_rows.ROW_MAPS)
3742
+
3743
+
3744
+ def _database_table_delta(before, after):
3745
+ """The definition keys and row-map entries that turn one table into the other, by name."""
3746
+ maps = _database_row_maps()
3747
+ before = before if isinstance(before, dict) else {}
3748
+ after = after if isinstance(after, dict) else {}
3749
+ definition, records = [], {}
3750
+ missing = object()
3751
+ for key in sorted(set(before) | set(after), key=str):
3752
+ old, new = before.get(key, missing), after.get(key, missing)
3753
+ if (key in maps and (old is missing or isinstance(old, dict))
3754
+ and (new is missing or isinstance(new, dict))):
3755
+ old = {} if old is missing else old
3756
+ new = {} if new is missing else new
3757
+ ids = sorted((rid for rid in set(old) | set(new)
3758
+ if old.get(rid, missing) != new.get(rid, missing)), key=str)
3759
+ if ids:
3760
+ records[str(key)] = [str(rid) for rid in ids]
3761
+ elif old != new:
3762
+ definition.append(str(key))
3763
+ return {"definition": definition, "records": records}
3764
+
3765
+
3766
+ def _database_table_guard(table, delta):
3767
+ """One table's definition and the records a delta names: what a confirm must find unchanged."""
3768
+ if not isinstance(table, dict):
3769
+ return None
3770
+ maps = _database_row_maps()
3771
+ # A row map is read only at the ids the delta names, so a record-mode table stays per record.
3772
+ view = {"definition": {str(key): value for key, value in table.items()
3773
+ if not (key in maps and isinstance(value, dict))},
3774
+ "records": {}}
3775
+ for name, ids in delta["records"].items():
3776
+ rows = table.get(name)
3777
+ view["records"][name] = (None if not isinstance(rows, dict)
3778
+ else {rid: rows[rid] for rid in ids if rid in rows})
3779
+ return view
3780
+
3781
+
3782
+ def _database_tables_digest(tables, deltas):
3783
+ return _json_digest({key: _database_table_guard(tables.get(key), deltas[key])
3784
+ for key in sorted(deltas)})
3785
+
3786
+
3787
+ def _apply_database_table_delta(document, key, delta, after):
3788
+ """Write one table's delta in place, so a row-mode store commits only those rows."""
3789
+ if not isinstance(after, dict):
3790
+ if key in document:
3791
+ del document[key]
3792
+ return
3793
+ table = document.get(key)
3794
+ if not isinstance(table, dict):
3795
+ document[key] = copy.deepcopy(after)
3796
+ return
3797
+ for name in delta["definition"]:
3798
+ if name in after:
3799
+ table[name] = copy.deepcopy(after[name])
3800
+ else:
3801
+ table.pop(name, None)
3802
+ for name, ids in delta["records"].items():
3803
+ new = after.get(name)
3804
+ if not isinstance(new, dict):
3805
+ table.pop(name, None)
3806
+ continue
3807
+ rows = table.get(name)
3808
+ if not isinstance(rows, dict):
3809
+ table[name] = {rid: copy.deepcopy(new[rid]) for rid in ids if rid in new}
3810
+ continue
3811
+ for rid in ids:
3812
+ if rid in new:
3813
+ rows[rid] = copy.deepcopy(new[rid])
3814
+ else:
3815
+ rows.pop(rid, None)
3816
+
3817
+
3818
+ def _database_write_plan(tables_before, tables_after, database):
3819
+ """The per-table commit plan: deltas, the before digest, and the target's changed record ids."""
3820
+ deltas = {key: _database_table_delta(tables_before.get(key), tables_after.get(key))
3821
+ for key in sorted(set(tables_before) | set(tables_after))}
3822
+ row_ids = sorted({rid for ids in deltas.get(database, {"records": {}})["records"].values()
3823
+ for rid in ids}, key=str)
3824
+ # `bucket_after` keeps its name for existing readers; it now holds the planned tables only.
3825
+ return {"bucket_before_digest": _database_tables_digest(tables_before, deltas),
3826
+ "table_deltas": deltas, "bucket_after": tables_after, "row_ids": row_ids}
3827
+
3828
+
3829
+ def _plan_database_shared_update(session, database, record_id, raw_values, g, revision):
3830
+ """D-11337: plan a record_update of shared fields on their overlay cell, or None for row fields."""
3831
+ from core import shared_overlay, user_tables
3832
+ import aios_grid
3833
+
3834
+ definitions = shared_overlay.fields(database, st=session.runtime)
3835
+ keys = [str(key) for key in raw_values]
3836
+ shared = [key for key in keys if shared_overlay.is_shared(database, key, st=session.runtime)]
3837
+ if not shared:
3838
+ return None
3839
+ if len(shared) != len(keys):
3840
+ raise err(400, "tool_arguments_invalid",
3841
+ "Change shared fields and other fields in separate calls.")
3842
+ # The grid's shared-cell door refuses an unresolved pid scope before any cell is read.
3843
+ if g.get("limits"):
3844
+ raise err(409, "pid_scope_unresolved",
3845
+ "The permitted record set could not be resolved. Nothing was saved.")
3846
+ by_key = {str(field.get("key")): field for field in g["fields"] if isinstance(field, dict)}
3847
+ from core import shares
3848
+ if not shares.may_edit("database", database, session.uname, is_admin=session.admin,
3849
+ st=session.runtime):
3850
+ raise err(403, "forbidden", "An edit share on this database is required to change a shared cell.")
3851
+ values = _database_value_plan(session, database, int(record_id), raw_values, g["fields"])
3852
+ for key in shared:
3853
+ # D-9735 item 3: `put_cells` stores any scalar, so a server-assigned kind refuses here.
3854
+ if any((field or {}).get("type") == "autonumber"
3855
+ or (field or {}).get("type") in user_tables.AUDIT_FIELD_TYPES
3856
+ for field in (definitions.get(key), by_key.get(key))):
3857
+ raise err(403, "computed_field", "Autonumber and record audit values are assigned by the server.")
3858
+ try:
3859
+ for value in values.values():
3860
+ shared_overlay._value(value)
3861
+ except ValueError as exc:
3862
+ raise err(400, "bad_value", str(exc)[:300]) from exc
3863
+ choices = {}
3864
+ for key, value in values.items():
3865
+ field = by_key.get(key) or definitions.get(key) or {}
3866
+ if field.get("type") in ("select", "multiselect") and value:
3867
+ known = {option.casefold() for option in aios_grid.merge_choice_options(
3868
+ (definitions.get(key) or {}).get("options") or field.get("options"))}
3869
+ parts = [value] if field["type"] == "select" else value.split(",")
3870
+ added = [part for part in parts if part.casefold() not in known]
3871
+ if added:
3872
+ choices[key] = added
3873
+ held = shared_overlay.cells(database, [int(record_id)], st=session.runtime).get(record_id) or {}
3874
+ before = {key: copy.deepcopy(held.get(key)) for key in values}
3875
+ changed, _ = _diff_documents({"databases/" + database: {
3876
+ "before": {"records": {record_id: before}}, "after": {"records": {record_id: values}}}})
3877
+ return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
3878
+ "warnings": [], "revision": {**revision, "shared_fields": _json_digest({
3879
+ key: definitions[key] for key in values})},
3880
+ "shared_cells": {"record_id": record_id, "before": before, "after": values,
3881
+ "fields": {key: copy.deepcopy(definitions[key]) for key in values},
3882
+ "choices": choices},
3883
+ "planned": {"database": database, "action": "record_update", "record_id": record_id}}
3884
+
3885
+
3886
+ def _commit_database_shared_update(session, database, shared):
3887
+ """Write a planned shared-cell update after checking the cell still holds its previewed value."""
3888
+ from core import shared_overlay
3889
+
3890
+ record_id = str(shared.get("record_id") or "")
3891
+ before, after = shared.get("before"), shared.get("after")
3892
+ if not record_id.isdigit() or not isinstance(before, dict) or not isinstance(after, dict) or not after:
3893
+ raise err(503, "confirmation_revision_unavailable",
3894
+ "The database version could not be checked. Nothing was saved.")
3895
+ held = shared_overlay.cells(database, [int(record_id)], st=session.runtime).get(record_id) or {}
3896
+ if any(held.get(key) != value for key, value in before.items()):
3897
+ raise err(409, "confirmation_stale", "The database changed after the preview, " + _STALE_FIX)
3898
+ try:
3899
+ expected = {"cells": before, "fields": shared["fields"]}
3900
+ if shared.get("choices"):
3901
+ shared_overlay.put_cells_and_choice_options(
3902
+ database, int(record_id), after, shared["choices"], st=session.runtime, expected=expected)
3903
+ else:
3904
+ shared_overlay.put_cells(database, int(record_id), after, st=session.runtime, expected=expected)
3905
+ except ValueError as exc:
3906
+ raise err(400, getattr(exc, "code", None) or "bad_value", str(exc)[:300]) from exc
3907
+
3908
+
3909
+ def _field_link_tables(*fields):
3910
+ """The tables a link field's reciprocal lives on, before and after a schema write."""
3911
+ tables = []
3912
+ for field in fields:
3913
+ bag = field.get("link") if isinstance(field, dict) else None
3914
+ if isinstance(bag, dict) and bag.get("table"):
3915
+ tables.append(str(bag["table"]))
3916
+ return tables
3917
 
3918
 
3919
  def _plan_database_write(session, args):
 
3930
  if action == "rename":
3931
  return _plan_database_rename(session, args)
3932
  routes_tables._managed_or_refuse(session, database, mutation=True)
3933
+ # D-11174: a database write reads, copies and diffs only the tables it changes.
3934
+ tables_before = _database_write_tables(session.runtime, (database,))
3935
+ table = copy.deepcopy(tables_before.get(database) or {})
 
3936
  if not table or not user_tables.is_user_table(database, st=session.runtime):
3937
  raise err(403, "connected_source_read_only",
3938
  "Records from connected sources are read only.")
3939
+ revision = _bucket_revision(_database_write_source(session.runtime), user_tables.STORE_KEY,
3940
+ tables_before)
3941
  if type(revision.get("rev")) is not int:
3942
  raise err(503, "confirmation_revision_unavailable",
3943
  "The database version could not be checked. Nothing was saved.")
3944
  warnings = []
 
3945
 
3946
  if action.startswith("record_"):
3947
  field_keys = None
 
3963
  record_id = str(args.get("record_id") or "").strip()
3964
  if not record_id.isdigit() or int(record_id) not in g["pids"]:
3965
  raise err(403, "record_unavailable", "That record is not available to this account.")
3966
+ if action == "record_update":
3967
+ shared_plan = _plan_database_shared_update(
3968
+ session, database, record_id, raw_values, g, revision)
3969
+ if shared_plan is not None:
3970
+ return shared_plan
3971
  before = copy.deepcopy(rows.get(record_id))
3972
  if before is None:
3973
  raise err(403, "record_unavailable", "That record is not available to this account.")
 
3984
  after[field["key"]] = ""
3985
  else:
3986
  after = None
3987
+ tables_after = copy.deepcopy(tables_before)
3988
+ target = tables_after[database]
3989
  if action == "record_delete":
3990
  (target.get("rows") or {}).pop(record_id, None)
3991
  else:
 
3999
  # This is part of the UI row writer's durable result, not an optional callback.
4000
  # Include newly encountered select values in the preview and the one guarded commit.
4001
  user_tables._hydrate_choice_options(target.get("fields") or [], [after])
4002
+ changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
4003
  if truncated:
4004
  warnings.append("The preview diff reached its display cap; split this change.")
4005
  return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
4006
  "warnings": warnings, "revision": revision,
4007
+ **_database_write_plan(tables_before, tables_after, database),
 
4008
  "planned": {"database": database, "action": action, "record_id": record_id}}
4009
 
4010
  field_key = str(args.get("field_key") or "").strip()
 
4058
  # Build the exact schema document through the core schema mechanics on an isolated runtime.
4059
  # This includes reciprocal-link cleanup/creation while keeping preview free of relation
4060
  # refreshes, audit calls, automation events and every other out-of-store side effect.
4061
+ # A link field's reciprocal lives on its target table, so that table joins the plan.
4062
+ tables_before.update(_database_write_tables(
4063
+ session.runtime, [key for key in _field_link_tables(before, after) if key not in tables_before]))
4064
  preview = _PreviewRuntime(session.runtime)
4065
+ _seed_preview(preview, user_tables.STORE_KEY, tables_before, revision)
4066
  if action == "field_create":
4067
  stored = user_tables.add_field(
4068
  database, after, st=preview, recurrence_validator=recurrence_validator)
 
4081
  warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
4082
  elif not user_tables.delete_field(database, field_key, st=preview):
4083
  raise err(403, "field_refused", "That field cannot be removed.")
4084
+ tables_after = preview.get(user_tables.STORE_KEY)
4085
+ changed, truncated = _diff_documents(_database_preview_documents(session, tables_before, tables_after))
4086
  if truncated:
4087
  warnings.append("The preview diff reached its display cap; split this change.")
4088
  affected_ids = [field_key]
 
4094
  return {"diff": changed,
4095
  "affected": {"ids": affected_ids, "count": len(affected_ids)},
4096
  "warnings": warnings, "revision": revision,
4097
+ **_database_write_plan(tables_before, tables_after, database),
 
4098
  "planned": {"database": database, "action": action, "field_key": field_key}}
4099
 
4100
 
 
4123
  "The database metadata changed after the preview, " + _STALE_FIX) from exc
4124
  return {"database": database, "action": action, "meta": result["meta"],
4125
  "affected": copy.deepcopy(plan.get("affected") or {})}
4126
+ if plan.get("shared_cells") is not None:
4127
+ # D-11337: a shared field's value lives in the overlay, never in the table's rows.
4128
+ _commit_database_shared_update(session, database, plan["shared_cells"])
4129
+ try:
4130
+ import routes_tables
4131
+ routes_tables._refresh_relations(session)
4132
+ except Exception as exc: # noqa: BLE001
4133
+ print(f"[external] relation refresh deferred: {type(exc).__name__}: {exc}")
4134
+ return {"database": database, **copy.deepcopy(plan.get("planned") or {}),
4135
+ "affected": copy.deepcopy(plan.get("affected") or {})}
4136
  expected_rev = plan.get("revision", {}).get("rev")
4137
  expected_digest = plan.get("bucket_before_digest")
4138
  replacement = copy.deepcopy(plan.get("bucket_after"))
4139
+ if type(expected_rev) is not int or not isinstance(replacement, dict) or not expected_digest:
4140
  raise err(503, "confirmation_revision_unavailable",
4141
  "The database version could not be checked. Nothing was saved.")
4142
+ deltas = copy.deepcopy(plan.get("table_deltas"))
4143
+ if not isinstance(deltas, dict):
4144
+ # A plan without deltas replaces its planned tables whole, under the same digest guard.
4145
+ deltas = {key: _database_table_delta(None, table) for key, table in replacement.items()}
4146
+ # The changed record ids keep a record-mode table per record; an older plan selects every record.
4147
+ row_ids = None if plan.get("row_ids") is None else list(plan["row_ids"])
4148
 
4149
  def apply_exact(current):
4150
+ # D-11174: read and write only the planned tables, in place, so no whole document moves.
4151
+ if _database_tables_digest(current, deltas) != expected_digest:
4152
  raise view_audit.ViewRevisionStale(
4153
  rev_sent=expected_rev, rev_current=expected_rev + 1,
4154
  message="The database changed after preview.")
4155
+ for key in sorted(deltas):
4156
+ _apply_database_table_delta(current, key, deltas[key], replacement.get(key))
4157
+ return current
4158
 
4159
  try:
4160
+ # One guarded read-modify-write over the planned tables. The expected revision is checked
4161
  # inside the PG lock where available; the closure's digest check also protects a backend
4162
  # that can only provide the runtime's documented guarded fallback.
4163
  user_tables.update_with_activity(
4164
  database, apply_exact, st=session.runtime, by=session.uname,
4165
+ via=_trusted_activity_via(session), row_ids=row_ids,
4166
+ field_keys=None, flush="sync", expect_rev=expected_rev)
4167
  except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
4168
  raise err(409, "confirmation_stale",
4169
  "The database changed after the preview, " + _STALE_FIX) from exc
 
4564
  )
4565
 
4566
 
4567
+ def _is_members_write(name, arguments):
4568
+ """A members permission write previews from the global account store, not the tenant runtime."""
4569
+ return name == "members" and isinstance(arguments, dict) and arguments.get("action") == "set_permissions"
4570
+
4571
+
4572
  def _strong_confirmation_required(name, args):
4573
+ if _is_members_write(name, args):
4574
+ # A business unit widening needs the strong phrase in place of the editor's confirm_widen.
4575
+ return bool((args.get("arguments") or {}).get("confirm_widen"))
4576
  if name not in external_tools.TOOLS:
4577
  import tool_groups
4578
  resolved = tool_groups.resolve(name, args)
 
4797
  "Part of this change could not be accepted. Nothing was previewed.")
4798
 
4799
  remote_note = _is_website_note(name, normalised)
4800
+ plan = result if name == "database_object" or remote_note or _is_members_write(name, normalised) else None
4801
  if plan is not None:
4802
  diff = copy.deepcopy(plan.get("diff") or [])
4803
  warnings = list(plan.get("warnings") or [])
 
4830
  normalised["arguments"]["expect"] = [
4831
  {"user": row["user"], "role": row["role"]}
4832
  for row in result["current_entries"]]
4833
+ if _is_members_write(name, normalised):
4834
+ # Pin the account state the preview read, and the widening it found, into the signed call.
4835
+ normalised["arguments"].update(expect=result["expect"], confirm_widen=bool(result["widening"]))
4836
 
4837
  _canonical_json(diff, capped=False)
4838
  diff_digest = _json_digest(diff)
 
4891
  response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
4892
  if name == "apps" and normalised.get("action") == "catalog_build":
4893
  response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
4894
+ if _is_members_write(name, normalised):
4895
+ response.update({key: copy.deepcopy(result[key]) for key in ("username", "database", "before", "after")})
4896
  if payload["strong"]:
4897
  response["confirmation_phrase"] = _CONFIRMATION_PHRASE
4898
  if issue_token:
 
5428
  if tool == "database_object":
5429
  result = _execute_database_object(
5430
  fresh_session, arguments, plan=recomputed.get("_plan"))
5431
+ elif _is_members_write(tool, arguments):
5432
+ # The account writer compares the pinned expect itself, so no runtime guard is needed.
5433
+ result = _dispatch_external_tool(_tool_context(fresh_session), tool, arguments, about=payload["about"])
5434
  else:
5435
  from datetime import datetime, timezone
5436
  commit_started = datetime.now(timezone.utc)
 
5461
  from core import user_tables
5462
  import routes_nav
5463
  bucket = routes_nav._NAV_META_KEY if arguments.get("action") == "rename" else user_tables.STORE_KEY
5464
+ # A failed revision read is replaced by the produced rev below, so no document is read here.
5465
+ after_revision = _bucket_revision(fresh_session.runtime, bucket, None)
5466
  produced_rev = before_revision["rev"] + 1
5467
  if after_revision.get("rev") != produced_rev:
5468
  after_revision = {"rev": produced_rev}
5469
+ elif _is_members_write(tool, arguments):
5470
+ after_revision = copy.deepcopy(result["revision"])
5471
  else:
5472
  after_revision = copy.deepcopy(guarded_session.runtime._expected)
5473
  after_revision_digest = _json_digest(after_revision)
 
5497
  response.update({key: copy.deepcopy(result[key]) for key in ("view", "audience")})
5498
  if tool == "apps" and arguments.get("action") == "catalog_build":
5499
  response.update({key: copy.deepcopy(result[key]) for key in _CATALOG_BUILD_RESULT_FIELDS})
5500
+ if _is_members_write(tool, arguments):
5501
+ response.update(username=result["username"], database=result["database"],
5502
+ after=copy.deepcopy(result["entry"]))
5503
  _confirmation_status(fresh_session, operation_id, "committed", response)
5504
  if isinstance(stored.get("arguments_ref"), dict):
5505
  _drop_operation_documents(fresh_session, [stored["arguments_ref"].get("key")])
5506
+ if (tool != "database_object" and not _is_members_write(tool, arguments)
5507
+ and receipt.get("persisted") is True):
5508
  try:
5509
  targets, native_ids = _committed_build_evidence(fresh_session, guarded_session.runtime, arguments,
5510
  build_kind=payload["buildKind"])
 
5636
  "query_filter_limit": "Filters allow 3 levels, 100 nodes and 50 conditions per group to bound query validation work. To reduce that work, " + _MCP_FIXES["query_filter_limit"] + ".",
5637
  "query_cursor_stale": "The query, data or permissions changed. To restart, " + _MCP_FIXES["query_cursor_stale"] + ".",
5638
  "query_cursor_invalid": "The query cursor is invalid. Repeat the original query without cursor for a fresh first page.",
5639
+ "query_source_unavailable": "This database's source could not be read for this query.",
5640
  "test_connection_production": "Test connections cannot write to production. Use a staging environment.",
5641
  "test_connection_environment_unknown": "This environment is unknown. Test connections need a named non-production environment.",
5642
  "external_test_connection_invalid": "Set test_connection to true or false.",
 
5670
  r"Traceback|stack trace|\bengine\b|\bmanifest\b|\brevision-guard\b|\bgate\b|"
5671
  r"\b(?:tenant store|dispatcher|document cleaner)\b|"
5672
  r"(?:[A-Za-z]:[\\/]|/(?:tmp|root|work|home|srv|app)/)|"
5673
+ # A file suffix ends at a word boundary: the views action view.share is no shell script.
5674
+ r"(?:[\w.-]+[/\\])*[\w.-]+\.(?:py|tsx?|pyc|ps1|sh|json)\b(?::\d+)?|"
5675
  r"\.(?:claude|agents|codex)/|~[/\\]", re.I)
5676
  _MCP_PROSE_REPLACEMENTS = (
5677
  (r"strict Assistant scope wall|scope wall|live UI walls|UI walls", "permissions"),
 
6276
  required.append("a named non-production environment for this test connection")
6277
  if group == "websites":
6278
  required.extend(_website_requirements(session, note=action == "add_internal_note"))
6279
+ if group == "members" and not deps.perms.is_admin(session.user):
6280
+ required.append("the Administrator role")
6281
  if legacy in {"describe_database", "query_database", "view_upsert", "view_delete", "view_share_preview", "view_object", "app_upsert", "app_delete"} or (
6282
  legacy == "database_object" and entry.get("legacy_action") != "list") or legacy == "sheet_set_binding":
6283
  if not rows:
 
6378
  return count if type(count) is int and count >= 0 else 0
6379
 
6380
 
6381
+ def _mcp_guide_group_note(requested, groups) -> str:
6382
+ """Name an unknown guide group and the groups that exist at call time, in one line."""
6383
+ shown = requested.strip() if isinstance(requested, str) else ""
6384
+ # Only a short identifier is echoed: a credential-shaped name would withhold the whole guide.
6385
+ named = (f"No guide group is named `{shown}`."
6386
+ if re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,59}", shown) and not _contains_credential(shown)
6387
+ else "No guide group has that name.")
6388
+ return (named + " The groups are " + ", ".join(f"`{name}`" for name in groups)
6389
+ + ". Read one with group, or use the index below.")
6390
+
6391
+
6392
  def _mcp_guide(session: Session, group=None) -> str:
6393
  """Return the group index or one group's action schemas without reading records."""
6394
+ try:
6395
+ return _mcp_guide_text(session, group)
6396
+ except HTTPException as exc:
6397
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
6398
+ if exc.status_code in (401, 403, 404, 429) or exc.status_code >= 500 or info.get("code") in MCP_RESULT_CODES:
6399
+ raise
6400
+ except sheets.SheetRefusal as exc:
6401
+ if exc.code in MCP_RESULT_CODES:
6402
+ raise
6403
+ # An unpublished refusal from a guide read would collapse to tool_refused, which names a rejected change.
6404
+ raise err(503, "tool_unavailable", "The guide could not be read right now. Try again shortly.")
6405
+
6406
+
6407
+ def _mcp_guide_text(session: Session, group=None) -> str:
6408
+ """Render the group index or one group's action schemas without reading records."""
6409
  import tool_groups
6410
  groups = tool_groups.groups()
6411
+ requested = group
6412
  if isinstance(group, str) and group not in groups and group != "data":
6413
  # A cased, plural or dotted name (Sheets, sheet.write_cells) reads the one group it names.
6414
  key = group.strip().lower().partition(".")[0]
 
6416
  if name in groups or name == "data"), group)
6417
  alias = group == "data"
6418
  group = tool_groups.canonical_name(group)
6419
+ note = None
6420
  if group is not None and group not in groups:
6421
+ # An unknown group answers the index, led by one line naming it and the real groups.
6422
+ note, group = _mcp_guide_group_note(requested, groups), None
6423
  # One rows-free definitions read per guide: a live runtime pays one pool checkout per database.
6424
  # The guide reads schemas only, so no reader here needs `rows`. On a failed read, keep the old path.
6425
  try:
 
6437
  rows = [row for row in rows if row["permitted"]]
6438
  capabilities = _mcp_capabilities(session, groups, [row for row in rows if row.get("schema")])
6439
  unread = _guide_unread_count(session)
6440
+ lines = [*([note, ""] if note else []), "# Loopable guide", "", f"Signed in as {session.uname}.", "", MCP_INSTRUCTIONS, "",
6441
  f"Scheduled results you have not read: {unread}.",
6442
  *(["Read schedules.inbox before answering."] if unread > 0 else []), "",
6443
  "Call a group with action and arguments. Each action keeps its own read or write rules.",
 
6589
  import tool_groups
6590
  resolved = tool_groups.resolve(name, arguments)
6591
  if resolved is None:
6592
+ raise sheets.SheetRefusal("tool_not_found", _unknown_tool_message(name))
6593
  entry, args = resolved
6594
  if call is not None and call["tool"] == name:
6595
  external_activity.note_action(arguments["action"])
 
6604
  return result
6605
 
6606
 
6607
+ def _unknown_tool_message(name):
6608
+ """Name the group's real actions for a dotted name such as table.call, never echoing the caller's text."""
6609
+ import tool_groups
6610
+ prefix = name.partition(".")[0] if isinstance(name, str) else ""
6611
+ group = tool_groups.groups().get(tool_groups.canonical_name(prefix)) if prefix else None
6612
+ if group is None:
6613
+ return "The requested tool is not available."
6614
+ return ("That is not a tool name. Call the " + group["name"] + " tool and set action to one of the "
6615
+ + group["name"] + " actions: " + ", ".join(group["actions"]) + ". Put that action's inputs "
6616
+ "inside arguments.")
6617
+
6618
+
6619
+ def _note_internal_code(out):
6620
+ """Keep the refusal code before _mcp_public_code collapses it, for the limitation mailbox."""
6621
+ call = external_activity._CALL.get()
6622
+ if call is None or not isinstance(out, dict):
6623
+ return
6624
+ data = out.get("data") if isinstance(out.get("data"), dict) else {}
6625
+ error = out.get("error") if not out.get("ok") else data.get("error")
6626
+ error = error if isinstance(error, dict) else {}
6627
+ code = error.get("code") or (data.get("code") if out.get("ok") and data.get("ok") is False else "")
6628
+ call["internal_code"] = code if isinstance(code, str) else ""
6629
+
6630
+
6631
  def _record_mcp_method(session, req, reply):
6632
  """Retain bounded method names for admitted protocol requests without request values."""
6633
  try:
 
6686
  code = error.get("code", "tool_refused") if error else ("tool_refused" if reply.get("error") else "")
6687
  if code and not call["limitation_recorded"]:
6688
  from routes_feedback import record_limitation
6689
+ # The reply's code is already public. The mailbox also keeps the one noted at dispatch.
6690
+ record_limitation(session, tool=limitation_tool, action=limitation_action, code=code,
6691
+ question=call["question"], internal_code=call.get("internal_code") or None)
6692
  except Exception:
6693
  pass
6694
  finally:
 
6901
  if code == "assistant_unknown_database":
6902
  # The read path's lowercase reason is replaced: the fix sentence names the missing key itself.
6903
  return fix
6904
+ if code in {"assistant_snapshot_unavailable", "assistant_store_not_ready"} and deps._assistant_source_cause_of(text):
6905
+ # S82-T125: a named cause carries its own retry line, or none where no retry can succeed.
6906
+ return text
6907
  return text if not fix or fix in text else (text + " " + fix).strip()
6908
 
6909
 
 
7121
  out = _tool_refusal(name, info.get("code", "tool_refused"), message, exc.status_code, **retry)
7122
  except (TypeError, ValueError, KeyError, AttributeError):
7123
  out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
7124
+ except Exception as exc:
7125
+ deps._assistant_log_cause("external tool", exc)
7126
  out = _tool_refusal(name, "tool_unavailable",
7127
  "The tool could not complete this operation, so try the same call again shortly.", 503)
7128
+ _note_internal_code(out)
7129
  return _mcp_content(rid, out, session)
7130
 
7131
 
 
7179
  "database_object", "view_object", "workbook_object", "worksheet_object",
7180
  "range_object", "calculation_object", "validated_batch",
7181
  "confirm_change",
7182
+ # Dotted names declare every grouped write with no legacy tool.
7183
+ "apps.catalog_build", "apps.apply_template", "apps.save_code", "apps.set_status",
7184
+ "automations.create", "automations.pause", "automations.resume", "automations.delete",
7185
+ "intelligence.propose", "intelligence.create_wiki",
7186
+ "members.set_permissions",
7187
+ "requests.send_message", "requests.ask_user", "requests.send", "requests.answer", "requests.complete",
7188
+ "schedules.create", "schedules.pause", "schedules.resume", "schedules.delete",
7189
+ "sheet.worksheet.rename", "sheet.worksheet.reorder", "sheet.worksheet.duplicate", "sheet.refresh_scope",
7190
+ "views.share", "websites.add_internal_note",
7191
  })
7192
 
7193
 
7194
  def _assert_write_allow_list(*, check_groups=False) -> None:
7195
  declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
7196
+ legacy = {name for name in _WRITE_TOOLS if "." not in name}
7197
+ undeclared = declared - legacy
7198
+ stale = legacy - declared
7199
  if undeclared or stale:
7200
  raise RuntimeError("External write allow-list mismatch: undeclared="
7201
  + repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
7202
  if check_groups:
7203
  import tool_groups
7204
+ found = tool_groups.groups()
7205
+ mapped = {entry.get("legacy_name") for group in found.values()
7206
  for entry in group["actions"].values()}
7207
  if set(external_tools.TOOLS) - mapped - {"confirm_change"}:
7208
  raise RuntimeError("Every legacy tool must have a grouped action")
7209
+ standalone = _WRITE_TOOLS - legacy
7210
+ grouped = {group + "." + action for group, definition in found.items()
7211
+ for action, entry in definition["actions"].items()
7212
+ if entry.get("read") is False and not entry.get("legacy_name")}
7213
+ undeclared = grouped - standalone
7214
+ stale = standalone - grouped
7215
+ if undeclared or stale:
7216
+ raise RuntimeError("Grouped write allow-list mismatch: undeclared="
7217
+ + repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
7218
 
7219
 
7220
  _assert_write_allow_list()
api/routes_feedback.py CHANGED
@@ -72,6 +72,9 @@ MAX_CHARS = 4000
72
  #: the newest, which is the one direction this store must never move in.
73
  MAX_ROWS = 5000
74
  QUESTION_MAX = 500
 
 
 
75
 
76
 
77
  def _limitation_address(tool, action):
@@ -97,19 +100,55 @@ def _limitation_address(tool, action):
97
  return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
98
 
99
 
100
- def record_limitation(session, *, tool, action, code, question=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  """Coalesce value-free AI limitations in the existing platform feedback document."""
102
  try:
103
  # Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
104
  if code == "request_method_unknown":
105
  return False
106
  from core import audit, freeze_notice
107
- from routes_external import _contains_credential, _mcp_public_code
108
  if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
109
  or not store.available()):
110
  return False
111
  tool, action = _limitation_address(tool, action)
 
112
  code = _mcp_public_code(code)
 
 
 
 
113
  fallback = f"I could not complete {tool}.{action}."
114
  # The caller supplies prose explicitly. Never derive it from arguments or results.
115
  explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
@@ -128,7 +167,7 @@ def record_limitation(session, *, tool, action, code, question=None):
128
  report = {"user": user, "ts": ts, "question": question}
129
  identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
130
  row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
131
- "user": user, "tool": tool, "action": action, "code": code,
132
  "question": question, "ts": ts, "firstTs": ts, "count": 1,
133
  "reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
134
 
@@ -145,7 +184,9 @@ def record_limitation(session, *, tool, action, code, question=None):
145
  filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
146
  filer = filers.get(user, {})
147
  # Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
148
- updated = {**old, "user": user, "ts": ts,
 
 
149
  "count": int(old.get("count", 1)) + 1, "reports": reports + [report],
150
  "filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
151
  "lastTs": ts}}}
 
72
  #: the newest, which is the one direction this store must never move in.
73
  MAX_ROWS = 5000
74
  QUESTION_MAX = 500
75
+ #: A report without a code borrows the filer's refusal at the same address from this recent window.
76
+ REFUSAL_WINDOW_SECONDS = 3600
77
+ _CODE = re.compile(r"[a-z][a-z0-9_]{0,99}")
78
 
79
 
80
  def _limitation_address(tool, action):
 
100
  return ("loopable_guide", "read") if tool == "loopable_guide" else (tool, action)
101
 
102
 
103
+ def refusal_code(session, tool, action, code=None):
104
+ """Return (internal, public) of this filer's latest recent refusal at the address, or None."""
105
+ try:
106
+ import tool_groups
107
+ from routes_external import _mcp_public_code
108
+ tool, action = _limitation_address(tool, action)
109
+ grouped = tool in tool_groups.groups()
110
+ user, tenant = str(session.uname or "").lower(), str(session.tenant or "").lower()
111
+ public = _mcp_public_code(code) if code else None
112
+ since = time.time() - REFUSAL_WINDOW_SECONDS
113
+ best = {}
114
+ for row in _rows():
115
+ filers = row.get("filers") if isinstance(row.get("filers"), dict) else {}
116
+ filer = {str(name).lower(): value for name, value in filers.items()}.get(user)
117
+ stamp = filer.get("lastTs") if isinstance(filer, dict) else (
118
+ row.get("ts") if str(row.get("user", "")).lower() == user else None)
119
+ if (row.get("category") != "limitation" or str(row.get("tenant", "")).lower() != tenant
120
+ or row.get("tool") != tool or grouped and row.get("action") != action
121
+ or type(stamp) not in (int, float) or stamp < since
122
+ or row.get("code") in (None, "capability_unavailable")
123
+ or public is not None and row.get("code") != public):
124
+ continue
125
+ # A group matches its exact action. A single tool such as confirm_change prefers it.
126
+ rank = (row.get("action") == action, stamp)
127
+ if not best or rank > best["rank"]:
128
+ best = {"rank": rank, "found": (row.get("internal_code") or row["code"], row["code"])}
129
+ return best.get("found")
130
+ except Exception:
131
+ return None
132
+
133
+
134
+ def record_limitation(session, *, tool, action, code, question=None, internal_code=None):
135
  """Coalesce value-free AI limitations in the existing platform feedback document."""
136
  try:
137
  # Protocol discovery belongs in method telemetry, outside the product limitation mailbox.
138
  if code == "request_method_unknown":
139
  return False
140
  from core import audit, freeze_notice
141
+ from routes_external import MCP_RESULT_CODES, _contains_credential, _mcp_public_code
142
  if (freeze_notice.enabled() or getattr(session.runtime, "preview_only", False)
143
  or not store.available()):
144
  return False
145
  tool, action = _limitation_address(tool, action)
146
+ internal = internal_code if internal_code is not None else code
147
  code = _mcp_public_code(code)
148
+ # The row keeps the refusal's own code beside the public one. The id stays over the public code.
149
+ if (not isinstance(internal, str) or not _CODE.fullmatch(internal)
150
+ or internal in MCP_RESULT_CODES and internal != code):
151
+ internal = code
152
  fallback = f"I could not complete {tool}.{action}."
153
  # The caller supplies prose explicitly. Never derive it from arguments or results.
154
  explicit = (isinstance(question, str) and 0 < len(question.strip()) <= QUESTION_MAX
 
167
  report = {"user": user, "ts": ts, "question": question}
168
  identity = hashlib.sha256(json.dumps([tenant, tool, action, code]).encode()).hexdigest()
169
  row = {"id": "lim_" + identity, "category": "limitation", "tenant": tenant,
170
+ "user": user, "tool": tool, "action": action, "code": code, "internal_code": internal,
171
  "question": question, "ts": ts, "firstTs": ts, "count": 1,
172
  "reports": [report], "filers": {user: {"count": 1, "lastTs": ts}}}
173
 
 
184
  filers = {old["user"]: {"count": int(old.get("count", 1)), "lastTs": old["ts"]}}
185
  filer = filers.get(user, {})
186
  # Reports have no cap. The feedback document remains one JSON value under the 128 MB store ceiling.
187
+ # A bare public code never overwrites a refusal code the row already holds.
188
+ kept = internal if internal != code else old.get("internal_code") or internal
189
+ updated = {**old, "user": user, "ts": ts, "internal_code": kept,
190
  "count": int(old.get("count", 1)) + 1, "reports": reports + [report],
191
  "filers": {**filers, user: {"count": int(filer.get("count", 0)) + 1,
192
  "lastTs": ts}}}
api/routes_grid.py CHANGED
@@ -405,6 +405,38 @@ def _workspace_values(values, session, module, fields):
405
  return {str(pid): cells(raw) for pid, raw in values.items()}
406
 
407
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
408
  def _workspace_filter_removals(workspace, fields, measures, hidden=()):
409
  """Clean saved display filters on a detached wire copy, retaining removal reasons."""
410
  from copy import deepcopy
@@ -534,6 +566,8 @@ def workspace(scope: str = "customer",
534
  workspace["limits"] = g.get("limits") or []
535
  if fields_only:
536
  workspace["fieldsOnly"] = True
 
 
537
  return {"workspace": _workspace_filter_removals(
538
  workspace, g['fields'], g['measures'], g.get('hidden') or ())}
539
 
@@ -594,6 +628,8 @@ def workspace(scope: str = "customer",
594
  workspace["mapSetup"] = map_setup
595
  if fields_only:
596
  workspace["fieldsOnly"] = True
 
 
597
  return {"workspace": _workspace_filter_removals(
598
  workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
599
 
@@ -698,6 +734,8 @@ def workspace(scope: str = "customer",
698
  workspace["cohortMode"] = True
699
  # The create pane offers "this cohort table only" (default) vs "all customer tables".
700
  workspace["scopeChoice"] = True
 
 
701
  return {"workspace": _workspace_filter_removals(
702
  workspace, g['fields'], g['measures'], g.get('hidden') or ())}
703
 
@@ -2076,7 +2114,7 @@ def grid_events_route(body: dict = Body(default=None),
2076
  Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
2077
  same sessions the read door (`/workspace`) admits, or a tenant without the customer module
2078
  can SEE its own user tables and not write to them."""
2079
- from core import grid_events, shared_overlay
2080
  from routes_customers import grid_assembly
2081
  from routes_tables import _app_write_via, _managed_or_refuse
2082
 
@@ -2180,7 +2218,11 @@ def grid_events_route(body: dict = Body(default=None),
2180
  # answer "the batch was refused" and leave the caller to guess which member, which is
2181
  # the same class of unfalsifiable answer the channel exists to end.
2182
  _before = len(ctx.out.refusals)
2183
- rerender = grid_events.handle_one(one, ctx)
 
 
 
 
2184
  row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
2185
  "rerender": bool(rerender)}
2186
  mine = ctx.out.refusals[_before:]
 
405
  return {str(pid): cells(raw) for pid, raw in values.items()}
406
 
407
 
408
+ def _published_rev(ws):
409
+ """S82-T156: the store revision the workspace was read at, which a view write stamps."""
410
+ rev = ws.get("rev") if isinstance(ws, dict) else None
411
+ return rev if isinstance(rev, int) and not isinstance(rev, bool) else None
412
+
413
+
414
+ def _settle_refused_view_write(ctx, event):
415
+ """Store a no-op receipt for a refused stale view write, so its resend re-reads and stops.
416
+
417
+ The refusal rolls back the event's own receipt, and the outbox resends the same id. Without
418
+ this receipt every resend is refused again and the browser's save queue never moves (D-634).
419
+ """
420
+ from core import edit_receipts, grid_events
421
+
422
+ if not getattr(ctx, "durable_receipts", False) or not isinstance(event, dict):
423
+ return
424
+ identity = event.get("opId", event.get("id"))
425
+ if not identity:
426
+ return
427
+ try:
428
+ identity = edit_receipts.op_id(identity)
429
+ dropped = {"rerender": True, "write": {"opId": identity}, "refusals": [],
430
+ "doc": None, "toast": None}
431
+ edit_receipts.run_once(
432
+ grid_events._store_of(ctx), ctx.uname,
433
+ edit_receipts.operation(ctx.scope_key, event.get("type"), event.get("pid")),
434
+ identity, lambda: dropped)
435
+ except Exception as exc: # noqa: BLE001
436
+ print(f"[grid/events] refused view write {identity!r}: resend receipt not saved "
437
+ f"({type(exc).__name__})")
438
+
439
+
440
  def _workspace_filter_removals(workspace, fields, measures, hidden=()):
441
  """Clean saved display filters on a detached wire copy, retaining removal reasons."""
442
  from copy import deepcopy
 
566
  workspace["limits"] = g.get("limits") or []
567
  if fields_only:
568
  workspace["fieldsOnly"] = True
569
+ if _published_rev(g["ws"]) is not None:
570
+ workspace["rev"] = _published_rev(g["ws"])
571
  return {"workspace": _workspace_filter_removals(
572
  workspace, g['fields'], g['measures'], g.get('hidden') or ())}
573
 
 
628
  workspace["mapSetup"] = map_setup
629
  if fields_only:
630
  workspace["fieldsOnly"] = True
631
+ if _published_rev(g["ws"]) is not None:
632
+ workspace["rev"] = _published_rev(g["ws"])
633
  return {"workspace": _workspace_filter_removals(
634
  workspace, g['fields'], g['measures'], g.get('_note_hidden') or ())}
635
 
 
734
  workspace["cohortMode"] = True
735
  # The create pane offers "this cohort table only" (default) vs "all customer tables".
736
  workspace["scopeChoice"] = True
737
+ if _published_rev(g["ws"]) is not None:
738
+ workspace["rev"] = _published_rev(g["ws"])
739
  return {"workspace": _workspace_filter_removals(
740
  workspace, g['fields'], g['measures'], g.get('hidden') or ())}
741
 
 
2114
  Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the
2115
  same sessions the read door (`/workspace`) admits, or a tenant without the customer module
2116
  can SEE its own user tables and not write to them."""
2117
+ from core import grid_events, shared_overlay, view_audit
2118
  from routes_customers import grid_assembly
2119
  from routes_tables import _app_write_via, _managed_or_refuse
2120
 
 
2218
  # answer "the batch was refused" and leave the caller to guess which member, which is
2219
  # the same class of unfalsifiable answer the channel exists to end.
2220
  _before = len(ctx.out.refusals)
2221
+ try:
2222
+ rerender = grid_events.handle_one(one, ctx)
2223
+ except view_audit.ViewRevisionStale:
2224
+ _settle_refused_view_write(ctx, one)
2225
+ raise
2226
  row = {**ctx.out.writes.get(eid, {}), "id": eid, "opId": eid,
2227
  "rerender": bool(rerender)}
2228
  mine = ctx.out.refusals[_before:]
api/routes_keychain.py CHANGED
@@ -12,7 +12,7 @@ the keychain cutover wave R3 staged.
12
  import os
13
  import time
14
 
15
- from fastapi import Body, Depends
16
  from fastapi import APIRouter
17
 
18
  from deps import Session, err, require_session
@@ -1066,6 +1066,8 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
1066
  raise err(503, "store_unavailable",
1067
  "the key was not saved; its sharing setting could not be stored, so "
1068
  "nothing was kept. Try again.")
 
 
1069
  return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
1070
 
1071
 
@@ -1275,8 +1277,18 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
1275
 
1276
  @router.post("/admin/keychain/{entry_id}/test")
1277
  def test_key(entry_id: str, session: Session = Depends(require_session)):
1278
- _own_row(session, entry_id) # 404 for an entry this caller may not see
1279
- return _kc().test_entry(session.runtime, entry_id)
 
 
 
 
 
 
 
 
 
 
1280
 
1281
 
1282
  def _unsynced_customer_records(session):
@@ -1486,6 +1498,144 @@ def _odoo_admin(session):
1486
  "configure it")
1487
 
1488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1489
  @router.get("/admin/connectors/odoo/config")
1490
  def odoo_config(session: Session = Depends(require_session)):
1491
  """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
@@ -1495,7 +1645,16 @@ def odoo_config(session: Session = Depends(require_session)):
1495
  entry = _odoo_entry(session)
1496
  server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
1497
  cfg = rel.read_config(session.runtime)
 
 
 
 
 
 
1498
  return {
 
 
 
1499
  "applicable": bool(rel.is_royal(session.tenant)),
1500
  "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
1501
  else "none"),
@@ -1520,6 +1679,74 @@ def odoo_config(session: Session = Depends(require_session)):
1520
  }
1521
 
1522
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1523
  @router.put("/admin/connectors/odoo/config")
1524
  def odoo_config_put(body: dict = Body(default=None),
1525
  session: Session = Depends(require_session)):
@@ -1584,12 +1811,22 @@ def odoo_config_put(body: dict = Body(default=None),
1584
  raise err(400, "missing_sync_tz",
1585
  "choose a time zone for the daily sync time")
1586
 
 
 
 
 
 
 
 
1587
  server_db = body.get("serverDb")
1588
  if server_db is not None:
1589
  server_db = " ".join(str(server_db).split())[:80]
1590
 
1591
  def _up(cur):
1592
  cur = cur if isinstance(cur, dict) else {}
 
 
 
1593
  if clean_grids is not None:
1594
  cur.setdefault("grids", {}).update(clean_grids)
1595
  if clean_every is not None:
@@ -1944,6 +2181,48 @@ def pause_connector(key: str, body: dict = Body(default=None),
1944
  return {"key": key, "paused": paused, "snapshots": snapshots}
1945
 
1946
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1947
  from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
1948
 
1949
  router.include_router(amazon_deletion_router)
 
12
  import os
13
  import time
14
 
15
+ from fastapi import Body, Depends, HTTPException
16
  from fastapi import APIRouter
17
 
18
  from deps import Session, err, require_session
 
1066
  raise err(503, "store_unavailable",
1067
  "the key was not saved; its sharing setting could not be stored, so "
1068
  "nothing was kept. Try again.")
1069
+ if etype == "odoo":
1070
+ _business_tz_on_connect(session) # S82-T107: default the zone on connect
1071
  return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
1072
 
1073
 
 
1277
 
1278
  @router.post("/admin/keychain/{entry_id}/test")
1279
  def test_key(entry_id: str, session: Session = Depends(require_session)):
1280
+ row = _own_row(session, entry_id) # 404 for an entry this caller may not see
1281
+ result = _kc().test_entry(session.runtime, entry_id)
1282
+ # S82-T107: a passing test of the connection that serves this tenant re-defaults the zone.
1283
+ if (isinstance(result, dict) and result.get("ok") and str(row.get("type") or "") == "odoo"
1284
+ and session.admin):
1285
+ try:
1286
+ serving = _odoo_entry(session)
1287
+ except Exception: # noqa: BLE001
1288
+ serving = None
1289
+ if serving is not None and str(serving.get("id")) == str(entry_id):
1290
+ _business_tz_on_connect(session)
1291
+ return result
1292
 
1293
 
1294
  def _unsynced_customer_records(session):
 
1498
  "configure it")
1499
 
1500
 
1501
+ # ═════════════════════════════════ S82-T107: THE TENANT'S BUSINESS TIMEZONE (`businessTz`, C10)
1502
+ #
1503
+ # One zone per tenant, stored beside the sync slot in the Odoo connector config. The business day
1504
+ # reads it. It is NEVER `syncTz`: that zone says when the sync runs, not where the business is.
1505
+ # It defaults from the connected Odoo user's own `tz` on connect, on test and on the first admin
1506
+ # config read. A blank Odoo `tz` leaves it unset, and the panel asks the admin to pick one. An
1507
+ # admin's pick wins over every later Odoo default.
1508
+ BUSINESS_TZ_KEY = "businessTz"
1509
+ #: Where the stored zone came from: "odoo", "admin", or "odoo-blank" (Odoo was asked, said nothing).
1510
+ BUSINESS_TZ_SOURCE_KEY = "businessTzSource"
1511
+
1512
+
1513
+ def _zone_ok(name):
1514
+ """True when `name` is an IANA zone this process can load."""
1515
+ if not isinstance(name, str) or not name.strip():
1516
+ return False
1517
+ try:
1518
+ from zoneinfo import ZoneInfo
1519
+ ZoneInfo(name.strip())
1520
+ return True
1521
+ except Exception: # noqa: BLE001
1522
+ return False
1523
+
1524
+
1525
+ def _odoo_cfg(rt):
1526
+ try:
1527
+ cur = rt.get(_rel().CONFIG_KEY) if rt is not None else None
1528
+ except Exception: # noqa: BLE001
1529
+ cur = None
1530
+ return cur if isinstance(cur, dict) else {}
1531
+
1532
+
1533
+ def stored_business_tz(rt):
1534
+ """The tenant's stored business timezone, or None. Reads `businessTz` only, never `syncTz`."""
1535
+ name = str(_odoo_cfg(rt).get(BUSINESS_TZ_KEY) or "").strip()
1536
+ return name if _zone_ok(name) else None
1537
+
1538
+
1539
+ def odoo_user_tz(creds, timeout=8):
1540
+ """The connected Odoo user's `tz` from `res.users`.
1541
+
1542
+ Returns the zone name, "" when Odoo holds none (or one this process cannot load), and None when
1543
+ Odoo could not be asked. One admin act over read-only XML-RPC: `authenticate`, then one `read`
1544
+ of the user's own row. It goes outside every limiter and the shared client on purpose."""
1545
+ import xmlrpc.client
1546
+ creds = creds or {}
1547
+ url = str(creds.get("url") or "").strip().rstrip("/")
1548
+ db, user, key = (str(creds.get(k) or "") for k in ("db", "user", "api_key"))
1549
+ if not (url.startswith(("http://", "https://")) and db and user and key):
1550
+ return None
1551
+ base = xmlrpc.client.SafeTransport if url.startswith("https") else xmlrpc.client.Transport
1552
+
1553
+ class _Timed(base):
1554
+ def make_connection(self, host):
1555
+ conn = super().make_connection(host)
1556
+ conn.timeout = timeout
1557
+ return conn
1558
+
1559
+ try:
1560
+ common = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/common", transport=_Timed(),
1561
+ allow_none=True)
1562
+ uid = common.authenticate(db, user, key, {})
1563
+ if not uid:
1564
+ return None
1565
+ models = xmlrpc.client.ServerProxy(f"{url}/xmlrpc/2/object", transport=_Timed(),
1566
+ allow_none=True)
1567
+ rows = models.execute_kw(db, uid, key, "res.users", "read", [[uid]], {"fields": ["tz"]})
1568
+ except Exception: # noqa: BLE001
1569
+ return None
1570
+ first = rows[0] if isinstance(rows, list) and rows and isinstance(rows[0], dict) else {}
1571
+ zone = str(first.get("tz") or "").strip() # Odoo sends False for a blank field
1572
+ return zone if _zone_ok(zone) else ""
1573
+
1574
+
1575
+ def _odoo_tz_creds(session, entry):
1576
+ """The fields of the connection serving this tenant's Odoo, or None. Never returned to a client."""
1577
+ if entry is None:
1578
+ if not env_odoo_available(session.runtime):
1579
+ return None
1580
+ got, _why = _env_odoo_fields()
1581
+ return got
1582
+ try:
1583
+ return _kc().read_fields(session.runtime, entry["id"]) or None
1584
+ except Exception: # noqa: BLE001
1585
+ return None
1586
+
1587
+
1588
+ def default_business_tz(session, entry=None, refresh=False):
1589
+ """Default `businessTz` from the connected Odoo user's `tz`, and return the zone in force.
1590
+
1591
+ `refresh=False` (the admin config read) asks Odoo only when no zone is stored and Odoo was not
1592
+ already asked. `refresh=True` (connect, test) asks again. An admin's pick is never replaced, a
1593
+ blank answer never clears a stored zone, and the default is never `syncTz` nor a literal."""
1594
+ rt = session.runtime
1595
+ cur = _odoo_cfg(rt)
1596
+ have = stored_business_tz(rt)
1597
+ source = str(cur.get(BUSINESS_TZ_SOURCE_KEY) or "")
1598
+ if have and source == "admin":
1599
+ return have
1600
+ if not refresh and (have or source == "odoo-blank"):
1601
+ return have
1602
+ creds = _odoo_tz_creds(session, entry)
1603
+ if not creds:
1604
+ return have
1605
+ zone = odoo_user_tz(creds)
1606
+ if zone is None or (not zone and have):
1607
+ return have
1608
+
1609
+ def _up(c):
1610
+ c = c if isinstance(c, dict) else {}
1611
+ if str(c.get(BUSINESS_TZ_SOURCE_KEY) or "") == "admin" and _zone_ok(c.get(BUSINESS_TZ_KEY)):
1612
+ return c # an admin picked a zone meanwhile
1613
+ if zone:
1614
+ c[BUSINESS_TZ_KEY] = zone
1615
+ c[BUSINESS_TZ_SOURCE_KEY] = "odoo"
1616
+ else:
1617
+ c.pop(BUSINESS_TZ_KEY, None)
1618
+ c[BUSINESS_TZ_SOURCE_KEY] = "odoo-blank"
1619
+ return c
1620
+
1621
+ try:
1622
+ rt.update(_rel().CONFIG_KEY, _up, flush="sync")
1623
+ except Exception: # noqa: BLE001
1624
+ return have
1625
+ return stored_business_tz(rt)
1626
+
1627
+
1628
+ def _business_tz_on_connect(session):
1629
+ """Connect and test: re-default the zone from the serving Odoo user. Never raises."""
1630
+ try:
1631
+ entry = _odoo_entry(session)
1632
+ if entry is None and not env_odoo_available(session.runtime):
1633
+ return None
1634
+ return default_business_tz(session, entry, refresh=True)
1635
+ except Exception: # noqa: BLE001
1636
+ return None
1637
+
1638
+
1639
  @router.get("/admin/connectors/odoo/config")
1640
  def odoo_config(session: Session = Depends(require_session)):
1641
  """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a
 
1645
  entry = _odoo_entry(session)
1646
  server_db, server_url, api_user, editable = _odoo_source_fields(session, entry)
1647
  cfg = rel.read_config(session.runtime)
1648
+ # S82-T107: the first admin read of a connection with no stored zone takes the Odoo user's tz.
1649
+ business_tz = cfg.get(BUSINESS_TZ_KEY) if _zone_ok(cfg.get(BUSINESS_TZ_KEY)) else None
1650
+ if business_tz is None:
1651
+ live = not cfg["frozen"] and (entry is not None or env_odoo_available(session.runtime))
1652
+ business_tz = (default_business_tz(session, entry) if live
1653
+ else stored_business_tz(session.runtime))
1654
  return {
1655
+ "businessTz": business_tz,
1656
+ "businessTzSource": str(_odoo_cfg(session.runtime).get(BUSINESS_TZ_SOURCE_KEY) or "")
1657
+ if business_tz else "",
1658
  "applicable": bool(rel.is_royal(session.tenant)),
1659
  "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime)
1660
  else "none"),
 
1679
  }
1680
 
1681
 
1682
+ def _odoo_field_name_tables(session):
1683
+ """Read only this connector's visible database definitions in the session's tenant."""
1684
+ from core import perm_scope, user_tables
1685
+ tables = {}
1686
+ for grid in _rel().grid_choices(session.runtime):
1687
+ key = grid["key"]
1688
+ if not perm_scope.may_read(session.user, key, st=session.runtime):
1689
+ continue
1690
+ definition = user_tables.get_one(key, st=session.runtime)
1691
+ if definition is not None:
1692
+ tables[key] = (definition, str(definition.get("name") or grid["label"]))
1693
+ return tables
1694
+
1695
+
1696
+ @router.get("/admin/connectors/odoo/field-names")
1697
+ def odoo_field_names(session: Session = Depends(require_session)):
1698
+ """List the connector fields this administrator may rename without changing their definitions."""
1699
+ _odoo_admin(session)
1700
+ from core import perm_scope, user_tables
1701
+ try:
1702
+ fields = []
1703
+ for key, (definition, name) in _odoo_field_name_tables(session).items():
1704
+ hidden = perm_scope.hidden_keys(session.user, key, definition.get("fields") or [],
1705
+ st=session.runtime)
1706
+ for field in user_tables.renamable_fields([key], st=session.runtime):
1707
+ if field["key"] not in hidden and user_tables.may_rename_field(
1708
+ key, field["key"], session.uname, session.admin, st=session.runtime):
1709
+ fields.append({**field, "tableLabel": name})
1710
+ return {"fields": fields}
1711
+ except HTTPException:
1712
+ raise
1713
+ except Exception:
1714
+ raise err(503, "store_unavailable", "Field names could not be read. Try again.") from None
1715
+
1716
+
1717
+ @router.put("/admin/connectors/odoo/field-names")
1718
+ def odoo_field_names_put(body: dict = Body(default=None),
1719
+ session: Session = Depends(require_session)):
1720
+ """Rename one connector field through the shared label-only authority."""
1721
+ _odoo_admin(session)
1722
+ from core import perm_scope, user_tables
1723
+ body = body or {}
1724
+ if not all(isinstance(body.get(key), str) and body[key].strip() for key in ("table", "key", "label")):
1725
+ raise err(400, "invalid_field_name", "Choose a database, column and nonblank field name.")
1726
+ table_key, fkey = body["table"], body["key"]
1727
+ try:
1728
+ tables = _odoo_field_name_tables(session)
1729
+ if table_key not in tables:
1730
+ raise err(404, "not_found", "This connector has no database by that name you may edit.")
1731
+ definition, name = tables[table_key]
1732
+ if fkey in perm_scope.hidden_keys(session.user, table_key, definition.get("fields") or [],
1733
+ st=session.runtime):
1734
+ raise err(403, "forbidden", "this database has no column by that name that you may edit")
1735
+ raw = {key: value for key, value in body.items() if key not in ("table", "key")}
1736
+ if not user_tables.may_rename_field(table_key, fkey, session.uname, session.admin,
1737
+ st=session.runtime, raw=raw):
1738
+ raise err(403, "preset_field_locked", "Only the name of a renamable pre-set column may change here.")
1739
+ saved = user_tables.rename_field(table_key, fkey, body["label"], session.uname,
1740
+ session.admin, st=session.runtime, by=session.uname, via="web")
1741
+ if saved is None:
1742
+ raise err(503, "store_unavailable", "The field name was not saved. Try again.")
1743
+ return {"field": {"table": table_key, "key": fkey, "label": saved["label"], "tableLabel": name}}
1744
+ except HTTPException:
1745
+ raise
1746
+ except Exception:
1747
+ raise err(503, "store_unavailable", "The field name was not saved. Try again.") from None
1748
+
1749
+
1750
  @router.put("/admin/connectors/odoo/config")
1751
  def odoo_config_put(body: dict = Body(default=None),
1752
  session: Session = Depends(require_session)):
 
1811
  raise err(400, "missing_sync_tz",
1812
  "choose a time zone for the daily sync time")
1813
 
1814
+ # S82-T107: the business timezone. An admin's pick, kept apart from the sync slot zone.
1815
+ clean_business_tz = None
1816
+ if BUSINESS_TZ_KEY in body:
1817
+ clean_business_tz = str(body.get(BUSINESS_TZ_KEY) or "").strip()
1818
+ if not _zone_ok(clean_business_tz):
1819
+ raise err(400, "invalid_business_tz", "choose a business timezone from the list")
1820
+
1821
  server_db = body.get("serverDb")
1822
  if server_db is not None:
1823
  server_db = " ".join(str(server_db).split())[:80]
1824
 
1825
  def _up(cur):
1826
  cur = cur if isinstance(cur, dict) else {}
1827
+ if clean_business_tz:
1828
+ cur[BUSINESS_TZ_KEY] = clean_business_tz
1829
+ cur[BUSINESS_TZ_SOURCE_KEY] = "admin"
1830
  if clean_grids is not None:
1831
  cur.setdefault("grids", {}).update(clean_grids)
1832
  if clean_every is not None:
 
2181
  return {"key": key, "paused": paused, "snapshots": snapshots}
2182
 
2183
 
2184
+ def _live_limits_admin(session):
2185
+ """S82-T101: live call limits are workspace wide, so only an administrator reads or sets them."""
2186
+ if not session.admin:
2187
+ raise err(403, "not_admin", "only an administrator can see or change live call limits")
2188
+
2189
+
2190
+ def _live_limits_payload(limits):
2191
+ from core import live_limiter
2192
+ return {**limits, "defaults": live_limiter.defaults(),
2193
+ "callsPerMinuteMin": live_limiter.CALLS_PER_MINUTE_MIN,
2194
+ "callsPerMinuteMax": live_limiter.CALLS_PER_MINUTE_MAX}
2195
+
2196
+
2197
+ @router.get("/admin/connectors/live-limits")
2198
+ def live_limits(session: Session = Depends(require_session)):
2199
+ """S82-T101 (R65): the tenant's AI and metric live call limits."""
2200
+ _live_limits_admin(session)
2201
+ from core import live_limiter
2202
+ try:
2203
+ limits = live_limiter.read_limits(session.runtime)
2204
+ except Exception:
2205
+ raise err(503, "store_unavailable", "the live call limits could not be read; try again")
2206
+ return _live_limits_payload(limits)
2207
+
2208
+
2209
+ @router.put("/admin/connectors/live-limits")
2210
+ def live_limits_put(body: dict = Body(default=None),
2211
+ session: Session = Depends(require_session)):
2212
+ """S82-T101 (R65): set calls per minute, one in flight, and max rows per call (null is no limit)."""
2213
+ _live_limits_admin(session)
2214
+ from core import live_limiter
2215
+ clean, errors = live_limiter.validate_limits(body if body is not None else {})
2216
+ if errors:
2217
+ raise err(400, "invalid_live_limits", "some live call limits are not valid",
2218
+ details={"fields": errors})
2219
+ try:
2220
+ limits = live_limiter.write_limits(session.runtime, clean, by=session.uname)
2221
+ except Exception:
2222
+ raise err(503, "store_unavailable", "the change was not saved; try again")
2223
+ return _live_limits_payload(limits)
2224
+
2225
+
2226
  from routes_amazon_deletion import router as amazon_deletion_router # noqa: E402
2227
 
2228
  router.include_router(amazon_deletion_router)
api/routes_nav.py CHANGED
@@ -596,18 +596,18 @@ def _granted_view_ids(session):
596
  #: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
597
  #: of the reserved words below is still a second connection, and its sibling must still be named.
598
  #:
599
- #: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
600
  #: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
601
- #: empty into the one Custom group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
602
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
603
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
604
- _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
605
 
606
- #: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "Custom";
607
  #: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
608
  CUSTOM_SOURCE_GROUP = "custom"
609
- #: Native provenance tokens, the legacy "Loopable" from `user_tables` included, fold into the key.
610
- _NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation"})
611
 
612
 
613
  def _native_group(group):
@@ -1165,7 +1165,7 @@ def _user_pref_now():
1165
 
1166
  @router.get("/nav/prefs/{key}")
1167
  def get_user_pref(key: str, session: Session = Depends(require_session)):
1168
- """C5: this user's stored value for one preference key, or 404 when unset."""
1169
  key = _user_pref_key(key)
1170
  try:
1171
  bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
@@ -1173,7 +1173,7 @@ def get_user_pref(key: str, session: Session = Depends(require_session)):
1173
  raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
1174
  row = bucket.get(key) if isinstance(bucket, dict) else None
1175
  if not isinstance(row, dict) or "value" not in row:
1176
- raise err(404, "pref_not_set", "That preference is not saved yet.")
1177
  return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
1178
 
1179
 
@@ -1427,6 +1427,10 @@ def nav_schema(key: str, session: Session = Depends(require_session)):
1427
  "means": means.get(f["key"], "")}
1428
  if f.get("options"):
1429
  entry["options"] = list(f["options"])
 
 
 
 
1430
  fields.append(entry)
1431
  except Exception:
1432
  fields = []
 
596
  #: ⚠ COUNTED ON THE CONNECTIONS, NOT ON THE USABLE LABELS. A second connection whose label is one
597
  #: of the reserved words below is still a second connection, and its sibling must still be named.
598
  #:
599
+ #: These reserved words are the rail's OWN vocabulary, not a connection's to take:
600
  #: `dataPath.ts::dataDatabaseOriginKey` folds `custom` / `loopable` / `blank` / `automation` /
601
+ #: `system` / empty into the one System group (S79-T60, key `CUSTOM_SOURCE_GROUP`). A connection someone named
602
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
603
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
604
+ _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "system", "odoo"})
605
 
606
+ #: S79-T60 (QI27): the stable key of the native origin group. The client shows it as "System";
607
  #: no logic compares that label. Must equal `dataPath.ts::CUSTOM_ORIGIN_KEY`.
608
  CUSTOM_SOURCE_GROUP = "custom"
609
+ #: Native provenance tokens fold into the key. Must equal `dataPath.ts::CUSTOM_SOURCE_TOKENS`.
610
+ _NATIVE_GROUP_TOKENS = frozenset({"", CUSTOM_SOURCE_GROUP, "loopable", "blank", "automation", "system"})
611
 
612
 
613
  def _native_group(group):
 
1165
 
1166
  @router.get("/nav/prefs/{key}")
1167
  def get_user_pref(key: str, session: Session = Depends(require_session)):
1168
+ """C5: this user's stored value for one preference key; an unset key answers 200 with null."""
1169
  key = _user_pref_key(key)
1170
  try:
1171
  bucket = (session.runtime.get(_NAV_USER_PREFS_KEY) or {}).get(session.uname) or {}
 
1173
  raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
1174
  row = bucket.get(key) if isinstance(bucket, dict) else None
1175
  if not isinstance(row, dict) or "value" not in row:
1176
+ return {"key": key, "value": None, "updated_at": None}
1177
  return {"key": key, "value": row.get("value"), "updated_at": row.get("updated_at")}
1178
 
1179
 
 
1427
  "means": means.get(f["key"], "")}
1428
  if f.get("options"):
1429
  entry["options"] = list(f["options"])
1430
+ elif f["key"] == "dba":
1431
+ # The dba vocabulary is the tenant's teams plus one multi-team value, never a contract list.
1432
+ from modules.customer_data import dba_options
1433
+ entry["options"] = dba_options()
1434
  fields.append(entry)
1435
  except Exception:
1436
  fields = []
api/routes_odoo_tables.py CHANGED
@@ -929,6 +929,7 @@ def _odoo_sources():
929
  "id": "so.id",
930
  "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set
931
  "default_wholesale_scope": True,
 
932
  "cols": {
933
  "order_no": ("so.name", _s),
934
  "odoo_id": ("so.id", _i),
@@ -990,6 +991,7 @@ def _odoo_sources():
990
  "id": "sol.id",
991
  "needs_excluded": True,
992
  "default_wholesale_scope": True,
 
993
  "cols": {
994
  "odoo_id": ("sol.id", _i),
995
  "order_no": ("so.name", _s),
@@ -1259,6 +1261,7 @@ def _odoo_sources():
1259
  },
1260
  }
1261
  GRID_SOURCES.update(rel.stock_source_specs())
 
1262
  GRID_SOURCES.update(rel.purchase_source_specs())
1263
  GRID_SOURCES.update(rel.payment_source_specs())
1264
  GRID_SOURCES[rel.SHIPMENTS_KEY] = rel.shipment_source_spec()
@@ -1658,6 +1661,11 @@ def _source_for(cur, table_key, *, allow_lookups=True):
1658
  import amazon_relational as amazon
1659
  if key in amazon.SOURCE:
1660
  return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
 
 
 
 
 
1661
  spec = _sources().get(key)
1662
  if not spec:
1663
  return None
@@ -1717,6 +1725,10 @@ def _source_for(cur, table_key, *, allow_lookups=True):
1717
  expr = spec['id']
1718
  cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
1719
  have_by_alias), cast)
 
 
 
 
1720
  if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
1721
  spec = {**spec, "page_key": spec["id"]}
1722
  return {**spec, "cols": cols}
@@ -1898,11 +1910,76 @@ def _filter_columns(spec, fields):
1898
  continue
1899
  ftype = str((by_key.get(key) or {}).get("type") or "text")
1900
  out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
 
 
1901
  if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
1902
  out[key]["json_tags"] = True
1903
  return out
1904
 
1905
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1906
  def _leaf_cols(nodes):
1907
  """Every `colId` a filter tree names, at any depth."""
1908
  seen = set()
@@ -4015,7 +4092,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
4015
  try:
4016
  cur, mismatch = (_mirror_cur(session, table_key)
4017
  if (table_key.startswith(("ut_amzn_", "ut_meta_"))
4018
- or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY)
 
4019
  else _mirror_cur(session))
4020
  except RuntimeError as e:
4021
  # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
@@ -4058,6 +4136,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
4058
  fields = list(declared.values())
4059
  spec = _if_read_spec(spec, fields, table_key, st)
4060
  cols = _filter_columns(spec, fields)
 
4061
 
4062
  # ── THE PERMANENT WALL (W36-T21 / R6), COMPILED FIRST AND FAIL-CLOSED ────────────────────
4063
  # An administrator's row filter is a `FilterTree` in the SAME vocabulary the client sends, so
@@ -4273,7 +4352,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
4273
  where = f"({where}) AND {pred.sql}" if where else pred.sql
4274
  params.extend(pred.params)
4275
  if str(search or "").strip():
4276
- got = _fs().compile_search(search.strip(), cols)
4277
  if got is not None:
4278
  where = f"({where}) AND {got.sql}" if where else got.sql
4279
  params.extend(got.params)
@@ -4288,7 +4367,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
4288
 
4289
 
4290
  @contextmanager
4291
- def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None):
 
4292
  """One compiler for native rows and their aggregate sibling, over the walled payload.
4293
 
4294
  The temporary connection contains only this request's already stripped, pinned rows and is
@@ -4334,6 +4414,36 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
4334
  else:
4335
  connection = datastore.bounded_connection(duckdb.connect(":memory:"))
4336
  with connection as cur:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4337
  cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
4338
  wanted = set(ids) if ids is not None else None
4339
  if streamed:
@@ -4386,6 +4496,7 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
4386
  spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
4387
  "cols": bindings, "where": None}
4388
  cols = _filter_columns(spec, fields)
 
4389
  limits = list(payload.get("limits") or [])
4390
  _had_cohort = bool(_active_cohort_leaves(query["filters"]))
4391
  tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
@@ -4405,7 +4516,7 @@ def _materialised_read(table_key, session, query, related, ids=None, *, payload=
4405
  if pred is not None and pred.uses_aggregate:
4406
  raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
4407
  where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
4408
- search = _fs().compile_search(query["search"], cols) if query["search"] else None
4409
  if search is not None:
4410
  where = f"({where}) AND {search.sql}" if where else search.sql
4411
  params.extend(search.params)
@@ -4439,7 +4550,10 @@ def _group_pin(cols, group_by, group_key):
4439
  key = json.loads(group_key) if group_key not in (None, "") else None
4440
  except ValueError:
4441
  raise err(400, "bad_group", "groupKey must be the JSON of a group key")
4442
- gexpr = _fs().group_sql(gspec)
 
 
 
4443
  if key is None:
4444
  return f"({gexpr}) IS NULL", []
4445
  if gspec["type"] in _fs().NUMERIC_TYPES:
@@ -4672,15 +4786,16 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
4672
  needed.add(group_field)
4673
  if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
4674
  needed = None
4675
- with _materialised_read(table_key, session, query, related, ids=ids, payload=payload, needed=needed) as read:
 
4676
  spec = read["spec"]
4677
- order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
4678
  where, params = read["where"], list(read["params"])
4679
  gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
4680
  if gsql:
4681
  where = f"({where}) AND {gsql}" if where else gsql
4682
  params.extend(gparams)
4683
- options = dict(select='_row_index', where=where, params=tuple(params),
4684
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
4685
  cur=read['cur'], **read['frm'])
4686
  if table_key == 'ut_activity':
@@ -4689,6 +4804,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
4689
  else:
4690
  import registry_source
4691
  revision = registry_source.generation(session, table_key)
 
 
4692
  question = _window_query(table_key, filters, filterConj, sorts, search, related,
4693
  groupBy, groupKey, ids, revision=revision)
4694
  win = _window_page(session, question, cursor, page_first=True,
@@ -4696,7 +4813,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
4696
  payload = read["payload"]
4697
  indexes = [int(row[0]) for row in win['rows']]
4698
  value_sql = read['spec']['id'] if read['streamed'] else 'payload'
4699
- values = (read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows '
 
4700
  'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
4701
  if indexes else [])
4702
  if read['streamed']:
@@ -4705,6 +4823,8 @@ def _materialised_window_read(table_key, session, offset, limit, filters, filter
4705
  page_ids = [by_index[index] for index in indexes]
4706
  if table_key in {'customer_data', 'product_data'}:
4707
  payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
 
 
4708
  else:
4709
  payload = registry_source.stored_page(session, table_key, page_ids, related=related)
4710
  rows = payload['rows']
@@ -4822,8 +4942,9 @@ def odoo_table_rows(table_key: str,
4822
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4823
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4824
  if (admitted is None if page is not None else _materialised_table(table_key)):
4825
- return _materialised_window(table_key, session, offset, limit, filters, filterConj,
4826
- sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor)
 
4827
 
4828
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
4829
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
@@ -4850,7 +4971,7 @@ def odoo_table_rows(table_key: str,
4850
  # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
4851
  # row on two pages and drop another entirely — a duplicate the user sees with no error
4852
  # anywhere. `compile_order_by`'s own docstring says so.
4853
- order = _fs().compile_order_by(sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"]
4854
 
4855
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
4856
  revision = _mirror_page_revision(cur, page_rev)
@@ -4977,14 +5098,23 @@ def odoo_table_rows(table_key: str,
4977
  for row in rows:
4978
  if str(row["pid"]) in notes:
4979
  row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
4980
- return {"ok": True, "fields": fields, "rows": rows,
4981
  # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
4982
  "total": win["total"], "totalUnfiltered": unfiltered,
4983
  "offset": win["offset"], "limit": win["limit"], "limits": limits,
4984
  "next_cursor": win['next_cursor'],
4985
  "today": _today(), "identity": {"pid": "pid"},
4986
  "scope": {"table": table_key, "readThrough": True},
4987
- "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)}
 
 
 
 
 
 
 
 
 
4988
 
4989
 
4990
  # ══════════════════════════════════════════════════════════
@@ -5297,7 +5427,7 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
5297
  "recommendation": "group by a column the grid reads from the connection, or ask "
5298
  "an administrator for access to this one"})
5299
  else:
5300
- gexpr = _fs().group_sql(gspec)
5301
 
5302
  # ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
5303
  # RECOVERED FROM THE GROUPS. A page may hold only some groups, and a `median` or a
@@ -5324,7 +5454,9 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
5324
  select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
5325
  f"THEN 1 ELSE 0 END"),
5326
  where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
5327
- projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj)
 
 
5328
  group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
5329
  rows = datastore.window(
5330
  select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
@@ -5333,7 +5465,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
5333
  for r in rows:
5334
  vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
5335
  for idx, (_slot, col_id, _expr) in enumerate(plan)}
5336
- groups.append({"key": _group_key(r[0]), "label": _group_label(r[0]),
 
5337
  "count": int(r[1] or 0), "aggregates": vals})
5338
  return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
5339
  "total_groups": distinct, "group_offset": group_offset,
@@ -5344,6 +5477,8 @@ def _aggregate_over(cur, spec, cols, where, params, frm, aggs, group_by=None, li
5344
  def _mirror_aggregate(read, tenant, aggs, group_by, group_offset, group_limit):
5345
  """Use an exact published summary or retain the full compiled SQL scan."""
5346
  from harness import summary_tables
 
 
5347
 
5348
  limits = list(read["limits"])
5349
  plan = _agg_plan(read["cols"], aggs, limits)
@@ -5691,8 +5826,31 @@ def _ut():
5691
 
5692
 
5693
  def _today():
5694
- import time
5695
- return time.strftime("%Y-%m-%d")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5696
 
5697
 
5698
  def _register_c1_window():
 
929
  "id": "so.id",
930
  "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set
931
  "default_wholesale_scope": True,
932
+ "business_days": ("order_date",), # stored UTC instants read as tenant days (`_source_for`)
933
  "cols": {
934
  "order_no": ("so.name", _s),
935
  "odoo_id": ("so.id", _i),
 
991
  "id": "sol.id",
992
  "needs_excluded": True,
993
  "default_wholesale_scope": True,
994
+ "business_days": ("order_date",),
995
  "cols": {
996
  "odoo_id": ("sol.id", _i),
997
  "order_no": ("so.name", _s),
 
1261
  },
1262
  }
1263
  GRID_SOURCES.update(rel.stock_source_specs())
1264
+ GRID_SOURCES[rel.LANDED_COST_LINES_KEY] = rel.landed_cost_line_source_spec()
1265
  GRID_SOURCES.update(rel.purchase_source_specs())
1266
  GRID_SOURCES.update(rel.payment_source_specs())
1267
  GRID_SOURCES[rel.SHIPMENTS_KEY] = rel.shipment_source_spec()
 
1661
  import amazon_relational as amazon
1662
  if key in amazon.SOURCE:
1663
  return _amazon_page_source(cur, key, binding=True, allow_lookups=allow_lookups)
1664
+ if _rel().base_table_key(key) == _rel().LANDED_COST_LINES_KEY:
1665
+ try:
1666
+ _rel().require_landed_cost_lines_mirror(cur)
1667
+ except _rel().Refused as exc:
1668
+ raise err(503, 'store_not_ready', str(exc)) from exc
1669
  spec = _sources().get(key)
1670
  if not spec:
1671
  return None
 
1725
  expr = spec['id']
1726
  cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr,
1727
  have_by_alias), cast)
1728
+ # Resolved per request: the zone is the bound tenant's, and the registry is shared.
1729
+ for name in (spec.get("business_days") or ()):
1730
+ if name in cols:
1731
+ cols[name] = (_business_day_sql(cols[name][0]), cols[name][1])
1732
  if str(table_key).startswith("ut_odoo_") and spec.get("table") and not spec.get("from_sql"):
1733
  spec = {**spec, "page_key": spec["id"]}
1734
  return {**spec, "cols": cols}
 
1910
  continue
1911
  ftype = str((by_key.get(key) or {}).get("type") or "text")
1912
  out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False}
1913
+ if (by_key.get(key) or {}).get('personal') is True:
1914
+ out[key].update(personal=True, column=key, encrypted=False)
1915
  if key == "customer_type" and (by_key.get(key) or {}).get("source") == "odoo":
1916
  out[key]["json_tags"] = True
1917
  return out
1918
 
1919
 
1920
+ def _query_compile(method, *args, **kwargs):
1921
+ try:
1922
+ return getattr(_fs(), method)(*args, **kwargs)
1923
+ except ValueError as exc:
1924
+ raise err(400, 'query_unsupported', str(exc)) from exc
1925
+
1926
+
1927
+ def _bind_personal_columns(cols, spec, cur, session, table_key):
1928
+ """Lend SQL only ciphertext indexes; legacy mirrors remain readable until keys are enabled."""
1929
+ from core import row_cipher, tenant_keys, user_table_rows, store_pg
1930
+ personal = [key for key, value in cols.items() if value.get('personal')]
1931
+ if not personal or tenant_keys.subkey(session.runtime.key, 'index') is None:
1932
+ return cols
1933
+ tenant = session.runtime.key
1934
+ lease = _RelationLease(cur)
1935
+ loaded = {}
1936
+
1937
+ def expression(column):
1938
+ if column in loaded:
1939
+ return loaded[column]
1940
+ import csv
1941
+ import tempfile
1942
+ from pathlib import Path
1943
+ from core.request_timing import remaining_budget
1944
+ name = lease.table('_personal')
1945
+ # Fetch ciphertext in bounded batches; DuckDB receives identities and HMACs only.
1946
+ with tempfile.TemporaryDirectory(prefix='personal-index-') as directory:
1947
+ path = Path(directory) / 'indexes.csv'
1948
+ with path.open('w', newline='', encoding='utf-8') as output:
1949
+ writer = csv.writer(output)
1950
+ with store_pg._pool().connection() as pg, pg.transaction():
1951
+ pg.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY')
1952
+ after = ''
1953
+ while True:
1954
+ batch = pg.execute(user_table_rows._sql(
1955
+ 'SELECT row_id, (value -> %s)::text FROM {recs} '
1956
+ 'WHERE table_key = %s AND row_id > %s ORDER BY row_id LIMIT 2048', tenant),
1957
+ (column, table_key, after)).fetchall()
1958
+ if not batch:
1959
+ break
1960
+ remaining_budget()
1961
+ for identity, raw in batch:
1962
+ value = json.loads(raw) if isinstance(raw, str) else raw
1963
+ if value is not None and not row_cipher.is_sealed(value):
1964
+ row_cipher.refuse(column, 'query before personal-field migration')
1965
+ writer.writerow((str(identity), value['bi'] if value is not None else None))
1966
+ after = str(batch[-1][0])
1967
+ cur.execute(f'CREATE TEMP TABLE "{name}" (pid VARCHAR, bi VARCHAR)')
1968
+ if path.stat().st_size:
1969
+ cur.execute(f'INSERT INTO "{name}" SELECT * FROM read_csv(?, header=false, '
1970
+ "columns={'pid':'VARCHAR','bi':'VARCHAR'})", [str(path)])
1971
+ identity = f'CAST({spec["id"]} AS VARCHAR)'
1972
+ found = f'SELECT bi FROM "{name}" pi WHERE pi.pid = {identity}'
1973
+ loaded[column] = (f'(CASE WHEN EXISTS (SELECT 1 FROM "{name}" pi WHERE pi.pid = {identity}) '
1974
+ f'THEN ({found}) ELSE error(\'Finish personal-field migration before querying this column.\') END)')
1975
+ return loaded[column]
1976
+
1977
+ for column in personal:
1978
+ cols[column].update(tenant=tenant, encrypted=True, _personal_lease=lease,
1979
+ personal_index_sql=lambda column=column: expression(column))
1980
+ return cols
1981
+
1982
+
1983
  def _leaf_cols(nodes):
1984
  """Every `colId` a filter tree names, at any depth."""
1985
  seen = set()
 
4092
  try:
4093
  cur, mismatch = (_mirror_cur(session, table_key)
4094
  if (table_key.startswith(("ut_amzn_", "ut_meta_"))
4095
+ or rel.base_table_key(table_key) == rel.SHIPMENTS_KEY
4096
+ or rel.base_table_key(table_key) in rel.stock_source_specs())
4097
  else _mirror_cur(session))
4098
  except RuntimeError as e:
4099
  # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
 
4136
  fields = list(declared.values())
4137
  spec = _if_read_spec(spec, fields, table_key, st)
4138
  cols = _filter_columns(spec, fields)
4139
+ cols = _bind_personal_columns(cols, spec, cur, session, table_key)
4140
 
4141
  # ── THE PERMANENT WALL (W36-T21 / R6), COMPILED FIRST AND FAIL-CLOSED ────────────────────
4142
  # An administrator's row filter is a `FilterTree` in the SAME vocabulary the client sends, so
 
4352
  where = f"({where}) AND {pred.sql}" if where else pred.sql
4353
  params.extend(pred.params)
4354
  if str(search or "").strip():
4355
+ got = _query_compile('compile_search', search.strip(), cols)
4356
  if got is not None:
4357
  where = f"({where}) AND {got.sql}" if where else got.sql
4358
  params.extend(got.params)
 
4367
 
4368
 
4369
  @contextmanager
4370
+ def _materialised_read(table_key, session, query, related, ids=None, *, payload=None, needed=None,
4371
+ sql_page=False):
4372
  """One compiler for native rows and their aggregate sibling, over the walled payload.
4373
 
4374
  The temporary connection contains only this request's already stripped, pinned rows and is
 
4414
  else:
4415
  connection = datastore.bounded_connection(duckdb.connect(":memory:"))
4416
  with connection as cur:
4417
+ if compiled and sql_page and not related:
4418
+ mirror_revision = _mirror_page_revision(cur, source_generation)
4419
+ with registry_source.canonical_selection(session, cur, table_key, metadata, needed, ids=ids) as spec:
4420
+ if spec is not None:
4421
+ cols = _filter_columns(dict(spec, cols={k: v for k, v in spec['cols'].items()
4422
+ if k in {f['key'] for f in fields}}), fields)
4423
+ limits = list(payload.get('limits') or [])
4424
+ had_cohort = bool(_active_cohort_leaves(query['filters']))
4425
+ tree, cohorts = _views_as_cohorts(query['filters'], table_key, session,
4426
+ spec, cols, None, [], cur, limits)
4427
+ tree, cohorts = _ranks_as_cohorts(tree, query['filterConj'], spec, cols,
4428
+ None, [], cur, cohorts, had_cohort, payload.get('today') or _today(), limits)
4429
+ if _leaf_cols(tree) - _PSEUDO_COLS - set(cols):
4430
+ raise err(400, 'filter_unsupported', 'A filter names a column unavailable to this account.')
4431
+ try:
4432
+ pred = _fs().compile_filter_tree(tree, conj=query['filterConj'], columns=cols,
4433
+ id_sql=spec['id'], today=payload.get('today') or _today(), cohort_sets=cohorts)
4434
+ except ValueError as exc:
4435
+ raise err(400, 'filter_unsupported', str(exc))
4436
+ if pred is not None and pred.uses_aggregate:
4437
+ raise err(400, 'filter_unsupported', 'This condition cannot be evaluated for individual rows.')
4438
+ yield {'cur': cur, 'spec': spec, 'cols': cols,
4439
+ 'where': pred.sql if pred else None, 'params': list(pred.params) if pred else [],
4440
+ 'frm': {'table': spec['table']}, 'limits': limits, 'payload': payload,
4441
+ 'source_count': cur.execute(f'SELECT count(*) FROM "{spec["table"]}"').fetchone()[0],
4442
+ 'streamed': True, 'selection': True}
4443
+ registry_source.require_generation(session, table_key, source_generation)
4444
+ if mirror_revision != _mirror_page_revision(cur, source_generation):
4445
+ raise err(409, 'window_population_changed', 'These records changed while loading. Reload this database.')
4446
+ return
4447
  cur.execute("CREATE TEMP TABLE pinned_rows (_row_index BIGINT, payload JSON)")
4448
  wanted = set(ids) if ids is not None else None
4449
  if streamed:
 
4496
  spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
4497
  "cols": bindings, "where": None}
4498
  cols = _filter_columns(spec, fields)
4499
+ cols = _bind_personal_columns(cols, spec, cur, session, table_key)
4500
  limits = list(payload.get("limits") or [])
4501
  _had_cohort = bool(_active_cohort_leaves(query["filters"]))
4502
  tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
 
4516
  if pred is not None and pred.uses_aggregate:
4517
  raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
4518
  where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
4519
+ search = _query_compile('compile_search', query["search"], cols) if query["search"] else None
4520
  if search is not None:
4521
  where = f"({where}) AND {search.sql}" if where else search.sql
4522
  params.extend(search.params)
 
4550
  key = json.loads(group_key) if group_key not in (None, "") else None
4551
  except ValueError:
4552
  raise err(400, "bad_group", "groupKey must be the JSON of a group key")
4553
+ gexpr = _query_compile('group_sql', gspec)
4554
+ if gspec.get('personal') and gspec.get('encrypted', True) and key is not None:
4555
+ from core import row_cipher
4556
+ key = row_cipher.blind_index(gspec['tenant'], key)
4557
  if key is None:
4558
  return f"({gexpr}) IS NULL", []
4559
  if gspec["type"] in _fs().NUMERIC_TYPES:
 
4786
  needed.add(group_field)
4787
  if query['search'] or _leaf_cols(query['filters']) & _PSEUDO_COLS:
4788
  needed = None
4789
+ with _materialised_read(table_key, session, query, related, ids=ids, payload=payload,
4790
+ needed=needed, sql_page=True) as read:
4791
  spec = read["spec"]
4792
+ order = _query_compile('compile_order_by', query["sorts"], read["cols"], tiebreak_sql=spec["id"])
4793
  where, params = read["where"], list(read["params"])
4794
  gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
4795
  if gsql:
4796
  where = f"({where}) AND {gsql}" if where else gsql
4797
  params.extend(gparams)
4798
+ options = dict(select=spec['id'] if read.get('selection') else '_row_index', where=where, params=tuple(params),
4799
  order_by=order or spec['id'], offset=window_offset, limit=limit or None,
4800
  cur=read['cur'], **read['frm'])
4801
  if table_key == 'ut_activity':
 
4804
  else:
4805
  import registry_source
4806
  revision = registry_source.generation(session, table_key)
4807
+ if read.get('selection'):
4808
+ revision = _mirror_page_revision(read['cur'], revision)
4809
  question = _window_query(table_key, filters, filterConj, sorts, search, related,
4810
  groupBy, groupKey, ids, revision=revision)
4811
  win = _window_page(session, question, cursor, page_first=True,
 
4813
  payload = read["payload"]
4814
  indexes = [int(row[0]) for row in win['rows']]
4815
  value_sql = read['spec']['id'] if read['streamed'] else 'payload'
4816
+ values = ([(pid, pid) for pid in indexes] if read.get('selection') else
4817
+ read['cur'].execute(f'SELECT _row_index, {value_sql} FROM pinned_rows '
4818
  'WHERE _row_index IN (SELECT unnest(?::BIGINT[]))', [indexes]).fetchall()
4819
  if indexes else [])
4820
  if read['streamed']:
 
4823
  page_ids = [by_index[index] for index in indexes]
4824
  if table_key in {'customer_data', 'product_data'}:
4825
  payload = registry_source.canonical_page(session, read['cur'], table_key, page_ids, related=related)
4826
+ if read.get('selection') and [int(row['pid']) for row in payload['rows']] != page_ids:
4827
+ raise err(409, 'window_population_changed', 'These records changed while loading. Reload this database.')
4828
  else:
4829
  payload = registry_source.stored_page(session, table_key, page_ids, related=related)
4830
  rows = payload['rows']
 
4942
  page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
4943
  admitted = require_amazon_mirror(table_key, session, st=page[0] if page else None)
4944
  if (admitted is None if page is not None else _materialised_table(table_key)):
4945
+ return _audit_personal_response(table_key, _materialised_window(
4946
+ table_key, session, offset, limit, filters, filterConj,
4947
+ sorts, search, related, groupBy, groupKey, ids=ids, cursor=cursor), session)
4948
 
4949
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
4950
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
 
4971
  # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
4972
  # row on two pages and drop another entirely — a duplicate the user sees with no error
4973
  # anywhere. `compile_order_by`'s own docstring says so.
4974
+ order = _query_compile('compile_order_by', sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"]
4975
 
4976
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
4977
  revision = _mirror_page_revision(cur, page_rev)
 
5098
  for row in rows:
5099
  if str(row["pid"]) in notes:
5100
  row[user_tables.CELL_NOTES_KEY] = notes[str(row["pid"])]
5101
+ return _audit_personal_response(table_key, {"ok": True, "fields": fields, "rows": rows,
5102
  # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`.
5103
  "total": win["total"], "totalUnfiltered": unfiltered,
5104
  "offset": win["offset"], "limit": win["limit"], "limits": limits,
5105
  "next_cursor": win['next_cursor'],
5106
  "today": _today(), "identity": {"pid": "pid"},
5107
  "scope": {"table": table_key, "readThrough": True},
5108
+ "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)}, session)
5109
+
5110
+
5111
+ def _audit_personal_response(table_key, payload, session):
5112
+ if not isinstance(payload, dict) or payload.get('ok') is False:
5113
+ return payload
5114
+ from core import personal_reads
5115
+ personal_reads.record_rows(table_key, payload.get('rows') or [], st=session.runtime,
5116
+ user=session.user, definition={'fields': payload.get('fields') or []})
5117
+ return payload
5118
 
5119
 
5120
  # ══════════════════════════════════════════════════════════
 
5427
  "recommendation": "group by a column the grid reads from the connection, or ask "
5428
  "an administrator for access to this one"})
5429
  else:
5430
+ gexpr = _query_compile('group_sql', gspec)
5431
 
5432
  # ⛔⛔ THE OVERALL SUMMARY IS COMPUTED WHETHER OR NOT THIS REQUEST GROUPS, AND IT IS **NOT**
5433
  # RECOVERED FROM THE GROUPS. A page may hold only some groups, and a `median` or a
 
5454
  select=(f"count(DISTINCT {gexpr}) + CASE WHEN count(*) - count({gexpr}) > 0 "
5455
  f"THEN 1 ELSE 0 END"),
5456
  where=where, params=tuple(params), order_by="1", limit=1, cur=cur, **frm)["rows"][0][0] or 0)
5457
+ personal_group = bool(gspec.get('personal') and gspec.get('encrypted', True))
5458
+ label_projection = [f'min({gspec["sql"]}) AS "_label"'] if personal_group else []
5459
+ projection = ", ".join([f'{gexpr} AS "_k"', 'count(*) AS "_c"'] + agg_proj + label_projection)
5460
  group_limit = min(group_limit if group_limit is not None else MAX_GROUPS, MAX_GROUPS)
5461
  rows = datastore.window(
5462
  select=projection, where=where, params=tuple(params), order_by="2 DESC, 1 ASC",
 
5465
  for r in rows:
5466
  vals = {col_id: (_agg_number(r[2 + idx]) if (2 + idx) < len(r) else None)
5467
  for idx, (_slot, col_id, _expr) in enumerate(plan)}
5468
+ value = r[-1] if personal_group else r[0]
5469
+ groups.append({"key": value if personal_group else _group_key(value), "label": _group_label(value),
5470
  "count": int(r[1] or 0), "aggregates": vals})
5471
  return {"total_rows": total_rows, "aggregates": answers, "groups": groups,
5472
  "total_groups": distinct, "group_offset": group_offset,
 
5477
  def _mirror_aggregate(read, tenant, aggs, group_by, group_offset, group_limit):
5478
  """Use an exact published summary or retain the full compiled SQL scan."""
5479
  from harness import summary_tables
5480
+ if any(col.get('personal') and col.get('encrypted', True) for col in read['cols'].values()):
5481
+ return None
5482
 
5483
  limits = list(read["limits"])
5484
  plan = _agg_plan(read["cols"], aggs, limits)
 
5826
 
5827
 
5828
  def _today():
5829
+ """The bound tenant's business day, ISO; UTC when the tenant has no zone."""
5830
+ from core import periods
5831
+ return periods.today().isoformat()
5832
+
5833
+
5834
+ def _business_day_sql(expr):
5835
+ """`expr`, a stored UTC instant, as its ISO business day in the bound tenant's zone.
5836
+
5837
+ ⛔ ONE expression serves the projection, the filter and the sort, so an `order_date` filter
5838
+ pushes down the same tenant-time bounds the grid shows. A bare date passes through. No zone
5839
+ or a degraded NULL keeps `expr` unchanged."""
5840
+ from core import periods
5841
+ zone = periods.business_tz()
5842
+ if not zone or str(expr).strip().upper() == "NULL":
5843
+ return expr
5844
+ try:
5845
+ from zoneinfo import ZoneInfo
5846
+ ZoneInfo(zone)
5847
+ except Exception: # noqa: BLE001
5848
+ return expr # `periods` reads an unknown zone as UTC; so does this
5849
+ name = zone.replace("'", "''")
5850
+ text = f"CAST({expr} AS VARCHAR)"
5851
+ return (f"(CASE WHEN LENGTH({text}) <= 10 THEN {text} ELSE COALESCE(CAST(CAST(timezone("
5852
+ f"'{name}', timezone('UTC', TRY_CAST({text} AS TIMESTAMP))) AS DATE) AS VARCHAR), "
5853
+ f"SUBSTR({text}, 1, 10)) END)")
5854
 
5855
 
5856
  def _register_c1_window():
api/routes_opportunity_finder.py CHANGED
@@ -7,8 +7,6 @@ from deps import Session, err, require_session
7
  router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
8
  SOURCE = "ut_amzn_brand_analytics"
9
  TEMPLATE_KIND = "opportunity_finder"
10
- #: Weeks in a list row's sparkline. Each week is one store read per /screen call.
11
- SPARK_WEEKS = 8
12
  SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
13
 
14
 
@@ -17,13 +15,14 @@ def _module():
17
  return opportunity_finder
18
 
19
 
20
- def _reader(session: Session = Depends(require_session)) -> Session:
21
- """Reads need read access to the Amazon brand analytics database."""
22
- from core import perm_scope
23
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
24
- raise err(403, "opportunity_finder_forbidden", "Your account cannot read Amazon brand analytics.")
25
- if perm_scope.wall_declared(session.user, SOURCE, st=session.runtime):
26
- raise err(403, "opportunity_finder_limited", "The finder is unavailable with your brand analytics access limit.")
 
27
  return session
28
 
29
 
@@ -31,6 +30,8 @@ def _writer(session: Session = Depends(_reader)) -> Session:
31
  """Writes also need a role that may edit."""
32
  if str(session.user.get("role") or "user").strip().lower() == "viewer":
33
  raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
 
 
34
  return session
35
 
36
 
@@ -46,7 +47,7 @@ def _call(fn, *args, **kwargs):
46
  try:
47
  return fn(*args, **kwargs)
48
  except PermissionError as exc:
49
- raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.") from exc
50
  except KeyError as exc:
51
  raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
52
  except ValueError as exc:
@@ -94,12 +95,32 @@ def _mirror_opener(session):
94
  return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  def _detail_options(session):
98
  """Auxiliary sources fail closed before any raw mirror projection or stored-row read."""
 
99
  from core import perm_scope
100
  of = _module()
 
 
101
 
102
  def admitted(key):
 
 
 
 
103
  return (perm_scope.may_read(session.user, key, st=session.runtime)
104
  and not perm_scope.wall_declared(session.user, key, st=session.runtime))
105
 
@@ -119,13 +140,10 @@ def _detail_options(session):
119
  "monthly_allowed": admitted("ut_amzn_search_query_performance")}
120
 
121
 
122
- def _with_spark(of, st, items):
123
- """Each page item gains `spark`: its rank in the newest SPARK_WEEKS held weeks, oldest first."""
124
- weeks = of.weeks(None, st=st)[-SPARK_WEEKS:] if SPARK_WEEKS > 0 else []
125
- held = [(week, of.week_point(None, week, st=st).get("terms") or {}) for week in weeks]
126
  for item in items:
127
- term = of.norm(item.get("term"))
128
- item["spark"] = [{"week": week, "rank": terms[term]} for week, terms in held if term in terms]
129
  return items
130
 
131
 
@@ -136,10 +154,12 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
136
  of = _module()
137
  record = _app(session, app)
138
  st = session.runtime
 
139
  listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
140
- opener=_mirror_opener(session))
141
- listed["items"] = _call(_with_spark, of, st, listed["items"])
142
- return {"canEdit": _may_edit(session, record), "week": listed["week"], "priorWeek": listed["priorWeek"],
 
143
  "weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
144
  "departments": listed["departments"], "counts": listed["counts"],
145
  "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
@@ -148,10 +168,13 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
148
 
149
 
150
  @router.get("/items/{item_id}")
151
- def item_detail(item_id: str, session: Session = Depends(_app_reader)):
152
  of = _module()
153
- return _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
154
- opener=_mirror_opener(session), **_detail_options(session))
 
 
 
155
 
156
 
157
  @router.patch("/items/{item_id}")
@@ -168,7 +191,7 @@ def term_trend(item_id: str, session: Session = Depends(_app_reader)):
168
  """A term's weekly rank trend and its newest top 3 clicked ASINs."""
169
  of = _module()
170
  return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session),
171
- **_detail_options(session))
172
 
173
 
174
  @router.get("/settings")
@@ -195,4 +218,4 @@ def validate(week: str | None = Query(None), start: str | None = Query(None), en
195
  session: Session = Depends(_app_reader)):
196
  of = _module()
197
  return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
198
- opener=_mirror_opener(session))
 
7
  router = APIRouter(prefix="/api/v1/app-kinds/opportunity_finder", tags=["opportunity_finder"])
8
  SOURCE = "ut_amzn_brand_analytics"
9
  TEMPLATE_KIND = "opportunity_finder"
 
 
10
  SETTING_NAMES = {"min_rise_pct": "min_rise_pct", "minRisePct": "min_rise_pct", "departments": "departments"}
11
 
12
 
 
15
  return opportunity_finder
16
 
17
 
18
+ def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
19
+ """R62: reads need the App grant plus the shared source check, never the source open grant."""
20
+ import application_registry
21
+ record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
22
+ _tenant(session), session.uname, session.admin, TEMPLATE_KIND)
23
+ application_registry.admit_app(session, record, (SOURCE, "ut_amzn_listings", "ut_amzn_search_query_performance"),
24
+ code="opportunity_finder")
25
+ session.source_app = record
26
  return session
27
 
28
 
 
30
  """Writes also need a role that may edit."""
31
  if str(session.user.get("role") or "user").strip().lower() == "viewer":
32
  raise err(403, "opportunity_finder_read_only", "You can view this App but not change it.")
33
+ if _source_scope(session) is not None:
34
+ raise err(403, "opportunity_finder_limited", "Saved workflow changes need unrestricted source rows and fields.")
35
  return session
36
 
37
 
 
47
  try:
48
  return fn(*args, **kwargs)
49
  except PermissionError as exc:
50
+ raise err(403, "opportunity_finder_limited", str(exc)) from exc
51
  except KeyError as exc:
52
  raise err(404, "opportunity_finder_not_found", "That search term is not in the opportunity finder.") from exc
53
  except ValueError as exc:
 
95
  return lambda: routes_odoo_tables._table_cursor(session.runtime, SOURCE)
96
 
97
 
98
+ def _source_scope(session):
99
+ """Lend the shared source evaluator to the provider only while a source wall applies."""
100
+ import application_registry
101
+ sources = (SOURCE, "ut_amzn_listings", "ut_amzn_search_query_performance")
102
+ walls = application_registry.app_access(session.user, session.runtime, None, sources)["narrow"]
103
+ active = {source for source, wall in walls.items() if wall["rows"] or wall["hidden"]}
104
+ if not active:
105
+ return None
106
+ from routes_app_dashboard import source_scope_for
107
+ scope = source_scope_for(session, session.source_app)
108
+ return lambda source, records: scope(source, records) if source in active else None
109
+
110
+
111
  def _detail_options(session):
112
  """Auxiliary sources fail closed before any raw mirror projection or stored-row read."""
113
+ import application_registry
114
  from core import perm_scope
115
  of = _module()
116
+ record = application_registry.readable_app(session.runtime, _tenant(session), session.uname,
117
+ session.admin, TEMPLATE_KIND)
118
 
119
  def admitted(key):
120
+ if key.startswith("ut_amzn_"):
121
+ access = application_registry.app_access(session.user, session.runtime, record, (key,))
122
+ wall = access["narrow"][key]
123
+ return access["app"] == "ready" and not wall["rows"] and not wall["hidden"]
124
  return (perm_scope.may_read(session.user, key, st=session.runtime)
125
  and not perm_scope.wall_declared(session.user, key, st=session.runtime))
126
 
 
140
  "monthly_allowed": admitted("ut_amzn_search_query_performance")}
141
 
142
 
143
+ def _with_spark(of, st, items, source_scope=None):
144
+ """Keep the legacy spark wire as an alias of the provider's complete, scoped page history."""
 
 
145
  for item in items:
146
+ item["spark"] = item["trendPoints"]
 
147
  return items
148
 
149
 
 
154
  of = _module()
155
  record = _app(session, app)
156
  st = session.runtime
157
+ scope = _call(_source_scope, session)
158
  listed = _call(of.items, _tenant(session), session.uname, status=status, cursor=cursor, st=st,
159
+ opener=_mirror_opener(session), source_scope=scope)
160
+ listed["items"] = _call(_with_spark, of, st, listed["items"], source_scope=scope)
161
+ return {**listed, "canEdit": scope is None and _may_edit(session, record),
162
+ "week": listed["week"], "priorWeek": listed["priorWeek"],
163
  "weeks": listed["weeks"], "windows": listed["windows"], "periodType": listed["periodType"],
164
  "departments": listed["departments"], "counts": listed["counts"],
165
  "total": listed["total"], "items": listed["items"], "cursor": listed["cursor"],
 
168
 
169
 
170
  @router.get("/items/{item_id}")
171
+ def item_detail(item_id: str, app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)):
172
  of = _module()
173
+ scope = _call(_source_scope, session)
174
+ detail = _call(of.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
175
+ opener=_mirror_opener(session), source_scope=scope, **_detail_options(session))
176
+ _call(_with_spark, of, session.runtime, [detail["item"]], source_scope=scope)
177
+ return {**detail, "canEdit": scope is None and _may_edit(session, _app(session, app))}
178
 
179
 
180
  @router.patch("/items/{item_id}")
 
191
  """A term's weekly rank trend and its newest top 3 clicked ASINs."""
192
  of = _module()
193
  return _call(of.term_detail, _tenant(session), item_id, st=session.runtime, opener=_mirror_opener(session),
194
+ source_scope=_call(_source_scope, session), **_detail_options(session))
195
 
196
 
197
  @router.get("/settings")
 
218
  session: Session = Depends(_app_reader)):
219
  of = _module()
220
  return _call(of.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
221
+ opener=_mirror_opener(session), source_scope=_call(_source_scope, session))
api/routes_payouts.py CHANGED
@@ -1,6 +1,6 @@
1
  """Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
2
 
3
- Every route names the App (`?app=`) and checks the caller may read it and the settlements database.
4
  Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
5
  router: `app_routes.mount_app_routers` does, from the App contract.
6
  """
@@ -18,11 +18,11 @@ def _module():
18
  return payouts
19
 
20
 
21
- def _reader(session: Session = Depends(require_session)) -> Session:
22
- """Reads need read access to the Amazon settlements database."""
23
- from core import perm_scope
24
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
25
- raise err(403, "payouts_forbidden", "Your account cannot read Amazon settlements.")
26
  return session
27
 
28
 
@@ -52,9 +52,12 @@ def _app(session, app_id):
52
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
  except Exception as exc: # noqa: BLE001
54
  raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
55
- if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
 
56
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
57
  raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
 
 
58
  return app
59
 
60
 
@@ -65,9 +68,9 @@ def _may_edit(session, app):
65
 
66
 
67
  def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
68
- """A read names the App the caller can read (404 first, so no App is confirmed), then the database."""
69
  _app(session, app)
70
- return _reader(session)
71
 
72
 
73
  def _settings_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
@@ -92,7 +95,7 @@ def _forecast_access(session):
92
  def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
93
  """A write names the App, and only an editor of that App may change it."""
94
  found = _app(session, app)
95
- _writer(_reader(session))
96
  if not _may_edit(session, found):
97
  raise err(403, "payouts_read_only", "You can view this App but not change it.")
98
  return session
@@ -104,6 +107,8 @@ def _call(fn, *args, missing="payouts_not_found", missing_message="That settleme
104
  return fn(*args, **kwargs)
105
  except PermissionError as exc:
106
  raise err(403, "payouts_read_only", str(exc)) from exc
 
 
107
  except pay.SourceUnavailable as exc:
108
  raise err(503, "payouts_no_source", str(exc)) from exc
109
  except ValueError as exc:
@@ -157,24 +162,34 @@ def update_item(item_id: str, status: str = Body(..., embed=True), note: str | N
157
 
158
 
159
  def _fee_access(session):
160
- """Fee comparisons require unrestricted settlement and estimate sources."""
161
  from core import perm_scope
 
162
  for key in (SOURCE, _module().FEE_ESTIMATE_SOURCE):
163
  team, agent = perm_scope.derive_pool_scope(session.user, key)
164
- if (not perm_scope.may_read(session.user, key, st=session.runtime)
165
- or perm_scope.wall_declared(session.user, key, st=session.runtime)
166
  or team is not None or agent is not None):
167
  raise err(403, "payouts_fee_forbidden", "Fee checks need access to settlements and fee estimates.")
168
 
169
 
170
  @router.get("/fee-checks")
171
  def read_fee_checks(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
172
- limit: int = Query(50, ge=1), q: str = Query(""), session: Session = Depends(_app_reader)):
 
173
  _fee_access(session)
174
- page = _call(_module().fee_checks, _tenant(session), st=session.runtime, cursor=cursor, limit=limit, q=q)
 
175
  return {**page, "canEdit": _may_edit(session, _app(session, app))}
176
 
177
 
 
 
 
 
 
 
 
 
178
  @router.patch("/fee-checks/{item_id}")
179
  def update_fee_check(item_id: str, status: str = Body(..., embed=True), session: Session = Depends(_app_writer)):
180
  _fee_access(session)
@@ -400,6 +415,14 @@ def read_screen(app: str = Query(..., min_length=1), currency: str | None = Quer
400
  return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]}
401
 
402
 
 
 
 
 
 
 
 
 
403
  @router.get("/screen/settlements/{settlement_id}")
404
  def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
405
  bucket: str | None = Query(None), cursor: str | None = Query(None),
 
1
  """Payouts App routes (C3): the overview, settlements with their waterfall and tie, owned items and settings.
2
 
3
+ Every route names the App (`?app=`) and checks its grant and source limits.
4
  Writes also need edit rights on that App and a role other than viewer. `main.py` does not mount this
5
  router: `app_routes.mount_app_routers` does, from the App contract.
6
  """
 
18
  return payouts
19
 
20
 
21
+ def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
22
+ """Bind the requested App before admitting its declared source contracts and limits."""
23
+ from routes_app_dashboard import admit_reader
24
+ record = _app(session, app) if isinstance(app, str) and app else None
25
+ admit_reader(session, 'payouts', record)
26
  return session
27
 
28
 
 
52
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
53
  except Exception as exc: # noqa: BLE001
54
  raise err(503, "payouts_unavailable", "The App could not be read. Try again shortly.") from exc
55
+ if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
56
+ or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
57
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
58
  raise err(404, "payouts_app_not_found", "That Payouts App does not exist.")
59
+ from routes_app_dashboard import admit_reader
60
+ admit_reader(session, 'payouts', app)
61
  return app
62
 
63
 
 
68
 
69
 
70
  def _app_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
71
+ """A read checks App identity before admitting its source scope."""
72
  _app(session, app)
73
+ return _reader(session, app=app)
74
 
75
 
76
  def _settings_reader(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
 
95
  def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(require_session)) -> Session:
96
  """A write names the App, and only an editor of that App may change it."""
97
  found = _app(session, app)
98
+ _writer(_reader(session, app=app))
99
  if not _may_edit(session, found):
100
  raise err(403, "payouts_read_only", "You can view this App but not change it.")
101
  return session
 
107
  return fn(*args, **kwargs)
108
  except PermissionError as exc:
109
  raise err(403, "payouts_read_only", str(exc)) from exc
110
+ except pay.TableRefused as exc:
111
+ raise err(409, "payouts_table_refused", str(exc)) from exc
112
  except pay.SourceUnavailable as exc:
113
  raise err(503, "payouts_no_source", str(exc)) from exc
114
  except ValueError as exc:
 
162
 
163
 
164
  def _fee_access(session):
165
+ """Fee comparisons use the App grant and refuse source or pool limits."""
166
  from core import perm_scope
167
+ _reader(session)
168
  for key in (SOURCE, _module().FEE_ESTIMATE_SOURCE):
169
  team, agent = perm_scope.derive_pool_scope(session.user, key)
170
+ if (perm_scope.wall_declared(session.user, key, st=session.runtime)
 
171
  or team is not None or agent is not None):
172
  raise err(403, "payouts_fee_forbidden", "Fee checks need access to settlements and fee estimates.")
173
 
174
 
175
  @router.get("/fee-checks")
176
  def read_fee_checks(app: str = Query(..., min_length=1), cursor: str | None = Query(None),
177
+ limit: int = Query(50, ge=1), q: str = Query(""), item: str | None = Query(None),
178
+ session: Session = Depends(_app_reader)):
179
  _fee_access(session)
180
+ page = _call(_module().fee_checks, _tenant(session), st=session.runtime, cursor=cursor, limit=limit, q=q,
181
+ item_id=item or None)
182
  return {**page, "canEdit": _may_edit(session, _app(session, app))}
183
 
184
 
185
+ @router.post("/fee-table")
186
+ def build_fee_table(session: Session = Depends(_app_reader)):
187
+ """Build the fee checks table when its projection moved; the Fee checks tab calls this before its Grid reads."""
188
+ _fee_access(session)
189
+ built = _call(_module().fee_table, _tenant(session), st=session.runtime)
190
+ return {k: built.get(k) for k in ("table", "rows", "stamp")}
191
+
192
+
193
  @router.patch("/fee-checks/{item_id}")
194
  def update_fee_check(item_id: str, status: str = Body(..., embed=True), session: Session = Depends(_app_writer)):
195
  _fee_access(session)
 
415
  return {**base, **page, "settlements": [{k: r.get(k) for k in LIST_FIELDS} for r in page["settlements"]]}
416
 
417
 
418
+ @router.post("/table")
419
+ def build_table(session: Session = Depends(_app_reader)):
420
+ """Build the settlements table when its source moved; the Settlements tab calls this before its Grid reads."""
421
+ pay = _module()
422
+ built = _call(pay.settlement_table, _tenant(session), st=session.runtime)
423
+ return {k: built.get(k) for k in ("table", "rows", "stamp")}
424
+
425
+
426
  @router.get("/screen/settlements/{settlement_id}")
427
  def read_screen_settlement(settlement_id: str, app: str = Query(..., min_length=1),
428
  bucket: str | None = Query(None), cursor: str | None = Query(None),
api/routes_products.py CHANGED
@@ -442,7 +442,7 @@ def _duplicate_code_limits(rows_src, fields):
442
  codes = ', '.join(report['codes'])
443
  return [{'subject': 'code', 'effect': 'duplicate_codes',
444
  'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
445
- 'recommendation': 'Ask the owner to name the canonical record for each shared code.'}]
446
 
447
 
448
  def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
@@ -515,6 +515,10 @@ def _compute_product_assembly(session: Session, scope: str = "product", storage_
515
  for _pid, _cells in _shared.items():
516
  _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
517
  ws["overlays"] = _ov
 
 
 
 
518
  workspace, fields, views, lists = aios_grid.workspace_wire(
519
  ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
520
  fields_base=fields_base, st=session.runtime)
@@ -704,13 +708,31 @@ def products(session: Session = Depends(module_gate(MODULE)), related: str = Non
704
  # report in there would red a gate outside this ticket's fence for a purely cosmetic nesting.
705
  # The envelope check beside it filters to the four `/customers` keys, so an additive key is
706
  # the shape this route already established with `identity` and `scope`.
707
- return {"fields": g["fields"], "rows": rows,
708
  "refusals": metric_refusals,
709
  "limits": field_limits + _duplicate_code_limits(g['rows_src'], g['fields']) + linked_metric_limits(metric_refusals),
710
  "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
711
  "identity": {"pid": "pid", "businessKey": "code"},
712
  "identity_report": pd.identity_report(g["rows_src"]),
713
  "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
714
 
715
 
716
  @router.post("/products/aggregates")
 
442
  codes = ', '.join(report['codes'])
443
  return [{'subject': 'code', 'effect': 'duplicate_codes',
444
  'cause': f"Shared SKU codes {codes}: {report['gap']} fewer rows than active Odoo products for these codes.",
445
+ 'recommendation': 'Ask the owner to choose the canonical Odoo record for each shared SKU.'}]
446
 
447
 
448
  def product_assembly(session: Session, scope: str = "product", storage_key: str = "",
 
515
  for _pid, _cells in _shared.items():
516
  _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells}
517
  ws["overlays"] = _ov
518
+ # D-7088: a saved lead overlay is the served lead, so coverage is recomputed from it here.
519
+ # This runs before the IF cells and the hidden strip, so every reader sees one coverage.
520
+ if with_rows:
521
+ rows_src = pd.apply_lead_overlay(rows_src, ws.get("overlays"))
522
  workspace, fields, views, lists = aios_grid.workspace_wire(
523
  ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key,
524
  fields_base=fields_base, st=session.runtime)
 
708
  # report in there would red a gate outside this ticket's fence for a purely cosmetic nesting.
709
  # The envelope check beside it filters to the four `/customers` keys, so an additive key is
710
  # the shape this route already established with `identity` and `scope`.
711
+ body = {"fields": g["fields"], "rows": rows,
712
  "refusals": metric_refusals,
713
  "limits": field_limits + _duplicate_code_limits(g['rows_src'], g['fields']) + linked_metric_limits(metric_refusals),
714
  "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
715
  "identity": {"pid": "pid", "businessKey": "code"},
716
  "identity_report": pd.identity_report(g["rows_src"]),
717
  "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
718
+ # D-11176: a paused Odoo serves the cached pool at any age, so the envelope names the pause.
719
+ # frozenAt is the served pool's epoch-seconds stamp. The keychain's frozenAt is naive local
720
+ # ISO text and never reaches the client. No stamp (a bounded pool) omits frozenAt.
721
+ import routes_keychain
722
+ rt = session.runtime
723
+ # A runtime that cannot answer the pause question (a gate double) serves no paused marks.
724
+ if callable(getattr(rt, "odoo_paused", None)) and routes_keychain.odoo_paused(rt):
725
+ body["paused"] = True
726
+ stamp = (_measure_stamp(rt, g["team_id"])
727
+ if getattr(rt, '_canonical_product_rows', None) is None else 0)
728
+ if stamp:
729
+ body["frozenAt"] = int(stamp)
730
+ # ProductsFrozenNotice renders only beside a frozen limit, and writes its own time.
731
+ body["limits"] = body["limits"] + [{
732
+ "subject": "products", "effect": "frozen",
733
+ "cause": "Odoo is paused, so these products show data as of the last sync.",
734
+ "recommendation": "Resume Odoo to refresh these products."}]
735
+ return body
736
 
737
 
738
  @router.post("/products/aggregates")
api/routes_query.py CHANGED
@@ -2381,11 +2381,17 @@ def conversation_log(before: str = None, limit: int = 50,
2381
 
2382
 
2383
  @router.get("/assistant/memory")
2384
- def conversation_memory(session: Session = Depends(require_session)):
2385
- entries = assistant_memory.recall(session.runtime, {"tenant": session.tenant, "username": session.uname},
2386
- limit=assistant_memory.MAX_ENTRIES)
 
 
 
 
 
2387
  doc = assistant_memory.document(session.runtime, session.uname)
2388
- return {"entries": entries, "total": len(entries), "dropped": doc["dropped"]}
 
2389
 
2390
 
2391
  @router.post("/assistant/confirm")
 
2381
 
2382
 
2383
  @router.get("/assistant/memory")
2384
+ def conversation_memory(session: Session = Depends(require_session), cursor: str | None = None,
2385
+ order: str = "newest", limit: int = assistant_memory.MAX_ENTRIES):
2386
+ """Page every authorized personal and company entry. `total` counts them all, `next` continues."""
2387
+ try:
2388
+ page = assistant_memory.inspect_page(session.runtime, {"tenant": session.tenant, "username": session.uname},
2389
+ cursor=cursor, limit=limit, order=order)
2390
+ except assistant_memory.InspectRequestError as exc:
2391
+ raise err(400, "bad_request", str(exc)) from exc
2392
  doc = assistant_memory.document(session.runtime, session.uname)
2393
+ return {"entries": page["entries"], "total": page["total"], "next": page["next"], "order": order,
2394
+ "dropped": doc["dropped"]}
2395
 
2396
 
2397
  @router.post("/assistant/confirm")
api/routes_ranking_diagnostic.py CHANGED
@@ -13,11 +13,14 @@ def _module():
13
  return ranking_diagnostic
14
 
15
 
16
- def _reader(session: Session = Depends(require_session)) -> Session:
17
- """Reads need read access to the Amazon Sales and Traffic database."""
18
- from core import perm_scope
19
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
20
- raise err(403, "ranking_diagnostic_forbidden", "Your account cannot read Amazon sales and traffic.")
 
 
 
21
  return session
22
 
23
 
@@ -25,6 +28,8 @@ def _writer(session: Session = Depends(_reader)) -> Session:
25
  """Writes also need a role that may edit."""
26
  if str(session.user.get("role") or "user").strip().lower() == "viewer":
27
  raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
 
 
28
  return session
29
 
30
 
@@ -40,7 +45,7 @@ def _call(fn, *args, **kwargs):
40
  try:
41
  return fn(*args, **kwargs)
42
  except PermissionError as exc:
43
- raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.") from exc
44
  except ValueError as exc:
45
  raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
46
  except KeyError as exc:
@@ -82,6 +87,19 @@ def _app_writer(app: str = Query(..., min_length=1), session: Session = Depends(
82
  return session
83
 
84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  @router.get("/screen")
86
  def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
87
  week: str | None = Query(None), cursor: str | None = Query(None),
@@ -91,20 +109,25 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
91
  rd = _module()
92
  record = _app(session, app)
93
  st = session.runtime
 
94
  held = _call(rd.weeks, _tenant(session), st=st)
95
  listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q,
96
- week=week or (held[-1] if held else None), st=st)
97
- return {"canEdit": _may_edit(session, record), "week": listed["week"], "weeks": listed["weeks"],
 
 
 
98
  "counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
99
  "problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"],
100
- "sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st),
101
  "settings": _call(rd.settings, _tenant(session), st=st)}
102
 
103
 
104
  @router.get("/items/{item_id}")
105
  def item_detail(item_id: str, session: Session = Depends(_app_reader)):
106
  rd = _module()
107
- return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime)
 
108
 
109
 
110
  @router.patch("/items/{item_id}")
@@ -123,7 +146,7 @@ def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None)
123
  """The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
124
  rd = _module()
125
  page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q,
126
- st=session.runtime)
127
  labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
128
  for row in page["asins"]:
129
  flagged = bool(row.get("cause") and page["week"])
@@ -135,34 +158,34 @@ def asin_picker(week: str | None = Query(None), cursor: str | None = Query(None)
135
  @router.get("/trend/{asin}")
136
  def asin_trend(asin: str, session: Session = Depends(_app_reader)):
137
  rd = _module()
138
- return _call(rd.trend, _tenant(session), asin, st=session.runtime)
139
 
140
 
141
  @router.get("/content-history/{asin}")
142
- def listing_content_history(asin: str, cursor: str | None = Query(None),
143
  limit: int = Query(200, ge=1, le=200), session: Session = Depends(_app_reader)):
144
- """Read one page of saved content changes behind the App and source permission walls."""
145
- from core import perm_scope
146
- source = "ut_amzn_listing_items"
147
- if not perm_scope.may_read(session.user, source, st=session.runtime):
148
- raise err(403, "ranking_diagnostic_content_forbidden", "Your account cannot read Amazon listing content.")
149
- if any(perm_scope.wall_declared(session.user, key, st=session.runtime) for key in (SOURCE, source)):
150
- raise err(403, "ranking_diagnostic_content_limited", "Content history is unavailable with your access limit.")
151
- return _call(_module().content_history, _tenant(session), asin, cursor=cursor, limit=limit, st=session.runtime)
152
-
153
-
154
- def _benchmark_reader(session: Session = Depends(_app_reader)) -> Session:
155
- """Require the SQP source grant and refuse unscoped row or field access limits."""
156
- from core import perm_scope
157
  from modules import ranking_benchmark
158
- if not perm_scope.may_read(session.user, ranking_benchmark.SOURCE, st=session.runtime):
159
- raise err(403, "ranking_benchmark_forbidden", "Your account cannot read Search Query Performance.")
160
- if any(perm_scope.wall_declared(session.user, key, st=session.runtime)
161
- for key in (SOURCE, ranking_benchmark.SOURCE)):
162
- raise err(403, "ranking_benchmark_limited", "The benchmark is unavailable with your access limit.")
163
  return session
164
 
165
 
 
 
 
 
 
166
  @router.get("/benchmark")
167
  def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
168
  limit: int = Query(200, ge=1, le=200), below_market: bool = Query(False),
@@ -170,7 +193,7 @@ def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
170
  """Read one weekly benchmark page, filtering below market before selecting the page."""
171
  from modules import ranking_benchmark
172
  return _call(ranking_benchmark.benchmark, _tenant(session), week, cursor=cursor, limit=limit,
173
- below_market=below_market, q=q, st=session.runtime)
174
 
175
 
176
  @router.get("/benchmark/validate")
@@ -180,7 +203,7 @@ def validate_benchmark(week: str | None = Query(None), cursor: str | None = Quer
180
  """Reconcile the selected benchmark page behind the same App and source permission walls."""
181
  from modules import ranking_benchmark
182
  return _call(ranking_benchmark.validate, _tenant(session), week, cursor=cursor, limit=limit,
183
- below_market=below_market, q=q, st=session.runtime)
184
 
185
 
186
  @router.get("/settings")
@@ -204,4 +227,5 @@ def write_settings(body: dict = Body(...), session: Session = Depends(_app_write
204
  def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
205
  session: Session = Depends(_app_reader)):
206
  rd = _module()
207
- return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime)
 
 
13
  return ranking_diagnostic
14
 
15
 
16
+ def _reader(app: str | None = Query(None), session: Session = Depends(require_session)) -> Session:
17
+ """R62: reads need the App grant plus the shared source check, never the source open grant."""
18
+ import application_registry
19
+ record = _app(session, app) if app else _call(application_registry.readable_app, session.runtime,
20
+ _tenant(session), session.uname, session.admin, TEMPLATE_KIND)
21
+ from routes_app_dashboard import admit_reader
22
+ admit_reader(session, TEMPLATE_KIND, record)
23
+ session.source_app = record
24
  return session
25
 
26
 
 
28
  """Writes also need a role that may edit."""
29
  if str(session.user.get("role") or "user").strip().lower() == "viewer":
30
  raise err(403, "ranking_diagnostic_read_only", "You can view this App but not change it.")
31
+ if _source_scope(session) is not None:
32
+ raise err(403, "ranking_diagnostic_limited", "Saved workflow changes need unrestricted source rows and fields.")
33
  return session
34
 
35
 
 
45
  try:
46
  return fn(*args, **kwargs)
47
  except PermissionError as exc:
48
+ raise err(403, "ranking_diagnostic_limited", str(exc)) from exc
49
  except ValueError as exc:
50
  raise err(400, "ranking_diagnostic_invalid", str(exc)) from exc
51
  except KeyError as exc:
 
87
  return session
88
 
89
 
90
+ def _source_scope(session):
91
+ """Pass the shared evaluator for every narrowed diagnostic source to the provider."""
92
+ import application_registry
93
+ sources = tuple("ut_" + source for source in _module().SOURCES)
94
+ walls = application_registry.app_access(session.user, session.runtime, None, sources)["narrow"]
95
+ active = {source for source, wall in walls.items() if wall["rows"] or wall["hidden"]}
96
+ if not active:
97
+ return None
98
+ from routes_app_dashboard import source_scope_for
99
+ scope = source_scope_for(session, session.source_app)
100
+ return lambda source, records: scope(source, records) if source in active else None
101
+
102
+
103
  @router.get("/screen")
104
  def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(None),
105
  week: str | None = Query(None), cursor: str | None = Query(None),
 
109
  rd = _module()
110
  record = _app(session, app)
111
  st = session.runtime
112
+ scope = _call(_source_scope, session)
113
  held = _call(rd.weeks, _tenant(session), st=st)
114
  listed = _call(rd.items, _tenant(session), session.uname, status=status, cursor=cursor, limit=limit, q=q,
115
+ week=week or (held[-1] if held else None), st=st, source_scope=scope)
116
+ return {"canEdit": _may_edit(session, record) and scope is None,
117
+ "readiness": listed["readiness"], "columns": listed["columns"],
118
+ **{key: listed["readiness"][key] for key in ("heldWeeks", "weeksNeeded", "nextSync")},
119
+ "week": listed["week"], "weeks": listed["weeks"],
120
  "counts": listed["counts"], "items": listed["items"], "cursor": listed["cursor"],
121
  "problems": listed["problems"], "statuses": list(rd.STATUSES), "total": listed["total"],
122
+ "sourceTotals": _call(rd.screen_totals, _tenant(session), listed["week"], st=st, source_scope=scope),
123
  "settings": _call(rd.settings, _tenant(session), st=st)}
124
 
125
 
126
  @router.get("/items/{item_id}")
127
  def item_detail(item_id: str, session: Session = Depends(_app_reader)):
128
  rd = _module()
129
+ return _call(rd.item_detail, _tenant(session), session.uname, item_id, st=session.runtime,
130
+ source_scope=_call(_source_scope, session))
131
 
132
 
133
  @router.patch("/items/{item_id}")
 
146
  """The ASIN picker page. A flagged ASIN carries its cause label and item id for the cause card."""
147
  rd = _module()
148
  page = _call(rd.asins, _tenant(session), session.uname, week=week, cursor=cursor, limit=limit, q=q,
149
+ st=session.runtime, source_scope=_call(_source_scope, session))
150
  labels = {key: label for key, label, _fix in (*rd.CAUSES, rd.UNEXPLAINED)}
151
  for row in page["asins"]:
152
  flagged = bool(row.get("cause") and page["week"])
 
158
  @router.get("/trend/{asin}")
159
  def asin_trend(asin: str, session: Session = Depends(_app_reader)):
160
  rd = _module()
161
+ return _call(rd.trend, _tenant(session), asin, st=session.runtime, source_scope=_call(_source_scope, session))
162
 
163
 
164
  @router.get("/content-history/{asin}")
165
+ def listing_content_history(asin: str, app: str = Query(..., min_length=1), cursor: str | None = Query(None),
166
  limit: int = Query(200, ge=1, le=200), session: Session = Depends(_app_reader)):
167
+ """Read one page of saved content changes behind the App grant and the shared source check (R62)."""
168
+ from routes_app_dashboard import admit_reader
169
+ admit_reader(session, TEMPLATE_KIND, _app(session, app), sources=(SOURCE, "ut_amzn_listing_items"))
170
+ return _call(_module().content_history, _tenant(session), asin, cursor=cursor, limit=limit, st=session.runtime,
171
+ source_scope=_call(_source_scope, session))
172
+
173
+
174
+ def _benchmark_reader(app: str = Query(..., min_length=1), session: Session = Depends(_app_reader)) -> Session:
175
+ """R62: the App grant plus the shared source check over SQP; a wall it cannot narrow refuses."""
 
 
 
 
176
  from modules import ranking_benchmark
177
+ from routes_app_dashboard import admit_reader
178
+ record = _app(session, app)
179
+ admit_reader(session, TEMPLATE_KIND, record, sources=(SOURCE, ranking_benchmark.SOURCE))
180
+ session.source_app = record
 
181
  return session
182
 
183
 
184
+ def _benchmark_scope(session):
185
+ from routes_app_dashboard import source_scope_for
186
+ return source_scope_for(session, session.source_app)
187
+
188
+
189
  @router.get("/benchmark")
190
  def benchmark(week: str | None = Query(None), cursor: str | None = Query(None),
191
  limit: int = Query(200, ge=1, le=200), below_market: bool = Query(False),
 
193
  """Read one weekly benchmark page, filtering below market before selecting the page."""
194
  from modules import ranking_benchmark
195
  return _call(ranking_benchmark.benchmark, _tenant(session), week, cursor=cursor, limit=limit,
196
+ below_market=below_market, q=q, st=session.runtime, source_scope=_benchmark_scope(session))
197
 
198
 
199
  @router.get("/benchmark/validate")
 
203
  """Reconcile the selected benchmark page behind the same App and source permission walls."""
204
  from modules import ranking_benchmark
205
  return _call(ranking_benchmark.validate, _tenant(session), week, cursor=cursor, limit=limit,
206
+ below_market=below_market, q=q, st=session.runtime, source_scope=_benchmark_scope(session))
207
 
208
 
209
  @router.get("/settings")
 
227
  def validate(week: str | None = Query(None), start: str | None = Query(None), end: str | None = Query(None),
228
  session: Session = Depends(_app_reader)):
229
  rd = _module()
230
+ return _call(rd.validate, _tenant(session), week=week, start=start, end=end, st=session.runtime,
231
+ source_scope=_call(_source_scope, session))
api/routes_reimbursements.py CHANGED
@@ -1,4 +1,7 @@
1
- """Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate."""
 
 
 
2
  import copy
3
  import hashlib
4
  import json
@@ -44,19 +47,53 @@ def _module():
44
  return reimbursements
45
 
46
 
47
- def _reader(session: Session = Depends(require_session)) -> Session:
48
- """Reads need read access to the Amazon reimbursements database."""
49
- from core import perm_scope
50
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
51
- raise err(403, "reimbursements_forbidden", "Your account cannot read Amazon reimbursements.")
52
- if perm_scope.wall_declared(session.user, SOURCE, st=session.runtime):
53
- raise err(403, "reimbursements_forbidden", "This App cannot apply your reimbursement source limits.")
 
 
 
 
 
 
 
 
 
 
 
54
  try:
55
- visible = {field["key"] for field in perm_scope.scoped_fields(session.user, SOURCE, st=session.runtime)}
56
- if not set(_module().REIMBURSEMENT_COLS) <= visible:
57
- raise PermissionError
58
- except Exception as exc:
59
- raise err(403, "reimbursements_forbidden", "This App needs access to the reimbursement source fields.") from exc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  return session
61
 
62
 
@@ -95,9 +132,12 @@ def _app(session, app_id):
95
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
96
  except Exception as exc: # noqa: BLE001
97
  raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
98
- if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
 
99
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
100
  raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
 
 
101
  return app
102
 
103
 
@@ -127,12 +167,13 @@ def app_screen(app: str = Query(..., min_length=1), status: str | None = Query(N
127
  session: Session = Depends(_app_reader)):
128
  rb = _module()
129
  record = _app(session, app)
130
- listed = _call(rb.items, _tenant(session), _user(session), status=status, search=search, cursor=cursor,
 
131
  held_cursor=held_cursor, limit=limit, st=session.runtime)
132
  return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
133
  "cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"],
134
- "limit": listed["limit"], "paidTotal": listed["paidTotal"],
135
- "problems": listed["problems"], "statuses": list(rb.STATUSES), "held": listed["held"]}
136
 
137
 
138
  @router.get("/items/{item_id}")
@@ -151,7 +192,7 @@ def case_text(item_id: str, app: str = Query(..., min_length=1), session: Sessio
151
  rb = _module()
152
 
153
  def facts_for_actor():
154
- _reader(session)
155
  _app(session, app)
156
  return _call(rb.case_facts, _tenant(session), _user(session), item_id,
157
  budget=case_drafts.MAX_FACT_CHARS, st=session.runtime)
@@ -186,6 +227,7 @@ def update_item(item_id: str, status: str = Body(...), caseId: str | None = Body
186
  def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
187
  session: Session = Depends(_app_reader)):
188
  rb = _module()
 
189
  return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime)
190
 
191
 
 
1
+ """Reimbursements App routes (C3): claim items, one item with its case text, status, paid rows, validate.
2
+
3
+ /screen and /paid refresh the App's module tables first (S82-T29), so both Grids read current rows.
4
+ /screen reads its counts from the stored claims table through the reader's row scope (S82-T30)."""
5
  import copy
6
  import hashlib
7
  import json
 
47
  return reimbursements
48
 
49
 
50
+ def _on_claim_event(evt):
51
+ """A Processed tick in the claims Grid becomes set_status as the person who ticked it (S82-T30)."""
52
+ rb = _module()
53
+ if (evt.get("type") != "event_field" or evt.get("table") != rb.CLAIMS_TABLE
54
+ or evt.get("field") != rb.PROCESSED or evt.get("st") is None):
55
+ return
56
+ rb.apply_processed(evt["st"], evt.get("rowId"), evt.get("after"), evt.get("user"),
57
+ allowed=_may_tick(evt["st"], evt.get("user")))
58
+
59
+
60
+ def _may_tick(st, username):
61
+ """A tick needs what the status PATCH needs: an editor of a live Reimbursements App."""
62
+ import application_registry
63
+ from core import perms
64
+ user = _module()._principal(st, username)
65
+ if not user or str(user.get("role") or "user").strip().lower() == "viewer":
66
+ return False
67
+ admin = perms.is_admin(user)
68
  try:
69
+ apps = application_registry.load(st)["applications"].values()
70
+ except Exception: # noqa: BLE001
71
+ return False
72
+ return any(isinstance(app, dict) and app.get("templateKind") == TEMPLATE_KIND and app.get("retired") is not True
73
+ and application_registry.may_edit(app, str(username), is_admin=admin) for app in apps)
74
+
75
+
76
+ def _register_row_hook():
77
+ """Register once, also across a module reload, which makes a new function object."""
78
+ from core import user_tables
79
+ mine = (_on_claim_event.__module__, _on_claim_event.__name__)
80
+ if not any((getattr(h, "__module__", None), getattr(h, "__name__", None)) == mine
81
+ for h in user_tables.ROW_HOOKS):
82
+ user_tables.ROW_HOOKS.append(_on_claim_event)
83
+
84
+
85
+ _register_row_hook()
86
+
87
+
88
+ def _reader(session: Session = Depends(require_session), app: str | None = Query(None)) -> Session:
89
+ """Bind the requested App before admitting its declared source contracts and limits."""
90
+ from routes_app_dashboard import admit_reader
91
+ record = _app(session, app) if isinstance(app, str) and app else None
92
+ admit_reader(session, 'reimbursements', record)
93
+ from core import user_tables
94
+ fields = (user_tables.get_definition(SOURCE, st=session.runtime) or {}).get('fields') or []
95
+ if not set(_module().REIMBURSEMENT_COLS) <= {field.get('key') for field in fields}:
96
+ raise err(503, 'reimbursements_source_contract', 'The reimbursement source fields are unavailable. Sync Amazon.')
97
  return session
98
 
99
 
 
132
  app = application_registry.load(session.runtime)["applications"].get(str(app_id or ""))
133
  except Exception as exc: # noqa: BLE001
134
  raise err(503, "reimbursements_unavailable", "The App could not be read. Try again shortly.") from exc
135
+ if (not isinstance(app, dict) or app.get("tenant") != _tenant(session)
136
+ or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
137
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
138
  raise err(404, "reimbursements_app_not_found", "That Reimbursements App does not exist.")
139
+ from routes_app_dashboard import admit_reader
140
+ admit_reader(session, 'reimbursements', app)
141
  return app
142
 
143
 
 
167
  session: Session = Depends(_app_reader)):
168
  rb = _module()
169
  record = _app(session, app)
170
+ # rb.screen refreshes the stored tables when stale, then reads them as the Grid does.
171
+ listed = _call(rb.screen, _tenant(session), _user(session), status=status, search=search, cursor=cursor,
172
  held_cursor=held_cursor, limit=limit, st=session.runtime)
173
  return {"canEdit": _may_edit(session, record), "counts": listed["counts"], "items": listed["items"],
174
  "cursor": listed["cursor"], "total": listed["total"], "offset": listed["offset"],
175
+ "limit": listed["limit"], "paidTotal": listed["paidTotal"], "problems": listed["problems"],
176
+ "statuses": list(rb.STATUSES), "held": listed["held"], "toFile": listed["toFile"]}
177
 
178
 
179
  @router.get("/items/{item_id}")
 
192
  rb = _module()
193
 
194
  def facts_for_actor():
195
+ _reader(session, app=app)
196
  _app(session, app)
197
  return _call(rb.case_facts, _tenant(session), _user(session), item_id,
198
  budget=case_drafts.MAX_FACT_CHARS, st=session.runtime)
 
227
  def paid_rows(cursor: str | None = Query(None), search: str = Query(""), limit: int = Query(200, ge=1),
228
  session: Session = Depends(_app_reader)):
229
  rb = _module()
230
+ rb.refresh_tables(_tenant(session), session.runtime)
231
  return _call(rb.paid, _tenant(session), session.uname, search=search, cursor=cursor, limit=limit, st=session.runtime)
232
 
233
 
api/routes_repeat_buyers.py CHANGED
@@ -55,17 +55,20 @@ def _app(session, app_id):
55
 
56
 
57
  def _read_sources(session, app_id, sources):
 
58
  _tenant(session)
59
- _app(session, app_id)
60
- from core import perm_scope
61
- for source in sources:
62
- if not perm_scope.may_read(session.user, source, st=session.runtime):
63
- raise err(403, "repeat_buyers_forbidden", "Your account cannot read this App source.")
64
- if perm_scope.wall_declared(session.user, source, st=session.runtime):
65
- raise err(403, "repeat_buyers_scoped", "This App cannot read a source with field or row restrictions.")
66
  return session
67
 
68
 
 
 
 
 
 
69
  def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
70
  """A read names the App and requires both Brand Analytics sources."""
71
  return _read_sources(session, app_id, SOURCES)
@@ -81,7 +84,8 @@ def read_summary(start: str | None = Query(None), end: str | None = Query(None),
81
  cursor: str | None = Query(None), limit: int = Query(24, ge=1),
82
  session: Session = Depends(_app_reader)):
83
  rr = _module()
84
- out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit, st=session.runtime)
 
85
  return {**out, "canEdit": False}
86
 
87
 
@@ -91,7 +95,7 @@ def list_skus(segment: str = Query("repeat"), start: str | None = Query(None), e
91
  session: Session = Depends(_app_reader)):
92
  rr = _module()
93
  return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
94
- limit=limit, q=q, st=session.runtime)
95
 
96
 
97
  @router.get("/{app_id}/sku")
@@ -99,7 +103,8 @@ def sku_detail(key: str = Query(..., min_length=1), start: str | None = Query(No
99
  end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1),
100
  session: Session = Depends(_app_reader)):
101
  rr = _module()
102
- return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit, st=session.runtime)
 
103
 
104
 
105
  @router.get("/{app_id}/subscribe-save")
@@ -107,5 +112,5 @@ def subscribe_save(start: str | None = Query(None), end: str | None = Query(None
107
  cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None),
108
  session: Session = Depends(_sns_reader)):
109
  return _call(_sns_module().per_sku, _tenant(session), start=start, end=end, cursor=cursor,
110
- limit=limit, q=q, st=session.runtime)
111
 
 
55
 
56
 
57
  def _read_sources(session, app_id, sources):
58
+ import application_registry
59
  _tenant(session)
60
+ # R62: the App grant plus the shared source check, never the source open grant.
61
+ record = _app(session, app_id)
62
+ application_registry.admit_app(session, record, sources, code="repeat_buyers")
63
+ session.source_app = record
 
 
 
64
  return session
65
 
66
 
67
+ def _source_scope(session):
68
+ from routes_app_dashboard import source_scope_for
69
+ return source_scope_for(session, session.source_app)
70
+
71
+
72
  def _app_reader(app_id: str = Path(..., min_length=1), session: Session = Depends(require_session)) -> Session:
73
  """A read names the App and requires both Brand Analytics sources."""
74
  return _read_sources(session, app_id, SOURCES)
 
84
  cursor: str | None = Query(None), limit: int = Query(24, ge=1),
85
  session: Session = Depends(_app_reader)):
86
  rr = _module()
87
+ out = _call(rr.summary, _tenant(session), start=start, end=end, cursor=cursor, limit=limit,
88
+ st=session.runtime, source_scope=_source_scope(session))
89
  return {**out, "canEdit": False}
90
 
91
 
 
95
  session: Session = Depends(_app_reader)):
96
  rr = _module()
97
  return _call(rr.skus, _tenant(session), segment=segment, start=start, end=end, cursor=cursor,
98
+ limit=limit, q=q, st=session.runtime, source_scope=_source_scope(session))
99
 
100
 
101
  @router.get("/{app_id}/sku")
 
103
  end: str | None = Query(None), cursor: str | None = Query(None), limit: int = Query(25, ge=1),
104
  session: Session = Depends(_app_reader)):
105
  rr = _module()
106
+ return _call(rr.detail, _tenant(session), key, start=start, end=end, cursor=cursor, limit=limit,
107
+ st=session.runtime, source_scope=_source_scope(session))
108
 
109
 
110
  @router.get("/{app_id}/subscribe-save")
 
112
  cursor: str | None = Query(None), limit: int = Query(100, ge=1), q: str | None = Query(None),
113
  session: Session = Depends(_sns_reader)):
114
  return _call(_sns_module().per_sku, _tenant(session), start=start, end=end, cursor=cursor,
115
+ limit=limit, q=q, st=session.runtime, source_scope=_source_scope(session))
116
 
api/routes_restock_planner.py CHANGED
@@ -50,10 +50,9 @@ def _module():
50
 
51
 
52
  def _reader(session: Session = Depends(require_session)) -> Session:
53
- """Reads need read access to the inventory planning database."""
54
- from core import perm_scope
55
- if not perm_scope.may_read(session.user, SOURCE, st=session.runtime):
56
- raise err(403, "restock_planner_forbidden", "Your account cannot read Amazon inventory planning.")
57
  return session
58
 
59
 
@@ -91,6 +90,8 @@ def _app(session, app_id):
91
  if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
92
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
93
  raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
 
 
94
  return app
95
 
96
 
@@ -147,7 +148,7 @@ def list_items(app: str = Query(..., min_length=1), band: str | None = Query(Non
147
  rp = _module()
148
  listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
149
  limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, low_inventory_fee=low_inventory_fee,
150
- st=session.runtime)
151
  return {**listed, "owners": owners(session), "thresholds": thresholds(session),
152
  "canEdit": _may_edit(session, _app(session, app))}
153
 
@@ -204,6 +205,18 @@ def read_capacity(session: Session = Depends(_app_reader)):
204
  return _call(_module().capacity, _tenant(session), st=session.runtime)
205
 
206
 
 
 
 
 
 
 
 
 
 
 
 
 
207
  @router.post("/items/{item_id}/cancel-plan")
208
  def cancel_plan(item_id: str, request: CancelPlanRequest | None = Body(None),
209
  session: Session = Depends(_app_writer)):
@@ -248,9 +261,86 @@ def _digest_module():
248
  return restock_digest
249
 
250
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
251
  def _deliver_digest(rt, kind, items):
252
  """Resolve C5 only when dispatching so a missing adapter cannot break API boot."""
253
- return importlib.import_module("notify_destinations").deliver(rt, kind, items)
254
 
255
 
256
  def _send_digest(rt, app_id, day=None, *, deliver=None):
 
50
 
51
 
52
  def _reader(session: Session = Depends(require_session)) -> Session:
53
+ """Admit a live App holder and refuse unsupported source row or field limits."""
54
+ from routes_app_dashboard import admit_reader
55
+ admit_reader(session, 'restock_planner')
 
56
  return session
57
 
58
 
 
90
  if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KIND or app.get("retired") is True
91
  or not application_registry.may_read(app, session.uname, is_admin=session.admin)):
92
  raise err(404, "restock_planner_app_not_found", "That Restock planner App does not exist.")
93
+ from routes_app_dashboard import admit_reader
94
+ admit_reader(session, 'restock_planner', app)
95
  return app
96
 
97
 
 
148
  rp = _module()
149
  listed = _call(rp.items, _tenant(session), session.uname, status=status, cursor=cursor,
150
  limit=min(limit, rp.MAX_PAGE_SIZE), band=band, q=q, low_inventory_fee=low_inventory_fee,
151
+ st=session.runtime, adopt=True)
152
  return {**listed, "owners": owners(session), "thresholds": thresholds(session),
153
  "canEdit": _may_edit(session, _app(session, app))}
154
 
 
205
  return _call(_module().capacity, _tenant(session), st=session.runtime)
206
 
207
 
208
+ @router.get("/tiles")
209
+ def read_tiles(days: int = Query(30, ge=1, le=60), session: Session = Depends(_app_reader)):
210
+ from core import perm_scope
211
+ from odoo_relational import PRODUCTS_KEY, PURCHASE_ORDERS_KEY, PURCHASE_LINES_KEY
212
+ sources = (SOURCE, "ut_amzn_restock_recommendations", _module().LINK_TABLE,
213
+ PRODUCTS_KEY, PURCHASE_ORDERS_KEY, PURCHASE_LINES_KEY)
214
+ if any(not perm_scope.may_read(session.user, source, st=session.runtime)
215
+ or perm_scope.wall_declared(session.user, source, st=session.runtime) for source in sources):
216
+ raise err(403, "restock_tiles_forbidden", "These tiles need access to their full source rows and fields")
217
+ return _call(_module().tiles, _tenant(session), st=session.runtime, days=days)
218
+
219
+
220
  @router.post("/items/{item_id}/cancel-plan")
221
  def cancel_plan(item_id: str, request: CancelPlanRequest | None = Body(None),
222
  session: Session = Depends(_app_writer)):
 
261
  return restock_digest
262
 
263
 
264
+ DIGEST_BODY_LIMIT = 8000
265
+ DIGEST_NO_DESTINATION = "The Restock owner has chosen no destination for the weekly digest."
266
+
267
+
268
+ def _digest_rows(rt, skus):
269
+ """{sku: Restock table row id} for the listed SKUs, so Slack can check the reader's access."""
270
+ from core import user_tables
271
+ want = set(skus)
272
+ if not want:
273
+ return {}
274
+ try:
275
+ table = user_tables.get_one(_module().TABLE_KEY, rt) or {}
276
+ except Exception: # noqa: BLE001
277
+ # Without row ids Slack refuses with its own named reason; email still delivers.
278
+ return {}
279
+ return {str(row.get("sku") or "").strip(): str(rid) for rid, row in (table.get("rows") or {}).items()
280
+ if isinstance(row, dict) and str(row.get("sku") or "").strip() in want}
281
+
282
+
283
+ def _digest_items(rt, envelope):
284
+ """D-12074: the digest envelope as C5 items that pass `clean_items`, split so no body is cut."""
285
+ lines = [("".join(c if c == "\t" or ord(c) >= 32 else " " for c in line))
286
+ for line in str(envelope.get("body") or "").split("\n")]
287
+ skus = [row.get("sku") for row in ((envelope.get("digest") or {}).get("items") or ())]
288
+ rows = _digest_rows(rt, [s for s in skus if s])
289
+ # A long line splits into pieces and a long body into numbered items: nothing is cut.
290
+ pieces = [(index, line[start:start + DIGEST_BODY_LIMIT]) for index, line in enumerate(lines)
291
+ for start in range(0, max(len(line), 1), DIGEST_BODY_LIMIT)]
292
+ chunks, current, size = [], [], 0
293
+ for index, text in pieces:
294
+ extra = len(text) + (1 if current else 0)
295
+ if current and size + extra > DIGEST_BODY_LIMIT:
296
+ chunks.append(current)
297
+ current, size, extra = [], 0, len(text)
298
+ current.append((index, text))
299
+ size += extra
300
+ chunks.append(current)
301
+ out = []
302
+ for number, chunk in enumerate(chunks, 1):
303
+ body = "\n".join(line for _i, line in chunk).strip() or "Nothing urgent"
304
+ item = {"id": str(envelope["id"]) if len(chunks) == 1 else f"{envelope['id']}-{number}",
305
+ "title": str(envelope.get("title") or "Weekly Restock digest")
306
+ + ("" if len(chunks) == 1 else f" ({number} of {len(chunks)})"),
307
+ "body": body}
308
+ if envelope.get("link"):
309
+ item["url"] = str(envelope["link"])
310
+ ids = [rows[skus[i]] for i, _line in chunk if i < len(skus) and skus[i] in rows]
311
+ if ids:
312
+ item["source"] = {"table": _module().TABLE_KEY, "rowIds": list(dict.fromkeys(ids))}
313
+ out.append(item)
314
+ return out
315
+
316
+
317
+ def _digest_receipt(delivery):
318
+ """D-12074: C5's delivery result in the shape `restock_digest._outcome` reads."""
319
+ if not isinstance(delivery, dict):
320
+ return delivery
321
+ results = [row for row in delivery.get("results") or () if isinstance(row, dict)]
322
+ destinations = [{"id": row.get("destination"), "state": row.get("state"),
323
+ "reason": row.get("message") or row.get("reason")} for row in results]
324
+ sent = sum(row["state"] == "sent" for row in destinations)
325
+ state = "sent" if destinations and sent == len(destinations) else "partial" if sent else "failed"
326
+ out = {"state": state, "destinations": destinations, "sent": sent,
327
+ "refused": len(destinations) - sent, "itemCount": delivery.get("itemCount")}
328
+ if not destinations:
329
+ out["reason"] = DIGEST_NO_DESTINATION
330
+ return out
331
+
332
+
333
+ def _deliver_digest_with(deliver):
334
+ """`restock_digest.send`'s deliver contract over one C5 `deliver` function."""
335
+ def _deliver(rt, kind, items):
336
+ envelope = items[0]
337
+ return _digest_receipt(deliver(rt, kind, _digest_items(rt, envelope), user=envelope["recipient"]))
338
+ return _deliver
339
+
340
+
341
  def _deliver_digest(rt, kind, items):
342
  """Resolve C5 only when dispatching so a missing adapter cannot break API boot."""
343
+ return _deliver_digest_with(importlib.import_module("notify_destinations").deliver)(rt, kind, items)
344
 
345
 
346
  def _send_digest(rt, app_id, day=None, *, deliver=None):